diff --git a/.azure-pipelines/docs.yml b/.azure-pipelines/docs.yml deleted file mode 100644 index b65fdfb4..00000000 --- a/.azure-pipelines/docs.yml +++ /dev/null @@ -1,21 +0,0 @@ -jobs: -- job: - displayName: Docs - pool: - vmImage: ubuntu-16.04 - - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - - - template: steps/install-dependencies.yml - - - bash: tox -e docs - displayName: Build docs - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: docs' - inputs: - pathToPublish: docs/build - artifactName: docs diff --git a/.azure-pipelines/jobs/run-manifest-check.yml b/.azure-pipelines/jobs/run-manifest-check.yml deleted file mode 100644 index 6aa63480..00000000 --- a/.azure-pipelines/jobs/run-manifest-check.yml +++ /dev/null @@ -1,15 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install --upgrade setuptools twine readme_renderer[md] - python setup.py sdist - twine check dist/* diff --git a/.azure-pipelines/jobs/run-tests-windows.yml b/.azure-pipelines/jobs/run-tests-windows.yml deleted file mode 100644 index 6b6f86fa..00000000 --- a/.azure-pipelines/jobs/run-tests-windows.yml +++ /dev/null @@ -1,12 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- template: ../steps/create-virtualenv.yml - -- template: ../steps/run-tests.yml diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml deleted file mode 100644 index 3544af41..00000000 --- a/.azure-pipelines/jobs/run-tests.yml +++ /dev/null @@ -1,38 +0,0 @@ -steps: -- task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - -- template: ../steps/install-dependencies.yml - -- bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - echo "Path $PATH" - echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade - pipenv install --deploy --dev - echo pipenv --venv && echo pipenv --py && echo pipenv run python --version - displayName: Make Virtualenv - -- script: | - # Fix Git SSL errors - export GIT_SSL_CAINFO="$(python -m certifi)" - export LANG="C.UTF-8" - export PIP_PROCESS_DEPENDENCY_LINKS="1" - git submodule sync && git submodule update --init --recursive - pipenv run pytest --junitxml=test-results.xml - displayName: Run integration tests - -- task: PublishTestResults@2 - displayName: Publish Test Results - inputs: - testResultsFiles: '**/test-results.xml' - testRunTitle: 'Python $(python.version)' - condition: succeededOrFailed() diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml deleted file mode 100644 index 5af6f866..00000000 --- a/.azure-pipelines/jobs/run-vendor-scripts.yml +++ /dev/null @@ -1,39 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Vendor_Scripts - displayName: Test Vendor Scripts - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - task: UsePythonVersion@0 - displayName: Use Python $(python.version) - inputs: - versionSpec: '$(python.version)' - architecture: '$(python.architecture)' - - - template: ../steps/install-dependencies.yml - - - bash: | - mkdir -p "$AGENT_HOMEDIRECTORY/.virtualenvs" - mkdir -p "$WORKON_HOME" - pip install certifi - export GIT_SSL_CAINFO=$(python -m certifi) - export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver bs4 vistir towncrier - python -m invoke vendoring.update - - - template: ./run-manifest-check.yml diff --git a/.azure-pipelines/jobs/test.yml b/.azure-pipelines/jobs/test.yml deleted file mode 100644 index 150937ef..00000000 --- a/.azure-pipelines/jobs/test.yml +++ /dev/null @@ -1,48 +0,0 @@ -parameters: - vmImage: - -jobs: -- job: Test_Primary - displayName: Test Primary - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - Python27: - python.version: '2.7' - python.architecture: x64 - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 - Python37: - python.version: '>= 3.7.0-b2' - python.architecture: x64 - ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: - Python37: - python.version: '>= 3.7' - python.architecture: x64 - steps: - - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests.yml - -- job: Test_Secondary - displayName: Test python3.6 - # Run after Test_Primary so we don't devour time and jobs if tests are going to fail - # dependsOn: Test_Primary - pool: - vmImage: ${{ parameters.vmImage }} - strategy: - maxParallel: 4 - matrix: - Python36: - python.version: '3.6' - python.architecture: x64 - steps: - - ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests-windows.yml - - - ${{ if ne(parameters.vmImage, 'vs2017-win2016') }}: - - template: ./run-tests.yml diff --git a/.azure-pipelines/linux.yml b/.azure-pipelines/linux.yml deleted file mode 100644 index a2f34d4c..00000000 --- a/.azure-pipelines/linux.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -jobs: -- template: jobs/test.yml - parameters: - vmImage: ubuntu-16.04 - -- template: jobs/run-vendor-scripts.yml - parameters: - vmImage: ubuntu-16.04 diff --git a/.azure-pipelines/scripts/New-Ramdisk.ps1 b/.azure-pipelines/scripts/New-Ramdisk.ps1 new file mode 100644 index 00000000..fb068e47 --- /dev/null +++ b/.azure-pipelines/scripts/New-Ramdisk.ps1 @@ -0,0 +1,75 @@ +# Taken from https://github.com/pypa/pip/blob/ceaf75b9ede9a9c25bcee84fe512fa6774889685/.azure-pipelines/scripts/New-RAMDisk.ps1 +[CmdletBinding()] +param( + [Parameter(Mandatory=$true, + HelpMessage="Drive letter to use for the RAMDisk")] + [String]$drive, + [Parameter(HelpMessage="Size to allocate to the RAMDisk")] + [UInt64]$size=1GB +) + +$ErrorActionPreference = "Stop" +Set-StrictMode -Version Latest + +Write-Output "Installing FS-iSCSITarget-Server" +Install-WindowsFeature -Name FS-iSCSITarget-Server + +Write-Output "Starting MSiSCSI" +Start-Service MSiSCSI +$retry = 10 +do { + $service = Get-Service MSiSCSI + if ($service.Status -eq "Running") { + break; + } + $retry-- + Start-Sleep -Milliseconds 500 +} until ($retry -eq 0) + +$service = Get-Service MSiSCSI +if ($service.Status -ne "Running") { + throw "MSiSCSI is not running" +} + +Write-Output "Configuring Firewall" +Get-NetFirewallServiceFilter -Service MSiSCSI | Enable-NetFirewallRule + +Write-Output "Configuring RAMDisk" +# Must use external-facing IP address, otherwise New-IscsiTargetPortal is +# unable to connect. +$ip = ( + Get-NetIPAddress -AddressFamily IPv4 | + Where-Object {$_.IPAddress -ne "127.0.0.1"} +)[0].IPAddress +if ( + -not (Get-IscsiServerTarget -ComputerName localhost | Where-Object {$_.TargetName -eq "ramdisks"}) +) { + New-IscsiServerTarget ` + -ComputerName localhost ` + -TargetName ramdisks ` + -InitiatorId IPAddress:$ip +} + +$newVirtualDisk = New-IscsiVirtualDisk ` + -ComputerName localhost ` + -Path ramdisk:local$drive.vhdx ` + -Size $size +Add-IscsiVirtualDiskTargetMapping ` + -ComputerName localhost ` + -TargetName ramdisks ` + -Path ramdisk:local$drive.vhdx + +Write-Output "Connecting to iSCSI" +New-IscsiTargetPortal -TargetPortalAddress $ip +Get-IscsiTarget | Where-Object {!$_.IsConnected} | Connect-IscsiTarget + +Write-Output "Configuring disk" +$newDisk = Get-IscsiConnection | + Get-Disk | + Where-Object {$_.SerialNumber -eq $newVirtualDisk.SerialNumber} + +Set-Disk -InputObject $newDisk -IsOffline $false +Initialize-Disk -InputObject $newDisk -PartitionStyle MBR +New-Partition -InputObject $newDisk -UseMaximumSize -DriveLetter $drive + +Format-Volume -DriveLetter $drive -NewFileSystemLabel Temp -FileSystem NTFS diff --git a/.azure-pipelines/steps/build-package.yml b/.azure-pipelines/steps/build-package.yml new file mode 100644 index 00000000..01f9fe0b --- /dev/null +++ b/.azure-pipelines/steps/build-package.yml @@ -0,0 +1,32 @@ +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- bash: | + PYTHON_PATH=$(python -c 'import sys; print(sys.executable)') + echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH" + displayName: Set Python Path + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version) + env: + PYTHON_VERSION: $(python.version) + +- template: install-dependencies.yml + +- script: | + python -m pip install --upgrade wheel pip setuptools twine readme_renderer[md] + python setup.py sdist bdist_wheel + twine check dist/* + displayName: Build and check package + env: + PY_EXE: $(PY_EXE) + GIT_SSL_CAINFO: $(GIT_SSL_CAINFO) + LANG: $(LANG) + PIPENV_DEFAULT_PYTHON_VERSION: $(PIPENV_DEFAULT_PYTHON_VERSION) + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml deleted file mode 100644 index 9d1a6903..00000000 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ /dev/null @@ -1,6 +0,0 @@ -steps: -- script: | - virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev - echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version - displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml index dfe733b6..fd5b31c7 100644 --- a/.azure-pipelines/steps/install-dependencies.yml +++ b/.azure-pipelines/steps/install-dependencies.yml @@ -1,3 +1,31 @@ +parameters: + python_version: '' + steps: -- script: 'python -m pip install --upgrade pip && python -m pip install -e .' + +- script: | + echo "##vso[task.setvariable variable=LANG]C.UTF-8" + echo "##vso[task.setvariable variable=PIP_PROCESS_DEPENDENCY_LINKS]1" + displayName: Set Environment Variables + +- script: | + echo "Python path: $(PY_EXE)" + echo "GIT_SSL_CAINFO: $(GIT_SSL_CAINFO)" + echo "PIPENV PYTHON VERSION: $(python.version)" + echo "python_version: ${{ parameters.python_version }}" + git submodule sync + git submodule update --init --recursive + $(PY_EXE) -m pip install --upgrade --upgrade-strategy=eager pip setuptools wheel + $(PY_EXE) -m pip install "virtualenv<20" + env: + PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }} + PYTHONWARNINGS: 'ignore:DEPRECATION' + PIPENV_NOSPIN: '1' + displayName: Make Virtualenv + +- script: | + $(PY_EXE) -m pip install -e . --upgrade + $(PY_EXE) -m pipenv install --deploy --dev --python="$(PY_EXE)" displayName: Upgrade Pip & Install Pipenv + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' diff --git a/.azure-pipelines/steps/run-tests-linux.yml b/.azure-pipelines/steps/run-tests-linux.yml new file mode 100644 index 00000000..dbb13f94 --- /dev/null +++ b/.azure-pipelines/steps/run-tests-linux.yml @@ -0,0 +1,30 @@ +parameters: + python_version: '' + +steps: + +- bash: | + pip install certifi + PYTHON_PATH=$(python -c 'import sys; print(sys.executable)') + CERTIFI_CONTENT=$(python -m certifi) + echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$CERTIFI_CONTENT" + echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH" + displayName: Set Python Path + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' + +- template: install-dependencies.yml + parameters: + python_version: ${{ parameters.python_version }} + +- script: | + # Fix Git SSL errors + echo "Using pipenv python version: $(PIPENV_DEFAULT_PYTHON_VERSION)" + git submodule sync && git submodule update --init --recursive + pipenv run pytest -n 4 --junitxml=junit/test-results.xml + displayName: Run integration tests + env: + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' + PIPENV_DEFAULT_PYTHON_VERSION: ${{ parameters.python_version }} + GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no diff --git a/.azure-pipelines/steps/run-tests-windows.yml b/.azure-pipelines/steps/run-tests-windows.yml new file mode 100644 index 00000000..34c4f772 --- /dev/null +++ b/.azure-pipelines/steps/run-tests-windows.yml @@ -0,0 +1,56 @@ +parameters: + python_version: '' + python_architecture: '' + +steps: + - task: PowerShell@2 + inputs: + filePath: .azure-pipelines/scripts/New-RAMDisk.ps1 + arguments: "-Drive R -Size 2GB" + displayName: Setup RAMDisk + + - powershell: | + mkdir R:\virtualenvs + $acl = Get-Acl "R:\" + $rule = New-Object System.Security.AccessControl.FileSystemAccessRule( + "Everyone", "FullControl", "ContainerInherit,ObjectInherit", "None", "Allow" + ) + $acl.AddAccessRule($rule) + Set-Acl "R:\" $acl + displayName: Set RAMDisk Permissions + + - powershell: | + Write-Host "##vso[task.setvariable variable=TEMP]R:\" + Write-Host "##vso[task.setvariable variable=TMP]R:\" + Write-Host "##vso[task.setvariable variable=WORKON_HOME]R:\virtualenvs" + Write-Host "##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]$env:PYTHON_VERSION" + Write-Host "##vso[task.setvariable variable=PIPENV_NOSPIN]1" + displayName: Fix Temp Variable + env: + PYTHON_VERSION: ${{ parameters.python_version }} + + - powershell: | + pip install certifi + $env:PYTHON_PATH=$(python -c "import sys; print(sys.executable)") + $env:CERTIFI_CONTENT=$(python -m certifi) + echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$env:CERTIFI_CONTENT" + echo "##vso[task.setvariable variable=PY_EXE]$env:PYTHON_PATH" + displayName: Set Python Path + env: + PYTHONWARNINGS: 'ignore:DEPRECATION' + + - template: install-dependencies.yml + parameters: + python_version: ${{ parameters.python_version }} + + - script: | + git submodule sync + git submodule update --init --recursive + pipenv run pytest -ra -n 4 --junit-xml=junit/test-results.xml tests/ + failOnStderr: false + displayName: Run integration tests + env: + TEMP: 'R:\' + PYTHONWARNINGS: 'ignore:DEPRECATION' + PIPENV_NOSPIN: 1 + GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no diff --git a/.azure-pipelines/steps/run-tests.yml b/.azure-pipelines/steps/run-tests.yml index f33523f1..41cb72fa 100644 --- a/.azure-pipelines/steps/run-tests.yml +++ b/.azure-pipelines/steps/run-tests.yml @@ -1,25 +1,28 @@ steps: -- powershell: | - # Fix Git SSL errors - pip install certifi - python -m certifi > cacert.txt - Write-Host "##vso[task.setvariable variable=GIT_SSL_CAINFO]$(Get-Content cacert.txt)" - $env:GIT_SSL_CAINFO="$(Get-Content cacert.txt)" - # Shorten paths to get under MAX_PATH or else integration tests will fail - # https://bugs.python.org/issue18199 - subst T: "$env:TEMP" - Write-Host "##vso[task.setvariable variable=TEMP]T:\" - $env:TEMP='T:\' - Write-Host "##vso[task.setvariable variable=TMP]T:\" - $env:TMP='T:\' - git submodule sync - git submodule update --init --recursive - D:\.venv\Scripts\pipenv run pytest -ra --ignore=pipenv\patched --ignore=pipenv\vendor --junitxml=test-results.xml tests - displayName: Run integration tests +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- script: | + echo '##vso[task.setvariable variable=PIPENV_DEFAULT_PYTHON_VERSION]'$(python.version) + env: + PYTHON_VERSION: $(python.version) + +- ${{ if eq(parameters.vmImage, 'windows-latest') }}: + - template: run-tests-windows.yml + parameters: + python_version: $(python.version) +- ${{ if ne(parameters.vmImage, 'windows-latest') }}: + - template: run-tests-linux.yml + parameters: + python_version: $(python.version) - task: PublishTestResults@2 displayName: Publish Test Results inputs: - testResultsFiles: '**/test-results.xml' + testResultsFiles: '**/junit/*.xml' testRunTitle: 'Python $(python.version)' condition: succeededOrFailed() diff --git a/.azure-pipelines/steps/run-vendor-scripts.yml b/.azure-pipelines/steps/run-vendor-scripts.yml new file mode 100644 index 00000000..d4580672 --- /dev/null +++ b/.azure-pipelines/steps/run-vendor-scripts.yml @@ -0,0 +1,32 @@ +parameters: + python_version: '' + +steps: +- task: UsePythonVersion@0 + inputs: + versionSpec: $(python.version) + architecture: '$(python.architecture)' + addToPath: true + displayName: Use Python $(python.version) + +- bash: | + python -m pip install --upgrade --upgrade-strategy=eager pip requests certifi wheel setuptools + PYTHON_PATH=$(python -c 'import sys; print(sys.executable)') + CERTIFI_CONTENT=$(python -m certifi) + echo "##vso[task.setvariable variable=PY_EXE]$PYTHON_PATH" + echo "##vso[task.setvariable variable=GIT_SSL_CAINFO]$CERTIFI_CONTENT" + displayName: Set Python Path + +- template: install-dependencies.yml + +- script: | + python -m pip install --upgrade invoke parver bs4 vistir towncrier --upgrade-strategy=eager + python -m invoke vendoring.update + displayName: Run Vendor Scripts + env: + PY_EXE: $(PY_EXE) + GIT_SSL_CAINFO: $(GIT_SSL_CAINFO) + LANG: $(LANG) + PIPENV_DEFAULT_PYTHON_VERSION: $(python.version) + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' diff --git a/.azure-pipelines/windows.yml b/.azure-pipelines/windows.yml deleted file mode 100644 index 35d5c16e..00000000 --- a/.azure-pipelines/windows.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: Pipenv Build Rules -trigger: - batch: true - branches: - include: - - master - paths: - exclude: - - docs/* - - news/* - - README.md - - pipenv/*.txt - - CHANGELOG.rst - - CONTRIBUTING.md - - CODE_OF_CONDUCT.md - - .gitignore - - .gitattributes - - .editorconfig - -jobs: -- template: jobs/test.yml - parameters: - vmImage: vs2017-win2016 diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 46b2e799..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,5 +0,0 @@ -steps: - - label: ":python:" - commands: - # - make - - ./run-tests.sh \ No newline at end of file diff --git a/.buildkite/project.json b/.buildkite/project.json deleted file mode 100644 index c6b1b270..00000000 --- a/.buildkite/project.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "Pipenv Test Suite", - "steps": [ - { - "type": "script", - "name": ":pipeline:", - "command": "buildkite-agent pipeline upload" - } - ] -} \ No newline at end of file diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..c9efdaea --- /dev/null +++ b/.dockerignore @@ -0,0 +1,11 @@ +./examples +./tests +./docs +./news +./pipenv +./.git +./.buildkite +./peeps +./.github +./tasks +./.azure-pipelines diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 18e12434..6723cc45 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -4,7 +4,7 @@ If you're requesting a new feature, please use the PEEP process: https://github.com/pypa/pipenv/blob/master/peeps/PEEP-000.md -Check the [diagnose documentation](https://docs.pipenv.org/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. +Check the [diagnose documentation](https://pipenv.pypa.io/en/latest/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. Make sure to mention your debugging experience if the documented solution failed. diff --git a/.github/ISSUE_TEMPLATE/Bug_report.md b/.github/ISSUE_TEMPLATE/Bug_report.md index c470a867..e516787f 100644 --- a/.github/ISSUE_TEMPLATE/Bug_report.md +++ b/.github/ISSUE_TEMPLATE/Bug_report.md @@ -5,7 +5,7 @@ about: Create a report to help us improve Be sure to check the existing issues (both open and closed!), and make sure you are running the latest version of Pipenv. -Check the [diagnose documentation](https://docs.pipenv.org/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. +Check the [diagnose documentation](https://pipenv.pypa.io/en/latest/diagnose/) for common issues before posting! We may close your issue if it is very similar to one of them. Please be considerate, or be on your way. Make sure to mention your debugging experience if the documented solution failed. diff --git a/.github/ISSUE_TEMPLATE/Feature_request.md b/.github/ISSUE_TEMPLATE/Feature_request.md index 8a30d5aa..e02f479c 100644 --- a/.github/ISSUE_TEMPLATE/Feature_request.md +++ b/.github/ISSUE_TEMPLATE/Feature_request.md @@ -5,7 +5,7 @@ about: Suggest an idea for this project Be sure to check the existing issues (both open and closed!), and make sure you are running the latest version of Pipenv. -Check the [diagnose documentation](https://docs.pipenv.org/diagnose/) for common issues and the [PEEP list](https://github.com/pypa/pipenv/blob/master/peeps/) before posting! We may close your issue if it is very similar to one of them. Please be considerate and follow the PEEP process, or be on your way. +Check the [diagnose documentation](https://pipenv.pypa.io/en/latest/diagnose/) for common issues and the [PEEP list](https://github.com/pypa/pipenv/blob/master/peeps/) before posting! We may close your issue if it is very similar to one of them. Please be considerate and follow the PEEP process, or be on your way. Make sure to mention your debugging experience if the documented solution failed. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml new file mode 100644 index 00000000..09be266d --- /dev/null +++ b/.github/workflows/ci.yaml @@ -0,0 +1,58 @@ +name: CI + +on: + push: + branches: + - master + pull_request: + +jobs: + build: + name: ${{matrix.os}} / ${{ matrix.python-version }} + runs-on: ${{ matrix.os }}-latest + strategy: + fail-fast: false + matrix: + python-version: [2.7, 3.6, 3.7, 3.8] + os: [MacOS, Ubuntu, Windows] + + steps: + - uses: actions/checkout@v1 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v1 + with: + python-version: ${{ matrix.python-version }} + + - name: Get python path + id: python-path + run: | + echo ::set-output name=path::$(python -c "import sys; print(sys.executable)") + + - name: Install latest pip, setuptools, wheel + run: | + python -m pip install --upgrade pip setuptools wheel --upgrade-strategy=eager + - name: Install dependencies + env: + PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} + PYTHONWARNINGS: ignore:DEPRECATION + PYTHONIOENCODING: 'utf-8' + GIT_ASK_YESNO: 'false' + run: | + git submodule sync + git submodule update --init --recursive + python -m pip install "virtualenv<20" + python -m pip install -e . --upgrade + pipenv install --deploy --dev --python=${{ steps.python-path.outputs.path }} + - name: Run tests + env: + PIPENV_DEFAULT_PYTHON_VERSION: ${{ matrix.python-version }} + PYTHONWARNINGS: ignore:DEPRECATION + PIPENV_NOSPIN: '1' + CI: '1' + GIT_ASK_YESNO: 'false' + PYPI_VENDOR_DIR: './tests/pypi/' + PYTHONIOENCODING: 'utf-8' + GIT_SSH_COMMAND: ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no + run: | + pipenv run pytest -ra -n 4 tests diff --git a/.gitignore b/.gitignore index 766ffe3a..bea15631 100644 --- a/.gitignore +++ b/.gitignore @@ -145,6 +145,10 @@ venv.bak/ # mypy .mypy_cache/ +# Temporarily generating these with pytype locally for type safety +typeshed/ +pytype.cfg + ### Python Patch ### .venv/ @@ -153,3 +157,13 @@ venv.bak/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) .vs/slnx.sqlite + +# mypy/typing section +typeshed/ +.dmypy.json +mypyhtml/ +XDG_CACHE_HOME/ +snap/ +prime/ +stage/ +pip-wheel-metadata/ diff --git a/.gitmodules b/.gitmodules index 04cdc997..4f0f9fa2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,7 +6,7 @@ url = https://github.com/pinax/pinax.git [submodule "tests/test_artifacts/git/requests"] path = tests/test_artifacts/git/requests - url = https://github.com/requests/requests.git + url = https://github.com/psf/requests.git [submodule "tests/test_artifacts/git/six"] path = tests/test_artifacts/git/six url = https://github.com/benjaminp/six.git @@ -24,4 +24,7 @@ url = https://github.com/pallets/flask.git [submodule "tests/test_artifacts/git/requests-2.18.4"] path = tests/test_artifacts/git/requests-2.18.4 - url = https://github.com/requests/requests + url = https://github.com/psf/requests +[submodule "tests/pypi"] + path = tests/pypi + url = https://github.com/sarugaku/pipenv-test-artifacts.git diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6e3ef65c..24f80589 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,99 +1,254 @@ -# Contribution Guidelines +Contributing to Pipenv +====================== -Before opening any issues or proposing any pull requests, please do the -following: +If you\'re reading this, you\'re probably interested in contributing to +Pipenv. Thank you very much! Open source projects live-and-die based on +the support they receive from others, and the fact that you\'re even +considering contributing to the Pipenv project is *very* generous of +you. -1. Read our [Contributor's Guide](https://docs.pipenv.org/dev/contributing/). -2. Understand our [development philosophy](https://docs.pipenv.org/dev/philosophy/). +This document lays out guidelines and advice for contributing to this +project. If you\'re thinking of contributing, please start by reading +this document and getting a feel for how contributing to this project +works. If you have any questions, feel free to reach out to either [Dan +Ryan](https://github.com/techalchemy), [Tzu-ping +Chung](https://github.com/uranusjr), or [Nate +Prewitt](https://github.com/nateprewitt), the primary maintainers. -To get the greatest chance of helpful responses, please also observe the -following additional notes. +The guide is split into sections based on the type of contribution +you\'re thinking of making, with a section that covers general +guidelines for all contributors. -## Questions +General Guidelines +------------------ -The GitHub issue tracker is for *bug reports* and *feature requests*. Please do -not use it to ask questions about how to use Pipenv. These questions should -instead be directed to [Stack Overflow](https://stackoverflow.com/). Make sure -that your question is tagged with the `pipenv` tag when asking it on -Stack Overflow, to ensure that it is answered promptly and accurately. +### Be Cordial -## Good Bug Reports +> **Be cordial or be on your way**. *---Kenneth Reitz* -Please be aware of the following things when filing bug reports: +Pipenv has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +\"[be cordial or be on your +way](https://www.kennethreitz.org/essays/be-cordial-or-be-on-your-way)\". -1. Avoid raising duplicate issues. *Please* use the GitHub issue search feature - to check whether your bug report or feature request has been mentioned in - the past. Duplicate bug reports and feature requests are a huge maintenance - burden on the limited resources of the project. If it is clear from your - report that you would have struggled to find the original, that's ok, but - if searching for a selection of words in your issue title would have found - the duplicate then the issue will likely be closed extremely abruptly. -2. When filing bug reports about exceptions or tracebacks, please include the - *complete* traceback. Partial tracebacks, or just the exception text, are - not helpful. Issues that do not contain complete tracebacks may be closed - without warning. -3. Make sure you provide a suitable amount of information to work with. This - means you should provide: +**All contributions are welcome**, as long as everyone involved is +treated with respect. - - Guidance on **how to reproduce the issue**. Ideally, this should be a - *small* code sample that can be run immediately by the maintainers. - Failing that, let us know what you're doing, how often it happens, what - environment you're using, etc. Be thorough: it prevents us needing to ask - further questions. - - Tell us **what you expected to happen**. When we run your example code, - what are we expecting to happen? What does "success" look like for your - code? - - Tell us **what actually happens**. It's not helpful for you to say "it - doesn't work" or "it fails". Tell us *how* it fails: do you get an - exception? A hang? The packages installed seem incorrect? - How was the actual result different from your expected result? - - Tell us **what version of Pipenv you're using**, and - **how you installed it**. Different versions of Pipenv behave - differently and have different bugs, and some distributors of Pipenv - ship patches on top of the code we supply. +### Get Early Feedback {#early-feedback} - If you do not provide all of these things, it will take us much longer to - fix your problem. If we ask you to clarify these and you never respond, we - will close your issue without fixing it. +If you are contributing, do not feel the need to sit on your +contribution until it is perfectly polished and complete. It helps +everyone involved for you to seek feedback as early as you possibly can. +Submitting an early, unfinished version of your contribution for +feedback in no way prejudices your chances of getting that contribution +accepted, and can save you from putting a lot of work into a +contribution that is not suitable for the project. -## Development Setup +### Contribution Suitability + +Our project maintainers have the last word on whether or not a +contribution is suitable for Pipenv. All contributions will be +considered carefully, but from time to time, contributions will be +rejected because they do not suit the current goals or needs of the +project. + +If your contribution is rejected, don\'t despair! As long as you +followed these guidelines, you will have a much better chance of getting +your next contribution accepted. + +Questions +--------- + +The GitHub issue tracker is for *bug reports* and *feature requests*. +Please do not use it to ask questions about how to use Pipenv. These +questions should instead be directed to [Stack +Overflow](https://stackoverflow.com/). Make sure that your question is +tagged with the `pipenv` tag when asking it on Stack Overflow, to ensure +that it is answered promptly and accurately. + +Code Contributions +------------------ + +### Steps for Submitting Code + +When contributing code, you\'ll want to follow this checklist: + +1. Understand our [development + philosophy](https://pipenv.pypa.io/en/latest/dev/philosophy/). +2. Fork the repository on GitHub. +3. Set up your `dev-setup`{.interpreted-text role="ref"} +4. Run the tests (`testing`{.interpreted-text role="ref"}) to confirm + they all pass on your system. If they don\'t, you\'ll need to + investigate why they fail. If you\'re unable to diagnose this + yourself, raise it as a bug report by following the guidelines in + this document: `bug-reports`{.interpreted-text role="ref"}. +5. Write tests that demonstrate your bug or feature. Ensure that they + fail. +6. Make your change. +7. Run the entire test suite again, confirming that all tests pass + *including the ones you just added*. +8. Send a GitHub Pull Request to the main repository\'s `master` + branch. GitHub Pull Requests are the expected method of code + collaboration on this project. + +The following sub-sections go into more detail on some of the points +above. + +### Development Setup {#dev-setup} To get your development environment setup, run: -```sh +``` {.sh} pip install -e . pipenv install --dev ``` -This will install the repo version of Pipenv and then install the development -dependencies. Once that has completed, you can start developing. +This will install the repo version of Pipenv and then install the +development dependencies. Once that has completed, you can start +developing. -The repo version of Pipenv must be installed over other global versions to -resolve conflicts with the `pipenv` folder being implicitly added to `sys.path`. -See [pypa/pipenv#2557](https://github.com/pypa/pipenv/issues/2557) for more details. +The repo version of Pipenv must be installed over other global versions +to resolve conflicts with the `pipenv` folder being implicitly added to +`sys.path`. See +[pypa/pipenv\#2557](https://github.com/pypa/pipenv/issues/2557) for more +details. ### Testing Tests are written in `pytest` style and can be run very simply: -```sh +``` {.sh} pytest ``` -This will run all Pipenv tests, which can take awhile. To run a subset of the -tests, the standard pytest filters are available, such as: +This will run all Pipenv tests, which can take awhile. To run a subset +of the tests, the standard pytest filters are available, such as: -- provide a directory or file: `pytest tests/unit` or `pytest tests/unit/test_cmdparse.py` -- provide a keyword expression: `pytest -k test_lock_editable_vcs_without_install` -- provide a nodeid: `pytest tests/unit/test_cmdparse.py::test_parse` -- provide a test marker: `pytest -m lock` +- provide a directory or file: `pytest tests/unit` or + `pytest tests/unit/test_cmdparse.py` +- provide a keyword expression: + `pytest -k test_lock_editable_vcs_without_install` +- provide a nodeid: `pytest tests/unit/test_cmdparse.py::test_parse` +- provide a test marker: `pytest -m lock` -#### Package Index +### Code Review + +Contributions will not be merged until they\'ve been code reviewed. You +should implement any code review feedback unless you strongly object to +it. In the event that you object to the code review feedback, you should +make your case clearly and calmly. If, after doing so, the feedback is +judged to still apply, you must either apply the feedback or withdraw +your contribution. + +### Package Index To speed up testing, tests that rely on a package index for locking and installing use a local server that contains vendored packages in the -`tests/pypi` directory. Each vendored package should have it's own folder -containing the necessary releases. When adding a release for a package, it is -easiest to use either the `.tar.gz` or universal wheels (ex: `py2.py3-none`). If -a `.tar.gz` or universal wheel is not available, add wheels for all available -architectures and platforms. +`tests/pypi` directory. Each vendored package should have it\'s own +folder containing the necessary releases. When adding a release for a +package, it is easiest to use either the `.tar.gz` or universal wheels +(ex: `py2.py3-none`). If a `.tar.gz` or universal wheel is not +available, add wheels for all available architectures and platforms. + +Documentation Contributions +--------------------------- + +Documentation improvements are always welcome! The documentation files +live in the `docs/` directory of the codebase. They\'re written in +[reStructuredText](http://docutils.sourceforge.net/rst.html), and use +[Sphinx](http://sphinx-doc.org/index.html) to generate the full suite of +documentation. + +When contributing documentation, please do your best to follow the style +of the documentation files. This means a soft-limit of 79 characters +wide in your text files and a semi-formal, yet friendly and +approachable, prose style. + +When presenting Python code, use single-quoted strings (`'hello'` +instead of `"hello"`). + +Bug Reports +----------- + +Bug reports are hugely important! They are recorded as [GitHub +issues](https://github.com/pypa/pipenv/issues). Please be aware of the +following things when filing bug reports: + +1. Avoid raising duplicate issues. *Please* use the GitHub issue search + feature to check whether your bug report or feature request has been + mentioned in the past. Duplicate bug reports and feature requests + are a huge maintenance burden on the limited resources of the + project. If it is clear from your report that you would have + struggled to find the original, that\'s ok, but if searching for a + selection of words in your issue title would have found the + duplicate then the issue will likely be closed extremely abruptly. + +2. When filing bug reports about exceptions or tracebacks, please + include the *complete* traceback. Partial tracebacks, or just the + exception text, are not helpful. Issues that do not contain complete + tracebacks may be closed without warning. + +3. Make sure you provide a suitable amount of information to work with. + This means you should provide: + + - Guidance on **how to reproduce the issue**. Ideally, this should + be a *small* code sample that can be run immediately by the + maintainers. Failing that, let us know what you\'re doing, how + often it happens, what environment you\'re using, etc. Be + thorough: it prevents us needing to ask further questions. + - Tell us **what you expected to happen**. When we run your + example code, what are we expecting to happen? What does + \"success\" look like for your code? + - Tell us **what actually happens**. It\'s not helpful for you to + say \"it doesn\'t work\" or \"it fails\". Tell us *how* it + fails: do you get an exception? A hang? The packages installed + seem incorrect? How was the actual result different from your + expected result? + - Tell us **what version of Pipenv you\'re using**, and **how you + installed it**. Different versions of Pipenv behave differently + and have different bugs, and some distributors of Pipenv ship + patches on top of the code we supply. + + If you do not provide all of these things, it will take us much + longer to fix your problem. If we ask you to clarify these and you + never respond, we will close your issue without fixing it. + +Run the tests +------------- + +Three ways of running the tests are as follows: + +1. `make test` (which uses `docker`) +2. `./run-tests.sh` or `run-tests.bat` +3. Using pipenv: + +``` {.console} +$ git clone https://github.com/pypa/pipenv.git +$ cd pipenv +$ git submodule sync && git submodule update --init --recursive +$ pipenv install --dev +$ pipenv run pytest +``` + +For the last two, it is important that your environment is setup +correctly, and this may take some work, for example, on a specific Mac +installation, the following steps may be needed: + + # Make sure the tests can access github + if [ "$SSH_AGENT_PID" = "" ] + then + eval `ssh-agent` + ssh-add + fi + + # Use unix like utilities, installed with brew, + # e.g. brew install coreutils + for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin + do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" + done + + export PATH + + # PIP_FIND_LINKS currently breaks test_uninstall.py + unset PIP_FIND_LINKS diff --git a/Dockerfile b/Dockerfile index c6b6fb7d..96ecfdeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,9 +3,11 @@ FROM heroku/heroku:18-build ENV DEBIAN_FRONTEND noninteractive ENV LC_ALL C.UTF-8 ENV LANG C.UTF-8 +# Python, don't write bytecode! +ENV PYTHONDONTWRITEBYTECODE 1 # -- Install Pipenv: -RUN apt update && apt upgrade -y && apt install python3.7-dev -y +RUN apt update && apt install python3.7-dev libffi-dev -y RUN curl --silent https://bootstrap.pypa.io/get-pip.py | python3.7 # Backwards compatility. diff --git a/LICENSE b/LICENSE index f5639bb0..ea28562e 100644 --- a/LICENSE +++ b/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright 2017 Kenneth Reitz +Copyright 2020 Python Packaging Authority Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/MANIFEST.in b/MANIFEST.in index 5e012535..d0f0719b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,5 +1,5 @@ include LICENSE README.md CONTRIBUTING.md CODE_OF_CONDUCT.md CHANGELOG.rst NOTICES HISTORY.txt -include Makefile pyproject.toml get-pipenv.py +include Makefile pyproject.toml get-pipenv.py .dockerignore *.yml include examples/Pipfil* recursive-include pipenv LICENSE LICENSE* *LICENSE* *COPYING* t32.exe t64.exe w32.exe w64.exe cacert.pem recursive-include pipenv *.cfg @@ -9,6 +9,7 @@ recursive-include pipenv Makefile recursive-include pipenv/vendor vendor.txt recursive-include pipenv README recursive-include pipenv *.json +recursive-include pipenv *.rst include pipenv/patched/notpip/_vendor/vendor.txt include pipenv/patched/safety.zip pipenv/patched/patched.txt include pipenv/vendor/pipreqs/stdlib pipenv/vendor/pipreqs/mapping @@ -37,3 +38,5 @@ prune docs/build prune news prune tasks prune tests +prune pipenv/vendor/importlib_metadata/tests +prune pipenv/vendor/importlib_resources/tests diff --git a/Makefile b/Makefile index 2722bdb4..cefc7fea 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,117 @@ +get_venv_dir:=$(shell mktemp -d 2>/dev/null || mktemp -d -t 'tmpvenv') +venv_dir := $(get_venv_dir)/pipenv_venv +venv_file := $(CURDIR)/.test_venv +get_venv_path =$(file < $(venv_file)) +# This is how we will build tag-specific wheels, e.g. py36 or py37 +PY_VERSIONS:= 2.7 3.5 3.6 3.7 3.8 +# This is how we will build generic wheels, e.g. py2 or py3 +INSTALL_TARGETS := $(addprefix install-py,$(PY_VERSIONS)) +CLEAN_TARGETS := $(addprefix clean-py,$(PY_VERSIONS)) +DATE_STRING := $(shell date +%Y.%m.%d) +THIS_MONTH_DATE := $(shell date +%Y.%m.01) +NEXT_MONTH_DATE := $(shell date -d "+1 month" +%Y.%m.01) + + format: black pipenv/*.py test: docker-compose up + +.PHONY: install +install: + pip install -e . + +install.stamp: install + @touch install.stamp + +.PHONY: install-py% +install-py%: install.stamp + @echo building for $(addprefix python, $(subst install-py,,$@)) + PIPENV_PYTHON=$(subst install-py,,$@) pipenv install --dev + +install-virtualenvs.stamp: ${INSTALL_TARGETS} + @touch install-virtualenvs.stamp + +.PHONY: ramdisk +ramdisk: + sudo mkdir -p /mnt/ramdisk + sudo mount -t tmpfs -o size=2g tmpfs /mnt/ramdisk + sudo chown -R ${USER}:${USER} /mnt/ramdisk + +.PHONY: ramdisk-virtualenv +ramdisk-virtualenv: ramdisk + [ ! -e "/mnt/ramdisk/.venv/bin/activate" ] && \ + python -m virtualenv /mnt/ramdisk/.venv + @echo "/mnt/ramdisk/.venv" >> $(venv_file) + +.PHONY: virtualenv +virtualenv: + [ ! -e $(venv_dir) ] && rm -rf $(venv_file) && python -m virtualenv $(venv_dir) + @echo $(venv_dir) >> $(venv_file) + +.PHONY: test-install +test-install: virtualenv + . $(get_venv_path)/bin/activate && \ + python -m pip install --upgrade pip virtualenv -e .[tests,dev] && \ + pipenv install --dev + +.PHONY: submodules +submodules: + git submodule sync + git submodule update --init --recursive + +.PHONY: tests +tests: virtualenv submodules test-install + . $(get_venv_path)/bin/activate && \ + pipenv run pytest -ra -vvv --full-trace --tb=long + +.PHONY: test-specific +test-specific: submodules virtualenv test-install + . $(get_venv_path)/bin/activate && pipenv run pytest -ra -k '$(tests)' + +.PHONY: retest +retest: virtualenv submodules test-install + . $(get_venv_path)/bin/activate && pipenv run pytest -ra -k 'test_check_unused or test_install_editable_git_tag or test_get_vcs_refs or test_skip_requirements_when_pipfile or test_editable_vcs_install or test_basic_vcs_install or test_git_vcs_install or test_ssh_vcs_install or test_vcs_can_use_markers' -vvv --full-trace --tb=long + +.PHONY: build +build: install-virtualenvs.stamp install.stamp + PIPENV_PYTHON=3.7 pipenv run python setup.py sdist bdist_wheel + +.PHONY: update-version +update-version: + @sed -i "s/^__version__ = .\+$\/__version__ = \"$(DATE_STRING)\"/g" ./pipenv/__version__.py + +.PHONY: update-prerelease-version +update-prerelease-version: + @sed -i "s/^__version__ = .\+$\/__version__ = \"$(THIS_MONTH_DATE).a1\"/g" ./pipenv/__version__.py + +.PHONY: pre-bump +pre-bump: + @sed -i "s/^__version__ = .\+$\/__version__ = \"$(NEXT_MONTH_DATE).dev0\"/g" ./pipenv/__version__.py + +.PHONY: lint +lint: + flake8 . + +.PHONY: check +check: build.stamp + pipenv run twine check dist/* && pipenv run check-manifest . + +.PHONY: upload-test +upload-test: build + twine upload --repository=testpypi dist/* + +.PHONY: clean-py% +clean-py%: + @echo "cleaning environment for $@..." + PIPENV_PYTHON="$(subst clean-py,,$@)" pipenv --rm + +.PHONY: cleanbuild +cleanbuild: + @echo "cleaning up existing builds..." + @rm -rf build/ dist/ + @rm -rf build.stamp + +.PHONY: clean +clean: + rm -rf install.stamp build.stamp install-virtualenvs.stamp diff --git a/Pipfile b/Pipfile index db952be7..826df207 100644 --- a/Pipfile +++ b/Pipfile @@ -1,27 +1,13 @@ [dev-packages] -pipenv = {path = ".", editable = true} -"flake8" = ">=3.3.0,<4" -pytest = "*" -mock = "*" -sphinx = "<=1.5.5" -twine = "*" +pipenv = {path = ".", editable = true, extras = ["tests", "dev"]} sphinx-click = "*" -pytest-xdist = "*" click = "*" -pytest-pypy = {path = "./tests/pytest-pypi", editable = true} -pytest-tap = "*" -flaky = "*" +pytest_pypi = {path = "./tests/pytest-pypi", editable = true} stdeb = {version="*", markers="sys_platform == 'linux'"} -black = {version="*", markers="python_version >= '3.6'"} -pytz = "*" -towncrier = {git = "https://github.com/hawkowl/towncrier.git", editable = true, ref = "master"} -parver = "*" -invoke = "*" jedi = "*" isort = "*" rope = "*" -passa = {editable = true, git = "https://github.com/sarugaku/passa.git"} -bs4 = "*" +passa = {git = "https://github.com/sarugaku/passa.git"} [packages] diff --git a/Pipfile.lock b/Pipfile.lock index 46115fdc..5d58f7dd 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,15 +1,15 @@ { "_meta": { "hash": { - "sha256": "f0a57ba6f180e7312f799a460a9d2790c9c26c3556eff2b5d1fe7d9b9174d05b" + "sha256": "f4d89c0aab5c4e865f8c96ba24613fb1e66bae803a3ceaeadb6abf0061898091" }, "pipfile-spec": 6, "requires": {}, "sources": [ { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true } ] }, @@ -27,6 +27,7 @@ "sha256:37228cda29411948b422fae072f57e31d3396d2ee1c9783775980ee9c9990af6", "sha256:58587dd4dc3daefad0487f6d9ae32b4542b185e1c36db6993290e7c41ca2b47c" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.5" }, "appdirs": { @@ -36,122 +37,113 @@ ], "version": "==1.4.3" }, - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, "arpeggio": { "hashes": [ - "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", - "sha256:dc5c0541e7cc2c6033dc0338133436abfac53655624784736e9bc8bd35e56583" + "sha256:948ce06163a48a72c97f4fe79ad3d1c1330b6fec4f22ece182fb60ef60bd022b", + "sha256:b9178917594bb9758002faed31e1e1c968b5ea7f2a8f78fd4a5b8fecaccfcfcd" ], - "version": "==1.9.0" + "version": "==1.9.2" }, "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ - "sha256:10cbf6e27dbce8c30807caf056c8eb50917e0eaafe86347671b57254006c3e69", - "sha256:ca4be454458f9dec299268d472aaa5a11f67a4ff70093396e1ceae9c76cf4bbb" + "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", + "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" ], - "version": "==18.2.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==19.3.0" }, "babel": { "hashes": [ - "sha256:6778d85147d5d85345c14a26aada5e478ab04e39b078b0745ee6870c2b5cf669", - "sha256:8cba50f48c529ca3fa18cf81fa9403be176d374ac4d60738b839122dfaaa3d23" + "sha256:1aac2ae2d0d8ea368fa90906567f5c08463d98ade155c0c4bfedd6a0f7160e38", + "sha256:d670ea0b10f8b723672d3a6abeb87b565b244da220d76b4dba1b66269ec152d4" ], - "version": "==2.6.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8.0" + }, + "backports.functools-lru-cache": { + "hashes": [ + "sha256:0bada4c2f8a43d533e4ecb7a12214d9420e66eb206d54bf2d682581ca4b80848", + "sha256:8fde5f188da2d593bd5bc0be98d9abc46c95bb8a9dde93429570192ee6cc2d4a" + ], + "markers": "python_version < '3.2'", + "version": "==1.6.1" }, "beautifulsoup4": { "hashes": [ - "sha256:194ec62a25438adcb3fdb06378b26559eda1ea8a747367d34c33cef9c7f48d57", - "sha256:90f8e61121d6ae58362ce3bed8cd997efb00c914eae0ff3d363c32f9a9822d10", - "sha256:f0abd31228055d698bb392a826528ea08ebb9959e6bea17c606fd9c9009db938" + "sha256:594ca51a10d2b3443cbac41214e12dbb2a1cd57e1a7344659849e2e20ba6a8d8", + "sha256:a4bbe77fd30670455c5296242967a123ec28c37e9702a8a81bd2f20a4baf0368", + "sha256:d4e96ac9b0c3a6d3f0caae2e4124e6055c5dcafde8e2f831ff194c104f0775a0" ], - "version": "==4.6.3" + "version": "==4.9.0" }, "black": { "hashes": [ - "sha256:817243426042db1d36617910df579a54f1afd659adb96fc5032fcf4b36209739", - "sha256:e030a9a28f542debc08acceb273f228ac422798e5215ba2a791a6ddeaaca22a5" + "sha256:1b30e59be925fafc1ee4565e5e08abef6b03fe455102883820fe5ee2e4734e0b", + "sha256:c2edb73a08e9e0e6f65a0e6af18b059b8b1cdd5bef997d7a0b181df93dc81539" ], - "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==18.9b0" + "version": "==19.10b0" }, "bleach": { "hashes": [ - "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", - "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + "sha256:cc8da25076a1fe56c3ac63671e2194458e0c4d9c7becfd52ca251650d517903c", + "sha256:e78e426105ac07026ba098f04de8abe9b6e3e98b5befbf89b51a5ef0a4292b03" ], - "version": "==3.0.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.1.4" }, "bs4": { "hashes": [ "sha256:36ecea1fd7cc5c0c6e4a1ff075df26d50da647b75376626cc186e2212886dd3a" ], - "index": "pypi", "version": "==0.0.1" }, - "cerberus": { - "hashes": [ - "sha256:f5c2e048fb15ecb3c088d192164316093fcfa602a74b3386eefb2983aa7e800a" - ], - "version": "==1.2" - }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:1d987a998c75633c40847cc966fcf5904906c920a7f17ef374f5aa4282abd304", + "sha256:51fcb31174be6e6664c5f69e3e1691a2d72a1a12e90f872cbdb1567eb47b6519" ], - "version": "==2018.10.15" + "version": "==2020.4.5.1" }, "cffi": { "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" + "sha256:001bf3242a1bb04d985d63e138230802c6c8d4db3668fb545fb5005ddf5bb5ff", + "sha256:00789914be39dffba161cfc5be31b55775de5ba2235fe49aa28c148236c4e06b", + "sha256:028a579fc9aed3af38f4892bdcc7390508adabc30c6af4a6e4f611b0c680e6ac", + "sha256:14491a910663bf9f13ddf2bc8f60562d6bc5315c1f09c704937ef17293fb85b0", + "sha256:1cae98a7054b5c9391eb3249b86e0e99ab1e02bb0cc0575da191aedadbdf4384", + "sha256:2089ed025da3919d2e75a4d963d008330c96751127dd6f73c8dc0c65041b4c26", + "sha256:2d384f4a127a15ba701207f7639d94106693b6cd64173d6c8988e2c25f3ac2b6", + "sha256:337d448e5a725bba2d8293c48d9353fc68d0e9e4088d62a9571def317797522b", + "sha256:399aed636c7d3749bbed55bc907c3288cb43c65c4389964ad5ff849b6370603e", + "sha256:3b911c2dbd4f423b4c4fcca138cadde747abdb20d196c4a48708b8a2d32b16dd", + "sha256:3d311bcc4a41408cf5854f06ef2c5cab88f9fded37a3b95936c9879c1640d4c2", + "sha256:62ae9af2d069ea2698bf536dcfe1e4eed9090211dbaafeeedf5cb6c41b352f66", + "sha256:66e41db66b47d0d8672d8ed2708ba91b2f2524ece3dee48b5dfb36be8c2f21dc", + "sha256:675686925a9fb403edba0114db74e741d8181683dcf216be697d208857e04ca8", + "sha256:7e63cbcf2429a8dbfe48dcc2322d5f2220b77b2e17b7ba023d6166d84655da55", + "sha256:8a6c688fefb4e1cd56feb6c511984a6c4f7ec7d2a1ff31a10254f3c817054ae4", + "sha256:8c0ffc886aea5df6a1762d0019e9cb05f825d0eec1f520c51be9d198701daee5", + "sha256:95cd16d3dee553f882540c1ffe331d085c9e629499ceadfbda4d4fde635f4b7d", + "sha256:99f748a7e71ff382613b4e1acc0ac83bf7ad167fb3802e35e90d9763daba4d78", + "sha256:b8c78301cefcf5fd914aad35d3c04c2b21ce8629b5e4f4e45ae6812e461910fa", + "sha256:c420917b188a5582a56d8b93bdd8e0f6eca08c84ff623a4c16e809152cd35793", + "sha256:c43866529f2f06fe0edc6246eb4faa34f03fe88b64a0a9a942561c8e22f4b71f", + "sha256:cab50b8c2250b46fe738c77dbd25ce017d5e6fb35d3407606e7a4180656a5a6a", + "sha256:cef128cb4d5e0b3493f058f10ce32365972c554572ff821e175dbc6f8ff6924f", + "sha256:cf16e3cf6c0a5fdd9bc10c21687e19d29ad1fe863372b5543deaec1039581a30", + "sha256:e56c744aa6ff427a607763346e4170629caf7e48ead6921745986db3692f987f", + "sha256:e577934fc5f8779c554639376beeaa5657d54349096ef24abe8c74c5d9c117c3", + "sha256:f2b0fa0c01d8a0c7483afd9f31d7ecf2d71760ca24499c8697aeb5ca37dc090c" ], - "version": "==1.11.5" + "version": "==1.14.0" }, "chardet": { "hashes": [ @@ -160,127 +152,148 @@ ], "version": "==3.0.4" }, + "check-manifest": { + "hashes": [ + "sha256:8754cc8efd7c062a3705b442d1c23ff702d4477b41a269c2e354b25e1f5535a4", + "sha256:a4c555f658a7c135b8a22bd26c2e55cfaf5876e4d5962d8c25652f2addd556bc" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.39" + }, "click": { "hashes": [ - "sha256:2335065e6395b9e67ca716de5f7526736bfa6ceead690adf616d925bdc622b13", - "sha256:5b94b49521f6456670fdb30cd82a4eca9412788a93fa6dd6df72c94d5a8ff2d7" + "sha256:8a18b4ea89d8820c5d0c7da8a64b2c324b4dabb695804dbfea19b9be9d88c0cc", + "sha256:e345d143d80bf5ee7534056164e5e112ea5e22716bbb1ce727941f4c8b471b9a" ], "index": "pypi", - "version": "==7.0" - }, - "cmarkgfm": { - "hashes": [ - "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", - "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", - "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", - "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", - "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", - "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", - "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", - "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", - "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", - "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", - "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", - "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", - "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", - "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", - "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", - "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", - "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", - "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", - "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", - "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", - "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", - "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", - "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", - "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", - "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", - "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", - "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", - "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" - ], - "version": "==0.4.2" + "version": "==7.1.1" }, "colorama": { "hashes": [ - "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", - "sha256:f8ac84de7840f5b9c4e3347b3c1eaa50f7e49c2b07596221daec5edaabbd7c48" + "sha256:7d73d2a99753107a36ac6b455ee49046802e59d9d076ef8e47b61499fa29afff", + "sha256:e96da0d330793e2cb9485e9ddfd918d456036c7149416295932478192f4436a1" ], - "version": "==0.4.1" + "version": "==0.4.3" }, "configparser": { "hashes": [ - "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" + "sha256:254c1d9c79f60c45dfde850850883d5aaa7f19a23f13561243a050d5a7c3fe4c", + "sha256:c7d282687a5308319bf3d2e7706e575c635b0a470342641c93bea0ea3b5331df" ], "markers": "python_version < '3.2'", - "version": "==3.5.0" + "version": "==4.0.2" }, - "cursor": { + "contextlib2": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:01f490098c18b19d2bd5bb5dc445b2054d2fa97f09a4280ba2c5f3c394c8162e", + "sha256:3355078a159fbb44ee60ea80abd0d87b80b78c248643b49aa6d94673b413609b" ], - "version": "==1.2.0" + "markers": "python_version < '3'", + "version": "==0.6.0.post1" + }, + "cryptography": { + "hashes": [ + "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", + "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", + "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", + "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", + "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", + "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", + "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", + "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", + "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", + "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", + "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", + "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", + "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", + "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", + "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", + "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", + "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", + "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", + "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", + "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", + "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.8" + }, + "decorator": { + "hashes": [ + "sha256:86156361c50488b84a3f148056ea716ca587df2f0de1d34750d35c21312725de", + "sha256:f069f3a01830ca754ba5258fde2278454a0b5b79e0d7f5c13b3b97e57d4acff6" + ], + "version": "==4.4.0" }, "distlib": { "hashes": [ - "sha256:57977cd7d9ea27986ec62f425630e4ddb42efe651ff80bc58ed8dbc3c7c21f19" + "sha256:2e166e231a26b36d6dfe35a48c4464346620f8645ed0ace01ee31822b288de21" ], - "version": "==0.2.8" + "version": "==0.3.0" }, "docutils": { "hashes": [ - "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", - "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", - "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + "sha256:0c5b78adfbf7762415433f5515cd5c9e762339e23369dbe8000d84a4bf4ab3af", + "sha256:c2de3a60e9e7d07be26b7f2b00ca0309c207e06c100f9cc2a94931fc75a478fc" ], - "version": "==0.14" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.16" + }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "markers": "python_version >= '2.7'", + "version": "==0.3" }, "enum34": { "hashes": [ - "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", - "sha256:644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a", - "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", - "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + "sha256:a98a201d6de3f2ab3db284e70a33b0f896fbf35f8086594e8c9e74b909058d53", + "sha256:c3858660960c984d6ab0ebad691265180da2b43f07e061c0f8dca9ef3cffd328", + "sha256:cce6a7477ed816bd2542d03d53db9f0db935dd013b70f336a95c73979289f248" ], - "markers": "python_version < '3.4'", - "version": "==1.1.6" + "markers": "python_version < '3'", + "version": "==1.1.10" }, "execnet": { "hashes": [ - "sha256:a7a84d5fa07a089186a329528f127c9d73b9de57f1a1131b82bb5320ee651f6a", - "sha256:fc155a6b553c66c838d1a22dba1dc9f5f505c43285a878c6f74a79c024750b83" + "sha256:cacb9df31c9680ec5f95553976c4da484d407e85e41c83cb812aa014f0eddc50", + "sha256:d4efd397930c46415f62f8a31388d6be4f27a91d7550eb79bc64a756e0056547" ], - "version": "==1.5.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.7.1" }, - "first": { + "filelock": { "hashes": [ - "sha256:3bb3de3582cb27071cfb514f00ed784dc444b7f96dc21e140de65fe00585c95e", - "sha256:41d5b64e70507d0c3ca742d68010a76060eea8a3d863e9b5130ab11a4a91aa0e" + "sha256:18d82244ee114f543149c66a6e0c14e9c4f8a1044b5cdaadd0f82159d6a6ff59", + "sha256:929b7d63ec5b7d6b71b0fa5ac14e030b3f70b75747cef1b10da9b879fef15836" ], - "version": "==2.0.1" + "version": "==3.0.12" }, "flake8": { "hashes": [ - "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", - "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" + "sha256:45681a117ecc81e870cbf1262835ae4af5e7a8b08e40b944a8a6e6b895914cfb", + "sha256:49356e766643ad15072a789a20915d3c91dc89fd313ccd71802303fd67e4deca" ], - "index": "pypi", - "version": "==3.5.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.7.9" }, "flaky": { "hashes": [ - "sha256:4ad7880aef8c35a34ddb394d4fa33047765bca1e3d67d182bf6eba9c8eabf3a2", - "sha256:d0533f473a46b916e6db6e84e20b06d8a70656600a0c14e819b0760b63f70226" + "sha256:5471615b32b0f8086573de924475b1f0d31e0e8655a089eb9c38a0fbff3f11aa", + "sha256:8cd5455bb00c677f787da424eaf8c4a58a922d0e97126d3085db5b279a98b698" ], - "index": "pypi", - "version": "==3.4.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.6.1" }, "flask": { "hashes": [ - "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", - "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + "sha256:4efa1ae2d7c9865af48986de8aeb8504bf32c7f3d6fdc9353d34b21f4b127060", + "sha256:8a4fdd8936eba2512e9c85df320a37e694c93945b33ef33c89946a340a238557" ], - "version": "==1.0.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.2" }, "funcsigs": { "hashes": [ @@ -290,33 +303,60 @@ "markers": "python_version < '3.3'", "version": "==1.0.2" }, + "functools32": { + "hashes": [ + "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", + "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" + ], + "markers": "python_version < '3.2'", + "version": "==3.2.3.post2" + }, "future": { "hashes": [ - "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + "sha256:67045236dcfd6816dc439556d009594abf643e5eb48992e36beac09c2ca659b8" ], - "version": "==0.16.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.17.1" }, "futures": { "hashes": [ - "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", - "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" + "sha256:49b3f5b064b6e3afc3316421a3f25f66c137ae88f068abbf72830170033c5e16", + "sha256:7e033af76a5e35f58e56da7a91e687706faf4e7bdfb2cbc3f2cca6b9bcda9794" ], - "markers": "python_version < '3' and python_version >= '2.6'", - "version": "==3.2.0" + "markers": "python_version < '3.2'", + "version": "==3.3.0" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:7588d1c14ae4c77d74036e8c22ff447b26d0fde8f007354fd48a7814db15b7cb", + "sha256:a068a21ceac8a4d63dbfd964670474107f541babbd2250d61922f029858365fa" ], - "version": "==2.7" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.9" }, "imagesize": { "hashes": [ - "sha256:3f349de3eb99145973fefb7dbe38554414e5c30abd0c8e4b970a7c9d09f3a1d8", - "sha256:f3832918bc3c66617f92e35f5d70729187676313caa60c187eb0f28b8fe5e3b5" + "sha256:6965f19a6a2039c7d48bca7dba2473069ff854c36ae6f19d2cde309d998228a1", + "sha256:b1f6b5a4eab1f73479a50fb79fcf729514a900c341d8503d62a62dbc4127a2b1" ], - "version": "==1.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.2.0" + }, + "importlib-metadata": { + "hashes": [ + "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", + "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" + ], + "markers": "python_version < '3.8'", + "version": "==1.6.0" + }, + "importlib-resources": { + "hashes": [ + "sha256:4019b6a9082d8ada9def02bece4a76b131518866790d58fdda0b5f8c603b36c2", + "sha256:dd98ceeef3f5ad2ef4cc287b8586da4ebad15877f351e9688987ad663a0a29b8" + ], + "markers": "python_version < '3.7'", + "version": "==1.4.0" }, "incremental": { "hashes": [ @@ -327,76 +367,82 @@ }, "invoke": { "hashes": [ - "sha256:4f4de934b15c2276caa4fbc5a3b8a61c0eb0b234f2be1780d2b793321995c2d6", - "sha256:dc492f8f17a0746e92081aec3f86ae0b4750bf41607ea2ad87e5a7b5705121b7", - "sha256:eb6f9262d4d25b40330fb21d1e99bf0f85011ccc3526980f8a3eaedd4b43892e" + "sha256:87b3ef9d72a1667e104f89b159eaf8a514dbf2f3576885b2bbdefe74c3fb2132", + "sha256:93e12876d88130c8e0d7fd6618dd5387d6b36da55ad541481dfa5e001656f134", + "sha256:de3f23bfe669e3db1085789fd859eb8ca8e0c5d9c20811e2407fa042e8a5e15d" ], - "index": "pypi", - "version": "==1.2.0" + "version": "==1.4.1" }, "isort": { "hashes": [ - "sha256:1153601da39a25b14ddc54955dbbacbb6b2d19135386699e2ad58517953b34af", - "sha256:b9c40e9750f3d77e6e4d441d8b0266cf555e7cdabdcff33c4fd06366ca761ef8", - "sha256:ec9ef8f4a9bc6f71eec99e1806bfa2de401650d996c59330782b89a5555c1497" + "sha256:54da7e92468955c4fceacd0c86bd0ec997b0e1ee80d97f67c35a78b719dccab1", + "sha256:6e811fcb295968434526407adb8796944f1988c5b65e8139058f2014cbe100fd" ], "index": "pypi", - "version": "==4.3.4" + "version": "==4.3.21" }, "itsdangerous": { "hashes": [ "sha256:321b033d07f2a4136d3ec762eac9f16a10ccd60f53c0c91af90217ace7ba1f19", "sha256:b12271b2047cb23eeb98c8b5622e2e5c5e9abd9784a153e9d8ef9cb4dd09d749" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.1.0" }, "jedi": { "hashes": [ - "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7", - "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148" + "sha256:cd60c93b71944d628ccac47df9a60fec53150de53d42dc10a7fc4b5ba6aae798", + "sha256:df40c97641cb943661d2db4c33c2e1ff75d491189423249e989bcea4464f3030" ], "index": "pypi", - "version": "==0.13.1" + "version": "==0.17.0" }, "jinja2": { "hashes": [ - "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", - "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + "sha256:89aab215427ef59c34ad58735269eb58b1a5808103067f7bb9d5836c651b3bb0", + "sha256:f0a4641d3cf955324a89c04f3d94663aa4d638abe8f733ecd3582848e1c37035" ], - "version": "==2.10" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.11.2" }, "markupsafe": { "hashes": [ - "sha256:048ef924c1623740e70204aa7143ec592504045ae4429b59c30054cb31e3c432", - "sha256:130f844e7f5bdd8e9f3f42e7102ef1d49b2e6fdf0d7526df3f87281a532d8c8b", - "sha256:19f637c2ac5ae9da8bfd98cef74d64b7e1bb8a63038a3505cd182c3fac5eb4d9", - "sha256:1b8a7a87ad1b92bd887568ce54b23565f3fd7018c4180136e1cf412b405a47af", - "sha256:1c25694ca680b6919de53a4bb3bdd0602beafc63ff001fea2f2fc16ec3a11834", - "sha256:1f19ef5d3908110e1e891deefb5586aae1b49a7440db952454b4e281b41620cd", - "sha256:1fa6058938190ebe8290e5cae6c351e14e7bb44505c4a7624555ce57fbbeba0d", - "sha256:31cbb1359e8c25f9f48e156e59e2eaad51cd5242c05ed18a8de6dbe85184e4b7", - "sha256:3e835d8841ae7863f64e40e19477f7eb398674da6a47f09871673742531e6f4b", - "sha256:4e97332c9ce444b0c2c38dd22ddc61c743eb208d916e4265a2a3b575bdccb1d3", - "sha256:525396ee324ee2da82919f2ee9c9e73b012f23e7640131dd1b53a90206a0f09c", - "sha256:52b07fbc32032c21ad4ab060fec137b76eb804c4b9a1c7c7dc562549306afad2", - "sha256:52ccb45e77a1085ec5461cde794e1aa037df79f473cbc69b974e73940655c8d7", - "sha256:5c3fbebd7de20ce93103cb3183b47671f2885307df4a17a0ad56a1dd51273d36", - "sha256:5e5851969aea17660e55f6a3be00037a25b96a9b44d2083651812c99d53b14d1", - "sha256:5edfa27b2d3eefa2210fb2f5d539fbed81722b49f083b2c6566455eb7422fd7e", - "sha256:7d263e5770efddf465a9e31b78362d84d015cc894ca2c131901a4445eaa61ee1", - "sha256:83381342bfc22b3c8c06f2dd93a505413888694302de25add756254beee8449c", - "sha256:857eebb2c1dc60e4219ec8e98dfa19553dae33608237e107db9c6078b1167856", - "sha256:98e439297f78fca3a6169fd330fbe88d78b3bb72f967ad9961bcac0d7fdd1550", - "sha256:bf54103892a83c64db58125b3f2a43df6d2cb2d28889f14c78519394feb41492", - "sha256:d9ac82be533394d341b41d78aca7ed0e0f4ba5a2231602e2f05aa87f25c51672", - "sha256:e982fe07ede9fada6ff6705af70514a52beb1b2c3d25d4e873e82114cf3c5401", - "sha256:edce2ea7f3dfc981c4ddc97add8a61381d9642dc3273737e756517cc03e84dd6", - "sha256:efdc45ef1afc238db84cb4963aa689c0408912a0239b0721cb172b4016eb31d6", - "sha256:f137c02498f8b935892d5c0172560d7ab54bc45039de8805075e19079c639a9c", - "sha256:f82e347a72f955b7017a39708a3667f106e6ad4d10b25f237396a7115d8ed5fd", - "sha256:fb7c206e01ad85ce57feeaaa0bf784b97fa3cad0d4a5737bc5295785f5c613a1" + "sha256:00bc623926325b26bb9605ae9eae8a215691f33cae5df11ca5424f06f2d1f473", + "sha256:09027a7803a62ca78792ad89403b1b7a73a01c8cb65909cd876f7fcebd79b161", + "sha256:09c4b7f37d6c648cb13f9230d847adf22f8171b1ccc4d5682398e77f40309235", + "sha256:1027c282dad077d0bae18be6794e6b6b8c91d58ed8a8d89a89d59693b9131db5", + "sha256:13d3144e1e340870b25e7b10b98d779608c02016d5184cfb9927a9f10c689f42", + "sha256:24982cc2533820871eba85ba648cd53d8623687ff11cbb805be4ff7b4c971aff", + "sha256:29872e92839765e546828bb7754a68c418d927cd064fd4708fab9fe9c8bb116b", + "sha256:43a55c2930bbc139570ac2452adf3d70cdbb3cfe5912c71cdce1c2c6bbd9c5d1", + "sha256:46c99d2de99945ec5cb54f23c8cd5689f6d7177305ebff350a58ce5f8de1669e", + "sha256:500d4957e52ddc3351cabf489e79c91c17f6e0899158447047588650b5e69183", + "sha256:535f6fc4d397c1563d08b88e485c3496cf5784e927af890fb3c3aac7f933ec66", + "sha256:596510de112c685489095da617b5bcbbac7dd6384aeebeda4df6025d0256a81b", + "sha256:62fe6c95e3ec8a7fad637b7f3d372c15ec1caa01ab47926cfdf7a75b40e0eac1", + "sha256:6788b695d50a51edb699cb55e35487e430fa21f1ed838122d722e0ff0ac5ba15", + "sha256:6dd73240d2af64df90aa7c4e7481e23825ea70af4b4922f8ede5b9e35f78a3b1", + "sha256:717ba8fe3ae9cc0006d7c451f0bb265ee07739daf76355d06366154ee68d221e", + "sha256:79855e1c5b8da654cf486b830bd42c06e8780cea587384cf6545b7d9ac013a0b", + "sha256:7c1699dfe0cf8ff607dbdcc1e9b9af1755371f92a68f706051cc8c37d447c905", + "sha256:88e5fcfb52ee7b911e8bb6d6aa2fd21fbecc674eadd44118a9cc3863f938e735", + "sha256:8defac2f2ccd6805ebf65f5eeb132adcf2ab57aa11fdf4c0dd5169a004710e7d", + "sha256:98c7086708b163d425c67c7a91bad6e466bb99d797aa64f965e9d25c12111a5e", + "sha256:9add70b36c5666a2ed02b43b335fe19002ee5235efd4b8a89bfcf9005bebac0d", + "sha256:9bf40443012702a1d2070043cb6291650a0841ece432556f784f004937f0f32c", + "sha256:ade5e387d2ad0d7ebf59146cc00c8044acbd863725f887353a10df825fc8ae21", + "sha256:b00c1de48212e4cc9603895652c5c410df699856a2853135b3967591e4beebc2", + "sha256:b1282f8c00509d99fef04d8ba936b156d419be841854fe901d8ae224c59f0be5", + "sha256:b2051432115498d3562c084a49bba65d97cf251f5a331c64a12ee7e04dacc51b", + "sha256:ba59edeaa2fc6114428f1637ffff42da1e311e29382d81b339c1817d37ec93c6", + "sha256:c8716a48d94b06bb3b2524c2b77e055fb313aeb4ea620c8dd03a105574ba704f", + "sha256:cd5df75523866410809ca100dc9681e301e3c27567cf498077e8551b6d20e42f", + "sha256:cdb132fc825c38e1aeec2c8aa9338310d29d337bebbd7baa06889d09a60a1fa2", + "sha256:e249096428b3ae81b08327a63a485ad0878de3fb939049038579ac0ef61e17e7", + "sha256:e8313f01ba26fbbe36c7be1966a7b7424942f670f38e666995b88d012765b9be" ], - "version": "==1.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.1" }, "mccabe": { "hashes": [ @@ -407,352 +453,436 @@ }, "mock": { "hashes": [ - "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", - "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" + "sha256:83657d894c90d5681d62155c82bda9c1187827525880eda8ff5df4ec813437c3", + "sha256:d157e52d4e5b938c550f39eb2fd15610db062441a9c2747d3dbfa9298211d0f8" ], - "index": "pypi", - "version": "==2.0.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==3.0.5" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", + "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", + "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" ], - "version": "==4.3.0" + "version": "==5.0.0" + }, + "orderedmultidict": { + "hashes": [ + "sha256:24e3b730cf84e4a6a68be5cc760864905cf66abc89851e724bd5b4e849eaa96b", + "sha256:b89895ba6438038d0bdf88020ceff876cf3eae0d5c66a69b526fab31125db2c5" + ], + "version": "==1.0" }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", + "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" ], - "version": "==18.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.3" }, "parso": { "hashes": [ - "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2", - "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24" + "sha256:158c140fc04112dc45bca311633ae5033c2c2a7b732fa33d0955bad8152a8dd0", + "sha256:908e9fae2144a076d72ae4e25539143d40b8e3eafbaeae03c1bfe226f4cdf12c" ], - "version": "==0.3.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.7.0" }, "parver": { "hashes": [ - "sha256:ac4afff688d19d5e1876bb68d4bccc1a1b6a5cc8bd6a646939a14d366695ba15", - "sha256:f025fba8f88a9c776971df6d62b6cf7f37d1108f84c163bda91e157d7d527075" + "sha256:b57d94e6f389f9db399bfc3ee4c4066f4cfb374ffef5727d5ae6a9c04eb8d228", + "sha256:bb9d19637c17819e276b5cf04e2dbfb81c4e2136da8873cc70dcd0e4fd3d14a3" ], - "index": "pypi", - "version": "==0.1.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.3.0" }, "passa": { - "editable": true, "git": "https://github.com/sarugaku/passa.git", - "ref": "4f3b8102f122cf0b75e5d7c513a2e61b0b093dcd" + "ref": "2ac00f16cd5a8f07d679a3ab02b7cc13c6f42bee", + "version": "==0.3.1.dev0" + }, + "pathlib2": { + "hashes": [ + "sha256:0ec8205a157c80d7acc301c0b18fbd5d44fe655968f5d947b6ecef5290fc35db", + "sha256:6cd9a47b597b37cc57de1c05e56fb1a1c9cc9fab04fe78c29acd090418529868" + ], + "markers": "python_version < '3'", + "version": "==2.3.5" + }, + "pathspec": { + "hashes": [ + "sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424", + "sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96" + ], + "version": "==0.7.0" }, "pbr": { "hashes": [ - "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", - "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" + "sha256:07f558fece33b05caf857474a366dfcc00562bca13dd8b47b2b3e22d9f9bf55c", + "sha256:579170e23f8e0c2f24b0de612f71f648eccb79fb1322c814ae6b3c07b5ba23e8" ], - "version": "==5.1.1" - }, - "pep517": { - "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" - ], - "version": "==0.3" - }, - "pip-shims": { - "hashes": [ - "sha256:3bc24ec050a6b9eea35419467237e4f47eaf806dadc9999bf887355c377edea7", - "sha256:edb4cf3c509eab2f36b55c1ac1a59a4c485ccd537cc87934d74950880f641256" - ], - "version": "==0.3.2" + "version": "==5.4.5" }, "pipenv": { "editable": true, + "extras": [ + "dev", + "tests" + ], "path": "." }, "pkginfo": { "hashes": [ - "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", - "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" ], - "version": "==1.4.2" - }, - "plette": { - "extras": [ - "validation" - ], - "hashes": [ - "sha256:c0e3553c1e581d8423daccbd825789c6e7f29b7d9e00e5331b12e1642a1a26d3", - "sha256:dde5d525cf5f0cbad4d938c83b93db17887918daf63c13eafed257c4f61b07b4" - ], - "version": "==0.2.2" + "version": "==1.5.0.1" }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", + "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], - "version": "==0.8.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.13.1" }, "py": { "hashes": [ - "sha256:bf92637198836372b520efcba9e020c330123be8ce527e535d185ed4b6f45694", - "sha256:e76826342cefe3c3d5f7e8ee4316b80d1dd8a300781612ddbc765c17ba25a6c6" + "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", + "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" ], - "version": "==1.7.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.8.1" }, "pycodestyle": { "hashes": [ - "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" ], - "version": "==2.4.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.5.0" }, "pycparser": { "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" + "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0", + "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705" ], - "version": "==2.19" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.20" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:17dbeb2e3f4d772725c777fabc446d5634d1038f234e77343108ce445ea69ce0", + "sha256:d976835886f8c5b31d47970ed689944a0262b5f3afa00a5a7b4dc81e5449f8a2" ], - "version": "==2.0.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.1.1" }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:2a3fe295e54a20164a9df49c75fa58526d3be48e14aceba6d6b1e8ac0bfd6f1b", + "sha256:98c8aa5a9f778fcd1026a17361ddaf7330d1b7c62ae97c3bb0ae73e0b9b6b0fe" ], - "version": "==2.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.5.2" }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b", + "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1" ], - "version": "==2.3.0" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.4.7" }, "pytest": { "hashes": [ - "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", - "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" + "sha256:19e8f75eac01dd3f211edd465b39efbcbdc8fc5f7866d7dd49fedb30d8adf339", + "sha256:c77a5f30a90e0ce24db9eaa14ddfd38d4afb5ea159309bdd2dae55b931bc9324" ], - "index": "pypi", - "version": "==3.8.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.6.9" }, "pytest-forked": { "hashes": [ - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" + "sha256:1805699ed9c9e60cb7a8179b8d4fa2b8898098e82d229b0825d8095f0f261100", + "sha256:1ae25dba8ee2e56fb47311c9638f9e58552691da87e82d25b0ce0e4bf52b7d87" ], - "version": "==0.2" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.3" }, - "pytest-pypy": { + "pytest-pypi": { "editable": true, "path": "./tests/pytest-pypi" }, - "pytest-tap": { + "pytest-timeout": { "hashes": [ - "sha256:3b05ec931424bbe44e944726b68f7ef185bb6d25ce9ce21ac52c9af7ffa9b506", - "sha256:ca063de56298034302f3cbce55c87a27d7bfa7af7de591cdb9ec6ce45fea5467" + "sha256:80faa19cd245a42b87a51699d640c00d937c02b749052bfca6bae8bdbe12c48e", + "sha256:95ca727d4a1dace6ec5f0534d2940b8417ff8b782f7eef0ea09240bdd94d95c2" ], - "index": "pypi", - "version": "==2.3" + "version": "==1.3.4" }, "pytest-xdist": { "hashes": [ - "sha256:06aa39361694c9365baaa03bec71159b59ad06c9826c6279ebba368cb3571561", - "sha256:1ef0d05c905cfa0c5442c90e9e350e65c6ada120e33a00a066ca51c89f5f869a" + "sha256:0f46020d3d9619e6d17a65b5b989c1ebbb58fc7b1da8fb126d70f4bac4dfeed1", + "sha256:7dc0d027d258cd0defc618fb97055fbd1002735ca7a6d17037018cf870e24011" ], - "index": "pypi", - "version": "==1.23.2" - }, - "pytoml": { - "hashes": [ - "sha256:ca2d0cb127c938b8b76a9a0d0f855cf930c1d50cc3a0af6d3595b566519a1013" - ], - "version": "==0.1.20" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.31.0" }, "pytz": { "hashes": [ - "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", - "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + "sha256:1c557d7d0e871de1f5ccd5833f60fb2550652da6be2693c1e02300743d21500d", + "sha256:b02c06db6cf09c12dd25137e563b31700d3b80fcc4ad23abb7a315f2789819be" ], - "index": "pypi", - "version": "==2018.5" + "version": "==2019.3" }, "readme-renderer": { "hashes": [ - "sha256:bb16f55b259f27f75f640acf5e00cf897845a8b3e4731b5c1a436e4b8529202f", - "sha256:c8532b79afc0375a85f10433eca157d6b50f7d6990f337fa498c96cd4bfc203d" + "sha256:1b6d8dd1673a0b293766b4106af766b6eff3654605f9c4f239e65de6076bc222", + "sha256:e67d64242f0174a63c3b727801a2fff4c1f38ebe5d71d95ff7ece081945a6cd4" ], - "version": "==24.0" + "version": "==25.0" + }, + "regex": { + "hashes": [ + "sha256:01b2d70cbaed11f72e57c1cfbaca71b02e3b98f739ce33f5f26f71859ad90431", + "sha256:046e83a8b160aff37e7034139a336b660b01dbfe58706f9d73f5cdc6b3460242", + "sha256:113309e819634f499d0006f6200700c8209a2a8bf6bd1bdc863a4d9d6776a5d1", + "sha256:200539b5124bc4721247a823a47d116a7a23e62cc6695744e3eb5454a8888e6d", + "sha256:25f4ce26b68425b80a233ce7b6218743c71cf7297dbe02feab1d711a2bf90045", + "sha256:269f0c5ff23639316b29f31df199f401e4cb87529eafff0c76828071635d417b", + "sha256:5de40649d4f88a15c9489ed37f88f053c15400257eeb18425ac7ed0a4e119400", + "sha256:7f78f963e62a61e294adb6ff5db901b629ef78cb2a1cfce3cf4eeba80c1c67aa", + "sha256:82469a0c1330a4beb3d42568f82dffa32226ced006e0b063719468dcd40ffdf0", + "sha256:8c2b7fa4d72781577ac45ab658da44c7518e6d96e2a50d04ecb0fd8f28b21d69", + "sha256:974535648f31c2b712a6b2595969f8ab370834080e00ab24e5dbb9d19b8bfb74", + "sha256:99272d6b6a68c7ae4391908fc15f6b8c9a6c345a46b632d7fdb7ef6c883a2bbb", + "sha256:9b64a4cc825ec4df262050c17e18f60252cdd94742b4ba1286bcfe481f1c0f26", + "sha256:9e9624440d754733eddbcd4614378c18713d2d9d0dc647cf9c72f64e39671be5", + "sha256:9ff16d994309b26a1cdf666a6309c1ef51ad4f72f99d3392bcd7b7139577a1f2", + "sha256:b33ebcd0222c1d77e61dbcd04a9fd139359bded86803063d3d2d197b796c63ce", + "sha256:bba52d72e16a554d1894a0cc74041da50eea99a8483e591a9edf1025a66843ab", + "sha256:bed7986547ce54d230fd8721aba6fd19459cdc6d315497b98686d0416efaff4e", + "sha256:c7f58a0e0e13fb44623b65b01052dae8e820ed9b8b654bb6296bc9c41f571b70", + "sha256:d58a4fa7910102500722defbde6e2816b0372a4fcc85c7e239323767c74f5cbc", + "sha256:f1ac2dc65105a53c1c2d72b1d3e98c2464a133b4067a51a3d2477b28449709a0" + ], + "version": "==2020.2.20" }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:43999036bfa82904b6af1d99e4882b560e5e2c68e5c4b0aa03b655f3d7d73fee", + "sha256:b3f43d496c6daba4493e7c431722aeb7dbc6288f52a6e04e7b6023b0247817e6" ], - "version": "==2.20.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==2.23.0" }, "requests-toolbelt": { "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" ], - "version": "==0.8.0" + "version": "==0.9.1" }, - "requirementslib": { + "retry": { "hashes": [ - "sha256:c2c00c7bd3bd4984c97d10cd4d143efbe33b5ed9e55961bea30ca7a9a4927289", - "sha256:dc6b692e8dee03d6e90c29db1e337b0bf8152cce84a57f0fb4765e596afde4e0" + "sha256:ccddf89761fa2c726ab29391837d4327f819ea14d244c232a1d24c67a2f98606", + "sha256:f8bfa8b99b69c4506d6f5bd3b0aabf77f98cdb17f3c9fc3f5ca820033336fba4" ], - "version": "==1.3.3" - }, - "resolvelib": { - "hashes": [ - "sha256:6c4c6690b0bdd78bcc002e1a5d1b6abbde58c694a6ea1838f165b20d2c943db7", - "sha256:8734e53271ef98f38a2c99324d5e7905bc00c97dc3fc5bb7d83c82a979e71c04" - ], - "version": "==0.2.2" + "version": "==0.9.2" }, "rope": { "hashes": [ - "sha256:a108c445e1cd897fe19272ab7877d172e7faf3d4148c80e7d20faba42ea8f7b2" + "sha256:52423a7eebb5306a6d63bdc91a7c657db51ac9babfb8341c9a1440831ecf3203", + "sha256:ae1fa2fd56f64f4cc9be46493ce54bed0dd12dee03980c61a4393d89d84029ad", + "sha256:d2830142c2e046f5fc26a022fe680675b6f48f81c7fc1f03a950706e746e9dfe" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.16.0" + }, + "scandir": { + "hashes": [ + "sha256:2586c94e907d99617887daed6c1d102b5ca28f1085f90446554abf1faf73123e", + "sha256:2ae41f43797ca0c11591c0c35f2f5875fa99f8797cb1a1fd440497ec0ae4b022", + "sha256:2b8e3888b11abb2217a32af0766bc06b65cc4a928d8727828ee68af5a967fa6f", + "sha256:2c712840c2e2ee8dfaf36034080108d30060d759c7b73a01a52251cc8989f11f", + "sha256:4d4631f6062e658e9007ab3149a9b914f3548cb38bfb021c64f39a025ce578ae", + "sha256:67f15b6f83e6507fdc6fca22fedf6ef8b334b399ca27c6b568cbfaa82a364173", + "sha256:7d2d7a06a252764061a020407b997dd036f7bd6a175a5ba2b345f0a357f0b3f4", + "sha256:8c5922863e44ffc00c5c693190648daa6d15e7c1207ed02d6f46a8dcc2869d32", + "sha256:92c85ac42f41ffdc35b6da57ed991575bdbe69db895507af88b9f499b701c188", + "sha256:b24086f2375c4a094a6b51e78b4cf7ca16c721dcee2eddd7aa6494b42d6d519d", + "sha256:cb925555f43060a1745d0a321cca94bcea927c50114b623d73179189a4e100ac" + ], + "markers": "python_version < '3.5'", + "version": "==1.10.0" + }, + "singledispatch": { + "hashes": [ + "sha256:5b06af87df13818d14f08a028e42f566640aef80805c3b50c5056b086e3c2b9c", + "sha256:833b46966687b3de7f438c761ac475213e53b306740f1abfaa86e1d1aae56aa8" + ], + "markers": "python_version < '3.4'", + "version": "==3.4.0.3" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", + "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" ], - "version": "==1.11.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.14.0" }, "snowballstemmer": { "hashes": [ - "sha256:919f26a68b2c17a7634da993d91339e288964f93c274f1343e3bbbe2096e1128", - "sha256:9f3bcd3c401c3e862ec0ebe6d2c069ebc012ce142cce209c098ccb5b09136e89" + "sha256:209f257d7533fdb3cb73bdbd24f436239ca3b2fa67d56f6ff88e86be08cc5ef0", + "sha256:df3bac3df4c2c01363f3dd2cfa78cce2840a79b9f1c2d2de9ce8d31683992f52" ], - "version": "==1.2.1" + "version": "==2.0.0" + }, + "soupsieve": { + "hashes": [ + "sha256:bdb0d917b03a1369ce964056fc195cfdff8819c40de04695a80bc813c3cfa1f5", + "sha256:e2c1c5dee4a1c36bcb790e0fabd5492d874b8ebd4617622c4f6a731701060dda" + ], + "version": "==1.9.5" }, "sphinx": { "hashes": [ - "sha256:11f271e7a9398385ed730e90f0bb41dc3815294bdcd395b46ed2d033bc2e7d87", - "sha256:4064ea6c56feeb268838cb8fbbee507d0c3d5d92fa63a7df935a916b52c9e2f5" + "sha256:9f3e17c64b34afc653d7c5ec95766e03043cc6d80b0de224f59b6b6e19d37c3c", + "sha256:c7658aab75c920288a8cf6f09f244c6cfdae30d82d803ac1634d9f223a80ca08" ], - "index": "pypi", - "version": "==1.5.5" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.8.5" }, "sphinx-click": { "hashes": [ - "sha256:0550d3e5dcd6244847bd0861ebe64101a2ef302913866e0ccd9095b2aa230051", - "sha256:404784f724504e3da2cb056767ba64955c4bfb9bfca8cfedd7142a962bafd70f" + "sha256:06952d5de6cbe2cb7d6dc656bc471652d2b484cf1e1b2d65edb7f4f2e867c7f6", + "sha256:1b649ebe9f7a85b78ef6545d1dc258da5abca850ac6375be104d484a6334a728" ], "index": "pypi", - "version": "==1.3.0" + "version": "==2.3.2" }, "sphinxcontrib-websupport": { "hashes": [ - "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", - "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" + "sha256:1501befb0fdf1d1c29a800fdbf4ef5dc5369377300ddbdd16d2cd40e54c6eefc", + "sha256:e02f717baf02d0b6c3dd62cf81232ffca4c9d5c331e03766982e3ff9f1d2bc3f" ], - "version": "==1.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.1.2" }, "stdeb": { "hashes": [ - "sha256:0ed2c2cc6b8ba21da7d646c6f37ca60b22e9e4950e3cec6bcd9c2e7e57e3747e" + "sha256:4d8351209dda2d26066980222e0d1855a315a68f9af48f0c10d743089afe7d4b" ], - "index": "pypi", "markers": "sys_platform == 'linux'", - "version": "==0.8.5" + "version": "==0.9.0" }, - "tap.py": { + "termcolor": { "hashes": [ - "sha256:8ad62ba6898fcef4913c67d468d0c4beae3109b74c03363538145e31b1840b29", - "sha256:f6532fd7483c5fdc2ed13575fa4494e7d037f797f8a2c6f8809a859be61271f5" + "sha256:1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b" ], - "version": "==2.5" + "version": "==1.1.0" }, "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", + "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" }, - "tomlkit": { - "hashes": [ - "sha256:d6506342615d051bc961f70bfcfa3d29b6616cc08a3ddfd4bc24196f16fd4ec2", - "sha256:f077456d35303e7908cc233b340f71e0bec96f63429997f38ca9272b7d64029e" - ], - "version": "==0.5.3" - }, "towncrier": { - "editable": true, - "git": "https://github.com/hawkowl/towncrier.git", - "ref": "47754a607a9b03f06affaf167d65b990786aae25" + "hashes": [ + "sha256:48251a1ae66d2cf7e6fa5552016386831b3e12bb3b2d08eb70374508c17a8196", + "sha256:de19da8b8cb44f18ea7ed3a3823087d2af8fcf497151bb9fd1e1b092ff56ed8d" + ], + "version": "==19.2.0" }, "tqdm": { "hashes": [ - "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", - "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + "sha256:00339634a22c10a7a22476ee946bbde2dbe48d042ded784e4d88e0236eca5d81", + "sha256:ea9e3fd6bd9a37e8783d75bfc4c1faf3c6813da6bd1c3e776488b41ec683af94" ], - "version": "==4.28.1" + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==4.45.0" }, "twine": { "hashes": [ - "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", - "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" + "sha256:630fadd6e342e725930be6c696537e3f9ccc54331742b16245dab292a17d0460", + "sha256:a3d22aab467b4682a22de4a422632e79d07eebd07ff2a7079effb13f8a693787" ], - "index": "pypi", - "version": "==1.12.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.15.0" + }, + "typed-ast": { + "hashes": [ + "sha256:0666aa36131496aed8f7be0410ff974562ab7eeac11ef351def9ea6fa28f6355", + "sha256:0c2c07682d61a629b68433afb159376e24e5b2fd4641d35424e462169c0a7919", + "sha256:249862707802d40f7f29f6e1aad8d84b5aa9e44552d2cc17384b209f091276aa", + "sha256:24995c843eb0ad11a4527b026b4dde3da70e1f2d8806c99b7b4a7cf491612652", + "sha256:269151951236b0f9a6f04015a9004084a5ab0d5f19b57de779f908621e7d8b75", + "sha256:4083861b0aa07990b619bd7ddc365eb7fa4b817e99cf5f8d9cf21a42780f6e01", + "sha256:498b0f36cc7054c1fead3d7fc59d2150f4d5c6c56ba7fb150c013fbc683a8d2d", + "sha256:4e3e5da80ccbebfff202a67bf900d081906c358ccc3d5e3c8aea42fdfdfd51c1", + "sha256:6daac9731f172c2a22ade6ed0c00197ee7cc1221aa84cfdf9c31defeb059a907", + "sha256:715ff2f2df46121071622063fc7543d9b1fd19ebfc4f5c8895af64a77a8c852c", + "sha256:73d785a950fc82dd2a25897d525d003f6378d1cb23ab305578394694202a58c3", + "sha256:8c8aaad94455178e3187ab22c8b01a3837f8ee50e09cf31f1ba129eb293ec30b", + "sha256:8ce678dbaf790dbdb3eba24056d5364fb45944f33553dd5869b7580cdbb83614", + "sha256:aaee9905aee35ba5905cfb3c62f3e83b3bec7b39413f0a7f19be4e547ea01ebb", + "sha256:bcd3b13b56ea479b3650b82cabd6b5343a625b0ced5429e4ccad28a8973f301b", + "sha256:c9e348e02e4d2b4a8b2eedb48210430658df6951fa484e59de33ff773fbd4b41", + "sha256:d205b1b46085271b4e15f670058ce182bd1199e56b317bf2ec004b6a44f911f6", + "sha256:d43943ef777f9a1c42bf4e552ba23ac77a6351de620aa9acf64ad54933ad4d34", + "sha256:d5d33e9e7af3b34a40dc05f498939f0ebf187f07c385fd58d591c533ad8562fe", + "sha256:fc0fea399acb12edbf8a628ba8d2312f583bdbdb3335635db062fa98cf71fca4", + "sha256:fe460b922ec15dd205595c9b5b99e2f056fd98ae8f9f56b888e7a17dc2b757e7" + ], + "markers": "python_version >= '3.4'", + "version": "==1.4.1" }, "typing": { "hashes": [ - "sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf", - "sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8", - "sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2" + "sha256:91dfe6f3f706ee8cc32d38edbbf304e9b7583fb37108fef38229617f8b3eba23", + "sha256:c8cabb5ab8945cd2f54917be357d134db9cc1eb039e59d1606dc1e60cb1d9d36", + "sha256:f38d83c5a7a7086543a0f649564d661859c5146a85775ab90c0d2f93ffaa9714" ], "markers": "python_version < '3.5'", - "version": "==3.6.4" + "version": "==3.7.4.1" }, "urllib3": { "hashes": [ - "sha256:61bf29cada3fc2fbefad4fdf059ea4bd1b4a86d2b6d15e1c7c0b582b9752fe39", - "sha256:de9529817c93f27c8ccbfead6985011db27bd0ddfcdb2d86f3f663385c6a9c22" + "sha256:88206b0eb87e6d677d424843ac5209e3fb9d0190d0ee169599165ec25e9d9115", + "sha256:3018294ebefce6572a474f0604c2021e33b3fd8006ecd11d62107a5d2a963527" ], - "version": "==1.24.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3' and python_version < '4'", + "version": "==1.25.9" }, "virtualenv": { "hashes": [ - "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", - "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" + "sha256:55059a7a676e4e19498f1aad09b8313a38fcc0cdbe4fdddc0e9b06946d21b4bb", + "sha256:0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3" ], - "version": "==16.1.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==16.7.9" }, "virtualenv-clone": { "hashes": [ - "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" + "sha256:07e74418b7cc64f4fda987bf5bc71ebd59af27a7bc9e8a8ee9fd54b1f2390a27", + "sha256:665e48dd54c84b98b71a657acb49104c54e7652bce9c1c4f6c6976ed4c827a29" ], - "version": "==0.4.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==0.5.4" }, - "vistir": { - "extras": [ - "spinner" - ], + "wcwidth": { "hashes": [ - "sha256:3a1020fb7be000b268af96641ced9ead844b1f75840c41e20e473647688fc630", - "sha256:6d2005ad670f77bd9c9b5415c4e2a4a20dce5b0cf0e0d11598eb463b2e0ebe44" + "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1", + "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" ], - "version": "==0.2.5" + "version": "==0.1.9" }, "webencodings": { "hashes": [ @@ -763,24 +893,19 @@ }, "werkzeug": { "hashes": [ - "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", - "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + "sha256:2de2a5db0baeae7b2d2664949077c2ac63fbd16d98da0ff71837f7d1dea3fd43", + "sha256:6c80b1e5ad3665290ea39320b91e1be1e0d5f60652b964a3070216de83d2e47c" ], - "version": "==0.14.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.0.1" }, - "wheel": { + "zipp": { "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + "sha256:c70410551488251b0fee67b460fb9a536af8d6f9f008ad10ac51f615b6a521b1", + "sha256:e0d9e63797e483a30d27e09fffd308c59a700d365ec34e93cc100844168bf921" ], - "version": "==0.32.3" - }, - "yaspin": { - "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" - ], - "version": "==0.14.0" + "markers": "python_version < '3.8'", + "version": "==1.2.0" } } } diff --git a/README.md b/README.md index c3a46889..f22c6776 100644 --- a/README.md +++ b/README.md @@ -3,11 +3,8 @@ Pipenv: Python Development Workflow for Humans [![image](https://img.shields.io/pypi/v/pipenv.svg)](https://python.org/pypi/pipenv) [![image](https://img.shields.io/pypi/l/pipenv.svg)](https://python.org/pypi/pipenv) -[![image](https://badge.buildkite.com/79c7eccf056b17c3151f3c4d0e4c4b8b724539d84f1e037b9b.svg?branch=master)](https://code.kennethreitz.org/source/pipenv/) -[![Azure Pipelines build status (Linux)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Linux)?branchName=master&label=Linux)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=13&branchName=master) -[![Azure Pipelines build status (Windows)](https://dev.azure.com/pypa/pipenv/_apis/build/status/pipenv%20CI%20(Windows)?branchName=master&label=Windows)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=12&branchName=master) +[![Azure Pipelines Build Status](https://dev.azure.com/pypa/pipenv/_apis/build/status/Pipenv%20CI?branchName=master)](https://dev.azure.com/pypa/pipenv/_build/latest?definitionId=16&branchName=master) [![image](https://img.shields.io/pypi/pyversions/pipenv.svg)](https://python.org/pypi/pipenv) -[![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz) ------------------------------------------------------------------------ @@ -20,7 +17,7 @@ well as adds/removes packages from your `Pipfile` as you install/uninstall packages. It also generates the ever-important `Pipfile.lock`, which is used to produce deterministic builds. -![image](http://media.kennethreitz.com.s3.amazonaws.com/pipenv.gif) +![GIF demonstrating Pipenv's usage](https://gist.githubusercontent.com/jlusk/855d611bbcfa2b159839db73d07f6ce9/raw/7f5743401809f7e630ee8ff458faa980e19924a0/pipenv.gif) The problems that Pipenv seeks to solve are multi-faceted: @@ -46,23 +43,29 @@ If you\'re on MacOS, you can install Pipenv easily with Homebrew: $ brew install pipenv -Or, if you\'re using Fedora 28: +Or, if you\'re using Debian Buster+: + + $ sudo apt install pipenv + +Or, if you\'re using Fedora: $ sudo dnf install pipenv + +Or, if you\'re using FreeBSD: -Otherwise, refer to the [documentation](https://docs.pipenv.org/install/) for instructions. + # pkg install py36-pipenv + +When none of the above is an option: + + $ pip install pipenv + +Otherwise, refer to the [documentation](https://pipenv.pypa.io/en/latest/#install-pipenv-today) for instructions. ✨🍰✨ ☤ User Testimonials ------------------- -**Jannis Leidel**, former pip maintainer--- - -: *Pipenv is the porcelain I always wanted to build for pip. It fits - my brain and mostly replaces virtualenvwrapper and manual pip calls - for me. Use it.* - **David Gang**--- : *This package manager is really awesome. For the first time I know @@ -297,4 +300,4 @@ Use the shell: ☤ Documentation --------------- -Documentation resides over at [pipenv.org](http://pipenv.org/). +Documentation resides over at [pipenv.pypa.io](https://pipenv.pypa.io/en/latest/). diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..021b4200 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,122 @@ +name: CI +trigger: + batch: true + branches: + include: + - master + paths: + exclude: + - docs/* + - news/* + - peeps/* + - examples/* + - pytest.ini + - README.md + - pipenv/*.txt + - CHANGELOG.rst + - CONTRIBUTING.md + - CODE_OF_CONDUCT.md + - .gitignore + - .gitattributes + - .editorconfig + +variables: +- group: CI + +jobs: +- job: TestVendoring + displayName: Vendoring + pool: + vmImage: 'Ubuntu-latest' + variables: + python.version: '3.7' + python.architecture: x64 + steps: + - template: .azure-pipelines/steps/run-vendor-scripts.yml + parameters: + vmImage: 'Ubuntu-latest' + +- job: TestPackaging + displayName: Packaging + pool: + vmImage: 'Ubuntu-latest' + variables: + python.version: '3.7' + python.architecture: x64 + steps: + - template: .azure-pipelines/steps/build-package.yml + parameters: + vmImage: 'Ubuntu-latest' + +- job: TestLinux + displayName: Linux / + pool: + vmImage: 'Ubuntu-latest' + strategy: + matrix: + "2.7": + python.version: '2.7' + python.architecture: x64 + "3.6": + python.version: '3.6' + python.architecture: x64 + "3.7": + python.version: '3.7' + python.architecture: x64 + "3.8": + python.version: '3.8' + python.architecture: x64 + maxParallel: 8 + steps: + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: 'Ubuntu-latest' + +- job: TestWindows + displayName: Windows / + timeoutInMinutes: 0 + pool: + vmImage: windows-latest + strategy: + matrix: + "2.7": + python.version: '2.7' + python.architecture: x64 + "3.6": + python.version: '3.6' + python.architecture: x64 + "3.7": + python.version: '3.7' + python.architecture: x64 + "3.8": + python.version: '3.8' + python.architecture: x64 + maxParallel: 8 + steps: + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: windows-latest + +- job: TestMacOS + displayName: MacOS / + pool: + vmImage: macOS-latest + strategy: + matrix: + "2.7": + python.version: '2.7' + python.architecture: x64 + "3.6": + python.version: '3.6' + python.architecture: x64 + "3.7": + python.version: '3.7' + python.architecture: x64 + "3.8": + python.version: '3.8' + python.architecture: x64 + maxParallel: 8 + steps: + - template: .azure-pipelines/steps/run-tests.yml + parameters: + vmImage: macOS-latest diff --git a/docs/_templates/hacks.html b/docs/_templates/hacks.html index 0ec542fa..fa7d60ed 100644 --- a/docs/_templates/hacks.html +++ b/docs/_templates/hacks.html @@ -18,7 +18,6 @@ /* Remain Responsive! */ @media screen and (max-width: 1008px) { - div.sphinxsidebar {display: none;} div.document {width: 100%!important;} /* Have code blocks escape the document right-margin. */ @@ -60,7 +59,7 @@ - + @@ -8,21 +8,37 @@

+ + + +

Pipenv is a production-ready tool that aims to bring the best of all packaging worlds to the Python world. It harnesses Pipfile, pip, and virtualenv into one single command.

It features very pretty terminal colors.

-

Stay Informed

Receive updates on new releases and upcoming projects.

-

- -

Follow @kennethreitz

-

Say Thanks!

-

Join Mailing List.

+

Follow @ThePyPA

+

Join Mailing List.

Other Projects

@@ -30,7 +46,7 @@
  • Pipenv-Pipes
    • -

    More Kenneth Reitz projects:

    +

    More projects founded by Kenneth Reitz:

    @@ -50,12 +66,11 @@

    -
  • Pipenv @ GitHub
    • -
  • Pipenv @ PyPI
    • +
  • Pipenv @ GitHub
    • +
  • Pipenv @ PyPI
  • Pipenv PPA (PyPA)
    • -
  • Issue Tracker
    • +
  • Issue Tracker
    • -
  • 日本語
    • +
  • 日本語
  • Español
    • - diff --git a/docs/_templates/sidebarlogo.html b/docs/_templates/sidebarlogo.html index 5107e4de..59e8641e 100644 --- a/docs/_templates/sidebarlogo.html +++ b/docs/_templates/sidebarlogo.html @@ -8,6 +8,27 @@

    + + + +

    Pipenv is a production-ready tool that aims to bring the best of all packaging worlds to the Python world. It harnesses Pipfile, pip, and virtualenv into one single command. @@ -17,12 +38,11 @@

    Stay Informed

    Receive updates on new releases and upcoming projects.

    -

    -

    Follow @kennethreitz

    -

    Say Thanks!

    -

    Join Mailing List.

    +

    Follow @ThePyPA

    +

    Join Mailing List.

    Other Projects

    @@ -30,7 +50,7 @@
  • Pipenv-Pipes
    • -

    More Kenneth Reitz projects:

    +

    More projects founded by Kenneth Reitz:

    @@ -51,11 +71,10 @@

  • Pipenv @ GitHub
    • -
  • Pipenv @ PyPI
    • +
  • Pipenv @ PyPI
  • Pipenv PPA (PyPA)
    • -
  • Issue Tracker
    • +
  • Issue Tracker
  • 日本語
  • Español
    • - diff --git a/docs/advanced.rst b/docs/advanced.rst index 3ac323e6..e6e31f62 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -20,7 +20,7 @@ This document covers some of Pipenv's more glorious and advanced features. If you'd like a specific package to be installed with a specific package index, you can do the following:: [[source]] - url = "https://pypi.python.org/simple" + url = "https://pypi.org/simple" verify_ssl = true name = "pypi" @@ -71,6 +71,11 @@ Luckily - pipenv will hash your Pipfile *before* expanding environment variables (and, helpfully, will substitute the environment variables again when you install from the lock file - so no need to commit any secrets! Woo!) +If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a ``ValueError, "No closing quotation"`` error while installing dependencies. :: + + [[source]] + url = "https://$USERNAME:'${PASSWORD}'@mypypi.example.com/simple" + ☤ Specifying Basically Anything ------------------------------- @@ -339,6 +344,21 @@ If a ``.env`` file is present in your project, ``$ pipenv shell`` and ``$ pipenv >>> os.environ['HELLO'] 'WORLD' +Shell like variable expansion is available in ``.env`` files using `${VARNAME}` syntax.:: + + $ cat .env + CONFIG_PATH=${HOME}/.config/foo + + $ pipenv run python + Loading .env environment variables… + Python 3.7.6 (default, Dec 19 2019, 22:52:49) + [GCC 9.2.1 20190827 (Red Hat 9.2.1-1)] on linux + Type "help", "copyright", "credits" or "license" for more information. + >>> import os + >>> os.environ['CONFIG_PATH'] + '/home/kennethreitz/.config/foo' + + This is very useful for keeping production credentials out of your codebase. We do not recommend committing ``.env`` files into source control! @@ -350,15 +370,19 @@ To prevent pipenv from loading the ``.env`` file, set the ``PIPENV_DONT_LOAD_ENV $ PIPENV_DONT_LOAD_ENV=1 pipenv shell +See `theskumar/python-dotenv `_ for more information on ``.env`` files. + ☤ Custom Script Shortcuts ------------------------- -Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. +Pipenv supports creating custom shortcuts in the (optional) ``[scripts]`` section of your Pipfile. You can then run ``pipenv run `` in your terminal to run the command in the -context of your pipenv virtual environment even if you have not activated the pipenv shell first. +context of your pipenv virtual environment even if you have not activated the pipenv shell first. -For example, in your Pipfile:: +For example, in your Pipfile: + +.. code-block:: toml [scripts] printspam = "python -c \"print('I am a silly example, no one would need to do this')\"" @@ -369,18 +393,26 @@ And then in your terminal:: I am a silly example, no one would need to do this Commands that expect arguments will also work. -For example:: +For example: + +.. code-block:: toml [scripts] echospam = "echo I am really a very silly example" +:: + $ pipenv run echospam "indeed" I am really a very silly example indeed ☤ Support for Environment Variables ----------------------------------- -Pipenv supports the usage of environment variables in values. For example:: +Pipenv supports the usage of environment variables in place of authentication fragments +in your Pipfile. These will only be parsed if they are present in the ``[[source]]`` +section. For example: + +.. code-block:: toml [[source]] url = "https://${PYPI_USERNAME}:${PYPI_PASSWORD}@my_private_repo.example.com/simple" @@ -395,8 +427,10 @@ Pipenv supports the usage of environment variables in values. For example:: records = "*" Environment variables may be specified as ``${MY_ENVAR}`` or ``$MY_ENVAR``. + On Windows, ``%MY_ENVAR%`` is supported in addition to ``${MY_ENVAR}`` or ``$MY_ENVAR``. +.. _configuration-with-environment-variables: ☤ Configuration With Environment Variables ------------------------------------------ @@ -467,7 +501,7 @@ and the corresponding Makefile:: pipenv install --dev test: - pipenv run py.test tests + pipenv run pytest tests Tox Automation Project @@ -483,7 +517,7 @@ and external testing:: deps = pipenv commands= pipenv install --dev - pipenv run py.test tests + pipenv run pytest tests [testenv:flake8-py3] basepython = python3.4 @@ -492,7 +526,7 @@ and external testing:: pipenv run flake8 --version pipenv run flake8 setup.py docs project test -Pipenv will automatically use the virtualenv provided by ``tox``. If ``pipenv install --dev`` installs e.g. ``pytest``, then installed command ``py.test`` will be present in given virtualenv and can be called directly by ``py.test tests`` instead of ``pipenv run py.test tests``. +Pipenv will automatically use the virtualenv provided by ``tox``. If ``pipenv install --dev`` installs e.g. ``pytest``, then installed command ``pytest`` will be present in given virtualenv and can be called directly by ``pytest tests`` instead of ``pipenv run pytest tests``. You might also want to add ``--ignore-pipfile`` to ``pipenv install``, as to not accidentally modify the lock-file on each test run. This causes Pipenv @@ -508,7 +542,7 @@ probably a good idea in any case. A 3rd party plugin, `tox-pipenv`_ is also available to use Pipenv natively with tox. -.. _Requests: https://github.com/kennethreitz/requests +.. _Requests: https://github.com/psf/requests .. _tox: https://tox.readthedocs.io/en/latest/ .. _tox-pipenv: https://tox-pipenv.readthedocs.io/en/latest/ .. _Travis-CI: https://travis-ci.org/ diff --git a/docs/basics.rst b/docs/basics.rst index 00f4c7d9..694fb0fa 100644 --- a/docs/basics.rst +++ b/docs/basics.rst @@ -10,6 +10,12 @@ This document covers some of Pipenv's more basic features. ☤ Example Pipfile & Pipfile.lock -------------------------------- +Pipfiles contain information for the dependencies of the project, and supersedes +the requirements.txt file used in most Python projects. You should add a Pipfile in the +Git repository letting users who clone the repository know the only thing required would be +installing Pipenv in the machine and typing ``pipenv install``. Pipenv is a reference +implementation for using Pipfile. + .. _example_files: Here is a simple example of a ``Pipfile`` and the resulting ``Pipfile.lock``. @@ -125,8 +131,9 @@ Example Pipfile.lock - Generally, keep both ``Pipfile`` and ``Pipfile.lock`` in version control. - Do not keep ``Pipfile.lock`` in version control if multiple versions of Python are being targeted. -- Specify your target Python version in your `Pipfile`'s ``[requires]`` section. Ideally, you should only have one target Python version, as this is a deployment tool. +- Specify your target Python version in your `Pipfile`'s ``[requires]`` section. Ideally, you should only have one target Python version, as this is a deployment tool. ``python_version`` should be in the format ``X.Y`` and ``python_full_version`` should be in ``X.Y.Z`` format. - ``pipenv install`` is fully compatible with ``pip install`` syntax, for which the full documentation can be found `here `_. +- Note that the ``Pipfile`` uses the `TOML Spec `_. @@ -182,12 +189,12 @@ in your ``Pipfile.lock`` for now, run ``pipenv lock --keep-outdated``. Make sur ☤ Specifying Versions of a Package ---------------------------------- -You can specify versions of a package using the `Semantic Versioning scheme `_ -(i.e. ``major.minor.micro``). +You can specify versions of a package using the `Semantic Versioning scheme `_ +(i.e. ``major.minor.micro``). For example, to install requests you can use: :: - $ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 + $ pipenv install requests~=1.2 Pipenv will install version ``1.2`` and any minor update, but not ``2.0``. @@ -195,19 +202,19 @@ This will update your ``Pipfile`` to reflect this requirement, automatically. In general, Pipenv uses the same specifier format as pip. However, note that according to `PEP 440`_ , you can't use versions containing a hyphen or a plus sign. -.. _`PEP 440`: +.. _`PEP 440`: https://www.python.org/dev/peps/pep-0440/ To make inclusive or exclusive version comparisons you can use: :: $ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 $ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 - $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 + $ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 -.. note:: The use of ``" "`` around the package and version specification is highly recommended +.. note:: The use of double quotes around the package and version specification (i.e. ``"requests>2.19"``) is highly recommended to avoid issues with `Input and output redirection `_ - in Unix-based operating systems. + in Unix-based operating systems. -The use of ``~=`` is preferred over the ``==`` identifier as the former prevents pipenv from updating the packages: :: +The use of ``~=`` is preferred over the ``==`` identifier as the latter prevents pipenv from updating the packages: :: $ pipenv install "requests~=2.2" # locks the major version of the package (this is equivalent to using ==2.*) @@ -215,7 +222,7 @@ To avoid installing a specific version you can use the ``!=`` identifier. For an in depth explanation of the valid identifiers and more complex use cases check `the relevant section of PEP-440`_. -.. _`the relevant section of PEP-440`: https://www.python.org/dev/peps/pep-0440/#version-specifiers> +.. _`the relevant section of PEP-440`: https://www.python.org/dev/peps/pep-0440/#version-specifiers ☤ Specifying Versions of Python ------------------------------- @@ -314,6 +321,7 @@ The user can provide these additional parameters: - ``--dev`` — Install both ``develop`` and ``default`` packages from ``Pipfile``. - ``--system`` — Use the system ``pip`` command rather than the one from your virtualenv. + - ``--deploy`` — Make sure the packages are properly locked in Pipfile.lock, and abort if the lock file is out-of-date. - ``--ignore-pipfile`` — Ignore the ``Pipfile`` and install from the ``Pipfile.lock``. - ``--skip-lock`` — Ignore the ``Pipfile.lock`` and install from the ``Pipfile``. In addition, do not write out a ``Pipfile.lock`` reflecting changes to the ``Pipfile``. @@ -398,10 +406,8 @@ environment into production. You can use ``pipenv lock`` to compile your depende your development environment and deploy the compiled ``Pipfile.lock`` to all of your production environments for reproducible builds. -.. note: +.. note:: If you'd like a ``requirements.txt`` output of the lockfile, run ``$ pipenv lock -r``. This will include all hashes, however (which is great!). To get a ``requirements.txt`` without hashes, use ``$ pipenv run pip freeze``. - -.. _configuration-with-environment-variables:https://docs.pipenv.org/advanced/#configuration-with-environment-variables diff --git a/docs/cli.rst b/docs/cli.rst new file mode 100644 index 00000000..873c67d7 --- /dev/null +++ b/docs/cli.rst @@ -0,0 +1,8 @@ +.. _cli: + +Pipenv CLI Reference +====================================== + +.. click:: pipenv:cli + :prog: pipenv + :show-nested: diff --git a/docs/conf.py b/docs/conf.py index c5d6fbe0..90e16afe 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -57,8 +57,8 @@ master_doc = 'index' # General information about the project. project = u'pipenv' -copyright = u'2017. A Kenneth Reitz Project' -author = u'Kenneth Reitz' +copyright = u'2020. A project founded by Kenneth Reitz' +author = u'Python Packaging Authority' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the diff --git a/docs/dev/contributing.rst b/docs/dev/contributing.rst index db7f14cc..fb116696 100644 --- a/docs/dev/contributing.rst +++ b/docs/dev/contributing.rst @@ -20,8 +20,12 @@ The guide is split into sections based on the type of contribution you're thinking of making, with a section that covers general guidelines for all contributors. + +General Guidelines +------------------ + Be Cordial ----------- +~~~~~~~~~~ **Be cordial or be on your way**. *—Kenneth Reitz* @@ -34,10 +38,11 @@ everyone involved is treated with respect. .. _be cordial or be on your way: https://www.kennethreitz.org/essays/be-cordial-or-be-on-your-way + .. _early-feedback: Get Early Feedback ------------------- +~~~~~~~~~~~~~~~~~~ If you are contributing, do not feel the need to sit on your contribution until it is perfectly polished and complete. It helps everyone involved for you to @@ -47,7 +52,7 @@ getting that contribution accepted, and can save you from putting a lot of work into a contribution that is not suitable for the project. Contribution Suitability ------------------------- +~~~~~~~~~~~~~~~~~~~~~~~~ Our project maintainers have the last word on whether or not a contribution is suitable for Pipenv. All contributions will be considered carefully, but from @@ -58,29 +63,92 @@ If your contribution is rejected, don't despair! As long as you followed these guidelines, you will have a much better chance of getting your next contribution accepted. + +Questions +--------- + +The GitHub issue tracker is for *bug reports* and *feature requests*. Please do +not use it to ask questions about how to use Pipenv. These questions should +instead be directed to `Stack Overflow`_. Make sure that your question is tagged +with the ``pipenv`` tag when asking it on Stack Overflow, to ensure that it is +answered promptly and accurately. + +.. _Stack Overflow: https://stackoverflow.com/ + + Code Contributions ------------------ + Steps for Submitting Code ~~~~~~~~~~~~~~~~~~~~~~~~~ When contributing code, you'll want to follow this checklist: -1. Fork the repository on GitHub. -2. `Run the tests`_ to confirm they all pass on your system. If they don't, you'll - need to investigate why they fail. If you're unable to diagnose this - yourself, raise it as a bug report by following the guidelines in this - document: :ref:`bug-reports`. -3. Write tests that demonstrate your bug or feature. Ensure that they fail. -4. Make your change. -5. Run the entire test suite again, confirming that all tests pass *including +#. Understand our `development philosophy`_. +#. Fork the repository on GitHub. +#. Set up your :ref:`dev-setup` +#. Run the tests (:ref:`testing`) to confirm they all pass on your system. + If they don't, you'll need to investigate why they fail. If you're unable + to diagnose this yourself, raise it as a bug report by following the guidelines + in this document: :ref:`bug-reports`. +#. Write tests that demonstrate your bug or feature. Ensure that they fail. +#. Make your change. +#. Run the entire test suite again, confirming that all tests pass *including the ones you just added*. -6. Send a GitHub Pull Request to the main repository's ``master`` branch. +#. Send a GitHub Pull Request to the main repository's ``master`` branch. GitHub Pull Requests are the expected method of code collaboration on this project. The following sub-sections go into more detail on some of the points above. +.. _development philosophy: https://pipenv.pypa.io/en/latest/dev/philosophy/ + + +.. _dev-setup: + +Development Setup +~~~~~~~~~~~~~~~~~ + +To get your development environment setup, run: + +.. code-block:: sh + + pip install -e . + pipenv install --dev + + +This will install the repo version of Pipenv and then install the development +dependencies. Once that has completed, you can start developing. + +The repo version of Pipenv must be installed over other global versions to +resolve conflicts with the ``pipenv`` folder being implicitly added to ``sys.path``. +See `pypa/pipenv#2557`_ for more details. + +.. _pypa/pipenv#2557: https://github.com/pypa/pipenv/issues/2557 + + +.. _testing: + +Testing +~~~~~~~ + +Tests are written in ``pytest`` style and can be run very simply: + +.. code-block:: sh + + pytest + + +This will run all Pipenv tests, which can take awhile. To run a subset of the +tests, the standard pytest filters are available, such as: + +- provide a directory or file: ``pytest tests/unit`` or ``pytest tests/unit/test_cmdparse.py`` +- provide a keyword expression: ``pytest -k test_lock_editable_vcs_without_install`` +- provide a nodeid: ``pytest tests/unit/test_cmdparse.py::test_parse`` +- provide a test marker: ``pytest -m lock`` + + Code Review ~~~~~~~~~~~ @@ -90,6 +158,20 @@ event that you object to the code review feedback, you should make your case clearly and calmly. If, after doing so, the feedback is judged to still apply, you must either apply the feedback or withdraw your contribution. + +Package Index +~~~~~~~~~~~~~ + +To speed up testing, tests that rely on a package index for locking and +installing use a local server that contains vendored packages in the +``tests/pypi`` directory. Each vendored package should have it's own folder +containing the necessary releases. When adding a release for a package, it is +easiest to use either the ``.tar.gz`` or universal wheels (ex: ``py2.py3-none``). If +a ``.tar.gz`` or universal wheel is not available, add wheels for all available +architectures and platforms. + + + Documentation Contributions --------------------------- @@ -114,13 +196,48 @@ When presenting Python code, use single-quoted strings (``'hello'`` instead of Bug Reports ----------- -Bug reports are hugely important! Before you raise one, though, please check -through the `GitHub issues`_, **both open and closed**, to confirm that the bug -hasn't been reported before. Duplicate bug reports are a huge drain on the time -of other contributors, and should be avoided as much as possible. +Bug reports are hugely important! They are recorded as `GitHub issues`_. Please +be aware of the following things when filing bug reports: .. _GitHub issues: https://github.com/pypa/pipenv/issues +1. Avoid raising duplicate issues. *Please* use the GitHub issue search feature + to check whether your bug report or feature request has been mentioned in + the past. Duplicate bug reports and feature requests are a huge maintenance + burden on the limited resources of the project. If it is clear from your + report that you would have struggled to find the original, that's ok, but + if searching for a selection of words in your issue title would have found + the duplicate then the issue will likely be closed extremely abruptly. +2. When filing bug reports about exceptions or tracebacks, please include the + *complete* traceback. Partial tracebacks, or just the exception text, are + not helpful. Issues that do not contain complete tracebacks may be closed + without warning. +3. Make sure you provide a suitable amount of information to work with. This + means you should provide: + + - Guidance on **how to reproduce the issue**. Ideally, this should be a + *small* code sample that can be run immediately by the maintainers. + Failing that, let us know what you're doing, how often it happens, what + environment you're using, etc. Be thorough: it prevents us needing to ask + further questions. + - Tell us **what you expected to happen**. When we run your example code, + what are we expecting to happen? What does "success" look like for your + code? + - Tell us **what actually happens**. It's not helpful for you to say "it + doesn't work" or "it fails". Tell us *how* it fails: do you get an + exception? A hang? The packages installed seem incorrect? + How was the actual result different from your expected result? + - Tell us **what version of Pipenv you're using**, and + **how you installed it**. Different versions of Pipenv behave + differently and have different bugs, and some distributors of Pipenv + ship patches on top of the code we supply. + + If you do not provide all of these things, it will take us much longer to + fix your problem. If we ask you to clarify these and you never respond, we + will close your issue without fixing it. + +.. _run-the-tests: + Run the tests ------------- @@ -128,10 +245,15 @@ Three ways of running the tests are as follows: 1. ``make test`` (which uses ``docker``) 2. ``./run-tests.sh`` or ``run-tests.bat`` -3. Using pipenv:: +3. Using pipenv: - pipenv install --dev - pipenv run pytest +.. code-block:: console + + $ git clone https://github.com/pypa/pipenv.git + $ cd pipenv + $ git submodule sync && git submodule update --init --recursive + $ pipenv install --dev + $ pipenv run pytest For the last two, it is important that your environment is setup correctly, and this may take some work, for example, on a specific Mac installation, the following diff --git a/docs/dev/philosophy.rst b/docs/dev/philosophy.rst index 9d364e84..3f4c0bd0 100644 --- a/docs/dev/philosophy.rst +++ b/docs/dev/philosophy.rst @@ -7,6 +7,8 @@ Pipenv is an open but opinionated tool, created by an open but opinionated devel Management Style ~~~~~~~~~~~~~~~~ + **To be updated (as of March 2020)**. + `Kenneth Reitz `__ is the BDFL. He has final say in any decision related to the Pipenv project. Kenneth is responsible for the direction and form of the library, as well as its presentation. In addition to making decisions based on technical merit, he is responsible for making decisions based on the development philosophy of Pipenv. `Dan Ryan `__, `Tzu-ping Chung `__, and `Nate Prewitt `__ are the core contributors. diff --git a/docs/index.rst b/docs/index.rst index a832ee8e..26bfde19 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,9 +15,6 @@ Pipenv: Python Dev Workflow for Humans .. image:: https://img.shields.io/pypi/pyversions/pipenv.svg :target: https://pypi.python.org/pypi/pipenv -.. image:: https://img.shields.io/badge/Say%20Thanks!-🦉-1EAEDB.svg - :target: https://saythanks.io/to/kennethreitz - --------------- **Pipenv** is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. *Windows is a first-class citizen, in our world.* @@ -26,9 +23,11 @@ It automatically creates and manages a virtualenv for your projects, as well as Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of ``setup.py`` vs ``Pipfile`` to define dependencies, see :ref:`pipfile-vs-setuppy`. -.. raw:: html - - +.. image:: https://gist.githubusercontent.com/jlusk/855d611bbcfa2b159839db73d07f6ce9/raw/7f5743401809f7e630ee8ff458faa980e19924a0/pipenv.gif + :height: 341px + :width: 654px + :scale: 100 % + :alt: a short animation of pipenv at work The problems that Pipenv seeks to solve are multi-faceted: @@ -70,9 +69,6 @@ Otherwise, refer to the :ref:`installing-pipenv` chapter for instructions. User Testimonials ----------------- -**Jannis Leidel**, former pip maintainer— - *Pipenv is the porcelain I always wanted to build for pip. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.* - **David Gang**— *This package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo*. @@ -121,6 +117,7 @@ Further Documentation Guides basics advanced + cli diagnose Contribution Guides @@ -132,13 +129,6 @@ Contribution Guides dev/philosophy dev/contributing -☤ Pipenv Usage --------------- - -.. click:: pipenv:cli - :prog: pipenv - :show-nested: - Indices and tables ================== diff --git a/docs/install.rst b/docs/install.rst index 4039cdb5..6aceba0c 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -146,7 +146,7 @@ To upgrade pipenv at any time:: If you don't even have pip installed, you can use this crude installation method, which will bootstrap your whole system:: - $ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get-pipenv.py | python + $ curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python ☤ Installing packages for your project @@ -223,7 +223,7 @@ have access to your installed packages with ``$ pipenv shell``. ☤ Virtualenv mapping caveat -============ +=========================== - Pipenv automatically maps projects to their specific virtualenvs. - The virtualenv is stored globally with the name of the project’s root directory plus the hash of the full path to the project's root (e.g., ``my_project-a3de50``). diff --git a/news/2317.doc.rst b/news/2317.doc.rst new file mode 100644 index 00000000..ff56fe4d --- /dev/null +++ b/news/2317.doc.rst @@ -0,0 +1 @@ +Added documenation about variable expansion in ``Pipfile`` entries. diff --git a/news/2553.behavior.rst b/news/2553.behavior.rst new file mode 100644 index 00000000..d66edfa2 --- /dev/null +++ b/news/2553.behavior.rst @@ -0,0 +1 @@ +Make conservative checks of known exceptions when subprocess returns output, so user won't see the whole traceback - just the error. \ No newline at end of file diff --git a/news/2722.bugfix.rst b/news/2722.bugfix.rst new file mode 100644 index 00000000..8c26df8d --- /dev/null +++ b/news/2722.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused editable package resolution to sometimes fail with an unhelpful setuptools-related error message. diff --git a/news/2783.bugfix.rst b/news/2783.bugfix.rst new file mode 100644 index 00000000..7fa3cfd1 --- /dev/null +++ b/news/2783.bugfix.rst @@ -0,0 +1,2 @@ +Fixed an issue which caused errors due to reliance on the system utilities ``which`` and ``where`` which may not always exist on some systems. +- Fixed a bug which caused periodic failures in python discovery when executables named ``python`` were not present on the target ``$PATH``. diff --git a/news/3053.bugfix.rst b/news/3053.bugfix.rst new file mode 100644 index 00000000..21134f59 --- /dev/null +++ b/news/3053.bugfix.rst @@ -0,0 +1 @@ +Dependency resolution now writes hashes for local and remote files to the lockfile. diff --git a/news/3071.bugfix.rst b/news/3071.bugfix.rst new file mode 100644 index 00000000..dd4145ea --- /dev/null +++ b/news/3071.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which prevented ``pipenv graph`` from correctly showing all dependencies when running from within ``pipenv shell``. diff --git a/news/3120.doc.rst b/news/3120.doc.rst new file mode 100644 index 00000000..a2f8ae6c --- /dev/null +++ b/news/3120.doc.rst @@ -0,0 +1 @@ +Consolidate all contributing docs in the rst file diff --git a/news/3148.bugfix.rst b/news/3148.bugfix.rst new file mode 100644 index 00000000..1f0f4a62 --- /dev/null +++ b/news/3148.bugfix.rst @@ -0,0 +1 @@ +Fixed resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3148.feature.rst b/news/3148.feature.rst new file mode 100644 index 00000000..e33434db --- /dev/null +++ b/news/3148.feature.rst @@ -0,0 +1 @@ +Added support for resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3246.doc.rst b/news/3246.doc.rst new file mode 100644 index 00000000..284ecd0a --- /dev/null +++ b/news/3246.doc.rst @@ -0,0 +1 @@ +Update the out-dated manual page. diff --git a/news/3292.trivial.rst b/news/3292.trivial.rst new file mode 100644 index 00000000..9cab5de1 --- /dev/null +++ b/news/3292.trivial.rst @@ -0,0 +1 @@ +Update pytest-pypi documentation not to be pytest-httpbin documentation. diff --git a/news/3298.bugfix.rst b/news/3298.bugfix.rst new file mode 100644 index 00000000..aa378723 --- /dev/null +++ b/news/3298.bugfix.rst @@ -0,0 +1,3 @@ +Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. + +- Fixed a bug with package discovery when running ``pipenv clean``. diff --git a/news/3298.feature.rst b/news/3298.feature.rst new file mode 100644 index 00000000..65a49424 --- /dev/null +++ b/news/3298.feature.rst @@ -0,0 +1,5 @@ +Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. + +- Improved error handling and formatting. + +- Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst new file mode 100644 index 00000000..cab9a50b --- /dev/null +++ b/news/3298.vendor.rst @@ -0,0 +1,34 @@ +Updated vendored dependencies: + + - **attrs**: ``18.2.0`` => ``19.1.0`` + - **certifi**: ``2018.10.15`` => ``2019.3.9`` + - **cached_property**: ``1.4.3`` => ``1.5.1`` + - **cerberus**: ``1.2.0`` => ``1.3.1`` + - **click-completion**: ``0.5.0`` => ``0.5.1`` + - **colorama**: ``0.3.9`` => ``0.4.1`` + - **distlib**: ``0.2.8`` => ``0.2.9`` + - **idna**: ``2.7`` => ``2.8`` + - **jinja2**: ``2.10.0`` => ``2.10.1`` + - **markupsafe**: ``1.0`` => ``1.1.1`` + - **orderedmultidict**: ``(new)`` => ``1.0`` + - **packaging**: ``18.0`` => ``19.0`` + - **parse**: ``1.9.0`` => ``1.12.0`` + - **pathlib2**: ``2.3.2`` => ``2.3.3`` + - **pep517**: ``(new)`` => ``0.5.0`` + - **pexpect**: ``4.6.0`` => ``4.7.0`` + - **pipdeptree**: ``0.13.0`` => ``0.13.2`` + - **pyparsing**: ``2.2.2`` => ``2.3.1`` + - **python-dotenv**: ``0.9.1`` => ``0.10.2`` + - **pythonfinder**: ``1.1.10`` => ``1.2.1`` + - **pytoml**: ``(new)`` => ``0.1.20`` + - **requests**: ``2.20.1`` => ``2.21.0`` + - **requirementslib**: ``1.3.3`` => ``1.5.0`` + - **scandir**: ``1.9.0`` => ``1.10.0`` + - **shellingham**: ``1.2.7`` => ``1.3.1`` + - **six**: ``1.11.0`` => ``1.12.0`` + - **tomlkit**: ``0.5.2`` => ``0.5.3`` + - **urllib3**: ``1.24`` => ``1.25.2`` + - **vistir**: ``0.3.0`` => ``0.4.1`` + - **yaspin**: ``0.14.0`` => ``0.14.3`` + +- Removed vendored dependency **cursor**. diff --git a/news/3328.feature.rst b/news/3328.feature.rst new file mode 100644 index 00000000..7e92d39f --- /dev/null +++ b/news/3328.feature.rst @@ -0,0 +1 @@ +Pipenv will now successfully recursively lock VCS sub-dependencies. diff --git a/news/3339.bugfix b/news/3339.bugfix new file mode 100644 index 00000000..8e67e36f --- /dev/null +++ b/news/3339.bugfix @@ -0,0 +1 @@ +Fix a bug where custom virtualenv can not be activated with pipenv shell diff --git a/news/3346.doc.rst b/news/3346.doc.rst new file mode 100644 index 00000000..c985f001 --- /dev/null +++ b/news/3346.doc.rst @@ -0,0 +1 @@ +Move CLI docs to its own page. diff --git a/news/3353.bugfix b/news/3353.bugfix new file mode 100644 index 00000000..23e2b6af --- /dev/null +++ b/news/3353.bugfix @@ -0,0 +1 @@ +Fix a bug where pipenv --clear is not working diff --git a/news/3368.feature.rst b/news/3368.feature.rst new file mode 100644 index 00000000..a998fce1 --- /dev/null +++ b/news/3368.feature.rst @@ -0,0 +1 @@ +Pipenv will now discover and resolve the intrinsic dependencies of **all** VCS dependencies, whether they are editable or not, to prevent resolution conflicts. diff --git a/news/3386.behavior.rst b/news/3386.behavior.rst new file mode 100644 index 00000000..8ddc27c6 --- /dev/null +++ b/news/3386.behavior.rst @@ -0,0 +1 @@ +Do not touch Pipfile early and rely on it so that one can do ``pipenv sync`` without a Pipfile. diff --git a/news/3404.bugfix.rst b/news/3404.bugfix.rst new file mode 100644 index 00000000..fa678d6a --- /dev/null +++ b/news/3404.bugfix.rst @@ -0,0 +1 @@ +Fixed a keyerror which could occur when locking VCS dependencies in some cases. diff --git a/news/3427.bugfix.rst b/news/3427.bugfix.rst index 42f659f0..76aeb489 100644 --- a/news/3427.bugfix.rst +++ b/news/3427.bugfix.rst @@ -1 +1 @@ -Fix a bug that ``ValidationError`` is thrown when some fields are missing in source section. +Fixed a bug that ``ValidationError`` is thrown when some fields are missing in source section. diff --git a/news/3434.trivial.rst b/news/3434.trivial.rst new file mode 100644 index 00000000..622b52db --- /dev/null +++ b/news/3434.trivial.rst @@ -0,0 +1 @@ +Improve the error message when one tries to initialize a Pipenv project under ``/``. diff --git a/news/3446.trivial.rst b/news/3446.trivial.rst index 1eb98bbb..c3f6a000 100644 --- a/news/3446.trivial.rst +++ b/news/3446.trivial.rst @@ -1 +1 @@ -Fix the wrong order of old and new hashes in message. +Fixed the wrong order of old and new hashes in message. diff --git a/news/3449.bugfix.rst b/news/3449.bugfix.rst index 72e8d228..4ed07046 100644 --- a/news/3449.bugfix.rst +++ b/news/3449.bugfix.rst @@ -1 +1 @@ -Update the index names in lock file when source name in Pipfile is changed. +Updated the index names in lock file when source name in Pipfile is changed. diff --git a/news/3479.bugfix.rst b/news/3479.bugfix.rst new file mode 100644 index 00000000..15e8e0f6 --- /dev/null +++ b/news/3479.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue which caused ``pipenv install --help`` to show duplicate entries for ``--pre``. diff --git a/news/3499.doc.rst b/news/3499.doc.rst new file mode 100644 index 00000000..d98b0f15 --- /dev/null +++ b/news/3499.doc.rst @@ -0,0 +1 @@ +Replace (non-existant) video on docs index.rst with equivalent gif. diff --git a/news/3502.bugfix.rst b/news/3502.bugfix.rst new file mode 100644 index 00000000..2700a740 --- /dev/null +++ b/news/3502.bugfix.rst @@ -0,0 +1 @@ +Fix bug causing ``[SSL: CERTIFICATE_VERIFY_FAILED]`` when Pipfile ``[[source]]`` has verify_ssl=false and url with custom port. diff --git a/news/3522.doc.rst b/news/3522.doc.rst new file mode 100644 index 00000000..3d71061f --- /dev/null +++ b/news/3522.doc.rst @@ -0,0 +1 @@ +Clarify wording in Basic Usage example on using double quotes to escape shell redirection diff --git a/news/3527.doc.rst b/news/3527.doc.rst new file mode 100644 index 00000000..b6043a08 --- /dev/null +++ b/news/3527.doc.rst @@ -0,0 +1 @@ +Ensure docs show navigation on small-screen devices diff --git a/news/3537.bugfix.rst b/news/3537.bugfix.rst new file mode 100644 index 00000000..779b9d7c --- /dev/null +++ b/news/3537.bugfix.rst @@ -0,0 +1 @@ +Fix ``sync --sequential`` ignoring ``pip install`` errors and logs. diff --git a/news/3577.feature.rst b/news/3577.feature.rst new file mode 100644 index 00000000..7944c098 --- /dev/null +++ b/news/3577.feature.rst @@ -0,0 +1 @@ +Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies. diff --git a/news/3584.bugfix.rst b/news/3584.bugfix.rst new file mode 100644 index 00000000..09684d1d --- /dev/null +++ b/news/3584.bugfix.rst @@ -0,0 +1 @@ +Fix the issue that lock file can't be created when ``PIPENV_PIPFILE`` is not under working directory. diff --git a/news/3595.feature.rst b/news/3595.feature.rst new file mode 100644 index 00000000..30b755b9 --- /dev/null +++ b/news/3595.feature.rst @@ -0,0 +1 @@ +Added the ability for Windows users to enable emojis by setting ``PIPENV_HIDE_EMOJIS=0``. diff --git a/news/3621.trivial.rst b/news/3621.trivial.rst new file mode 100644 index 00000000..4d38a31e --- /dev/null +++ b/news/3621.trivial.rst @@ -0,0 +1 @@ +Removed unused vendored package shutilwhich diff --git a/news/3629.doc.rst b/news/3629.doc.rst new file mode 100644 index 00000000..4d878c40 --- /dev/null +++ b/news/3629.doc.rst @@ -0,0 +1 @@ +Added a link to the TOML Spec under General Recommendations & Version Control to clarify how Pipfiles should be written. diff --git a/news/3640.trivial.rst b/news/3640.trivial.rst new file mode 100644 index 00000000..eb9b718d --- /dev/null +++ b/news/3640.trivial.rst @@ -0,0 +1 @@ +Removed unused vendored package blindspin diff --git a/news/3644.trivial.rst b/news/3644.trivial.rst new file mode 100644 index 00000000..5a7db2e1 --- /dev/null +++ b/news/3644.trivial.rst @@ -0,0 +1 @@ +Use tablib instead of requests in tests to avoid failures when vendored diff --git a/news/3647.bugfix.rst b/news/3647.bugfix.rst new file mode 100644 index 00000000..cb64edc1 --- /dev/null +++ b/news/3647.bugfix.rst @@ -0,0 +1 @@ +Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies. diff --git a/news/3652.feature.rst b/news/3652.feature.rst new file mode 100644 index 00000000..7e5becb9 --- /dev/null +++ b/news/3652.feature.rst @@ -0,0 +1 @@ +Allow overriding PIPENV_INSTALL_TIMEOUT environment variable (in seconds). diff --git a/news/3656.bugfix.rst b/news/3656.bugfix.rst new file mode 100644 index 00000000..58df2020 --- /dev/null +++ b/news/3656.bugfix.rst @@ -0,0 +1,2 @@ +The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies. +- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated. diff --git a/news/3669.trivial.rst b/news/3669.trivial.rst new file mode 100644 index 00000000..86ff9280 --- /dev/null +++ b/news/3669.trivial.rst @@ -0,0 +1 @@ +Allow KeyboardInterrupt to cancel test suite checks for working internet and ssh diff --git a/news/3684.trivial.rst b/news/3684.trivial.rst new file mode 100644 index 00000000..64561ec7 --- /dev/null +++ b/news/3684.trivial.rst @@ -0,0 +1 @@ +Cleaned up some conditional logic that would always evaluate ``True``. diff --git a/news/3711.trivial.rst b/news/3711.trivial.rst new file mode 100644 index 00000000..48c531b2 --- /dev/null +++ b/news/3711.trivial.rst @@ -0,0 +1 @@ +Add installation instructions for Debian Buster+ in README diff --git a/news/3718.bugfix.rst b/news/3718.bugfix.rst new file mode 100644 index 00000000..7a90ea50 --- /dev/null +++ b/news/3718.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which sometimes caused pipenv to fail to respect the ``--site-packages`` flag when passed with ``pipenv install``. diff --git a/news/3724.trivial.rst b/news/3724.trivial.rst new file mode 100644 index 00000000..63a55013 --- /dev/null +++ b/news/3724.trivial.rst @@ -0,0 +1 @@ +Update pytest configuration to support pytest 4. diff --git a/news/3738.feature.rst b/news/3738.feature.rst new file mode 100644 index 00000000..bb8237e7 --- /dev/null +++ b/news/3738.feature.rst @@ -0,0 +1,3 @@ +Allow overriding PIP_EXISTS_ACTION evironment variable (value is passed to pip install). +Possible values here: https://pip.pypa.io/en/stable/reference/pip/#exists-action-option +Useful when you need to `PIP_EXISTS_ACTION=i` (ignore existing packages) - great for CI environments, where you need really fast setup. diff --git a/news/3745.bugfix.rst b/news/3745.bugfix.rst new file mode 100644 index 00000000..229047a4 --- /dev/null +++ b/news/3745.bugfix.rst @@ -0,0 +1 @@ +Normalize the package names to lowercase when comparing used and in-Pipfile packages. diff --git a/news/3753.trivial.rst b/news/3753.trivial.rst new file mode 100644 index 00000000..2ab71d38 --- /dev/null +++ b/news/3753.trivial.rst @@ -0,0 +1 @@ +Improve the error message of ``pipenv --py`` when virtualenv can't be found. diff --git a/news/3759.doc.rst b/news/3759.doc.rst new file mode 100644 index 00000000..5aebd29e --- /dev/null +++ b/news/3759.doc.rst @@ -0,0 +1 @@ +Updated the documentation with the new ``pytest`` entrypoint. diff --git a/news/3763.feature.rst b/news/3763.feature.rst new file mode 100644 index 00000000..544a1ace --- /dev/null +++ b/news/3763.feature.rst @@ -0,0 +1 @@ +Pipenv will no longer forcibly override ``PIP_NO_DEPS`` on all vcs and file dependencies as resolution happens on these in a pre-lock step. diff --git a/news/3766.bugfix.rst b/news/3766.bugfix.rst new file mode 100644 index 00000000..f7f7d304 --- /dev/null +++ b/news/3766.bugfix.rst @@ -0,0 +1 @@ +``pipenv update --outdated`` will now correctly handle comparisons between pre/post-releases and normal releases. diff --git a/news/3766.vendor.rst b/news/3766.vendor.rst new file mode 100644 index 00000000..16ebbed9 --- /dev/null +++ b/news/3766.vendor.rst @@ -0,0 +1 @@ +Updated ``pip_shims`` to support ``--outdated`` with new pip versions. diff --git a/news/3768.bugfix.rst b/news/3768.bugfix.rst new file mode 100644 index 00000000..8efe0197 --- /dev/null +++ b/news/3768.bugfix.rst @@ -0,0 +1 @@ +Fixed a ``KeyError`` which could occur when pinning outdated VCS dependencies via ``pipenv lock --keep-outdated``. diff --git a/news/3786.bugfix.rst b/news/3786.bugfix.rst new file mode 100644 index 00000000..210f7973 --- /dev/null +++ b/news/3786.bugfix.rst @@ -0,0 +1 @@ +Resolved an issue which caused resolution to fail when encountering poorly formatted ``python_version`` markers in ``setup.py`` and ``setup.cfg`` files. diff --git a/news/3794.bugfix.rst b/news/3794.bugfix.rst new file mode 100644 index 00000000..a2999fdd --- /dev/null +++ b/news/3794.bugfix.rst @@ -0,0 +1 @@ +Fix a bug that installation errors are displayed as a list. diff --git a/news/3807.bugfix.rst b/news/3807.bugfix.rst new file mode 100644 index 00000000..6330c6bc --- /dev/null +++ b/news/3807.bugfix.rst @@ -0,0 +1,2 @@ +Update ``pythonfinder`` to fix a problem that ``python.exe`` will be mistakenly chosen for +virtualenv creation under WSL. diff --git a/news/3809.bugfix.rst b/news/3809.bugfix.rst new file mode 100644 index 00000000..bd603aaf --- /dev/null +++ b/news/3809.bugfix.rst @@ -0,0 +1 @@ +Fixed several bugs which could prevent editable VCS dependencies from being installed into target environments, even when reporting successful installation. diff --git a/news/3810.feature.rst b/news/3810.feature.rst new file mode 100644 index 00000000..33503779 --- /dev/null +++ b/news/3810.feature.rst @@ -0,0 +1 @@ +Improved verbose logging output during ``pipenv lock`` will now stream output to the console while maintaining a spinner. diff --git a/news/3819.bugfix.rst b/news/3819.bugfix.rst new file mode 100644 index 00000000..a6e05fb5 --- /dev/null +++ b/news/3819.bugfix.rst @@ -0,0 +1 @@ +``pipenv check --system`` should find the correct Python interpreter when ``python`` does not exist on the system. diff --git a/news/3842.bugfix.rst b/news/3842.bugfix.rst new file mode 100644 index 00000000..fb21be89 --- /dev/null +++ b/news/3842.bugfix.rst @@ -0,0 +1 @@ +Resolve the symlinks when the path is absolute. diff --git a/news/3844.behavior.rst b/news/3844.behavior.rst new file mode 100644 index 00000000..2648e839 --- /dev/null +++ b/news/3844.behavior.rst @@ -0,0 +1 @@ +Re-enable ``--help`` option for ``pipenv run`` command. diff --git a/news/3879.bugfix.rst b/news/3879.bugfix.rst new file mode 100644 index 00000000..95413ca5 --- /dev/null +++ b/news/3879.bugfix.rst @@ -0,0 +1 @@ +Pass ``--pre`` and ``--clear`` options to ``pipenv update --outdated``. diff --git a/news/3885.trivial.rst b/news/3885.trivial.rst new file mode 100644 index 00000000..7782e0c9 --- /dev/null +++ b/news/3885.trivial.rst @@ -0,0 +1 @@ +Remove a misleading code comment from Specifying Versions documentation. diff --git a/news/3911.doc.rst b/news/3911.doc.rst new file mode 100644 index 00000000..a5ab134f --- /dev/null +++ b/news/3911.doc.rst @@ -0,0 +1 @@ +Fix link to GIF in README.md demonstrating Pipenv's usage, and add descriptive alt text. diff --git a/news/3912.doc.rst b/news/3912.doc.rst new file mode 100644 index 00000000..24598d19 --- /dev/null +++ b/news/3912.doc.rst @@ -0,0 +1 @@ +Added a line describing potential issues in fancy extension. diff --git a/news/3913.doc.rst b/news/3913.doc.rst new file mode 100644 index 00000000..54fbbfe8 --- /dev/null +++ b/news/3913.doc.rst @@ -0,0 +1 @@ +Documental description of how Pipfile works and association with Pipenv. diff --git a/news/3914.doc.rst b/news/3914.doc.rst new file mode 100644 index 00000000..ae37d61d --- /dev/null +++ b/news/3914.doc.rst @@ -0,0 +1 @@ +Clarify the proper value of `python_version` and `python_full_version`. diff --git a/news/3915.doc.rst b/news/3915.doc.rst new file mode 100644 index 00000000..2cc94a20 --- /dev/null +++ b/news/3915.doc.rst @@ -0,0 +1 @@ +Write description for --deploy extension and few extensions differences. diff --git a/news/4018.feature.rst b/news/4018.feature.rst new file mode 100644 index 00000000..fcd6a2a9 --- /dev/null +++ b/news/4018.feature.rst @@ -0,0 +1 @@ +Added support for automatic python installs via ``asdf`` and associated ``PIPENV_DONT_USE_ASDF`` environment variable. diff --git a/news/4045.bugfix.rst b/news/4045.bugfix.rst new file mode 100644 index 00000000..6558018c --- /dev/null +++ b/news/4045.bugfix.rst @@ -0,0 +1 @@ +Honor PIPENV_SPINNER environment variable diff --git a/news/4100.doc.rst b/news/4100.doc.rst new file mode 100644 index 00000000..050bdcca --- /dev/null +++ b/news/4100.doc.rst @@ -0,0 +1 @@ +More documentation for ``.env`` files diff --git a/news/4137.doc b/news/4137.doc new file mode 100644 index 00000000..45de74d2 --- /dev/null +++ b/news/4137.doc @@ -0,0 +1 @@ +Updated documentation to point to working links. \ No newline at end of file diff --git a/news/4167.doc.rst b/news/4167.doc.rst new file mode 100644 index 00000000..da71ef02 --- /dev/null +++ b/news/4167.doc.rst @@ -0,0 +1 @@ +Replace docs.pipenv.org with pipenv.pypa.io diff --git a/news/4169.vendor.rst b/news/4169.vendor.rst new file mode 100644 index 00000000..d128f56c --- /dev/null +++ b/news/4169.vendor.rst @@ -0,0 +1,50 @@ +Update vendored dependencies and invocations + +- Update vendored and patched dependencies + - Update patches on `piptools`, `pip`, `pip-shims`, `tomlkit` +- Fix invocations of dependencies + - Fix custom `InstallCommand` instantiation + - Update `PackageFinder` usage + - Fix `Bool` stringify attempts from `tomlkit` + +Updated vendored dependencies: + - **attrs**: ``18.2.0`` => ``19.1.0`` + - **certifi**: ``2018.10.15`` => ``2019.3.9`` + - **cached_property**: ``1.4.3`` => ``1.5.1`` + - **cerberus**: ``1.2.0`` => ``1.3.1`` + - **click**: ``7.0.0`` => ``7.1.1`` + - **click-completion**: ``0.5.0`` => ``0.5.1`` + - **colorama**: ``0.3.9`` => ``0.4.3`` + - **contextlib2**: ``(new)`` => ``0.6.0.post1`` + - **distlib**: ``0.2.8`` => ``0.2.9`` + - **funcsigs**: ``(new)`` => ``1.0.2`` + - **importlib_metadata** ``1.3.0`` => ``1.5.1`` + - **importlib-resources**: ``(new)`` => ``1.4.0`` + - **idna**: ``2.7`` => ``2.9`` + - **jinja2**: ``2.10.0`` => ``2.11.1`` + - **markupsafe**: ``1.0`` => ``1.1.1`` + - **more-itertools**: ``(new)`` => ``5.0.0`` + - **orderedmultidict**: ``(new)`` => ``1.0`` + - **packaging**: ``18.0`` => ``19.0`` + - **parse**: ``1.9.0`` => ``1.15.0`` + - **pathlib2**: ``2.3.2`` => ``2.3.3`` + - **pep517**: ``(new)`` => ``0.5.0`` + - **pexpect**: ``4.6.0`` => ``4.8.0`` + - **pip-shims**: ``0.2.0`` => ``0.5.1`` + - **pipdeptree**: ``0.13.0`` => ``0.13.2`` + - **pyparsing**: ``2.2.2`` => ``2.4.6`` + - **python-dotenv**: ``0.9.1`` => ``0.10.2`` + - **pythonfinder**: ``1.1.10`` => ``1.2.2`` + - **pytoml**: ``(new)`` => ``0.1.20`` + - **requests**: ``2.20.1`` => ``2.23.0`` + - **requirementslib**: ``1.3.3`` => ``1.5.4`` + - **scandir**: ``1.9.0`` => ``1.10.0`` + - **shellingham**: ``1.2.7`` => ``1.3.2`` + - **six**: ``1.11.0`` => ``1.14.0`` + - **tomlkit**: ``0.5.2`` => ``0.5.11`` + - **urllib3**: ``1.24`` => ``1.25.8`` + - **vistir**: ``0.3.0`` => ``0.5.0`` + - **yaspin**: ``0.14.0`` => ``0.14.3`` + - **zipp**: ``0.6.0`` + +- Removed vendored dependency **cursor**. diff --git a/news/4188.bugfix.rst b/news/4188.bugfix.rst new file mode 100644 index 00000000..0ea2c943 --- /dev/null +++ b/news/4188.bugfix.rst @@ -0,0 +1 @@ +Fixed an issue with ``pipenv check`` failing due to an invalid API key from ``pyup.io``. diff --git a/news/4188.vendor.rst b/news/4188.vendor.rst new file mode 100644 index 00000000..e24a45fd --- /dev/null +++ b/news/4188.vendor.rst @@ -0,0 +1,11 @@ +Add and update vendored dependencies to accommodate ``safety`` vendoring: +- **safety** ``(none)`` => ``1.8.7`` +- **dparse** ``(none)`` => ``0.5.0`` +- **pyyaml** ``(none)`` => ``5.3.1`` +- **urllib3** ``1.25.8`` => ``1.25.9`` +- **certifi** ``2019.11.28`` => ``2020.4.5.1`` +- **pyparsing** ``2.4.6`` => ``2.4.7`` +- **resolvelib** ``0.2.2`` => ``0.3.0`` +- **importlib-metadata** ``1.5.1`` => ``1.6.0`` +- **pip-shims** ``0.5.1`` => ``0.5.2`` +- **requirementslib** ``1.5.5`` => ``1.5.6`` diff --git a/news/4199.behavior.rst b/news/4199.behavior.rst new file mode 100644 index 00000000..eb1dae3c --- /dev/null +++ b/news/4199.behavior.rst @@ -0,0 +1 @@ +Make sure `pipenv lock -r --pypi-mirror {MIRROR_URL}` will respect the pypi-mirror in requirements output. diff --git a/news/towncrier_template.rst b/news/towncrier_template.rst index 55505f82..0fca0c3f 100644 --- a/news/towncrier_template.rst +++ b/news/towncrier_template.rst @@ -13,7 +13,7 @@ {% if definitions[category]['showcontent'] %} {% for text, values in sections[section][category]|dictsort(by='value') %} -- {{ text }}{% if category != 'process' %}{{ values|sort|join(',\n ') }}{% endif %} +- {{ text }} {% if category != 'process' %}{{ values|sort|join(',\n ') }}{% endif %} {% endfor %} {% else %} diff --git a/peeps/PEEP-004.md b/peeps/PEEP-004.md new file mode 100644 index 00000000..a3dc8c33 --- /dev/null +++ b/peeps/PEEP-004.md @@ -0,0 +1,9 @@ +## PEEP-004: Subcommands + +NOT YET ACCEPTED + +Pipenv will automatically run commands like "pipenv deploy" if the "pipenv-deploy" binary is available on the path. + +These subcommands cannot overwrite built-in commands. + +These subcommands will receive environment variables with contextual information. diff --git a/peeps/PEEP-005.md b/peeps/PEEP-005.md new file mode 100644 index 00000000..2cc0279f --- /dev/null +++ b/peeps/PEEP-005.md @@ -0,0 +1,65 @@ +# PEEP-005: Do Not Remove Entries from the Lockfile When Using `--keep-outdated` + +**PROPOSED** + +This PEEP describes a change that would retain entries in the Lockfile even if they were not returned during resolution when the user passes the `--keep-outdated` flag. + +☤ + +The `--keep-outdated` flag is currently provided by Pipenv for the purpose of holding back outdated dependencies (i.e. dependencies that are not newly introduced). This proposal attempts to identify the reasoning behind the flag and identifies a need for a project-wide scoping. Finally, this proposal outlines the expected behavior of `--keep-outdated` under the specified circumstances, as well as the required changes to achieve full implementation. + +## Retaining Outdated Dependencies + +The purpose of retaining outdated dependencies is to allow the user to introduce a new package to their environment with a minimal impact on their existing environment. In an effort to achieve this, `keep_outdated` was proposed as both a flag and a Pipfile setting [in this issue](https://github.com/pypa/pipenv/issues/1255#issuecomment-354585775), originally described as follows: + +> pipenv lock --keep-outdated to request a minimal update that only adjusts the lock file to account for Pipfile changes (additions, removals, and changes to version constraints)... and pipenv install --keep-outdated needed to request only the minimal changes required to satisfy the installation request + +However, the current implementation always fully re-locks, rather than only locking the new dependencies. As a result, dependencies in the `Pipfile.lock` with markers for a python version different from that of the running interpreter will be removed, even if they have nothing to do with the current changeset. For instance, say you have the following dependency in your `Pipfile.lock`: + +```json +{ + "default": { + "backports.weakref": { + "hashes": [...], + "version": "==1.5", + "markers": "python_version<='3.4'" + } + } +} +``` + +If this lockfile were to be re-generated with Python 3, even with `--keep-outdated`, this entry would be removed. This makes it very difficult to maintain lockfiles which are compatible across major python versions, yet all that would be required to correct this would be a tweak to the implementation of `keep-outdated`. I believe this was the goal to begin with, but I feel this behavior should be documented and clarified before moving forward. + +## Desired Behavior + +1. The only changes that should occur in `Pipfile.lock` when `--keep-outdated` is passed should be changes resulting from new packages added or pin changes in the project `Pipfile`; +2. Existing packages in the project `Pipfile.lock` should remain in place, even if they are not returned during resolution; +3. New dependencies should be written to the lockfile; +4. Conflicts should be resolved as outlined below. + +## Conflict Resolution + +If a conflict should occur due to the presence in the `Pipfile.lock` of a dependency of a new package, the following steps should be undertaken before alerting the user: + +1. Determine whether the previously locked version of the dependency meets the constraints required of the new package; if so, pin that version; +2. If the previously locked version is not present in the `Pipfile` and is not a dependency of any other dependencies (i.e. has no presence in `pipenv graph`, etc), update the lockfile with the new version; +3. If there is a new or existing dependency which has a conflict with existing entries in the lockfile, perform an intermediate resolution step by checking: + a. If the new dependency can be satisfied by existing installs; + b. Whether conflicts can be upgraded without affecting locked dependencies; + c. If locked dependencies must be upgraded, whether those dependencies ultimately have any dependencies in the `Pipfile`; + d. If a traversal up the graph lands in the `Pipfile`, create _abstract dependencies_ from the `Pipfile` entries and determine whether they will still be satisfied by the new version; + e. If a new pin is required, ensure that any subdependencies of the newly pinned dependencies are therefore also re-pinned (simply prefer the updated lockfile instead of the cached version); + +4. Raise an Exception alerting the user that they either need to do a full lock or manually pin a version. + +## Necessary Changes + +In order to make these changes, we will need to modify the dependency resolution process. Overall, locking will require the following implementation changes: + +1. The ability to restore any entries that would otherwise be removed when the `--keep-outdated` flag is passed. The process already provides a caching mechanism, so we simply need to restore missing cache keys; +2. Conflict resolution steps: + a. Check an abstract dependency/candidate against a lockfile entry; + b. Requirements mapping for each dependency in the environment to determine if a lockfile entry is a descendent of any other entries; + + +Author: Dan Ryan diff --git a/peeps/PEEP-006.md b/peeps/PEEP-006.md new file mode 100644 index 00000000..5a3739e1 --- /dev/null +++ b/peeps/PEEP-006.md @@ -0,0 +1,62 @@ +# PEEP-006: Include all deps in output of `pipenv lock -r --dev` + +This proposal makes the behavior of `pipenv lock --requirements --dev` +consistent with the behaviour of other commands: converting all dependencies, +not just the development dependencies. + +☤ + +If you type `pipenv lock --help` the help document says: + +```bash +-d, --dev Install both develop and default packages. [env var:PIPENV_DEV] +``` + +That is not accurate and confusing for `pipenv lock -r`, which only produces the develop requirments. + +This PEEP proposes to change the behavior of `pipenv lock -r -d` to produce **all** requirements, both develop +and default. The help string of `-d/--dev` will be changed to **"Generate both develop and default requirements"**. + +As the existing behaviour was intended to support generating traditional `dev-requirements.txt` +files, a new flag, `--dev-only`, will be introduced to restrict output to development requirements only. + +When the new `pipenv lock` specific flag is used, the common `-d/--dev` flag is redundant, but +ignored (i.e. `pipenv lock -r --dev-only` and `pipenv lock -r --dev --dev-only` do the same thing). +If `--dev-only` is specified without `-r/--requirements`, then `PipenvOptionsError` will be thrown. + +As part of this change, `pipenv lock --requirements` will be updated to emit a comment header +indicating that the file was autogenerated, and the options passed to `pipenv lock`. This will use +the following `pip-compile` inspired format: + + # + # These requirements were autogenerated by pipenv + # To regenerate from the project's Pipfile, run: + # + # pipenv lock --requirements + # + +`--dev` or `--dev-only` will be append to the emitted regeneration command if +those options are set. + +To allow this new header to be turned off, `pipenv lock --requirements` will also support the same +`--header/--no-header` options that `pip-compile` offers. + +In the first release including this change, and in releases for at least 6 months from that date, +the emitted header will include the following note when the `--dev` option is set: + + # Note: in pipenv 2020.x, "--dev" changed to emit both default and development + # requirements. To emit only development requirements, pass "--dev-only". + +## Impact + +The users relying on the old behavior will get more requirements listed in the +``dev-requirements.txt`` file, which in most cases is harmless. They can pass +the `--dev-only` flag after updating `pipenv` to achieve the same thing as before. + +## Related issues: + +- #3316 + +## Related pull requests: + +- #4183 diff --git a/pipenv/__init__.py b/pipenv/__init__.py index 7a12158a..624397c5 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -8,7 +8,7 @@ import os import sys import warnings -from .__version__ import __version__ +from .__version__ import __version__ # noqa PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) @@ -26,16 +26,6 @@ warnings.filterwarnings("ignore", category=DependencyWarning) warnings.filterwarnings("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=UserWarning) -if sys.version_info >= (3, 1) and sys.version_info <= (3, 6): - if sys.stdout.isatty() and sys.stderr.isatty(): - import io - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = fs_str("1") @@ -46,8 +36,24 @@ try: except Exception: pass +from pipenv.vendor.vistir.misc import get_text_stream + +stdout = get_text_stream("stdout") +stderr = get_text_stream("stderr") + +if os.name == "nt": + from pipenv.vendor.vistir.misc import _can_use_color, _wrap_for_color + + if _can_use_color(stdout): + stdout = _wrap_for_color(stdout) + if _can_use_color(stderr): + stderr = _wrap_for_color(stderr) + +sys.stdout = stdout +sys.stderr = stderr + from .cli import cli -from . import resolver +from . import resolver # noqa if __name__ == "__main__": cli() diff --git a/pipenv/__version__.py b/pipenv/__version__.py index a745170a..1258976a 100644 --- a/pipenv/__version__.py +++ b/pipenv/__version__.py @@ -2,4 +2,4 @@ # // ) ) / / // ) ) //___) ) // ) ) || / / # //___/ / / / //___/ / // // / / || / / # // / / // ((____ // / / ||/ / -__version__ = "2018.11.27.dev0" +__version__ = "2020.04.01.a1" diff --git a/pipenv/_compat.py b/pipenv/_compat.py index ca3bbd0f..fdcca7f5 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -4,11 +4,6 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ - -import functools -import importlib -import io -import os import sys import warnings @@ -122,6 +117,12 @@ UNICODE_TO_ASCII_TRANSLATION_MAP = { } +def decode_for_output(output, target=sys.stdout): + return vistir.misc.decode_for_output( + output, sys.stdout, translation_map=UNICODE_TO_ASCII_TRANSLATION_MAP + ) + + def decode_output(output): if not isinstance(output, six.string_types): return output @@ -129,7 +130,7 @@ def decode_output(output): output = output.encode(DEFAULT_ENCODING) except (AttributeError, UnicodeDecodeError, UnicodeEncodeError): if six.PY2: - output = unicode.translate(vistir.misc.to_text(output), + output = unicode.translate(vistir.misc.to_text(output), # noqa UNICODE_TO_ASCII_TRANSLATION_MAP) else: output = output.translate(UNICODE_TO_ASCII_TRANSLATION_MAP) @@ -144,5 +145,5 @@ def fix_utf8(text): text = decode_output(text) except UnicodeDecodeError: if six.PY2: - text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) + text = unicode.translate(vistir.misc.to_text(text), UNICODE_TO_ASCII_TRANSLATION_MAP) # noqa return text diff --git a/pipenv/cli/__init__.py b/pipenv/cli/__init__.py index d1819953..0f57e306 100644 --- a/pipenv/cli/__init__.py +++ b/pipenv/cli/__init__.py @@ -1,4 +1,4 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import -from .command import cli +from .command import cli # noqa diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index a0f721ab..c433ec08 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -8,19 +8,15 @@ from click import ( argument, echo, edit, group, option, pass_context, secho, version_option, Choice ) -import click_completion -import crayons -import delegator - -from click_didyoumean import DYMCommandCollection - from ..__version__ import __version__ +from ..patched import crayons +from ..vendor import click_completion, delegator from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, general_options, install_options, lock_options, pass_state, - pypi_mirror_option, python_option, requirementstxt_option, - skip_lock_option, sync_options, system_option, three_option, - uninstall_options, verbose_option + pypi_mirror_option, python_option, site_packages_option, skip_lock_option, + sync_options, system_option, three_option, uninstall_options, + verbose_option ) @@ -64,18 +60,15 @@ def cli( state, where=False, venv=False, - rm=False, - bare=False, - three=False, - python=False, - help=False, py=False, envs=False, - man=False, + rm=False, + bare=False, completion=False, - pypi_mirror=None, + man=False, support=None, - clear=False, + help=False, + site_packages=None, **kwargs ): # Handle this ASAP to make shell startup fast. @@ -109,7 +102,7 @@ def cli( if man: if system_which("man"): - path = os.sep.join([os.path.dirname(__file__), "pipenv.1"]) + path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "pipenv.1") os.execle(system_which("man"), "man", path, os.environ) return 0 else: @@ -124,7 +117,7 @@ def cli( echo( "\nYou can learn more at:\n {0}".format( crayons.green( - "http://docs.pipenv.org/advanced/#configuration-with-environment-variables" + "https://pipenv.pypa.io/en/latest/advanced/#configuration-with-environment-variables" ) ) ) @@ -145,16 +138,19 @@ def cli( get_pipenv_diagnostics() return 0 # --clear was passed… - elif clear: + elif state.clear: do_clear() return 0 - # --venv was passed… elif venv: # There is no virtualenv yet. if not project.virtualenv_exists: echo( - crayons.red("No virtualenv has been created for this project yet!"), + "{}({}){}".format( + crayons.red("No virtualenv has been created for this project"), + crayons.white(project.project_directory, bold=True), + crayons.red(" yet!") + ), err=True, ) ctx.abort() @@ -219,6 +215,7 @@ def cli( @system_option @code_option @deploy_option +@site_packages_option @skip_lock_option @install_options @pass_state @@ -251,6 +248,7 @@ def install( extra_index_url=state.extra_index_urls, packages=state.installstate.packages, editable_packages=state.installstate.editables, + site_packages=state.site_packages ) if retcode: ctx.abort() @@ -300,6 +298,7 @@ def uninstall( if retcode: sys.exit(retcode) + @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS) @lock_options @pass_state @@ -311,9 +310,13 @@ def lock( ): """Generates Pipfile.lock.""" from ..core import ensure_project, do_init, do_lock - # Ensure that virtualenv is available. - ensure_project(three=state.three, python=state.python, pypi_mirror=state.pypi_mirror) + # Note that we don't pass clear on to ensure_project as it is also + # handled in do_lock + ensure_project( + three=state.three, python=state.python, pypi_mirror=state.pypi_mirror, + warn=(not state.quiet), site_packages=state.site_packages, + ) if state.installstate.requirementstxt: do_init( dev=state.installstate.dev, @@ -327,6 +330,7 @@ def lock( pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, + write=not state.quiet, ) @@ -338,7 +342,8 @@ def lock( "--fancy", is_flag=True, default=False, - help="Run in shell in fancy mode (for elegantly configured shells).", + help="Run in shell in fancy mode. Make sure the shell have no path manipulating" + " scripts. Run $pipenv shell for issues with compatibility mode.", ) @option( "--anyway", @@ -389,7 +394,6 @@ def shell( @cli.command( - add_help_option=False, short_help="Spawns a command installed into the virtualenv.", context_settings=subcommand_context_no_interspersion, ) @@ -400,7 +404,6 @@ def shell( def run(state, command, args): """Spawns a command installed into the virtualenv.""" from ..core import do_run - do_run( command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) @@ -496,12 +499,14 @@ def update( do_sync, project, ) - - ensure_project(three=state.three, python=state.python, warn=True, pypi_mirror=state.pypi_mirror) + ensure_project( + three=state.three, python=state.python, pypi_mirror=state.pypi_mirror, + warn=(not state.quiet), site_packages=state.site_packages, clear=state.clear + ) if not outdated: outdated = bool(dry_run) if outdated: - do_outdated(pypi_mirror=state.pypi_mirror) + do_outdated(clear=state.clear, pre=state.installstate.pre, pypi_mirror=state.pypi_mirror) packages = [p for p in state.installstate.packages if p] editable = [p for p in state.installstate.editables if p] if not packages: @@ -526,12 +531,13 @@ def update( err=True, ) ctx.abort() - do_lock( + ctx=ctx, clear=state.clear, pre=state.installstate.pre, keep_outdated=state.installstate.keep_outdated, pypi_mirror=state.pypi_mirror, + write=not state.quiet, ) do_sync( ctx=ctx, @@ -578,7 +584,7 @@ def run_open(state, module, *args, **kwargs): EDITOR=atom pipenv open requests """ - from ..core import which, ensure_project + from ..core import which, ensure_project, inline_activate_virtual_environment # Ensure that virtualenv is available. ensure_project( @@ -598,6 +604,7 @@ def run_open(state, module, *args, **kwargs): else: p = c.out.strip().rstrip("cdo") echo(crayons.normal("Opening {0!r} in your EDITOR.".format(p), bold=True)) + inline_activate_virtual_environment() edit(filename=p) return 0 @@ -652,11 +659,9 @@ def sync( def clean(ctx, state, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" from ..core import do_clean - do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run, + system=state.system) -# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). -if "-" not in "".join(sys.argv) and len(sys.argv) > 1: - cli = DYMCommandCollection(sources=[cli]) if __name__ == "__main__": cli() diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 745275dd..fc45256f 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -6,8 +6,9 @@ import os import click.types from click import ( - BadParameter, Group, Option, argument, echo, make_pass_decorator, option + BadParameter, BadArgumentUsage, Group, Option, argument, echo, make_pass_decorator, option ) +from click_didyoumean import DYMMixin from .. import environments from ..utils import is_valid_url @@ -19,7 +20,7 @@ CONTEXT_SETTINGS = { } -class PipenvGroup(Group): +class PipenvGroup(DYMMixin, Group): """Custom Group class provides formatted main help""" def get_help_option(self, ctx): @@ -55,11 +56,12 @@ class State(object): self.index = None self.extra_index_urls = [] self.verbose = False + self.quiet = False self.pypi_mirror = None self.python = None self.two = None self.three = None - self.site_packages = False + self.site_packages = None self.clear = False self.system = False self.installstate = InstallState() @@ -230,18 +232,39 @@ def verbose_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) if value: + if state.quiet: + raise BadArgumentUsage( + "--verbose and --quiet are mutually exclusive! Please choose one!", + ctx=ctx + ) state.verbose = True - setup_verbosity(ctx, param, value) + setup_verbosity(ctx, param, 1) return option("--verbose", "-v", is_flag=True, expose_value=False, callback=callback, help="Verbose mode.", type=click.types.BOOL)(f) +def quiet_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + if value: + if state.verbose: + raise BadArgumentUsage( + "--verbose and --quiet are mutually exclusive! Please choose one!", + ctx=ctx + ) + state.quiet = True + setup_verbosity(ctx, param, -1) + return option("--quiet", "-q", is_flag=True, expose_value=False, + callback=callback, help="Quiet mode.", type=click.types.BOOL)(f) + + def site_packages_option(f): def callback(ctx, param, value): state = ctx.ensure_object(State) + validate_bool_or_none(ctx, param, value) state.site_packages = value return value - return option("--site-packages", is_flag=True, default=False, type=click.types.BOOL, + return option("--site-packages/--no-site-packages", is_flag=True, default=None, help="Enable site-packages for the virtualenv.", callback=callback, expose_value=False, show_envvar=True)(f) @@ -293,8 +316,9 @@ def code_option(f): if value: state.installstate.code = value return value - return option("--code", "-c", nargs=1, default=False, help="Import from codebase.", - callback=callback, expose_value=False)(f) + return option("--code", "-c", nargs=1, default=False, help="Install packages " + "automatically discovered from import statements.", callback=callback, + expose_value=False)(f) def deploy_option(f): @@ -311,8 +335,14 @@ def setup_verbosity(ctx, param, value): if not value: return import logging - logging.getLogger("pip").setLevel(logging.INFO) - environments.PIPENV_VERBOSITY = 1 + loggers = ("pip", "piptools") + if value == 1: + for logger in loggers: + logging.getLogger(logger).setLevel(logging.INFO) + elif value == -1: + for logger in loggers: + logging.getLogger(logger).setLevel(logging.CRITICAL) + environments.PIPENV_VERBOSITY = value def validate_python_path(ctx, param, value): @@ -327,6 +357,12 @@ def validate_python_path(ctx, param, value): return value +def validate_bool_or_none(ctx, param, value): + if value is not None: + return click.types.BOOL(value) + return False + + def validate_pypi_mirror(ctx, param, value): if value and not is_valid_url(value): raise BadParameter("Invalid PyPI mirror URL: %s" % value) @@ -375,7 +411,6 @@ def install_options(f): f = index_option(f) f = extra_index_option(f) f = requirementstxt_option(f) - f = pre_option(f) f = selective_upgrade_option(f) f = ignore_pipfile_option(f) f = editable_option(f) diff --git a/pipenv/cmdparse.py b/pipenv/cmdparse.py index 21aba77e..2e550e22 100644 --- a/pipenv/cmdparse.py +++ b/pipenv/cmdparse.py @@ -10,7 +10,7 @@ class ScriptEmptyError(ValueError): def _quote_if_contains(value, pattern): - if next(re.finditer(pattern, value), None): + if next(iter(re.finditer(pattern, value)), None): return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) return value diff --git a/pipenv/core.py b/pipenv/core.py index 981d0a4c..ff8cc502 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,43 +1,52 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function +import io import json as simplejson import logging import os -import shutil import sys import time import warnings import click import six -import urllib3.util as urllib3_util -import vistir -import click_completion -import crayons import delegator import dotenv import pipfile +import vistir + +from click_completion import init as init_completion from . import environments, exceptions, pep508checker, progress -from ._compat import fix_utf8 +from ._compat import decode_for_output, fix_utf8 from .cmdparse import Script from .environments import ( - PIPENV_CACHE_DIR, PIPENV_COLORBLIND, PIPENV_DEFAULT_PYTHON_VERSION, - PIPENV_DONT_USE_PYENV, PIPENV_HIDE_EMOJIS, PIPENV_MAX_SUBPROCESS, - PIPENV_PYUP_API_KEY, PIPENV_SHELL_FANCY, PIPENV_SKIP_VALIDATION, - PIPENV_YES, SESSION_IS_INTERACTIVE + PIP_EXISTS_ACTION, PIPENV_CACHE_DIR, PIPENV_COLORBLIND, + PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_DONT_USE_PYENV, PIPENV_DONT_USE_ASDF, + PIPENV_HIDE_EMOJIS, PIPENV_MAX_SUBPROCESS, PIPENV_PYUP_API_KEY, + PIPENV_RESOLVE_VCS, PIPENV_SHELL_FANCY, PIPENV_SKIP_VALIDATION, PIPENV_YES, + SESSION_IS_INTERACTIVE, is_type_checking ) -from .project import Project, SourceNotFound +from .patched import crayons +from .project import Project from .utils import ( - convert_deps_to_pip, create_mirror_source, create_spinner, download_file, - escape_cmd, escape_grouped_arguments, find_windows_executable, - get_canonical_names, is_pinned, is_pypi_url, is_required_version, is_star, - is_valid_url, parse_indexes, pep423_name, prepare_pip_source_args, - proper_case, python_version, venv_resolve_deps + convert_deps_to_pip, create_spinner, download_file, + escape_grouped_arguments, find_python, find_windows_executable, + get_canonical_names, get_source_list, interrupt_handled_subprocess, + is_pinned, is_python_command, is_required_version, is_star, is_valid_url, + parse_indexes, pep423_name, prepare_pip_source_args, proper_case, + python_version, run_command, venv_resolve_deps ) +if is_type_checking(): + from typing import Dict, List, Optional, Union, Text + from pipenv.vendor.requirementslib.models.requirements import Requirement + TSourceDict = Dict[Text, Union[Text, bool]] + + # Packages that should be ignored later. BAD_PACKAGES = ( "distribute", @@ -72,7 +81,7 @@ else: INSTALL_LABEL2 = " " STARTING_LABEL = " " # Enable shell completion. -click_completion.init() +init_completion() # Disable colors, for the color blind and others who do not prefer colors. if PIPENV_COLORBLIND: crayons.disable() @@ -86,16 +95,19 @@ def which(command, location=None, allow_global=False): location = os.environ.get("VIRTUAL_ENV", None) if not (location and os.path.exists(location)) and not allow_global: raise RuntimeError("location not created nor specified") + + version_str = "python{0}".format(".".join([str(v) for v in sys.version_info[:2]])) + is_python = command in ("python", os.path.basename(sys.executable), version_str) if not allow_global: if os.name == "nt": p = find_windows_executable(os.path.join(location, "Scripts"), command) else: p = os.path.join(location, "bin", command) else: - if command == "python": + if is_python: p = sys.executable if not os.path.exists(p): - if command == "python": + if is_python: p = sys.executable or system_which("python") else: p = system_which(command) @@ -113,8 +125,10 @@ def do_clear(): from pip import locations try: - vistir.path.rmtree(PIPENV_CACHE_DIR) - vistir.path.rmtree(locations.USER_CACHE_DIR) + vistir.path.rmtree(PIPENV_CACHE_DIR, onerror=vistir.path.handle_remove_readonly) + vistir.path.rmtree( + locations.USER_CACHE_DIR, onerror=vistir.path.handle_remove_readonly + ) except OSError as e: # Ignore FileNotFoundError. This is needed for Python 2.7. import errno @@ -238,7 +252,9 @@ def import_from_code(path="."): rs = [] try: - for r in pipreqs.get_all_imports(path): + for r in pipreqs.get_all_imports( + path, encoding="utf-8", extra_ignore_dirs=[".venv"] + ): if r not in BAD_PACKAGES: rs.append(r) pkg_names = pipreqs.get_pkg_names(rs) @@ -323,26 +339,16 @@ def find_a_system_python(line): * Search for "python" and "pythonX.Y" executables in PATH to find a match. * Nothing fits, return None. """ - if not line: - return None - if os.path.isabs(line): - return line - from .vendor.pythonfinder import Finder + from .vendor.pythonfinder import Finder finder = Finder(system=False, global_search=True) + if not line: + return next(iter(finder.find_all_python_versions()), None) + # Use the windows finder executable if (line.startswith("py ") or line.startswith("py.exe ")) and os.name == "nt": line = line.split(" ", 1)[1].lstrip("-") - elif line.startswith("py"): - python_entry = finder.which(line) - if python_entry: - return python_entry.path.as_posix() - return None - python_entry = finder.find_python_version(line) - if not python_entry: - python_entry = finder.which("python{0}".format(line)) - if python_entry: - return python_entry.path.as_posix() - return None + python_entry = find_python(finder, line) + return python_entry def ensure_python(three=None, python=None): @@ -375,6 +381,9 @@ def ensure_python(three=None, python=None): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) + if environments.is_verbose(): + click.echo(u"Using python: {0}".format(python), err=True) + click.echo(u"Path to python: {0}".format(path_to_python), err=True) if not path_to_python and python is not None: # We need to install Python. click.echo( @@ -385,28 +394,36 @@ def ensure_python(three=None, python=None): ), err=True, ) - # Pyenv is installed - from .vendor.pythonfinder.environment import PYENV_INSTALLED + # check for python installers + from .vendor.pythonfinder.environment import PYENV_INSTALLED, ASDF_INSTALLED + from .installers import Pyenv, Asdf, InstallerError - if not PYENV_INSTALLED: + # prefer pyenv if both pyenv and asdf are installed as it's + # dedicated to python installs so probably the preferred + # method of the user for new python installs. + if PYENV_INSTALLED and not PIPENV_DONT_USE_PYENV: + installer = Pyenv("pyenv") + elif ASDF_INSTALLED and not PIPENV_DONT_USE_ASDF: + installer = Asdf("asdf") + else: + installer = None + + if not installer: abort() else: - if (not PIPENV_DONT_USE_PYENV) and (SESSION_IS_INTERACTIVE or PIPENV_YES): - from .pyenv import Runner, PyenvError - - pyenv = Runner("pyenv") + if SESSION_IS_INTERACTIVE or PIPENV_YES: try: - version = pyenv.find_version_to_install(python) + version = installer.find_version_to_install(python) except ValueError: abort() - except PyenvError as e: + except InstallerError as e: click.echo(fix_utf8("Something went wrong…")) click.echo(crayons.blue(e.err), err=True) abort() s = "{0} {1} {2}".format( "Would you like us to install", crayons.green("CPython {0}".format(version)), - "with pyenv?", + "with {0}?".format(installer), ) # Prompt the user to continue… if not (PIPENV_YES or click.confirm(s, default=True)): @@ -417,15 +434,15 @@ def ensure_python(three=None, python=None): u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Installing", bold=True), crayons.green(u"CPython {0}".format(version), bold=True), - crayons.normal(u"with pyenv", bold=True), + crayons.normal(u"with {0}".format(installer), bold=True), crayons.normal(u"(this may take a few minutes)"), crayons.normal(fix_utf8("…"), bold=True), ) ) with create_spinner("Installing python...") as sp: try: - c = pyenv.install(version) - except PyenvError as e: + c = installer.install(version) + except InstallerError as e: sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Failed...") ) @@ -458,7 +475,7 @@ def ensure_python(three=None, python=None): return path_to_python -def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror=None): +def ensure_virtualenv(three=None, python=None, site_packages=None, pypi_mirror=None): """Creates a virtualenv, if one doesn't exist.""" from .environments import PIPENV_USE_SYSTEM @@ -472,6 +489,8 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= ensure_environment() # Ensure Python is available. python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() # Create the virtualenv. # Abort if --system (or running in a virtualenv). if PIPENV_USE_SYSTEM: @@ -490,10 +509,13 @@ def ensure_virtualenv(three=None, python=None, site_packages=False, pypi_mirror= cleanup_virtualenv(bare=False) sys.exit(1) # If --three, --two, or --python were passed… - elif (python) or (three is not None) or (site_packages is not False): + elif (python) or (three is not None) or (site_packages is not None): USING_DEFAULT_PYTHON = False # Ensure python is installed before deleting existing virtual env - ensure_python(three=three, python=python) + python = ensure_python(three=three, python=python) + if python is not None and not isinstance(python, six.string_types): + python = python.path.as_posix() + click.echo(crayons.red("Virtualenv already exists!"), err=True) # If VIRTUAL_ENV is set, there is a possibility that we are # going to remove the active virtualenv that the user cares @@ -523,7 +545,7 @@ def ensure_project( validate=True, system=False, warn=True, - site_packages=False, + site_packages=None, deploy=False, skip_requirements=False, pypi_mirror=None, @@ -540,11 +562,16 @@ def ensure_project( # Automatically use an activated virtualenv. if PIPENV_USE_SYSTEM: system = True - if not project.pipfile_exists: - if deploy is True: - raise exceptions.PipfileNotFound - else: - project.touch_pipfile() + if not project.pipfile_exists and deploy: + raise exceptions.PipfileNotFound + # Fail if working under / + if not project.name: + click.echo( + "{0}: Pipenv is not intended to work under the root directory, " + "please choose another path.".format(crayons.red("ERROR")), + err=True + ) + sys.exit(1) # Skip virtualenv creation when --system was used. if not system: ensure_virtualenv( @@ -566,7 +593,7 @@ def ensure_project( crayons.red("Warning", bold=True), crayons.normal("python_version", bold=True), crayons.blue(project.required_python_version), - crayons.blue(python_version(path_to_python)), + crayons.blue(python_version(path_to_python) or "unknown"), crayons.green(shorten_path(path_to_python)), ), err=True, @@ -607,24 +634,23 @@ def shorten_path(location, bold=False): def do_where(virtualenv=False, bare=True): """Executes the where functionality.""" if not virtualenv: - location = project.pipfile_location - # Shorten the virtual display of the path to the virtualenv. - if not bare: - location = shorten_path(location) - if not location: + if not project.pipfile_exists: click.echo( "No Pipfile present at project home. Consider running " "{0} first to automatically generate a Pipfile for you." "".format(crayons.green("`pipenv install`")), err=True, ) - elif not bare: + return + location = project.pipfile_location + # Shorten the virtual display of the path to the virtualenv. + if not bare: + location = shorten_path(location) click.echo( "Pipfile found at {0}.\n Considering this to be the project home." "".format(crayons.green(location)), err=True, ) - pass else: click.echo(project.project_directory) else: @@ -637,10 +663,10 @@ def do_where(virtualenv=False, bare=True): click.echo(location) -def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): +def _cleanup_procs(procs, failed_deps_queue, retry=True): while not procs.empty(): c = procs.get() - if concurrent: + if not c.blocking: c.block() failed = False if c.return_code != 0: @@ -651,96 +677,113 @@ def _cleanup_procs(procs, concurrent, failed_deps_queue, retry=True): click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… if failed: - if not retry: + if "does not match installed location" in c.err: + project.environment.expand_egg_links() + click.echo("{0}".format( + crayons.yellow( + "Failed initial installation: Failed to overwrite existing " + "package, likely due to path aliasing. Expanding and trying " + "again!" + ) + )) + dep = c.dep.copy() + elif not retry: # The Installation failed… # We echo both c.out and c.err because pip returns error details on out. err = c.err.strip().splitlines() if c.err else [] out = c.out.strip().splitlines() if c.out else [] - err_lines = [line for line in [out, err]] + err_lines = [line for message in [out, err] for line in message] # Return the subprocess' return code. raise exceptions.InstallError(c.dep.name, extra=err_lines) + else: + # Alert the user. + dep = c.dep.copy() + click.echo( + "{0} {1}! Will try again.".format( + crayons.red("An error occurred while installing"), + crayons.green(dep.as_line()), + ), err=True + ) # Save the Failed Dependency for later. - dep = c.dep.copy() failed_deps_queue.put(dep) - # Alert the user. - click.echo( - "{0} {1}! Will try again.".format( - crayons.red("An error occurred while installing"), - crayons.green(dep.as_line()), - ), err=True - ) def batch_install(deps_list, procs, failed_deps_queue, - requirements_dir, no_deps=False, ignore_hashes=False, + requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, blocking=False, pypi_mirror=None, - nprocs=PIPENV_MAX_SUBPROCESS, retry=True): - + retry=True, sequential_deps=None): + from .vendor.requirementslib.models.utils import strip_extras_markers_from_requirement + if sequential_deps is None: + sequential_deps = [] failed = (not retry) + install_deps = not no_deps if not failed: - label = INSTALL_LABEL if os.name != "nt" else "" + label = INSTALL_LABEL if not PIPENV_HIDE_EMOJIS else "" else: label = INSTALL_LABEL2 + deps_to_install = deps_list[:] + deps_to_install.extend(sequential_deps) + sequential_dep_names = [d.name for d in sequential_deps] + deps_list_bar = progress.bar( - deps_list, width=32, + deps_to_install, width=32, label=label ) - indexes = [] + trusted_hosts = [] # Install these because for dep in deps_list_bar: - index = None - if dep.index: - index = project.find_source(dep.index) - indexes.append(index) - if not index.get("verify_ssl", False): - trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) + extra_indexes = [] + if dep.req.req: + dep.req.req = strip_extras_markers_from_requirement(dep.req.req) + if dep.markers: + dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers())) # Install the module. is_artifact = False if dep.is_file_or_url and (dep.is_direct_url or any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] )): is_artifact = True - - extra_indexes = [] - if not index and indexes: - index = next(iter(indexes)) - if len(indexes) > 1: - extra_indexes = indexes[1:] + elif dep.is_vcs: + is_artifact = True + if not PIPENV_RESOLVE_VCS and is_artifact and not dep.editable: + install_deps = True + no_deps = False with vistir.contextmanagers.temp_environ(): if not allow_global: os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] - if no_deps: - link = getattr(dep.req, "link", None) - is_wheel = False - if link: - is_wheel = link.is_wheel - is_non_editable_vcs = (dep.is_vcs and not dep.editable) - no_deps = not (dep.is_file_or_url and not (is_wheel or dep.editable)) + if "GIT_CONFIG" in os.environ and dep.is_vcs: + del os.environ["GIT_CONFIG"] + use_pep517 = True + if failed and not dep.is_vcs: + use_pep517 = False + c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, + no_deps=not install_deps, block=any([dep.editable, dep.is_vcs, blocking]), - index=index, + index=dep.index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes + extra_indexes=extra_indexes, + use_pep517=use_pep517, ) - if dep.is_vcs or dep.editable: + c.dep = dep + # if dep.is_vcs or dep.editable: + is_sequential = sequential_deps and dep.name in sequential_dep_names + if is_sequential: c.block() - if procs.qsize() < nprocs: - c.dep = dep - procs.put(c) - if procs.full() or procs.qsize() == len(deps_list): - _cleanup_procs(procs, not blocking, failed_deps_queue, retry=retry) + procs.put(c) + if procs.full() or procs.qsize() == len(deps_list) or is_sequential: + _cleanup_procs(procs, failed_deps_queue, retry=retry) def do_install_dependencies( @@ -753,7 +796,7 @@ def do_install_dependencies( skip_lock=False, concurrent=True, requirements_dir=None, - pypi_mirror=False, + pypi_mirror=None, ): """" Executes the install functionality. @@ -764,7 +807,6 @@ def do_install_dependencies( from six.moves import queue if requirements: bare = True - blocking = not concurrent # Load the lockfile if it exists, or if only is being used (e.g. lock is being used). if skip_lock or only or not project.lockfile_exists: if not bare: @@ -785,10 +827,10 @@ def do_install_dependencies( ) ) # Allow pip to resolve dependencies when in skip-lock mode. - no_deps = not skip_lock + no_deps = not skip_lock # skip_lock true, no_deps False, pip resolves deps deps_list = list(lockfile.get_requirements(dev=dev, only=requirements)) if requirements: - index_args = prepare_pip_source_args(project.sources) + index_args = prepare_pip_source_args(get_source_list(pypi_mirror=pypi_mirror, project=project)) index_args = " ".join(index_args).replace(" -", "\n-") deps = [ req.as_line(sources=False, include_hashes=False) for req in deps_list @@ -800,26 +842,40 @@ def do_install_dependencies( ) sys.exit(0) - procs = queue.Queue(maxsize=PIPENV_MAX_SUBPROCESS) + if concurrent: + nprocs = PIPENV_MAX_SUBPROCESS + else: + nprocs = 1 + procs = queue.Queue(maxsize=nprocs) failed_deps_queue = queue.Queue() if skip_lock: ignore_hashes = True - + editable_or_vcs_deps = [dep for dep in deps_list if (dep.editable or dep.vcs)] + normal_deps = [dep for dep in deps_list if not (dep.editable or dep.vcs)] install_kwargs = { "no_deps": no_deps, "ignore_hashes": ignore_hashes, "allow_global": allow_global, - "blocking": blocking, "pypi_mirror": pypi_mirror + "blocking": not concurrent, "pypi_mirror": pypi_mirror, + "sequential_deps": editable_or_vcs_deps } - if concurrent: - install_kwargs["nprocs"] = PIPENV_MAX_SUBPROCESS - else: - install_kwargs["nprocs"] = 1 batch_install( - deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs + normal_deps, procs, failed_deps_queue, requirements_dir, **install_kwargs ) if not procs.empty(): - _cleanup_procs(procs, concurrent, failed_deps_queue) + _cleanup_procs(procs, failed_deps_queue) + + # click.echo(crayons.normal( + # decode_for_output("Installing editable and vcs dependencies…"), bold=True + # )) + + # install_kwargs.update({"blocking": True}) + # # XXX: All failed and editable/vcs deps should be installed in sequential mode! + # procs = queue.Queue(maxsize=1) + # batch_install( + # editable_or_vcs_deps, procs, failed_deps_queue, requirements_dir, + # **install_kwargs + # ) # Iterate over the hopefully-poorly-packaged dependencies… if not failed_deps_queue.empty(): @@ -830,16 +886,12 @@ def do_install_dependencies( while not failed_deps_queue.empty(): failed_dep = failed_deps_queue.get() retry_list.append(failed_dep) - install_kwargs.update({ - "nprocs": 1, - "retry": False, - "blocking": True, - }) + install_kwargs.update({"retry": False}) batch_install( retry_list, procs, failed_deps_queue, requirements_dir, **install_kwargs ) if not procs.empty(): - _cleanup_procs(procs, False, failed_deps_queue, retry=False) + _cleanup_procs(procs, failed_deps_queue, retry=False) def convert_three_to_python(three, python): @@ -857,8 +909,9 @@ def convert_three_to_python(three, python): return python -def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): +def do_create_virtualenv(python=None, site_packages=None, pypi_mirror=None): """Creates a virtualenv.""" + click.echo( crayons.normal(fix_utf8("Creating a virtualenv for this project…"), bold=True), err=True ) @@ -868,11 +921,13 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): ) # Default to using sys.executable, if Python wasn't provided. + using_string = u"Using" if not python: python = sys.executable + using_string = "Using default python from" click.echo( u"{0} {1} {3} {2}".format( - crayons.normal("Using", bold=True), + crayons.normal(using_string, bold=True), crayons.red(python, bold=True), crayons.normal(fix_utf8("to create virtualenv…"), bold=True), crayons.green("({0})".format(python_version(python))), @@ -902,20 +957,19 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): pip_config = {} # Actually create the virtualenv. - nospin = environments.PIPENV_NOSPIN - with create_spinner("Creating virtual environment...") as sp: - c = vistir.misc.run( - cmd, verbose=False, return_object=True, write_to_stdout=False, - combine_stderr=False, block=True, nospin=True, env=pip_config, + error = None + with create_spinner(u"Creating virtual environment...") as sp: + with interrupt_handled_subprocess(cmd, combine_stderr=False, env=pip_config) as c: + click.echo(crayons.blue(u"{0}".format(c.out)), err=True) + if c.returncode != 0: + error = c.err if environments.is_verbose() else exceptions.prettify_exc(c.err) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format(u"Failed creating virtual environment")) + else: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format(u"Successfully created virtual environment!")) + if error is not None: + raise exceptions.VirtualenvCreationException( + extra=crayons.red("{0}".format(error)) ) - click.echo(crayons.blue("{0}".format(c.out)), err=True) - if c.returncode != 0: - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Failed creating virtual environment")) - raise exceptions.VirtualenvCreationException( - extra=[crayons.blue("{0}".format(c.err)),] - ) - else: - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Successfully created virtual environment!")) # Associate project directory with the environment. # This mimics Pew's "setproject". @@ -1016,9 +1070,11 @@ def do_lock( dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. for is_dev in [True, False]: - pipfile_section = "dev_packages" if is_dev else "packages" - lockfile_section = "develop" if is_dev else "default" - packages = getattr(project, pipfile_section) + pipfile_section = "dev-packages" if is_dev else "packages" + if project.pipfile_exists: + packages = project.parsed_pipfile.get(pipfile_section, {}) + else: + packages = getattr(project, pipfile_section.replace("-", "_")) if write: # Alert the user of progress. @@ -1031,12 +1087,9 @@ def do_lock( err=True, ) - deps = convert_deps_to_pip( - packages, project, r=False, include_index=True - ) # Mutates the lockfile venv_resolve_deps( - deps, + packages, which=which, project=project, dev=is_dev, @@ -1045,7 +1098,8 @@ def do_lock( allow_global=system, pypi_mirror=pypi_mirror, pipfile=packages, - lockfile=lockfile + lockfile=lockfile, + keep_outdated=keep_outdated ) # Support for --keep-outdated… @@ -1062,6 +1116,12 @@ def do_lock( lockfile[section_name][canonical_name] = cached_lockfile[ section_name ][canonical_name].copy() + for key in ["default", "develop"]: + packages = set(cached_lockfile[key].keys()) + new_lockfile = set(lockfile[key].keys()) + missing = packages - new_lockfile + for missing_pkg in missing: + lockfile[key][missing_pkg] = cached_lockfile[key][missing_pkg].copy() # Overwrite any develop packages with default packages. lockfile["develop"].update(overwrite_dev(lockfile.get("default", {}), lockfile["develop"])) if write: @@ -1140,12 +1200,19 @@ def do_init( pypi_mirror=None, ): """Executes the init functionality.""" - from .environments import PIPENV_VIRTUALENV + from .environments import ( + PIPENV_VIRTUALENV, PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_PYTHON, PIPENV_USE_SYSTEM + ) + python = None + if PIPENV_PYTHON is not None: + python = PIPENV_PYTHON + elif PIPENV_DEFAULT_PYTHON_VERSION is not None: + python = PIPENV_DEFAULT_PYTHON_VERSION - if not system: + if not system and not PIPENV_USE_SYSTEM: if not project.virtualenv_exists: try: - do_create_virtualenv(pypi_mirror=pypi_mirror) + do_create_virtualenv(python=python, three=None, pypi_mirror=pypi_mirror) except KeyboardInterrupt: cleanup_virtualenv(bare=False) sys.exit(1) @@ -1244,12 +1311,110 @@ def do_init( ) +def get_pip_args( + pre=False, # type: bool + verbose=False, # type: bool, + upgrade=False, # type: bool, + require_hashes=False, # type: bool, + no_build_isolation=False, # type: bool, + no_use_pep517=False, # type: bool, + no_deps=False, # type: bool, + selective_upgrade=False, # type: bool + src_dir=None, # type: Optional[str] +): + # type: (...) -> List[str] + from .vendor.packaging.version import parse as parse_version + arg_map = { + "pre": ["--pre"], + "verbose": ["--verbose"], + "upgrade": ["--upgrade"], + "require_hashes": ["--require-hashes"], + "no_build_isolation": ["--no-build-isolation"], + "no_use_pep517": [], + "no_deps": ["--no-deps"], + "selective_upgrade": [ + "--upgrade-strategy=only-if-needed", + "--exists-action={0}".format(PIP_EXISTS_ACTION or "i") + ], + "src_dir": src_dir, + } + if project.environment.pip_version >= parse_version("19.0"): + arg_map["no_use_pep517"].append("--no-use-pep517") + if project.environment.pip_version < parse_version("19.1"): + arg_map["no_use_pep517"].append("--no-build-isolation") + arg_set = [] + for key in arg_map.keys(): + if key in locals() and locals().get(key): + arg_set.extend(arg_map.get(key)) + elif key == "selective_upgrade" and not locals().get(key): + arg_set.append("--exists-action=i") + return list(vistir.misc.dedup(arg_set)) + + +def get_requirement_line( + requirement, # type: Requirement + src_dir=None, # type: Optional[str] + include_hashes=True, # type: bool + format_for_file=False, # type: bool +): + # type: (...) -> Union[List[str], str] + line = None + if requirement.vcs or requirement.is_file_or_url: + if src_dir and requirement.line_instance.wheel_kwargs: + requirement.line_instance._wheel_kwargs.update({ + "src_dir": src_dir + }) + requirement.line_instance.vcsrepo + line = requirement.line_instance.line + if requirement.line_instance.markers: + line = '{0}; {1}'.format(line, requirement.line_instance.markers) + if not format_for_file: + line = '"{0}"'.format(line) + if requirement.editable: + if not format_for_file: + return ["-e", line] + return '-e {0}'.format(line) + if not format_for_file: + return [line] + return line + return requirement.as_line(include_hashes=include_hashes, as_list=not format_for_file) + + +def write_requirement_to_file( + requirement, # type: Requirement + requirements_dir=None, # type: Optional[str] + src_dir=None, # type: Optional[str] + include_hashes=True # type: bool +): + # type: (...) -> str + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + line = requirement.line_instance.get_line( + with_prefix=True, with_hashes=include_hashes, with_markers=True, as_list=False + ) + + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + if environments.is_verbose(): + click.echo( + "Writing supplied requirement line to temporary file: {0!r}".format(line), + err=True + ) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + return r + + def pip_install( requirement=None, r=None, allow_global=False, ignore_hashes=False, - no_deps=True, + no_deps=None, block=True, index=None, pre=False, @@ -1257,148 +1422,100 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, - trusted_hosts=None + trusted_hosts=None, + use_pep517=True ): - from pipenv.patched.notpip._internal import logger as piplogger - from .utils import Mapping - from .vendor.urllib3.util import parse_url - - src = [] - write_to_tmpfile = False - if requirement: - needs_hashes = not requirement.editable and not ignore_hashes and r is None - has_subdir = requirement.is_vcs and requirement.req.subdirectory - write_to_tmpfile = needs_hashes or has_subdir - + piplogger = logging.getLogger("pipenv.patched.notpip._internal.commands.install") + src_dir = None if not trusted_hosts: trusted_hosts = [] + trusted_hosts.extend(os.environ.get("PIP_TRUSTED_HOSTS", [])) + if not allow_global: + src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR", project.virtualenv_src_location)) + else: + src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR")) + if requirement: + if requirement.editable or not requirement.hashes: + ignore_hashes = True + elif not (requirement.is_vcs or requirement.editable or requirement.vcs): + ignore_hashes = False + line = None + # Try installing for each source in project.sources. + if not index and requirement.index: + index = requirement.index + if index and not extra_indexes: + extra_indexes = list(project.sources) + if requirement and requirement.vcs or requirement.editable: + requirement.index = None + # Install dependencies when a package is a non-editable VCS dependency. + # Don't specify a source directory when using --system. + if not requirement.editable and no_deps is not True: + # Leave this off becauase old lockfiles don't have all deps included + # TODO: When can it be turned back on? + no_deps = False + elif requirement.editable and no_deps is None: + no_deps = True + + r = write_requirement_to_file( + requirement, requirements_dir=requirements_dir, src_dir=src_dir, + include_hashes=not ignore_hashes + ) + sources = get_source_list( + index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, + pypi_mirror=pypi_mirror + ) + if r: + with io.open(r, "r") as fh: + if "--hash" not in fh.read(): + ignore_hashes = True if environments.is_verbose(): - piplogger.setLevel(logging.INFO) + piplogger.setLevel(logging.WARN) if requirement: click.echo( crayons.normal("Installing {0!r}".format(requirement.name), bold=True), err=True, ) - # Create files for hash mode. - if write_to_tmpfile: - if not requirements_dir: - requirements_dir = vistir.path.create_tracked_tempdir( - prefix="pipenv", suffix="requirements") - f = vistir.compat.NamedTemporaryFile( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, - delete=False - ) - f.write(vistir.misc.to_bytes(requirement.as_line())) - r = f.name - f.close() - # Install dependencies when a package is a VCS dependency. - if requirement and requirement.vcs: - no_deps = False - # Don't specify a source directory when using --system. - if not allow_global and ("PIP_SRC" not in os.environ): - src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) - # Try installing for each source in project.sources. - if index: - if isinstance(index, (Mapping, dict)): - index_source = index - else: - try: - index_source = project.find_source(index) - index_source = index_source.copy() - except SourceNotFound: - src_name = project.src_name_from_url(index) - index_url = parse_url(index) - verify_ssl = index_url.host not in trusted_hosts - index_source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} - sources = [index_source.copy(),] - if extra_indexes: - if isinstance(extra_indexes, six.string_types): - extra_indexes = [extra_indexes,] - for idx in extra_indexes: - extra_src = None - if isinstance(idx, (Mapping, dict)): - extra_src = idx - try: - extra_src = project.find_source(idx) if not extra_src else extra_src - except SourceNotFound: - src_name = project.src_name_from_url(idx) - src_url = parse_url(idx) - verify_ssl = src_url.host not in trusted_hosts - extra_src = {"url": idx, "verify_ssl": verify_ssl, "name": extra_src} - if extra_src["url"] != index_source["url"]: - sources.append(extra_src) - else: - for idx in project.pipfile_sources: - if idx["url"] != sources[0]["url"]: - sources.append(idx) - else: - sources = project.pipfile_sources - if pypi_mirror: - sources = [ - create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source - for source in sources - ] - if (requirement and requirement.editable) and not r: - line_kwargs = {"as_list": True} - if requirement.markers: - line_kwargs["include_markers"] = False - install_reqs = requirement.as_line(**line_kwargs) - if requirement.editable and install_reqs[0].startswith("-e "): - req, install_reqs = install_reqs[0], install_reqs[1:] - editable_opt, req = req.split(" ", 1) - install_reqs = [editable_opt, req] + install_reqs - if not all(item.startswith("--hash") for item in install_reqs): - ignore_hashes = True - elif r: - install_reqs = ["-r", r] - with open(r) as f: - if "--hash" not in f.read(): - ignore_hashes = True - else: - ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) - if not requirement.markers: - install_reqs = [escape_cmd(r) for r in install_reqs] - elif len(install_reqs) > 1: - install_reqs = install_reqs[0] + [escape_cmd(r) for r in install_reqs[1:]] pip_command = [which_pip(allow_global=allow_global), "install"] - if pre: - pip_command.append("--pre") - if src: - pip_command.extend(src) - if environments.is_verbose(): - pip_command.append("--verbose") - pip_command.append("--upgrade") - if selective_upgrade: - pip_command.append("--upgrade-strategy=only-if-needed") - if no_deps: - pip_command.append("--no-deps") - pip_command.extend(install_reqs) + pip_args = get_pip_args( + pre=pre, verbose=environments.is_verbose(), upgrade=True, + selective_upgrade=selective_upgrade, no_use_pep517=not use_pep517, + no_deps=no_deps, require_hashes=not ignore_hashes + ) + pip_command.extend(pip_args) + if r: + pip_command.extend(["-r", vistir.path.normalize_path(r)]) + elif line: + pip_command.extend(line) pip_command.extend(prepare_pip_source_args(sources)) - if not ignore_hashes: - pip_command.append("--require-hashes") - if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) + DEFAULT_EXISTS_ACTION = "w" + if selective_upgrade: + DEFAULT_EXISTS_ACTION = "i" + exists_action = vistir.misc.fs_str(PIP_EXISTS_ACTION or DEFAULT_EXISTS_ACTION) pip_config = { "PIP_CACHE_DIR": vistir.misc.fs_str(cache_dir.as_posix()), "PIP_WHEEL_DIR": vistir.misc.fs_str(cache_dir.joinpath("wheels").as_posix()), "PIP_DESTINATION_DIR": vistir.misc.fs_str( cache_dir.joinpath("pkgs").as_posix() ), - "PIP_EXISTS_ACTION": vistir.misc.fs_str("w"), + "PIP_EXISTS_ACTION": exists_action, "PATH": vistir.misc.fs_str(os.environ.get("PATH")), } - if src: + if src_dir: + if environments.is_verbose(): + click.echo("Using source directory: {0!r}".format(src_dir), err=True) pip_config.update( - {"PIP_SRC": vistir.misc.fs_str(project.virtualenv_src_location)} + {"PIP_SRC": vistir.misc.fs_str(src_dir)} ) cmd = Script.parse(pip_command) pip_command = cmd.cmdify() + c = None c = delegator.run(pip_command, block=block, env=pip_config) + c.env = pip_config return c @@ -1425,18 +1542,61 @@ def pip_download(package_name): return c +def fallback_which(command, location=None, allow_global=False, system=False): + """ + A fallback implementation of the `which` utility command that relies exclusively on + searching the path for commands. + + :param str command: The command to search for, optional + :param str location: The search location to prioritize (prepend to path), defaults to None + :param bool allow_global: Whether to search the global path, defaults to False + :param bool system: Whether to use the system python instead of pipenv's python, defaults to False + :raises ValueError: Raised if no command is provided + :raises TypeError: Raised if the command provided is not a string + :return: A path to the discovered command location + :rtype: str + """ + + from .vendor.pythonfinder import Finder + if not command: + raise ValueError("fallback_which: Must provide a command to search for...") + if not isinstance(command, six.string_types): + raise TypeError("Provided command must be a string, received {0!r}".format(command)) + global_search = system or allow_global + if location is None: + global_search = True + finder = Finder(system=False, global_search=global_search, path=location) + if is_python_command(command): + result = find_python(finder, command) + if result: + return result + result = finder.which(command) + if result: + return result.path.as_posix() + return "" + + def which_pip(allow_global=False): """Returns the location of virtualenv-installed pip.""" + + location = None + if "VIRTUAL_ENV" in os.environ: + location = os.environ["VIRTUAL_ENV"] if allow_global: - if "VIRTUAL_ENV" in os.environ: - return which("pip", location=os.environ["VIRTUAL_ENV"]) + if location: + pip = which("pip", location=location) + if pip: + return pip for p in ("pip", "pip3", "pip2"): where = system_which(p) if where: return where - return which("pip") + pip = which("pip") + if not pip: + pip = fallback_which("pip", allow_global=allow_global, location=location) + return pip def system_which(command, mult=False): @@ -1446,6 +1606,7 @@ def system_which(command, mult=False): vistir.compat.fs_str(k): vistir.compat.fs_str(val) for k, val in os.environ.items() } + result = None try: c = delegator.run("{0} {1}".format(_which, command)) try: @@ -1460,21 +1621,19 @@ def system_which(command, mult=False): ) assert c.return_code == 0 except AssertionError: - return None if not mult else [] + result = fallback_which(command, allow_global=True) except TypeError: - from .vendor.pythonfinder import Finder - finder = Finder() - result = finder.which(command) - if result: - return result.path.as_posix() - return + if not result: + result = fallback_which(command, allow_global=True) else: - result = c.out.strip() or c.err.strip() - if mult: - return result.split("\n") - - else: - return result.split("\n")[0] + if not result: + result = next(iter([c.out, c.err]), "").split("\n") + result = next(iter(result)) if not mult else result + return result + if not result: + result = fallback_which(command, allow_global=True) + result = [result] if mult else result + return result def format_help(help): @@ -1611,16 +1770,29 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): def do_py(system=False): + if not project.virtualenv_exists: + click.echo( + "{}({}){}".format( + crayons.red("No virtualenv has been created for this project "), + crayons.white(project.project_directory, bold=True), + crayons.red(" yet!") + ), + err=True, + ) + return + try: click.echo(which("python", allow_global=system)) except AttributeError: click.echo(crayons.red("No project found!")) -def do_outdated(pypi_mirror=None): +def do_outdated(pypi_mirror=None, pre=False, clear=False): # TODO: Allow --skip-lock here? from .vendor.requirementslib.models.requirements import Requirement + from .vendor.requirementslib.models.utils import get_version from .vendor.packaging.utils import canonicalize_name + from .vendor.vistir.compat import Mapping from collections import namedtuple packages = {} @@ -1632,11 +1804,12 @@ def do_outdated(pypi_mirror=None): (pkg.project_name, pkg.parsed_version, pkg.latest_version) for pkg in project.environment.get_outdated_packages() } + reverse_deps = project.environment.reverse_dependencies() for result in installed_packages: dep = Requirement.from_line(str(result.as_requirement())) packages.update(dep.as_pipfile()) updated_packages = {} - lockfile = do_lock(write=False, pypi_mirror=pypi_mirror) + lockfile = do_lock(clear=clear, pre=pre, write=False, pypi_mirror=pypi_mirror) for section in ("develop", "default"): for package in lockfile[section]: try: @@ -1655,10 +1828,26 @@ def do_outdated(pypi_mirror=None): elif canonicalize_name(package) in outdated_packages: skipped.append(outdated_packages[canonicalize_name(package)]) for package, old_version, new_version in skipped: - click.echo(crayons.yellow( - "Skipped Update of Package {0!s}: {1!s} installed, {2!s} available.".format( - package, old_version, new_version - )), err=True + name_in_pipfile = project.get_package_name_in_pipfile(package) + pipfile_version_text = "" + required = "" + version = None + if name_in_pipfile: + version = get_version(project.packages[name_in_pipfile]) + reverse_deps = reverse_deps.get(name_in_pipfile) + if isinstance(reverse_deps, Mapping) and "required" in reverse_deps: + required = " {0} required".format(reverse_deps["required"]) + if version: + pipfile_version_text = " ({0} set in Pipfile)".format(version) + else: + pipfile_version_text = " (Unpinned in Pipfile)" + click.echo( + crayons.yellow( + "Skipped Update of Package {0!s}: {1!s} installed,{2!s}{3!s}, " + "{4!s} available.".format( + package, old_version, required, pipfile_version_text, new_version + ) + ), err=True ) if not outdated: click.echo(crayons.green("All packages are up to date!", bold=True)) @@ -1692,6 +1881,7 @@ def do_install( deploy=False, keep_outdated=False, selective_upgrade=False, + site_packages=None, ): from .environments import PIPENV_VIRTUALENV, PIPENV_USE_SYSTEM from .vendor.pip_shims.shims import PipError @@ -1719,6 +1909,7 @@ def do_install( deploy=deploy, skip_requirements=skip_requirements, pypi_mirror=pypi_mirror, + site_packages=site_packages, ) # Don't attempt to install develop and default packages if Pipfile is missing if not project.pipfile_exists and not (package_args or dev) and not code: @@ -1856,10 +2047,10 @@ def do_install( # This is for if the user passed in dependencies, then we want to make sure we else: - from .vendor.requirementslib import Requirement + from .vendor.requirementslib.models.requirements import Requirement # make a tuple of (display_name, entry) - pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] + pkg_list = packages + ['-e {0}'.format(pkg) for pkg in editable_packages] if not system and not project.virtualenv_exists: do_init( dev=dev, @@ -1872,6 +2063,7 @@ def do_install( pypi_mirror=pypi_mirror, skip_lock=skip_lock, ) + pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) for pkg_line in pkg_list: click.echo( crayons.normal( @@ -1885,21 +2077,27 @@ def do_install( os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] + sp.text = "Resolving {0}...".format(pkg_line) try: pkg_requirement = Requirement.from_line(pkg_line) except ValueError as e: sp.write_err(vistir.compat.fs_str("{0}: {1}".format(crayons.red("WARNING"), e))) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) sys.exit(1) if index_url: pkg_requirement.index = index_url + no_deps = False + sp.text = "Installing..." try: + sp.text = "Installing {0}...".format(pkg_requirement.name) + if environments.is_verbose(): + sp.hide_and_write("Installing package: {0}".format(pkg_requirement.as_line(include_hashes=False))) c = pip_install( pkg_requirement, ignore_hashes=True, allow_global=system, selective_upgrade=selective_upgrade, - no_deps=False, + no_deps=no_deps, pre=pre, requirements_dir=requirements_directory, index=index_url, @@ -1907,17 +2105,36 @@ def do_install( pypi_mirror=pypi_mirror, ) if not c.ok: - raise RuntimeError(c.err) + sp.write_err( + u"{0} An error occurred while installing {1}!".format( + crayons.red(u"Error: ", bold=True), crayons.green(pkg_line) + ), + ) + sp.write_err( + vistir.compat.fs_str(u"Error text: {0}".format(c.out)) + ) + sp.write_err(crayons.blue(vistir.compat.fs_str(format_pip_error(c.err)))) + if environments.is_verbose(): + sp.write_err(crayons.blue(vistir.compat.fs_str(format_pip_output(c.out)))) + if "setup.py egg_info" in c.err: + sp.write_err(vistir.compat.fs_str( + "This is likely caused by a bug in {0}. " + "Report this to its maintainers.".format( + crayons.green(pkg_requirement.name) + ) + )) + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) + sys.exit(1) except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str( "{0}: {1}".format(crayons.red("WARNING"), e), )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Installation Failed", )) sys.exit(1) # Warn if --editable wasn't passed. - if pkg_requirement.is_vcs and not pkg_requirement.editable: + if pkg_requirement.is_vcs and not pkg_requirement.editable and not PIPENV_RESOLVE_VCS: sp.write_err( "{0}: You installed a VCS dependency in non-editable mode. " "This will work fine, but sub-dependencies will not be resolved by {1}." @@ -1927,24 +2144,6 @@ def do_install( crayons.red("$ pipenv lock"), ) ) - click.echo(crayons.blue(format_pip_output(c.out))) - # Ensure that package was successfully installed. - if c.return_code != 0: - sp.write_err(vistir.compat.fs_str( - "{0} An error occurred while installing {1}!".format( - crayons.red("Error: ", bold=True), crayons.green(pkg_line) - ), - )) - sp.write_err(vistir.compat.fs_str(crayons.blue(format_pip_error(c.err)))) - if "setup.py egg_info" in c.err: - sp.write_err(vistir.compat.fs_str( - "This is likely caused by a bug in {0}. " - "Report this to its maintainers.".format( - crayons.green(pkg_requirement.name) - ) - )) - sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Installation Failed")) - sys.exit(1) sp.write(vistir.compat.fs_str( u"{0} {1} {2} {3}{4}".format( crayons.normal(u"Adding", bold=True), @@ -1954,15 +2153,25 @@ def do_install( crayons.normal(fix_utf8("…"), bold=True), ) )) + # Add the package to the Pipfile. + try: + project.add_package_to_pipfile(pkg_requirement, dev) + except ValueError: + import traceback + sp.write_err( + "{0} {1}".format( + crayons.red("Error:", bold=True), traceback.format_exc() + ) + ) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed adding package to Pipfile" + )) sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) - # Add the package to the Pipfile. - try: - project.add_package_to_pipfile(pkg_requirement, dev) - except ValueError as e: - raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) + if pip_shims_module: + os.environ["PIP_SHIMS_BASE_MODULE"] = pip_shims_module do_init( dev=dev, system=system, @@ -2074,6 +2283,7 @@ def do_uninstall( p for normalized, p in selected_pkg_map.items() if normalized in (used_packages - bad_pkgs) ] + pip_path = None for normalized, package_name in selected_pkg_map.items(): click.echo( crayons.white( @@ -2083,12 +2293,10 @@ def do_uninstall( # Uninstall the package. if package_name in packages_to_remove: with project.environment.activated(): - cmd = "{0} uninstall {1} -y".format( - escape_grouped_arguments(which_pip(allow_global=system)), package_name, - ) - if environments.is_verbose(): - click.echo("$ {0}".format(cmd)) - c = delegator.run(cmd) + if pip_path is None: + pip_path = which_pip(allow_global=system) + cmd = [pip_path, "uninstall", package_name, "-y"] + c = run_command(cmd) click.echo(crayons.blue(c.out)) if c.return_code != 0: failure = True @@ -2297,6 +2505,8 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): load_dot_env() + previous_pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() @@ -2304,6 +2514,7 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): # Only set PIPENV_ACTIVE after finishing reading virtualenv_location # such as in inline_activate_virtual_environment # otherwise its value will be changed + previous_pipenv_active_value = os.environ.get("PIPENV_ACTIVE") os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") os.environ.pop("PIP_SHIMS_BASE_MODULE", None) @@ -2315,10 +2526,21 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): click.echo(crayons.normal("$ {0}".format(cmd_string)), err=True) except ScriptEmptyError: click.echo("Can't run script {0!r}-it's empty?", err=True) + run_args = [script] + run_kwargs = {} if os.name == "nt": - do_run_nt(script) + run_fn = do_run_nt else: - do_run_posix(script, command=command) + run_fn = do_run_posix + run_kwargs = {"command": command} + try: + run_fn(*run_args, **run_kwargs) + finally: + os.environ.pop("PIPENV_ACTIVE", None) + if previous_pipenv_active_value is not None: + os.environ["PIPENV_ACTIVE"] = previous_pipenv_active_value + if previous_pip_shims_module is not None: + os.environ["PIP_SHIMS_BASE_MODULE"] = previous_pip_shims_module def do_check( @@ -2333,6 +2555,9 @@ def do_check( args=None, pypi_mirror=None ): + from pipenv.vendor.vistir.compat import JSONDecodeError + from pipenv.vendor.first import first + if not system: # Ensure that virtualenv is available. ensure_project( @@ -2345,8 +2570,8 @@ def do_check( if not args: args = [] if unused: - deps_required = [k for k in project.packages.keys()] - deps_needed = import_from_code(unused) + deps_required = [k.lower() for k in project.packages.keys()] + deps_needed = [k.lower() for k in import_from_code(unused)] for dep in deps_needed: try: deps_required.remove(dep) @@ -2365,18 +2590,32 @@ def do_check( else: sys.exit(0) if not quiet and not environments.is_quiet(): - click.echo(crayons.normal(fix_utf8("Checking PEP 508 requirements…"), bold=True)) - if system: - python = system_which("python") - else: - python = which("python") - # Run the PEP 508 checker in the virtualenv. - c = delegator.run( - '"{0}" {1}'.format( - python, escape_grouped_arguments(pep508checker.__file__.rstrip("cdo")) - ) + click.echo(crayons.normal(decode_for_output("Checking PEP 508 requirements…"), bold=True)) + pep508checker_path = pep508checker.__file__.rstrip("cdo") + safety_path = os.path.join( + os.path.dirname(os.path.abspath(__file__)), "patched", "safety" ) - results = simplejson.loads(c.out) + if not system: + python = which("python") + else: + python = first(system_which(p) for p in ("python", "python3", "python2")) + if not python: + click.echo(crayons.red("The Python interpreter can't be found."), err=True) + sys.exit(1) + _cmd = [vistir.compat.Path(python).as_posix()] + # Run the PEP 508 checker in the virtualenv. + cmd = _cmd + [vistir.compat.Path(pep508checker_path).as_posix()] + c = run_command(cmd) + if c.return_code is not None: + try: + results = simplejson.loads(c.out.strip()) + except JSONDecodeError: + click.echo("{0}\n{1}\n{2}".format( + crayons.white(decode_for_output("Failed parsing pep508 results: "), bold=True), + c.out.strip(), + c.err.strip() + )) + sys.exit(1) # Load the pipfile. p = pipfile.Pipfile.load(project.pipfile_location) failed = False @@ -2403,42 +2642,45 @@ def do_check( if not quiet and not environments.is_quiet(): click.echo(crayons.green("Passed!")) if not quiet and not environments.is_quiet(): - click.echo(crayons.normal(fix_utf8("Checking installed package safety…"), bold=True)) - path = pep508checker.__file__.rstrip("cdo") - path = os.sep.join(__file__.split(os.sep)[:-1] + ["patched", "safety.zip"]) - ignored = "" - if ignore: - ignored = "--ignore {0}".format(" --ignore ".join(ignore)) - click.echo( - crayons.normal( - "Notice: Ignoring CVE(s) {0}".format(crayons.yellow(", ".join(ignore))) - ), - err=True, + click.echo(crayons.normal( + decode_for_output("Checking installed package safety…"), bold=True) ) - - key = "--key={0}".format(PIPENV_PYUP_API_KEY) - if db: + if ignore: + if not isinstance(ignore, (tuple, list)): + ignore = [ignore] + ignored = [["--ignore", cve] for cve in ignore] if not quiet and not environments.is_quiet(): - click.echo(crayons.normal("Using local database {}".format(db))) - key = "--db {0}".format(db) + click.echo( + crayons.normal( + "Notice: Ignoring CVE(s) {0}".format(crayons.yellow(", ".join(ignore))) + ), + err=True, + ) + else: + ignored = [] switch = output if output == "default": switch = "json" - check_string = '"{0}" {1} check --{2} {3} {4}'.format( - python, escape_grouped_arguments(path), switch, ignored, key - ) - - c = delegator.run(check_string) - + cmd = _cmd + [safety_path, "check", "--{0}".format(switch)] + if db: + if not quiet and not environments.is_quiet(): + click.echo(crayons.normal("Using local database {}".format(db))) + cmd.append("--db={0}".format(db)) + if PIPENV_PYUP_API_KEY and not db: + cmd = cmd + ["--key={0}".format(PIPENV_PYUP_API_KEY)] + if ignored: + for cve in ignored: + cmd += cve + c = run_command(cmd, catch_exceptions=False) if output == "default": try: results = simplejson.loads(c.out) - except ValueError: - click.echo("An error occurred:", err=True) - click.echo(c.err if len(c.err) > 0 else c.out, err=True) - sys.exit(1) + except (ValueError, JSONDecodeError): + raise exceptions.JSONParseError(c.out, c.err) + except Exception: + raise exceptions.PipenvCmdError(c.cmd, c.out, c.err, c.return_code) for (package, resolved, installed, description, vuln) in results: click.echo( "{0}: {1} {2} resolved ({3} installed)!".format( @@ -2450,17 +2692,20 @@ def do_check( ) click.echo("{0}".format(description)) click.echo() - if not results: + if c.ok: click.echo(crayons.green("All good!")) + sys.exit(0) else: sys.exit(1) else: click.echo(c.out) - sys.exit(1) + sys.exit(c.return_code) def do_graph(bare=False, json=False, json_tree=False, reverse=False): + from pipenv.vendor.vistir.compat import JSONDecodeError import pipdeptree + pipdeptree_path = pipdeptree.__file__.rstrip("cdo") try: python_path = which("python") except AttributeError: @@ -2475,6 +2720,9 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): sys.exit(1) except RuntimeError: pass + else: + python_path = vistir.compat.Path(python_path).as_posix() + pipdeptree_path = vistir.compat.Path(pipdeptree_path).as_posix() if reverse and json: click.echo( @@ -2525,17 +2773,20 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): err=True, ) sys.exit(1) - cmd = '"{0}" {1} {2} -l'.format( - python_path, escape_grouped_arguments(pipdeptree.__file__.rstrip("cdo")), flag - ) + cmd_args = [python_path, pipdeptree_path, flag, "-l"] + c = run_command(cmd_args) # Run dep-tree. - c = delegator.run(cmd) if not bare: if json: data = [] - for d in simplejson.loads(c.out): - if d["package"]["key"] not in BAD_PACKAGES: - data.append(d) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + for d in parsed: + if d["package"]["key"] not in BAD_PACKAGES: + data.append(d) click.echo(simplejson.dumps(data, indent=4)) sys.exit(0) elif json_tree: @@ -2551,12 +2802,18 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): obj["dependencies"] = traverse(obj["dependencies"]) return obj - data = traverse(simplejson.loads(c.out)) - click.echo(simplejson.dumps(data, indent=4)) - sys.exit(0) + try: + parsed = simplejson.loads(c.out.strip()) + except JSONDecodeError: + raise exceptions.JSONParseError(c.out, c.err) + else: + data = traverse(parsed) + click.echo(simplejson.dumps(data, indent=4)) + sys.exit(0) else: for line in c.out.strip().split("\n"): # Ignore bad packages as top level. + # TODO: This should probably be a "==" in + line.partition if line.split("==")[0] in BAD_PACKAGES and not reverse: continue @@ -2625,34 +2882,32 @@ def do_sync( click.echo(crayons.green("All dependencies are now up-to-date!")) -def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): +def do_clean( + ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None, + system=False +): # Ensure that virtualenv is available. from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) # Make sure that the virtualenv's site packages are configured correctly # otherwise we may end up removing from the global site packages directory - installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg - in project.environment.get_installed_packages() - ] + installed_package_names = project.installed_package_names.copy() # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): click.echo("Ignoring {0}.".format(bad_package), err=True) - del installed_package_names[installed_package_names.index( - canonicalize_name(bad_package) - )] + installed_package_names.remove(canonicalize_name(bad_package)) # Intelligently detect if --dev should be used or not. - develop = [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] - default = [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] - for used_package in set(develop + default): + locked_packages = { + canonicalize_name(pkg) for pkg in project.lockfile_package_names["combined"] + } + for used_package in locked_packages: if used_package in installed_package_names: - del installed_package_names[installed_package_names.index( - canonicalize_name(used_package) - )] + installed_package_names.remove(used_package) failure = False + cmd = [which_pip(allow_global=system), "uninstall", "-y", "-qq"] for apparent_bad_package in installed_package_names: if dry_run and not bare: click.echo(apparent_bad_package) @@ -2664,9 +2919,8 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro ) ) # Uninstall the package. - c = delegator.run( - "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) - ) + cmd = [which_pip(), "uninstall", apparent_bad_package, "-y"] + c = run_command(cmd) if c.return_code != 0: failure = True sys.exit(int(failure)) diff --git a/pipenv/environment.py b/pipenv/environment.py index 06757938..7538ea9e 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -1,25 +1,27 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import contextlib import importlib +import io import json import operator import os import site import sys -from distutils.sysconfig import get_python_lib -from sysconfig import get_paths +from sysconfig import get_paths, get_python_version +import itertools import pkg_resources import six -import vistir import pipenv -from cached_property import cached_property +from .vendor.cached_property import cached_property +from .vendor import vistir -from .utils import normalize_path +from .utils import normalize_path, make_posix BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) @@ -45,6 +47,9 @@ class Environment(object): self.extra_dists = [] prefix = prefix if prefix else sys.prefix self.prefix = vistir.compat.Path(prefix) + self._base_paths = {} + if self.is_venv: + self._base_paths = self.get_paths() self.sys_paths = get_paths() def safe_import(self, name): @@ -85,29 +90,45 @@ class Environment(object): deps.add(dist) try: reqs = dist.requires() - except (AttributeError, OSError, IOError): # The METADATA file can't be found + # KeyError = limited metadata can be found + except (KeyError, AttributeError, OSError, IOError): # The METADATA file can't be found return deps for req in reqs: dist = working_set.find(req) deps |= cls.resolve_dist(dist, working_set) return deps - def add_dist(self, dist_name): - dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + def extend_dists(self, dist): extras = self.resolve_dist(dist, self.base_working_set) + self.extra_dists.append(dist) if extras: self.extra_dists.extend(extras) + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + self.extend_dists(dist) + @cached_property def python_version(self): with self.activated(): - from sysconfig import get_python_version - py_version = get_python_version() + sysconfig = self.safe_import("sysconfig") + py_version = sysconfig.get_python_version() return py_version + def find_libdir(self): + libdir = self.prefix / "lib" + return next(iter(list(libdir.iterdir())), None) + @property def python_info(self): include_dir = self.prefix / "include" + if not os.path.exists(include_dir): + include_dirs = self.get_include_path() + if include_dirs: + include_path = include_dirs.get("include", include_dirs.get("platinclude")) + if not include_path: + return {} + include_dir = vistir.compat.Path(include_path) python_path = next(iter(list(include_dir.iterdir())), None) if python_path and python_path.name.startswith("python"): python_version = python_path.name.replace("python", "") @@ -115,6 +136,16 @@ class Environment(object): return {"py_version_short": py_version_short, "abiflags": abiflags} return {} + def _replace_parent_version(self, path, replace_version): + if not os.path.exists(path): + base, leaf = os.path.split(path) + base, parent = os.path.split(base) + leaf = os.path.join(parent, leaf).replace( + replace_version, self.python_info.get("py_version_short", get_python_version()) + ) + return os.path.join(base, leaf) + return path + @cached_property def base_paths(self): """ @@ -145,50 +176,83 @@ class Environment(object): 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} """ - prefix = self.prefix.as_posix() - install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' - paths = get_paths(install_scheme, vars={ - 'base': prefix, - 'platbase': prefix, - }) + prefix = make_posix(self.prefix.as_posix()) + paths = {} + if self._base_paths: + paths = self._base_paths.copy() + else: + try: + paths = self.get_paths() + except Exception: + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + current_version = get_python_version() + try: + for k in list(paths.keys()): + if not os.path.exists(paths[k]): + paths[k] = self._replace_parent_version(paths[k], current_version) + except OSError: + # Sometimes virtualenvs are made using virtualenv interpreters and there is no + # include directory, which will cause this approach to fail. This failsafe + # will make sure we fall back to the shell execution to find the real include path + paths = self.get_include_path() + paths.update(self.get_lib_paths()) + paths["scripts"] = self.script_basedir + if not paths: + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + if not os.path.exists(paths["purelib"]) and not os.path.exists(paths["platlib"]): + lib_paths = self.get_lib_paths() + paths.update(lib_paths) paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath if "prefix" not in paths: paths["prefix"] = prefix - purelib = get_python_lib(plat_specific=0, prefix=prefix) - platlib = get_python_lib(plat_specific=1, prefix=prefix) + purelib = paths["purelib"] = make_posix(paths["purelib"]) + platlib = paths["platlib"] = make_posix(paths["platlib"]) if purelib == platlib: lib_dirs = purelib else: lib_dirs = purelib + os.pathsep + platlib paths["libdir"] = purelib - paths["purelib"] = purelib - paths["platlib"] = platlib - paths['PYTHONPATH'] = lib_dirs + paths['PYTHONPATH'] = os.pathsep.join(["", ".", lib_dirs]) paths["libdirs"] = lib_dirs return paths @cached_property def script_basedir(self): """Path to the environment scripts dir""" - script_dir = self.base_paths["scripts"] - return script_dir + prefix = make_posix(self.prefix.as_posix()) + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + paths = get_paths(install_scheme, vars={ + 'base': prefix, + 'platbase': prefix, + }) + return paths["scripts"] @property def python(self): """Path to the environment python""" - py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + py = vistir.compat.Path(self.script_basedir).joinpath("python").absolute().as_posix() if not py: return vistir.compat.Path(sys.executable).as_posix() return py @cached_property def sys_path(self): - """The system path inside the environment + """ + The system path inside the environment :return: The :data:`sys.path` from the environment :rtype: list """ + from .vendor.vistir.compat import JSONDecodeError current_executable = vistir.compat.Path(sys.executable).as_posix() if not self.python or self.python == current_executable: return sys.path @@ -196,18 +260,173 @@ class Environment(object): return sys.path cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False, write_to_stdout=False) - path = json.loads(path.strip()) + try: + path = json.loads(path.strip()) + except JSONDecodeError: + path = sys.path return path + def build_command(self, python_lib=False, python_inc=False, scripts=False, py_version=False): + """Build the text for running a command in the given environment + + :param python_lib: Whether to include the python lib dir commands, defaults to False + :type python_lib: bool, optional + :param python_inc: Whether to include the python include dir commands, defaults to False + :type python_inc: bool, optional + :param scripts: Whether to include the scripts directory, defaults to False + :type scripts: bool, optional + :param py_version: Whether to include the python version info, defaults to False + :type py_version: bool, optional + :return: A string representing the command to run + """ + pylib_lines = [] + pyinc_lines = [] + py_command = ( + "import sysconfig, distutils.sysconfig, io, json, sys; paths = {{" + "%s }}; value = u'{{0}}'.format(json.dumps(paths));" + "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + ) + distutils_line = "distutils.sysconfig.get_python_{0}(plat_specific={1})" + sysconfig_line = "sysconfig.get_path('{0}')" + if python_lib: + for key, var, val in (("pure", "lib", "0"), ("plat", "lib", "1")): + dist_prefix = "{0}lib".format(key) + # XXX: We need to get 'stdlib' or 'platstdlib' + sys_prefix = "{0}stdlib".format("" if key == "pure" else key) + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (dist_prefix, distutils_line.format(var, val))) + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (sys_prefix, sysconfig_line.format(sys_prefix))) + if python_inc: + for key, var, val in (("include", "inc", "0"), ("platinclude", "inc", "1")): + pylib_lines.append("u'%s': u'{{0}}'.format(%s)" % (key, distutils_line.format(var, val))) + lines = pylib_lines + pyinc_lines + if scripts: + lines.append("u'scripts': u'{{0}}'.format(%s)" % sysconfig_line.format("scripts")) + if py_version: + lines.append("u'py_version_short': u'{{0}}'.format(distutils.sysconfig.get_python_version()),") + lines_as_str = u",".join(lines) + py_command = py_command % lines_as_str + return py_command + + def get_paths(self): + """ + Get the paths for the environment by running a subcommand + + :return: The python paths for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = self.build_command(python_lib=True, python_inc=True, scripts=True, py_version=True) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + if c.returncode == 0: + paths = {} + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + if "purelib" in paths: + paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) + for key in ("platlib", "scripts", "platstdlib", "stdlib", "include", "platinclude"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + return None + + def get_lib_paths(self): + """Get the include path for the environment + + :return: The python include path for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = self.build_command(python_lib=True) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + paths = None + if c.returncode == 0: + paths = {} + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + if "purelib" in paths: + paths["libdir"] = paths["purelib"] = make_posix(paths["purelib"]) + for key in ("platlib", "platstdlib", "stdlib"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + if not paths: + if not self.prefix.joinpath("lib").exists(): + return {} + stdlib_path = next(iter([ + p for p in self.prefix.joinpath("lib").iterdir() + if p.name.startswith("python") + ]), None) + lib_path = None + if stdlib_path: + lib_path = next(iter([ + p.as_posix() for p in stdlib_path.iterdir() + if p.name == "site-packages" + ])) + paths = {"stdlib": stdlib_path.as_posix()} + if lib_path: + paths["purelib"] = lib_path + return paths + return {} + + def get_include_path(self): + """Get the include path for the environment + + :return: The python include path for the environment + :rtype: Dict[str, str] + """ + tmpfile = vistir.path.create_tracked_tempfile(suffix=".json") + tmpfile.close() + tmpfile_path = make_posix(tmpfile.name) + py_command = ( + "import distutils.sysconfig, io, json, sys; paths = {{u'include': " + "u'{{0}}'.format(distutils.sysconfig.get_python_inc(plat_specific=0)), " + "u'platinclude': u'{{0}}'.format(distutils.sysconfig.get_python_inc(" + "plat_specific=1)) }}; value = u'{{0}}'.format(json.dumps(paths));" + "fh = io.open('{0}', 'w'); fh.write(value); fh.close()" + ) + command = [self.python, "-c", py_command.format(tmpfile_path)] + c = vistir.misc.run( + command, return_object=True, block=True, nospin=True, write_to_stdout=False + ) + if c.returncode == 0: + paths = [] + with io.open(tmpfile_path, "r", encoding="utf-8") as fh: + paths = json.load(fh) + for key in ("include", "platinclude"): + if key in paths: + paths[key] = make_posix(paths[key]) + return paths + else: + vistir.misc.echo("Failed to load paths: {0}".format(c.err), fg="yellow") + vistir.misc.echo("Output: {0}".format(c.out), fg="yellow") + return None + @cached_property def sys_prefix(self): - """The prefix run inside the context of the environment + """ + The prefix run inside the context of the environment :return: The python prefix inside the environment :rtype: :data:`sys.prefix` """ - command = [self.python, "-c" "import sys; print(sys.prefix)"] + command = [self.python, "-c", "import sys; print(sys.prefix)"] c = vistir.misc.run(command, return_object=True, block=True, nospin=True, write_to_stdout=False) sys_prefix = vistir.compat.Path(vistir.misc.to_text(c.out).strip()).as_posix() return sys_prefix @@ -236,15 +455,55 @@ class Environment(object): return "purelib", purelib return "platlib", self.paths["platlib"] + @property + def pip_version(self): + """ + Get the pip version in the environment. Useful for knowing which args we can use + when installing. + """ + from .vendor.packaging.version import parse as parse_version + pip = next(iter( + pkg for pkg in self.get_installed_packages() if pkg.key == "pip" + ), None) + if pip is not None: + return parse_version(pip.version) + return parse_version("19.3") + + def expand_egg_links(self): + """ + Expand paths specified in egg-link files to prevent pip errors during + reinstall + """ + prefixes = [ + vistir.compat.Path(prefix) + for prefix in self.base_paths["libdirs"].split(os.pathsep) + if vistir.path.is_in_path(prefix, self.prefix.as_posix()) + ] + for loc in prefixes: + if not loc.exists(): + continue + for pth in loc.iterdir(): + if not pth.suffix == ".egg-link": + continue + contents = [ + vistir.path.normalize_path(line.strip()) + for line in pth.read_text().splitlines() + ] + pth.write_text("\n".join(contents)) + def get_distributions(self): - """Retrives the distributions installed on the library path of the environment + """ + Retrives the distributions installed on the library path of the environment :return: A set of distributions found on the library path :rtype: iterator """ pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + libdirs = self.base_paths["libdirs"].split(os.pathsep) + dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs) + for dist in itertools.chain.from_iterable(dists): + yield dist def find_egg(self, egg_dist): """Find an egg by name in the given environment""" @@ -271,58 +530,56 @@ class Environment(object): def dist_is_in_project(self, dist): """Determine whether the supplied distribution is in the environment.""" from .project import _normalized - prefix = _normalized(self.base_paths["prefix"]) + prefixes = [ + _normalized(prefix) for prefix in self.base_paths["libdirs"].split(os.pathsep) + if _normalized(prefix).startswith(_normalized(self.prefix.as_posix())) + ] location = self.locate_dist(dist) if not location: return False - return _normalized(location).startswith(prefix) + location = _normalized(make_posix(location)) + return any(location.startswith(prefix) for prefix in prefixes) def get_installed_packages(self): """Returns all of the installed packages in a given environment""" workingset = self.get_working_set() - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + packages = [ + pkg for pkg in workingset + if self.dist_is_in_project(pkg) and pkg.key != "python" + ] return packages @contextlib.contextmanager def get_finder(self, pre=False): - from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder - from .environments import PIPENV_CACHE_DIR - index_urls = [source.get("url") for source in self.sources] - - class PipCommand(Command): - name = "PipCommand" - - pip_command = PipCommand() - index_opts = cmdoptions.make_option_group( - index_group, pip_command.parser + from .vendor.pip_shims.shims import ( + InstallCommand, get_package_finder ) - cmd_opts = pip_command.cmd_opts - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.insert_option_group(0, cmd_opts) + from .environments import PIPENV_CACHE_DIR + + pip_command = InstallCommand() pip_args = self._modules["pipenv"].utils.prepare_pip_source_args(self.sources) pip_options, _ = pip_command.parser.parse_args(pip_args) pip_options.cache_dir = PIPENV_CACHE_DIR pip_options.pre = self.pipfile.get("pre", pre) with pip_command._build_session(pip_options) as session: - finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, allow_all_prereleases=pip_options.pre, - trusted_hosts=pip_options.trusted_hosts, - process_dependency_links=pip_options.process_dependency_links, - session=session - ) + finder = get_package_finder(install_cmd=pip_command, options=pip_options, session=session) yield finder def get_package_info(self, pre=False): + from .vendor.pip_shims.shims import pip_version, parse_version, CandidateEvaluator dependency_links = [] packages = self.get_installed_packages() # This code is borrowed from pip's current implementation - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend(dist.get_metadata_lines('dependency_links.txt')) + if parse_version(pip_version) < parse_version("19.0"): + for dist in packages: + if dist.has_metadata('dependency_links.txt'): + dependency_links.extend( + dist.get_metadata_lines('dependency_links.txt') + ) with self.get_finder() as finder: - finder.add_dependency_links(dependency_links) + if parse_version(pip_version) < parse_version("19.0"): + finder.add_dependency_links(dependency_links) for dist in packages: typ = 'unknown' @@ -336,9 +593,10 @@ class Environment(object): if not all_candidates: continue - best_candidate = max(all_candidates, key=finder._candidate_sort_key) - remote_version = best_candidate.version - if best_candidate.location.is_wheel: + candidate_evaluator = finder.make_candidate_evaluator(project_name=dist.key) + best_candidate_result = candidate_evaluator.compute_best_candidate(all_candidates) + remote_version = best_candidate_result.best_candidate.version + if best_candidate_result.best_candidate.link.is_wheel: typ = 'wheel' else: typ = 'sdist' @@ -350,36 +608,89 @@ class Environment(object): def get_outdated_packages(self, pre=False): return [ pkg for pkg in self.get_package_info(pre=pre) - if pkg.latest_version._version > pkg.parsed_version._version + if pkg.latest_version._key > pkg.parsed_version._key ] - def get_package_requirements(self): + @classmethod + def _get_requirements_for_package(cls, node, key_tree, parent=None, chain=None): + if chain is None: + chain = [node.project_name] + + d = node.as_dict() + if parent: + d['required_version'] = node.version_spec if node.version_spec else 'Any' + else: + d['required_version'] = d['installed_version'] + + get_children = lambda n: key_tree.get(n.key, []) # noqa + + d['dependencies'] = [ + cls._get_requirements_for_package(c, key_tree, parent=node, + chain=chain+[c.project_name]) + for c in get_children(node) + if c.project_name not in chain + ] + + return d + + def get_package_requirements(self, pkg=None): from .vendor.pipdeptree import flatten, sorted_tree, build_dist_index, construct_tree - dist_index = build_dist_index(self.get_installed_packages()) + packages = self.get_installed_packages() + if pkg: + packages = [p for p in packages if p.key == pkg] + dist_index = build_dist_index(packages) tree = sorted_tree(construct_tree(dist_index)) branch_keys = set(r.key for r in flatten(tree.values())) - nodes = [p for p in tree.keys() if p.key not in branch_keys] + if pkg is not None: + nodes = [p for p in tree.keys() if p.key == pkg] + else: + nodes = [p for p in tree.keys() if p.key not in branch_keys] key_tree = dict((k.key, v) for k, v in tree.items()) - get_children = lambda n: key_tree.get(n.key, []) - def aux(node, parent=None, chain=None): - if chain is None: - chain = [node.project_name] + return [self._get_requirements_for_package(p, key_tree) for p in nodes] - d = node.as_dict() - if parent: - d['required_version'] = node.version_spec if node.version_spec else 'Any' - else: - d['required_version'] = d['installed_version'] + @classmethod + def reverse_dependency(cls, node): + new_node = { + "package_name": node["package_name"], + "installed_version": node["installed_version"], + "required_version": node["required_version"] + } + for dependency in node.get("dependencies", []): + for dep in cls.reverse_dependency(dependency): + new_dep = dep.copy() + new_dep["parent"] = (node["package_name"], node["installed_version"]) + yield new_dep + yield new_node - d['dependencies'] = [ - aux(c, parent=node, chain=chain+[c.project_name]) - for c in get_children(node) - if c.project_name not in chain - ] - - return d - return [aux(p) for p in nodes] + def reverse_dependencies(self): + from vistir.misc import unnest, chunked + rdeps = {} + for req in self.get_package_requirements(): + for d in self.reverse_dependency(req): + parents = None + name = d["package_name"] + pkg = { + name: { + "installed": d["installed_version"], + "required": d["required_version"] + } + } + parents = tuple(d.get("parent", ())) + pkg[name]["parents"] = parents + if rdeps.get(name): + if not (rdeps[name].get("required") or rdeps[name].get("installed")): + rdeps[name].update(pkg[name]) + rdeps[name]["parents"] = rdeps[name].get("parents", ()) + parents + else: + rdeps[name] = pkg[name] + for k in list(rdeps.keys()): + entry = rdeps[k] + if entry.get("parents"): + rdeps[k]["parents"] = set([ + p for p, version in chunked(2, unnest(entry["parents"])) + ]) + return rdeps def get_working_set(self): """Retrieve the working set of installed packages for the environment. @@ -475,21 +786,34 @@ class Environment(object): vendor_dir = parent_path.joinpath("vendor").as_posix() patched_dir = parent_path.joinpath("patched").as_posix() parent_path = parent_path.as_posix() + self.add_dist("pip") prefix = self.prefix.as_posix() with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): os.environ["PATH"] = os.pathsep.join([ - vistir.compat.fs_str(self.scripts_dir), + vistir.compat.fs_str(self.script_basedir), vistir.compat.fs_str(self.prefix.as_posix()), os.environ.get("PATH", "") ]) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + from .environments import PIPENV_USE_SYSTEM if self.is_venv: + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + else: + if not PIPENV_USE_SYSTEM and not os.environ.get("VIRTUAL_ENV"): + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + os.environ.pop("PYTHONHOME", None) sys.path = self.sys_path sys.prefix = self.sys_prefix site.addsitedir(self.base_paths["purelib"]) + pip = self.safe_import("pip") # noqa + pip_vendor = self.safe_import("pip._vendor") + pep517_dir = os.path.join(os.path.dirname(pip_vendor.__file__), "pep517") + site.addsitedir(pep517_dir) + os.environ["PYTHONPATH"] = os.pathsep.join([ + os.environ.get("PYTHONPATH", self.base_paths["PYTHONPATH"]), pep517_dir + ]) if include_extras: site.addsitedir(parent_path) sys.path.extend([parent_path, patched_dir, vendor_dir]) @@ -543,7 +867,7 @@ class Environment(object): def install(self, requirements): if not isinstance(requirements, (tuple, list)): - requirements = [requirements,] + requirements = [requirements] with self.get_finder() as finder: args = [] for format_control in ('no_binary', 'only_binary'): @@ -593,12 +917,9 @@ class Environment(object): monkey_patch.activate() pip_shims = self.safe_import("pip_shims") pathset_base = pip_shims.UninstallPathSet - import recursive_monkey_patch - recursive_monkey_patch.monkey_patch( - PatchedUninstaller, pathset_base - ) + pathset_base._permitted = PatchedUninstaller._permitted dist = next( - iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), + iter(d for d in self.get_working_set() if d.project_name == pkgname), None ) pathset = pathset_base.from_dist(dist) @@ -606,7 +927,7 @@ class Environment(object): pathset.remove(auto_confirm=auto_confirm, verbose=verbose) try: yield pathset - except Exception as e: + except Exception: if pathset is not None: pathset.rollback() else: diff --git a/pipenv/environments.py b/pipenv/environments.py index 445e3b63..848fec87 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -3,16 +3,27 @@ import os import sys +from io import UnsupportedOperation + from appdirs import user_cache_dir from ._compat import fix_utf8 -from .vendor.vistir.misc import fs_str +from .vendor.vistir.misc import _isatty, fs_str # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. os.environ["PYTHONDONTWRITEBYTECODE"] = fs_str("1") + +def _is_env_truthy(name): + """An environment variable is truthy if it exists and isn't one of (0, false, no, off) + """ + if name not in os.environ: + return False + return os.environ.get(name).lower() not in ("0", "false", "no", "off") + + PIPENV_IS_CI = bool("CI" in os.environ or "TF_BUILD" in os.environ) # HACK: Prevent invalid shebangs with Homebrew-installed Python: @@ -57,6 +68,12 @@ PIPENV_DONT_USE_PYENV = bool(os.environ.get("PIPENV_DONT_USE_PYENV")) Default is to install Python automatically via pyenv when needed, if possible. """ +PIPENV_DONT_USE_ASDF = bool(os.environ.get("PIPENV_DONT_USE_ASDF")) +"""If set, Pipenv does not attempt to install Python with asdf. + +Default is to install Python automatically via asdf when needed, if possible. +""" + PIPENV_DOTENV_LOCATION = os.environ.get("PIPENV_DOTENV_LOCATION") """If set, Pipenv loads the ``.env`` file at the specified location. @@ -70,13 +87,15 @@ Default is to detect emulators automatically. This should be set if your emulator, e.g. Cmder, cannot be detected correctly. """ -PIPENV_HIDE_EMOJIS = bool(os.environ.get("PIPENV_HIDE_EMOJIS")) +PIPENV_HIDE_EMOJIS = ( + os.environ.get("PIPENV_HIDE_EMOJIS") is None + and (os.name == "nt" or PIPENV_IS_CI) + or _is_env_truthy("PIPENV_HIDE_EMOJIS") +) """Disable emojis in output. Default is to show emojis. This is automatically set on Windows. """ -if os.name == "nt" or PIPENV_IS_CI: - PIPENV_HIDE_EMOJIS = True PIPENV_IGNORE_VIRTUALENVS = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS")) """If set, Pipenv will always assign a virtual environment for this project. @@ -86,7 +105,7 @@ environment, and reuses it if possible. This is usually the desired behavior, and enables the user to use any user-built environments with Pipenv. """ -PIPENV_INSTALL_TIMEOUT = 60 * 15 +PIPENV_INSTALL_TIMEOUT = int(os.environ.get("PIPENV_INSTALL_TIMEOUT", 60 * 15)) """Max number of seconds to wait for package installation. Defaults to 900 (15 minutes), a very long arbitrary time. @@ -113,7 +132,7 @@ PIPENV_MAX_ROUNDS = int(os.environ.get("PIPENV_MAX_ROUNDS", "16")) Default is 16, an arbitrary number that works most of the time. """ -PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "16")) +PIPENV_MAX_SUBPROCESS = int(os.environ.get("PIPENV_MAX_SUBPROCESS", "8")) """How many subprocesses should Pipenv use when installing. Default is 16, an arbitrary number that seems to work. @@ -137,14 +156,13 @@ environments. if PIPENV_IS_CI: PIPENV_NOSPIN = True -PIPENV_SPINNER = "dots" +PIPENV_SPINNER = "dots" if not os.name == "nt" else "bouncingBar" +PIPENV_SPINNER = os.environ.get("PIPENV_SPINNER", PIPENV_SPINNER) """Sets the default spinner type. Spinners are identitcal to the node.js spinners and can be found at https://github.com/sindresorhus/cli-spinners """ -if os.name == "nt": - PIPENV_SPINNER = "bouncingBar" PIPENV_PIPFILE = os.environ.get("PIPENV_PIPFILE") """If set, this specifies a custom Pipfile location. @@ -223,8 +241,26 @@ Default is to lock dependencies and update ``Pipfile.lock`` on each run. NOTE: This only affects the ``install`` and ``uninstall`` commands. """ +PIP_EXISTS_ACTION = os.environ.get("PIP_EXISTS_ACTION", "w") +"""Specifies the value for pip's --exists-action option + +Defaullts to (w)ipe +""" + +PIPENV_RESOLVE_VCS = ( + os.environ.get("PIPENV_RESOLVE_VCS") is None + or _is_env_truthy("PIPENV_RESOLVE_VCS") +) + +"""Tells Pipenv whether to resolve all VCS dependencies in full. + +As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full. +To retain this behavior and avoid handling any conflicts that arise from the new +approach, you may set this to '0', 'off', or 'false'. +""" + PIPENV_PYUP_API_KEY = os.environ.get( - "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0" + "PIPENV_PYUP_API_KEY", None ) # Internal, support running in a different Python from sys.executable. @@ -252,7 +288,10 @@ PIPENV_SHELL = ( ) # Internal, to tell whether the command line session is interactive. -SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) +try: + SESSION_IS_INTERACTIVE = _isatty(sys.stdout.fileno()) +except UnsupportedOperation: + SESSION_IS_INTERACTIVE = _isatty(sys.stdout) # Internal, consolidated verbosity representation as an integer. The default @@ -280,13 +319,35 @@ def is_quiet(threshold=-1): def is_in_virtualenv(): - pipenv_active = os.environ.get("PIPENV_ACTIVE") - virtual_env = os.environ.get("VIRTUAL_ENV") - return (PIPENV_USE_SYSTEM or virtual_env) and not ( - pipenv_active or PIPENV_IGNORE_VIRTUALENVS - ) + """ + Check virtualenv membership dynamically + + :return: True or false depending on whether we are in a regular virtualenv or not + :rtype: bool + """ + + pipenv_active = os.environ.get("PIPENV_ACTIVE", False) + virtual_env = None + use_system = False + ignore_virtualenvs = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS", False)) + + if not pipenv_active and not ignore_virtualenvs: + virtual_env = os.environ.get("VIRTUAL_ENV") + use_system = bool(virtual_env) + return (use_system or virtual_env) and not (pipenv_active or ignore_virtualenvs) PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = is_type_checking() diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 0da42fe7..919442f1 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -1,42 +1,67 @@ # -*- coding=utf-8 -*- import itertools +import re import sys -from pprint import pformat -from traceback import format_exception +from collections import namedtuple +from traceback import format_tb import six from . import environments -from ._compat import fix_utf8 +from ._compat import decode_for_output from .patched import crayons -from .vendor.click._compat import get_text_stderr from .vendor.click.exceptions import ( - Abort, BadOptionUsage, BadParameter, ClickException, Exit, FileError, - MissingParameter, UsageError + ClickException, FileError, UsageError ) -from .vendor.click.types import Path -from .vendor.click.utils import echo as click_echo +from .vendor.vistir.misc import echo as click_echo +import vistir + +ANSI_REMOVAL_RE = re.compile(r"\033\[((?:\d|;)*)([a-zA-Z])", re.MULTILINE) +STRING_TYPES = (six.string_types, crayons.ColoredString) + +if sys.version_info[:2] >= (3, 7): + KnownException = namedtuple( + 'KnownException', ['exception_name', 'match_string', 'show_from_string', 'prefix'], + defaults=[None, None, None, ""] + ) +else: + KnownException = namedtuple( + 'KnownException', ['exception_name', 'match_string', 'show_from_string', 'prefix'], + ) + KnownException.__new__.__defaults__ = (None, None, None, "") + +KNOWN_EXCEPTIONS = [ + KnownException("PermissionError", prefix="Permission Denied:"), + KnownException( + "VirtualenvCreationException", + match_string="do_create_virtualenv", + show_from_string=None + ) +] def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): if environments.is_verbose() or not issubclass(exc_type, ClickException): hook(exc_type, exception, traceback) else: - exc = format_exception(exc_type, exception, traceback) - lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) - lines = list(lines)[-11:-1] + tb = format_tb(traceback, limit=-6) + lines = itertools.chain.from_iterable([frame.splitlines() for frame in tb]) + formatted_lines = [] for line in lines: line = line.strip("'").strip('"').strip("\n").strip() if not line.startswith("File"): line = " {0}".format(line) else: line = " {0}".format(line) - line = "[pipenv.exceptions.{0!s}]: {1}".format( + line = "[{0!s}]: {1}".format( exception.__class__.__name__, line ) - click_echo(fix_utf8(line), err=True) + formatted_lines.append(line) + # use new exception prettification rules to format exceptions according to + # UX rules + click_echo(decode_for_output(prettify_exc("\n".join(formatted_lines))), err=True) exception.show() @@ -56,16 +81,65 @@ class PipenvException(ClickException): def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() if self.extra: - if isinstance(self.extra, six.string_types): - self.extra = [self.extra,] + if isinstance(self.extra, STRING_TYPES): + self.extra = [self.extra] for extra in self.extra: extra = "[pipenv.exceptions.{0!s}]: {1}".format( self.__class__.__name__, extra ) + extra = decode_for_output(extra, file) click_echo(extra, file=file) - click_echo(fix_utf8("{0}".format(self.message)), file=file) + click_echo(decode_for_output("{0}".format(self.message), file), file=file) + + +class PipenvCmdError(PipenvException): + def __init__(self, cmd, out="", err="", exit_code=1): + self.cmd = cmd + self.out = out + self.err = err + self.exit_code = exit_code + message = "Error running command: {0}".format(cmd) + PipenvException.__init__(self, message) + + def show(self, file=None): + if file is None: + file = vistir.misc.get_text_stderr() + click_echo("{0} {1}".format( + crayons.red("Error running command: "), + crayons.white(decode_for_output("$ {0}".format(self.cmd), file), bold=True) + ), err=True) + if self.out: + click_echo("{0} {1}".format( + crayons.white("OUTPUT: "), + decode_for_output(self.out, file) + ), err=True) + if self.err: + click_echo("{0} {1}".format( + crayons.white("STDERR: "), + decode_for_output(self.err, file) + ), err=True) + + +class JSONParseError(PipenvException): + def __init__(self, contents="", error_text=""): + self.error_text = error_text + PipenvException.__init__(self, contents) + + def show(self, file=None): + if file is None: + file = vistir.misc.get_text_stderr() + message = "{0}\n{1}".format( + crayons.white("Failed parsing JSON results:", bold=True), + decode_for_output(self.message.strip(), file) + ) + click_echo(message, err=True) + if self.error_text: + click_echo("{0} {1}".format( + crayons.white("ERROR TEXT:", bold=True), + decode_for_output(self.error_text, file) + ), err=True) class PipenvUsageError(UsageError): @@ -78,25 +152,24 @@ class PipenvUsageError(UsageError): message = formatted_message.format(msg_prefix, crayons.white(message, bold=True)) self.message = message extra = kwargs.pop("extra", []) - UsageError.__init__(self, fix_utf8(message), ctx) + UsageError.__init__(self, decode_for_output(message), ctx) self.extra = extra def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() color = None if self.ctx is not None: color = self.ctx.color if self.extra: - if isinstance(self.extra, six.string_types): - self.extra = [self.extra,] + if isinstance(self.extra, STRING_TYPES): + self.extra = [self.extra] for extra in self.extra: if color: extra = getattr(crayons, color, "blue")(extra) - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) hint = '' - if (self.cmd is not None and - self.cmd.get_help_option(self.ctx) is not None): + if self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None: hint = ('Try "%s %s" for help.\n' % (self.ctx.command_path, self.ctx.help_option_names[0])) if self.ctx is not None: @@ -117,30 +190,33 @@ class PipenvFileError(FileError): crayons.white("{0} not found!".format(filename), bold=True), message ) - FileError.__init__(self, filename=filename, hint=fix_utf8(message), **kwargs) + FileError.__init__(self, filename=filename, hint=decode_for_output(message), **kwargs) self.extra = extra def show(self, file=None): if file is None: - file = get_text_stderr() + file = vistir.misc.get_text_stderr() if self.extra: - if isinstance(self.extra, six.string_types): - self.extra = [self.extra,] + if isinstance(self.extra, STRING_TYPES): + self.extra = [self.extra] for extra in self.extra: - click_echo(fix_utf8(extra), file=file) + click_echo(decode_for_output(extra, file), file=file) click_echo(self.message, file=file) class PipfileNotFound(PipenvFileError): def __init__(self, filename="Pipfile", extra=None, **kwargs): extra = kwargs.pop("extra", []) - message = ("{0} {1}".format( + message = ( + "{0} {1}".format( crayons.red("Aborting!", bold=True), - crayons.white("Please ensure that the file exists and is located in your" - " project root directory.", bold=True) + crayons.white( + "Please ensure that the file exists and is located in your" + " project root directory.", bold=True + ) ) ) - super(PipfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(PipfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) class LockfileNotFound(PipenvFileError): @@ -151,21 +227,21 @@ class LockfileNotFound(PipenvFileError): crayons.red("$ pipenv lock", bold=True), crayons.white("before you can continue.", bold=True) ) - super(LockfileNotFound, self).__init__(filename, message=fix_utf8(message), extra=extra, **kwargs) + super(LockfileNotFound, self).__init__(filename, message=message, extra=extra, **kwargs) class DeployException(PipenvUsageError): def __init__(self, message=None, **kwargs): if not message: - message = crayons.normal("Aborting deploy", bold=True) + message = str(crayons.normal("Aborting deploy", bold=True)) extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + PipenvUsageError.__init__(self, message=message, extra=extra, **kwargs) class PipenvOptionsError(PipenvUsageError): def __init__(self, option_name, message=None, ctx=None, **kwargs): extra = kwargs.pop("extra", []) - PipenvUsageError.__init__(self, message=fix_utf8(message), ctx=ctx, **kwargs) + PipenvUsageError.__init__(self, message=message, ctx=ctx, **kwargs) self.extra = extra self.option_name = option_name @@ -179,7 +255,10 @@ class SystemUsageError(PipenvOptionsError): crayons.red("Warning", bold=True) ), ] - message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + if message is None: + message = str( + crayons.blue("See also: {0}".format(crayons.white("--deploy flag."))) + ) super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) @@ -191,7 +270,7 @@ class PipfileException(PipenvFileError): hint = "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), hint) filename = project.pipfile_location extra = kwargs.pop("extra", []) - PipenvFileError.__init__(self, filename, fix_utf8(hint), extra=extra, **kwargs) + PipenvFileError.__init__(self, filename, hint, extra=extra, **kwargs) class SetupException(PipenvException): @@ -207,7 +286,7 @@ class VirtualenvException(PipenvException): "There was an unexpected error while activating your virtualenv. " "Continuing anyway..." ) - PipenvException.__init__(self, fix_utf8(message), **kwargs) + PipenvException.__init__(self, message, **kwargs) class VirtualenvActivationException(VirtualenvException): @@ -218,7 +297,7 @@ class VirtualenvActivationException(VirtualenvException): "not activated. Continuing anyway…" ) self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + VirtualenvException.__init__(self, message, **kwargs) class VirtualenvCreationException(VirtualenvException): @@ -226,45 +305,72 @@ class VirtualenvCreationException(VirtualenvException): if not message: message = "Failed to create virtual environment." self.message = message - VirtualenvException.__init__(self, fix_utf8(message), **kwargs) + extra = kwargs.pop("extra", None) + if extra is not None and isinstance(extra, STRING_TYPES): + # note we need the format interpolation because ``crayons.ColoredString`` + # is not an actual string type but is only a preparation for interpolation + # so replacement or parsing requires this step + extra = ANSI_REMOVAL_RE.sub("", "{0}".format(extra)) + if "KeyboardInterrupt" in extra: + extra = str( + crayons.red("Virtualenv creation interrupted by user", bold=True) + ) + self.extra = extra = [extra] + VirtualenvException.__init__(self, message, extra=extra) class UninstallError(PipenvException): def __init__(self, package, command, return_values, return_code, **kwargs): - extra = [crayons.blue("Attempted to run command: {0}".format( - crayons.yellow("$ {0}".format(command), bold=True) - )),] + extra = [ + "{0} {1}".format( + crayons.blue("Attempted to run command: "), + crayons.yellow("$ {0!r}".format(command), bold=True) + ) + ] extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) - if isinstance(package, (tuple, list)): + if isinstance(package, (tuple, list, set)): package = " ".join(package) - message = "{0} {1}...".format( + message = "{0!s} {1!s}...".format( crayons.normal("Failed to uninstall package(s)"), - crayons.yellow(package, bold=True) + crayons.yellow("{0}!s".format(package), bold=True) ) self.exit_code = return_code - PipenvException.__init__(self, message=fix_utf8(message), extra=extra) + PipenvException.__init__(self, message=message, extra=extra) self.extra = extra class InstallError(PipenvException): def __init__(self, package, **kwargs): + package_message = "" + if package is not None: + package_message = "Couldn't install package: {0}\n".format( + crayons.white("{0!s}".format(package), bold=True) + ) message = "{0} {1}".format( - crayons.red("ERROR:", bold=True), + "{0}".format(package_message), crayons.yellow("Package installation failed...") ) extra = kwargs.pop("extra", []) - PipenvException.__init__(self, message=fix_utf8(message), extra=extra, **kwargs) + PipenvException.__init__(self, message=message, extra=extra, **kwargs) class CacheError(PipenvException): def __init__(self, path, **kwargs): - message = "{0} {1} {2}\n{0}".format( - crayons.red("ERROR:", bold=True), + message = "{0} {1}\n{2}".format( crayons.blue("Corrupt cache file"), - crayons.white(path), + crayons.white("{0!s}".format(path)), crayons.white('Consider trying "pipenv lock --clear" to clear the cache.') ) - super(PipenvException, self).__init__(message=fix_utf8(message)) + PipenvException.__init__(self, message=message) + + +class DependencyConflict(PipenvException): + def __init__(self, message): + extra = ["{0} {1}".format( + crayons.red("The operation failed...", bold=True), + crayons.red("A dependency conflict was detected and could not be resolved."), + )] + PipenvException.__init__(self, message, extra=extra) class ResolutionFailure(PipenvException): @@ -286,9 +392,7 @@ class ResolutionFailure(PipenvException): ) if "no version found at all" in message: no_version_found = True - message = "{0} {1}".format( - crayons.red("ERROR:", bold=True), crayons.yellow(message) - ) + message = crayons.yellow("{0}".format(message)) if no_version_found: message = "{0}\n{1}".format( message, @@ -297,4 +401,67 @@ class ResolutionFailure(PipenvException): "See PEP440 for more information." ) ) - super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) + PipenvException.__init__(self, message, extra=extra) + + +class RequirementError(PipenvException): + + def __init__(self, req=None): + from .utils import VCS_LIST + keys = ("name", "path",) + VCS_LIST + ("line", "uri", "url", "relpath") + if req is not None: + possible_display_values = [getattr(req, value, None) for value in keys] + req_value = next(iter( + val for val in possible_display_values if val is not None + ), None) + if not req_value: + getstate_fn = getattr(req, "__getstate__", None) + slots = getattr(req, "__slots__", None) + keys_fn = getattr(req, "keys", None) + if getstate_fn: + req_value = getstate_fn() + elif slots: + slot_vals = [ + (k, getattr(req, k, None)) for k in slots + if getattr(req, k, None) + ] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in slot_vals + ]) + elif keys_fn: + values = [(k, req.get(k)) for k in keys_fn() if req.get(k)] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in values + ]) + else: + req_value = getattr(req.line_instance, "line", None) + message = "{0} {1}".format( + crayons.normal(decode_for_output("Failed creating requirement instance")), + crayons.white(decode_for_output("{0!r}".format(req_value))) + ) + extra = [str(req)] + PipenvException.__init__(self, message, extra=extra) + + +def prettify_exc(error): + """Catch known errors and prettify them instead of showing the + entire traceback, for better UX""" + errors = [] + for exc in KNOWN_EXCEPTIONS: + search_string = exc.match_string if exc.match_string else exc.exception_name + split_string = exc.show_from_string if exc.show_from_string else exc.exception_name + if search_string in error: + # for known exceptions with no display rules and no prefix + # we should simply show nothing + if not exc.show_from_string and not exc.prefix: + errors.append("") + continue + elif exc.prefix and exc.prefix in error: + _, error, info = error.rpartition(exc.prefix) + else: + _, error, info = error.rpartition(split_string) + errors.append("{0} {1}".format(error, info)) + if not errors: + return "{}".format(vistir.misc.decode_for_output(error)) + + return "\n".join(errors) diff --git a/pipenv/pyenv.py b/pipenv/installers.py similarity index 61% rename from pipenv/pyenv.py rename to pipenv/installers.py index 941e5991..cd8e49a2 100644 --- a/pipenv/pyenv.py +++ b/pipenv/installers.py @@ -48,19 +48,22 @@ class Version(object): return (self.major, self.minor) == (other.major, other.minor) -class PyenvError(RuntimeError): +class InstallerError(RuntimeError): def __init__(self, desc, c): - super(PyenvError, self).__init__(desc) + super(InstallerError, self).__init__(desc) self.out = c.out self.err = c.err -class Runner(object): +class Installer(object): - def __init__(self, pyenv): - self._cmd = pyenv + def __init__(self, cmd): + self._cmd = cmd - def _pyenv(self, *args, **kwargs): + def __str__(self): + return self._cmd + + def _run(self, *args, **kwargs): timeout = kwargs.pop('timeout', delegator.TIMEOUT) if kwargs: k = list(kwargs.keys())[0] @@ -69,21 +72,16 @@ class Runner(object): c = delegator.run(args, block=False, timeout=timeout) c.block() if c.return_code != 0: - raise PyenvError('faild to run {0}'.format(args), c) + raise InstallerError('faild to run {0}'.format(args), c) return c def iter_installable_versions(self): """Iterate through CPython versions available for Pipenv to install. """ - for name in self._pyenv('install', '--list').out.splitlines(): - try: - version = Version.parse(name.strip()) - except ValueError: - continue - yield version + raise NotImplementedError def find_version_to_install(self, name): - """Find a version in pyenv from the version supplied. + """Find a version in the installer from the version supplied. A ValueError is raised if a matching version cannot be found. """ @@ -103,16 +101,64 @@ class Runner(object): return best_match def install(self, version): - """Install the given version with pyenv. + """Install the given version with runner implementation. The version must be a ``Version`` instance representing a version - found in pyenv. + found in the Installer. A ValueError is raised if the given version does not have a match in - pyenv. A PyenvError is raised if the pyenv command fails. + the runner. A InstallerError is raised if the runner command fails. """ - c = self._pyenv( + raise NotImplementedError + + +class Pyenv(Installer): + + def iter_installable_versions(self): + """Iterate through CPython versions available for Pipenv to install. + """ + for name in self._run('install', '--list').out.splitlines(): + try: + version = Version.parse(name.strip()) + except ValueError: + continue + yield version + + def install(self, version): + """Install the given version with pyenv. + The version must be a ``Version`` instance representing a version + found in pyenv. + A ValueError is raised if the given version does not have a match in + pyenv. A InstallerError is raised if the pyenv command fails. + """ + c = self._run( 'install', '-s', str(version), timeout=PIPENV_INSTALL_TIMEOUT, ) return c + + +class Asdf(Installer): + + def iter_installable_versions(self): + """Iterate through CPython versions available for asdf to install. + """ + for name in self._run('list-all', 'python').out.splitlines(): + try: + version = Version.parse(name.strip()) + except ValueError: + continue + yield version + + def install(self, version): + """Install the given version with asdf. + The version must be a ``Version`` instance representing a version + found in asdf. + A ValueError is raised if the given version does not have a match in + asdf. A InstallerError is raised if the asdf command fails. + """ + c = self._run( + 'install', 'python', str(version), + timeout=PIPENV_INSTALL_TIMEOUT, + ) + return c diff --git a/pipenv/patched/notpip/LICENSE.txt b/pipenv/patched/notpip/LICENSE.txt index d3379fac..737fec5c 100644 --- a/pipenv/patched/notpip/LICENSE.txt +++ b/pipenv/patched/notpip/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright (c) 2008-2018 The pip developers (see AUTHORS.txt file) +Copyright (c) 2008-2019 The pip developers (see AUTHORS.txt file) Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the diff --git a/pipenv/patched/notpip/__init__.py b/pipenv/patched/notpip/__init__.py index ae265fa7..a487794a 100644 --- a/pipenv/patched/notpip/__init__.py +++ b/pipenv/patched/notpip/__init__.py @@ -1 +1 @@ -__version__ = "18.1" +__version__ = "19.3.1" diff --git a/pipenv/patched/notpip/__main__.py b/pipenv/patched/notpip/__main__.py index a4879980..36a4800f 100644 --- a/pipenv/patched/notpip/__main__.py +++ b/pipenv/patched/notpip/__main__.py @@ -13,7 +13,7 @@ if __package__ == '': path = os.path.dirname(os.path.dirname(__file__)) sys.path.insert(0, path) -from pipenv.patched.notpip._internal import main as _main # isort:skip # noqa +from pipenv.patched.notpip._internal.main import main as _main # isort:skip # noqa if __name__ == '__main__': sys.exit(_main()) diff --git a/pipenv/patched/notpip/_internal/__init__.py b/pipenv/patched/notpip/_internal/__init__.py index 6d223928..18d727b6 100644 --- a/pipenv/patched/notpip/_internal/__init__.py +++ b/pipenv/patched/notpip/_internal/__init__.py @@ -1,78 +1,2 @@ #!/usr/bin/env python -from __future__ import absolute_import - -import locale -import logging -import os -import warnings - -import sys - -# 2016-06-17 barry@debian.org: urllib3 1.14 added optional support for socks, -# but if invoked (i.e. imported), it will issue a warning to stderr if socks -# isn't available. requests unconditionally imports urllib3's socks contrib -# module, triggering this warning. The warning breaks DEP-8 tests (because of -# the stderr output) and is just plain annoying in normal usage. I don't want -# to add socks as yet another dependency for pip, nor do I want to allow-stder -# in the DEP-8 tests, so just suppress the warning. pdb tells me this has to -# be done before the import of pip.vcs. -from pipenv.patched.notpip._vendor.urllib3.exceptions import DependencyWarning -warnings.filterwarnings("ignore", category=DependencyWarning) # noqa - -# We want to inject the use of SecureTransport as early as possible so that any -# references or sessions or what have you are ensured to have it, however we -# only want to do this in the case that we're running on macOS and the linked -# OpenSSL is too old to handle TLSv1.2 -try: - import ssl -except ImportError: - pass -else: - # Checks for OpenSSL 1.0.1 on MacOS - if sys.platform == "darwin" and ssl.OPENSSL_VERSION_NUMBER < 0x1000100f: - try: - from pipenv.patched.notpip._vendor.urllib3.contrib import securetransport - except (ImportError, OSError): - pass - else: - securetransport.inject_into_urllib3() - -from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete -from pipenv.patched.notpip._internal.cli.main_parser import parse_command -from pipenv.patched.notpip._internal.commands import commands_dict -from pipenv.patched.notpip._internal.exceptions import PipError -from pipenv.patched.notpip._internal.utils import deprecation -from pipenv.patched.notpip._internal.vcs import git, mercurial, subversion, bazaar # noqa -from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning - -logger = logging.getLogger(__name__) - -# Hide the InsecureRequestWarning from urllib3 -warnings.filterwarnings("ignore", category=InsecureRequestWarning) - - -def main(args=None): - if args is None: - args = sys.argv[1:] - - # Configure our deprecation warnings to be sent through loggers - deprecation.install_warning_logger() - - autocomplete() - - try: - cmd_name, cmd_args = parse_command(args) - except PipError as exc: - sys.stderr.write("ERROR: %s" % exc) - sys.stderr.write(os.linesep) - sys.exit(1) - - # Needed for locale.getpreferredencoding(False) to work - # in pip._internal.utils.encoding.auto_decode - try: - locale.setlocale(locale.LC_ALL, '') - except locale.Error as e: - # setlocale can apparently crash if locale are uninitialized - logger.debug("Ignoring error %s when setting locale", e) - command = commands_dict[cmd_name](isolated=("--isolated" in cmd_args)) - return command.main(cmd_args) +import pipenv.patched.notpip._internal.utils.inject_securetransport # noqa diff --git a/pipenv/patched/notpip/_internal/build_env.py b/pipenv/patched/notpip/_internal/build_env.py index 6d696fbd..7760b521 100644 --- a/pipenv/patched/notpip/_internal/build_env.py +++ b/pipenv/patched/notpip/_internal/build_env.py @@ -1,125 +1,205 @@ """Build Environment used for isolation during sdist building """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + import logging import os import sys +import textwrap +from collections import OrderedDict from distutils.sysconfig import get_python_lib from sysconfig import get_paths from pipenv.patched.notpip._vendor.pkg_resources import Requirement, VersionConflict, WorkingSet -from pipenv.patched.notpip._internal.utils.misc import call_subprocess +from pipenv.patched.notpip import __file__ as pip_location +from pipenv.patched.notpip._internal.utils.subprocess import call_subprocess from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner +if MYPY_CHECK_RUNNING: + from typing import Tuple, Set, Iterable, Optional, List + from pipenv.patched.notpip._internal.index import PackageFinder + logger = logging.getLogger(__name__) +class _Prefix: + + def __init__(self, path): + # type: (str) -> None + self.path = path + self.setup = False + self.bin_dir = get_paths( + 'nt' if os.name == 'nt' else 'posix_prefix', + vars={'base': path, 'platbase': path} + )['scripts'] + # Note: prefer distutils' sysconfig to get the + # library paths so PyPy is correctly supported. + purelib = get_python_lib(plat_specific=False, prefix=path) + platlib = get_python_lib(plat_specific=True, prefix=path) + if purelib == platlib: + self.lib_dirs = [purelib] + else: + self.lib_dirs = [purelib, platlib] + + class BuildEnvironment(object): """Creates and manages an isolated environment to install build deps """ def __init__(self): + # type: () -> None self._temp_dir = TempDirectory(kind="build-env") - self._temp_dir.create() - @property - def path(self): - return self._temp_dir.path + self._prefixes = OrderedDict(( + (name, _Prefix(os.path.join(self._temp_dir.path, name))) + for name in ('normal', 'overlay') + )) + + self._bin_dirs = [] # type: List[str] + self._lib_dirs = [] # type: List[str] + for prefix in reversed(list(self._prefixes.values())): + self._bin_dirs.append(prefix.bin_dir) + self._lib_dirs.extend(prefix.lib_dirs) + + # Customize site to: + # - ensure .pth files are honored + # - prevent access to system site packages + system_sites = { + os.path.normcase(site) for site in ( + get_python_lib(plat_specific=False), + get_python_lib(plat_specific=True), + ) + } + self._site_dir = os.path.join(self._temp_dir.path, 'site') + if not os.path.exists(self._site_dir): + os.mkdir(self._site_dir) + with open(os.path.join(self._site_dir, 'sitecustomize.py'), 'w') as fp: + fp.write(textwrap.dedent( + ''' + import os, site, sys + + # First, drop system-sites related paths. + original_sys_path = sys.path[:] + known_paths = set() + for path in {system_sites!r}: + site.addsitedir(path, known_paths=known_paths) + system_paths = set( + os.path.normcase(path) + for path in sys.path[len(original_sys_path):] + ) + original_sys_path = [ + path for path in original_sys_path + if os.path.normcase(path) not in system_paths + ] + sys.path = original_sys_path + + # Second, add lib directories. + # ensuring .pth file are processed. + for path in {lib_dirs!r}: + assert not path in sys.path + site.addsitedir(path) + ''' + ).format(system_sites=system_sites, lib_dirs=self._lib_dirs)) def __enter__(self): - self.save_path = os.environ.get('PATH', None) - self.save_pythonpath = os.environ.get('PYTHONPATH', None) - self.save_nousersite = os.environ.get('PYTHONNOUSERSITE', None) + self._save_env = { + name: os.environ.get(name, None) + for name in ('PATH', 'PYTHONNOUSERSITE', 'PYTHONPATH') + } - install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' - install_dirs = get_paths(install_scheme, vars={ - 'base': self.path, - 'platbase': self.path, + path = self._bin_dirs[:] + old_path = self._save_env['PATH'] + if old_path: + path.extend(old_path.split(os.pathsep)) + + pythonpath = [self._site_dir] + + os.environ.update({ + 'PATH': os.pathsep.join(path), + 'PYTHONNOUSERSITE': '1', + 'PYTHONPATH': os.pathsep.join(pythonpath), }) - scripts = install_dirs['scripts'] - if self.save_path: - os.environ['PATH'] = scripts + os.pathsep + self.save_path - else: - os.environ['PATH'] = scripts + os.pathsep + os.defpath - - # Note: prefer distutils' sysconfig to get the - # library paths so PyPy is correctly supported. - purelib = get_python_lib(plat_specific=0, prefix=self.path) - platlib = get_python_lib(plat_specific=1, prefix=self.path) - if purelib == platlib: - lib_dirs = purelib - else: - lib_dirs = purelib + os.pathsep + platlib - if self.save_pythonpath: - os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ - self.save_pythonpath - else: - os.environ['PYTHONPATH'] = lib_dirs - - os.environ['PYTHONNOUSERSITE'] = '1' - - return self.path - def __exit__(self, exc_type, exc_val, exc_tb): - def restore_var(varname, old_value): + for varname, old_value in self._save_env.items(): if old_value is None: os.environ.pop(varname, None) else: os.environ[varname] = old_value - restore_var('PATH', self.save_path) - restore_var('PYTHONPATH', self.save_pythonpath) - restore_var('PYTHONNOUSERSITE', self.save_nousersite) - def cleanup(self): + # type: () -> None self._temp_dir.cleanup() - def missing_requirements(self, reqs): - """Return a list of the requirements from reqs that are not present + def check_requirements(self, reqs): + # type: (Iterable[str]) -> Tuple[Set[Tuple[str, str]], Set[str]] + """Return 2 sets: + - conflicting requirements: set of (installed, wanted) reqs tuples + - missing requirements: set of reqs """ - missing = [] - with self: - ws = WorkingSet(os.environ["PYTHONPATH"].split(os.pathsep)) + missing = set() + conflicting = set() + if reqs: + ws = WorkingSet(self._lib_dirs) for req in reqs: try: if ws.find(Requirement.parse(req)) is None: - missing.append(req) - except VersionConflict: - missing.append(req) - return missing + missing.add(req) + except VersionConflict as e: + conflicting.add((str(e.args[0].as_requirement()), + str(e.args[1]))) + return conflicting, missing - def install_requirements(self, finder, requirements, message): + def install_requirements( + self, + finder, # type: PackageFinder + requirements, # type: Iterable[str] + prefix_as_string, # type: str + message # type: Optional[str] + ): + # type: (...) -> None + prefix = self._prefixes[prefix_as_string] + assert not prefix.setup + prefix.setup = True + if not requirements: + return + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) args = [ - sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--no-user', '--prefix', self.path, '--no-warn-script-location', - ] + sys_executable, os.path.dirname(pip_location), 'install', + '--ignore-installed', '--no-user', '--prefix', prefix.path, + '--no-warn-script-location', + ] # type: List[str] if logger.getEffectiveLevel() <= logging.DEBUG: args.append('-v') for format_control in ('no_binary', 'only_binary'): formats = getattr(finder.format_control, format_control) args.extend(('--' + format_control.replace('_', '-'), ','.join(sorted(formats or {':none:'})))) - if finder.index_urls: - args.extend(['-i', finder.index_urls[0]]) - for extra_index in finder.index_urls[1:]: + + index_urls = finder.index_urls + if index_urls: + args.extend(['-i', index_urls[0]]) + for extra_index in index_urls[1:]: args.extend(['--extra-index-url', extra_index]) else: args.append('--no-index') for link in finder.find_links: args.extend(['--find-links', link]) - for _, host, _ in finder.secure_origins: + + for host in finder.trusted_hosts: args.extend(['--trusted-host', host]) if finder.allow_all_prereleases: args.append('--pre') - if finder.process_dependency_links: - args.append('--process-dependency-links') args.append('--') args.extend(requirements) with open_spinner(message) as spinner: - call_subprocess(args, show_stdout=False, spinner=spinner) + call_subprocess(args, spinner=spinner) class NoOpBuildEnvironment(BuildEnvironment): @@ -138,5 +218,5 @@ class NoOpBuildEnvironment(BuildEnvironment): def cleanup(self): pass - def install_requirements(self, finder, requirements, message): + def install_requirements(self, finder, requirements, prefix, message): raise NotImplementedError() diff --git a/pipenv/patched/notpip/_internal/cache.py b/pipenv/patched/notpip/_internal/cache.py index d91b8170..9d241eca 100644 --- a/pipenv/patched/notpip/_internal/cache.py +++ b/pipenv/patched/notpip/_internal/cache.py @@ -1,6 +1,9 @@ """Cache Management """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + import errno import hashlib import logging @@ -8,12 +11,18 @@ import os from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.notpip._internal.download import path_to_url from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url from pipenv.patched.notpip._internal.wheel import InvalidWheelFilename, Wheel +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, List, Any + from pipenv.patched.notpip._internal.index import FormatControl + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag + logger = logging.getLogger(__name__) @@ -29,6 +38,7 @@ class Cache(object): """ def __init__(self, cache_dir, format_control, allowed_formats): + # type: (str, FormatControl, Set[str]) -> None super(Cache, self).__init__() self.cache_dir = expanduser(cache_dir) if cache_dir else None self.format_control = format_control @@ -38,6 +48,7 @@ class Cache(object): assert self.allowed_formats.union(_valid_formats) == _valid_formats def _get_cache_path_parts(self, link): + # type: (Link) -> List[str] """Get parts of part that must be os.path.joined with cache_dir """ @@ -63,6 +74,7 @@ class Cache(object): return parts def _get_candidates(self, link, package_name): + # type: (Link, Optional[str]) -> List[Any] can_not_cache = ( not self.cache_dir or not package_name or @@ -87,23 +99,32 @@ class Cache(object): raise def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached items in for link. """ raise NotImplementedError() - def get(self, link, package_name): + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Pep425Tag] + ): + # type: (...) -> Link """Returns a link to a cached item if it exists, otherwise returns the passed link. """ raise NotImplementedError() def _link_for_candidate(self, link, candidate): + # type: (Link, str) -> Link root = self.get_path_for_link(link) path = os.path.join(root, candidate) return Link(path_to_url(path)) def cleanup(self): + # type: () -> None pass @@ -112,11 +133,13 @@ class SimpleWheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(SimpleWheelCache, self).__init__( cache_dir, format_control, {"binary"} ) def get_path_for_link(self, link): + # type: (Link) -> str """Return a directory to store cached wheels for link Because there are M wheels for any one sdist, we provide a directory @@ -136,7 +159,13 @@ class SimpleWheelCache(Cache): # Store wheels within the root cache_dir return os.path.join(self.cache_dir, "wheels", *parts) - def get(self, link, package_name): + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Pep425Tag] + ): + # type: (...) -> Link candidates = [] for wheel_name in self._get_candidates(link, package_name): @@ -144,10 +173,12 @@ class SimpleWheelCache(Cache): wheel = Wheel(wheel_name) except InvalidWheelFilename: continue - if not wheel.supported(): + if not wheel.supported(supported_tags): # Built for a different python/arch/etc continue - candidates.append((wheel.support_index_min(), wheel_name)) + candidates.append( + (wheel.support_index_min(supported_tags), wheel_name) + ) if not candidates: return link @@ -160,14 +191,15 @@ class EphemWheelCache(SimpleWheelCache): """ def __init__(self, format_control): + # type: (FormatControl) -> None self._temp_dir = TempDirectory(kind="ephem-wheel-cache") - self._temp_dir.create() super(EphemWheelCache, self).__init__( self._temp_dir.path, format_control ) def cleanup(self): + # type: () -> None self._temp_dir.cleanup() @@ -179,6 +211,7 @@ class WheelCache(Cache): """ def __init__(self, cache_dir, format_control): + # type: (str, FormatControl) -> None super(WheelCache, self).__init__( cache_dir, format_control, {'binary'} ) @@ -186,17 +219,35 @@ class WheelCache(Cache): self._ephem_cache = EphemWheelCache(format_control) def get_path_for_link(self, link): + # type: (Link) -> str return self._wheel_cache.get_path_for_link(link) def get_ephem_path_for_link(self, link): + # type: (Link) -> str return self._ephem_cache.get_path_for_link(link) - def get(self, link, package_name): - retval = self._wheel_cache.get(link, package_name) - if retval is link: - retval = self._ephem_cache.get(link, package_name) - return retval + def get( + self, + link, # type: Link + package_name, # type: Optional[str] + supported_tags, # type: List[Pep425Tag] + ): + # type: (...) -> Link + retval = self._wheel_cache.get( + link=link, + package_name=package_name, + supported_tags=supported_tags, + ) + if retval is not link: + return retval + + return self._ephem_cache.get( + link=link, + package_name=package_name, + supported_tags=supported_tags, + ) def cleanup(self): + # type: () -> None self._wheel_cache.cleanup() self._ephem_cache.cleanup() diff --git a/pipenv/patched/notpip/_internal/cli/autocompletion.py b/pipenv/patched/notpip/_internal/cli/autocompletion.py index 15b560a1..d8e65709 100644 --- a/pipenv/patched/notpip/_internal/cli/autocompletion.py +++ b/pipenv/patched/notpip/_internal/cli/autocompletion.py @@ -1,12 +1,15 @@ """Logic that powers autocompletion installed by ``pip completion``. """ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + import optparse import os import sys from pipenv.patched.notpip._internal.cli.main_parser import create_main_parser -from pipenv.patched.notpip._internal.commands import commands_dict, get_summaries +from pipenv.patched.notpip._internal.commands import commands_dict, create_command from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions @@ -23,7 +26,7 @@ def autocomplete(): except IndexError: current = '' - subcommands = [cmd for cmd, summary in get_summaries()] + subcommands = list(commands_dict) options = [] # subcommand try: @@ -54,7 +57,7 @@ def autocomplete(): print(dist) sys.exit(1) - subcommand = commands_dict[subcommand_name]() + subcommand = create_command(subcommand_name) for opt in subcommand.parser.option_list_all: if opt.help != optparse.SUPPRESS_HELP: diff --git a/pipenv/patched/notpip/_internal/cli/base_command.py b/pipenv/patched/notpip/_internal/cli/base_command.py index 229831f2..dd818fe0 100644 --- a/pipenv/patched/notpip/_internal/cli/base_command.py +++ b/pipenv/patched/notpip/_internal/cli/base_command.py @@ -1,61 +1,69 @@ """Base Command class, and related routines""" -from __future__ import absolute_import + +from __future__ import absolute_import, print_function import logging import logging.config import optparse import os +import platform import sys +import traceback from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.command_context import CommandContextMixIn from pipenv.patched.notpip._internal.cli.parser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, + ConfigOptionParser, + UpdatingDefaultsHelpFormatter, ) from pipenv.patched.notpip._internal.cli.status_codes import ( - ERROR, PREVIOUS_BUILD_DIR_ERROR, SUCCESS, UNKNOWN_ERROR, + ERROR, + PREVIOUS_BUILD_DIR_ERROR, + SUCCESS, + UNKNOWN_ERROR, VIRTUALENV_NOT_FOUND, ) -from pipenv.patched.notpip._internal.download import PipSession from pipenv.patched.notpip._internal.exceptions import ( - BadCommand, CommandError, InstallationError, PreviousBuildDirError, + BadCommand, + CommandError, + InstallationError, + PreviousBuildDirError, UninstallationError, ) -from pipenv.patched.notpip._internal.index import PackageFinder -from pipenv.patched.notpip._internal.locations import running_under_virtualenv -from pipenv.patched.notpip._internal.req.constructors import ( - install_req_from_editable, install_req_from_line, -) -from pipenv.patched.notpip._internal.req.req_file import parse_requirements -from pipenv.patched.notpip._internal.utils.logging import setup_logging -from pipenv.patched.notpip._internal.utils.misc import get_prog, normalize_path -from pipenv.patched.notpip._internal.utils.outdated import pip_version_check +from pipenv.patched.notpip._internal.utils.deprecation import deprecated +from pipenv.patched.notpip._internal.utils.logging import BrokenStdoutLoggingError, setup_logging +from pipenv.patched.notpip._internal.utils.misc import get_prog from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.virtualenv import running_under_virtualenv if MYPY_CHECK_RUNNING: - from typing import Optional # noqa: F401 + from typing import List, Tuple, Any + from optparse import Values __all__ = ['Command'] logger = logging.getLogger(__name__) -class Command(object): - name = None # type: Optional[str] - usage = None # type: Optional[str] - hidden = False # type: bool +class Command(CommandContextMixIn): + usage = None # type: str ignore_require_venv = False # type: bool - def __init__(self, isolated=False): + def __init__(self, name, summary, isolated=False): + # type: (str, str, bool) -> None + super(Command, self).__init__() parser_kw = { 'usage': self.usage, - 'prog': '%s %s' % (get_prog(), self.name), + 'prog': '%s %s' % (get_prog(), name), 'formatter': UpdatingDefaultsHelpFormatter(), 'add_help_option': False, - 'name': self.name, + 'name': name, 'description': self.__doc__, 'isolated': isolated, } + self.name = name + self.summary = summary self.parser = ConfigOptionParser(**parser_kw) # Commands should add options to this option group @@ -69,58 +77,60 @@ class Command(object): ) self.parser.add_option_group(gen_opts) - def _build_session(self, options, retries=None, timeout=None): - session = PipSession( - cache=( - normalize_path(os.path.join(options.cache_dir, "http")) - if options.cache_dir else None - ), - retries=retries if retries is not None else options.retries, - insecure_hosts=options.trusted_hosts, - ) + def handle_pip_version_check(self, options): + # type: (Values) -> None + """ + This is a no-op so that commands by default do not do the pip version + check. + """ + # Make sure we do the pip version check if the index_group options + # are present. + assert not hasattr(options, 'no_index') - # Handle custom ca-bundles from the user - if options.cert: - session.verify = options.cert - - # Handle SSL client certificate - if options.client_cert: - session.cert = options.client_cert - - # Handle timeouts - if options.timeout or timeout: - session.timeout = ( - timeout if timeout is not None else options.timeout - ) - - # Handle configured proxies - if options.proxy: - session.proxies = { - "http": options.proxy, - "https": options.proxy, - } - - # Determine if we can prompt the user for authentication or not - session.auth.prompting = not options.no_input - - return session + def run(self, options, args): + # type: (Values, List[Any]) -> Any + raise NotImplementedError def parse_args(self, args): + # type: (List[str]) -> Tuple # factored out for testability return self.parser.parse_args(args) def main(self, args): + # type: (List[str]) -> int + try: + with self.main_context(): + return self._main(args) + finally: + logging.shutdown() + + def _main(self, args): + # type: (List[str]) -> int options, args = self.parse_args(args) # Set verbosity so that it can be used elsewhere. self.verbosity = options.verbose - options.quiet - setup_logging( + level_number = setup_logging( verbosity=self.verbosity, no_color=options.no_color, user_log_file=options.log, ) + if sys.version_info[:2] == (2, 7): + message = ( + "A future version of pip will drop support for Python 2.7. " + "More details about Python 2 support in pip, can be found at " + "https://pip.pypa.io/en/latest/development/release-process/#python-2-support" # noqa + ) + if platform.python_implementation() == "CPython": + message = ( + "Python 2.7 will reach the end of its life on January " + "1st, 2020. Please upgrade your Python as Python 2.7 " + "won't be maintained after that date. " + ) + message + deprecated(message, replacement=None, gone_in=None) + # TODO: Try to get these passing down from the command? # without resorting to os.environ to hold these. # This also affects isolated builds and it should. @@ -156,9 +166,17 @@ class Command(object): return ERROR except CommandError as exc: - logger.critical('ERROR: %s', exc) + logger.critical('%s', exc) logger.debug('Exception information:', exc_info=True) + return ERROR + except BrokenStdoutLoggingError: + # Bypass our logger and write any remaining messages to stderr + # because stdout no longer works. + print('ERROR: Pipe to stdout was broken', file=sys.stderr) + if level_number <= logging.DEBUG: + traceback.print_exc(file=sys.stderr) + return ERROR except KeyboardInterrupt: logger.critical('Operation cancelled by user') @@ -170,109 +188,6 @@ class Command(object): return UNKNOWN_ERROR finally: - allow_version_check = ( - # Does this command have the index_group options? - hasattr(options, "no_index") and - # Is this command allowed to perform this check? - not (options.disable_pip_version_check or options.no_index) - ) - # Check if we're using the latest version of pip available - if allow_version_check: - session = self._build_session( - options, - retries=0, - timeout=min(5, options.timeout) - ) - with session: - pip_version_check(session, options) - - # Shutdown the logging module - logging.shutdown() + self.handle_pip_version_check(options) return SUCCESS - - -class RequirementCommand(Command): - - @staticmethod - def populate_requirement_set(requirement_set, args, options, finder, - session, name, wheel_cache): - """ - Marshal cmd line args into a requirement set. - """ - # NOTE: As a side-effect, options.require_hashes and - # requirement_set.require_hashes may be updated - - for filename in options.constraints: - for req_to_add in parse_requirements( - filename, - constraint=True, finder=finder, options=options, - session=session, wheel_cache=wheel_cache): - req_to_add.is_direct = True - requirement_set.add_requirement(req_to_add) - - for req in args: - req_to_add = install_req_from_line( - req, None, isolated=options.isolated_mode, - wheel_cache=wheel_cache - ) - req_to_add.is_direct = True - requirement_set.add_requirement(req_to_add) - - for req in options.editables: - req_to_add = install_req_from_editable( - req, - isolated=options.isolated_mode, - wheel_cache=wheel_cache - ) - req_to_add.is_direct = True - requirement_set.add_requirement(req_to_add) - - for filename in options.requirements: - for req_to_add in parse_requirements( - filename, - finder=finder, options=options, session=session, - wheel_cache=wheel_cache): - req_to_add.is_direct = True - requirement_set.add_requirement(req_to_add) - # If --require-hashes was a line in a requirements file, tell - # RequirementSet about it: - requirement_set.require_hashes = options.require_hashes - - if not (args or options.editables or options.requirements): - opts = {'name': name} - if options.find_links: - raise CommandError( - 'You must give at least one requirement to %(name)s ' - '(maybe you meant "pip %(name)s %(links)s"?)' % - dict(opts, links=' '.join(options.find_links))) - else: - raise CommandError( - 'You must give at least one requirement to %(name)s ' - '(see "pip help %(name)s")' % opts) - - def _build_package_finder(self, options, session, - platform=None, python_versions=None, - abi=None, implementation=None): - """ - Create a package finder appropriate to this requirement command. - """ - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) - index_urls = [] - - return PackageFinder( - find_links=options.find_links, - format_control=options.format_control, - index_urls=index_urls, - trusted_hosts=options.trusted_hosts, - allow_all_prereleases=options.pre, - process_dependency_links=options.process_dependency_links, - session=session, - platform=platform, - versions=python_versions, - abi=abi, - implementation=implementation, - prefer_binary=options.prefer_binary, - ) diff --git a/pipenv/patched/notpip/_internal/cli/cmdoptions.py b/pipenv/patched/notpip/_internal/cli/cmdoptions.py index a075a67e..ba6166d9 100644 --- a/pipenv/patched/notpip/_internal/cli/cmdoptions.py +++ b/pipenv/patched/notpip/_internal/cli/cmdoptions.py @@ -5,27 +5,55 @@ The principle here is to define options once, but *not* instantiate them globally. One reason being that options with action='append' can carry state between parses. pip parses general options twice internally, and shouldn't pass on state. To be consistent, all options will follow this design. - """ + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import +import logging +import textwrap import warnings +from distutils.util import strtobool from functools import partial from optparse import SUPPRESS_HELP, Option, OptionGroup +from textwrap import dedent from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, src_prefix +from pipenv.patched.notpip._internal.locations import USER_CACHE_DIR, get_src_prefix from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.models.target_python import TargetPython from pipenv.patched.notpip._internal.utils.hashes import STRONG_HASHES from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import BAR_TYPES if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Callable, Dict, Optional, Tuple + from optparse import OptionParser, Values + from pipenv.patched.notpip._internal.cli.parser import ConfigOptionParser + +logger = logging.getLogger(__name__) + + +def raise_option_error(parser, option, msg): + """ + Raise an option parsing error using parser.error(). + + Args: + parser: an OptionParser instance. + option: an Option instance. + msg: the error text. + """ + msg = '{} error: {}'.format(option, msg) + msg = textwrap.fill(' '.join(msg.split())) + parser.error(msg) def make_option_group(group, parser): + # type: (Dict[str, Any], ConfigOptionParser) -> OptionGroup """ Return an OptionGroup object group -- assumed to be dict with 'name' and 'options' keys @@ -38,6 +66,7 @@ def make_option_group(group, parser): def check_install_build_global(options, check_options=None): + # type: (Values, Optional[Values]) -> None """Disable wheels if per-setup.py call options are set. :param options: The OptionParser options to update. @@ -60,6 +89,7 @@ def check_install_build_global(options, check_options=None): def check_dist_restriction(options, check_target=False): + # type: (Values, bool) -> None """Function for determining if custom platform options are allowed. :param options: The OptionParser options. @@ -80,7 +110,7 @@ def check_dist_restriction(options, check_target=False): # Installations or downloads using dist restrictions must not combine # source distributions and dist-specific wheels, as they are not - # gauranteed to be locally compatible. + # guaranteed to be locally compatible. if dist_restriction_set and sdist_dependencies_allowed: raise CommandError( "When restricting platform and interpreter constraints using " @@ -108,7 +138,7 @@ help_ = partial( dest='help', action='help', help='Show help.', -) # type: Any +) # type: Callable[..., Option] isolated_mode = partial( Option, @@ -120,7 +150,7 @@ isolated_mode = partial( "Run pip in an isolated mode, ignoring environment variables and user " "configuration." ), -) +) # type: Callable[..., Option] require_virtualenv = partial( Option, @@ -130,7 +160,7 @@ require_virtualenv = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] verbose = partial( Option, @@ -139,7 +169,7 @@ verbose = partial( action='count', default=0, help='Give more output. Option is additive, and can be used up to 3 times.' -) +) # type: Callable[..., Option] no_color = partial( Option, @@ -148,7 +178,7 @@ no_color = partial( action='store_true', default=False, help="Suppress colored output", -) +) # type: Callable[..., Option] version = partial( Option, @@ -156,7 +186,7 @@ version = partial( dest='version', action='store_true', help='Show version and exit.', -) # type: Any +) # type: Callable[..., Option] quiet = partial( Option, @@ -169,7 +199,7 @@ quiet = partial( ' times (corresponding to WARNING, ERROR, and CRITICAL logging' ' levels).' ), -) # type: Any +) # type: Callable[..., Option] progress_bar = partial( Option, @@ -182,7 +212,7 @@ progress_bar = partial( 'Specify type of progress to be displayed [' + '|'.join(BAR_TYPES.keys()) + '] (default: %default)' ), -) # type: Any +) # type: Callable[..., Option] log = partial( Option, @@ -190,7 +220,7 @@ log = partial( dest="log", metavar="path", help="Path to a verbose appending log." -) # type: Any +) # type: Callable[..., Option] no_input = partial( Option, @@ -200,7 +230,7 @@ no_input = partial( action='store_true', default=False, help=SUPPRESS_HELP -) # type: Any +) # type: Callable[..., Option] proxy = partial( Option, @@ -209,7 +239,7 @@ proxy = partial( type='str', default='', help="Specify a proxy in the form [user:passwd@]proxy.server:port." -) # type: Any +) # type: Callable[..., Option] retries = partial( Option, @@ -219,7 +249,7 @@ retries = partial( default=5, help="Maximum number of retries each connection should attempt " "(default %default times).", -) # type: Any +) # type: Callable[..., Option] timeout = partial( Option, @@ -229,7 +259,7 @@ timeout = partial( type='float', default=15, help='Set the socket timeout (default %default seconds).', -) # type: Any +) # type: Callable[..., Option] skip_requirements_regex = partial( Option, @@ -239,10 +269,11 @@ skip_requirements_regex = partial( type='str', default='', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] def exists_action(): + # type: () -> Option return Option( # Option when path already exist '--exists-action', @@ -253,7 +284,7 @@ def exists_action(): action='append', metavar='action', help="Default action when a path already exists: " - "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort).", + "(s)witch, (i)gnore, (w)ipe, (b)ackup, (a)bort.", ) @@ -264,7 +295,7 @@ cert = partial( type='str', metavar='path', help="Path to alternate CA bundle.", -) # type: Any +) # type: Callable[..., Option] client_cert = partial( Option, @@ -275,7 +306,7 @@ client_cert = partial( metavar='path', help="Path to SSL client certificate, a single file containing the " "private key and the certificate in PEM format.", -) # type: Any +) # type: Callable[..., Option] index_url = partial( Option, @@ -283,11 +314,11 @@ index_url = partial( dest='index_url', metavar='URL', default=PyPI.simple_url, - help="Base URL of Python Package Index (default %default). " + help="Base URL of the Python Package Index (default %default). " "This should point to a repository compliant with PEP 503 " "(the simple repository API) or a local directory laid out " "in the same format.", -) # type: Any +) # type: Callable[..., Option] def extra_index_url(): @@ -310,10 +341,11 @@ no_index = partial( action='store_true', default=False, help='Ignore package index (only looking at --find-links URLs instead).', -) # type: Any +) # type: Callable[..., Option] def find_links(): + # type: () -> Option return Option( '-f', '--find-links', dest='find_links', @@ -327,29 +359,20 @@ def find_links(): def trusted_host(): + # type: () -> Option return Option( "--trusted-host", dest="trusted_hosts", action="append", metavar="HOSTNAME", default=[], - help="Mark this host as trusted, even though it does not have valid " - "or any HTTPS.", + help="Mark this host or host:port pair as trusted, even though it " + "does not have valid or any HTTPS.", ) -# Remove after 1.5 -process_dependency_links = partial( - Option, - "--process-dependency-links", - dest="process_dependency_links", - action="store_true", - default=False, - help="Enable the processing of dependency links.", -) # type: Any - - def constraints(): + # type: () -> Option return Option( '-c', '--constraint', dest='constraints', @@ -362,6 +385,7 @@ def constraints(): def requirements(): + # type: () -> Option return Option( '-r', '--requirement', dest='requirements', @@ -374,6 +398,7 @@ def requirements(): def editable(): + # type: () -> Option return Option( '-e', '--editable', dest='editables', @@ -390,19 +415,21 @@ src = partial( '--src', '--source', '--source-dir', '--source-directory', dest='src_dir', metavar='dir', - default=src_prefix, + default=get_src_prefix(), help='Directory to check out editable projects into. ' 'The default in a virtualenv is "/src". ' 'The default for global installs is "/src".' -) # type: Any +) # type: Callable[..., Option] def _get_format_control(values, option): + # type: (Values, Option) -> Any """Get a format_control object.""" return getattr(values, option.dest) def _handle_no_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.no_binary, existing.only_binary, @@ -410,6 +437,7 @@ def _handle_no_binary(option, opt_str, value, parser): def _handle_only_binary(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None existing = _get_format_control(parser.values, option) FormatControl.handle_mutual_excludes( value, existing.only_binary, existing.no_binary, @@ -417,6 +445,7 @@ def _handle_only_binary(option, opt_str, value, parser): def no_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--no-binary", dest="format_control", action="callback", @@ -425,13 +454,14 @@ def no_binary(): help="Do not use binary packages. Can be supplied multiple times, and " "each time adds to the existing value. Accepts either :all: to " "disable all binary packages, :none: to empty the set, or one or " - "more package names with commas between them. Note that some " - "packages are tricky to compile and may fail to install when " - "this option is used on them.", + "more package names with commas between them (no colons). Note " + "that some packages are tricky to compile and may fail to " + "install when this option is used on them.", ) def only_binary(): + # type: () -> Option format_control = FormatControl(set(), set()) return Option( "--only-binary", dest="format_control", action="callback", @@ -454,7 +484,55 @@ platform = partial( default=None, help=("Only use wheels compatible with . " "Defaults to the platform of the running system."), -) +) # type: Callable[..., Option] + + +# This was made a separate function for unit-testing purposes. +def _convert_python_version(value): + # type: (str) -> Tuple[Tuple[int, ...], Optional[str]] + """ + Convert a version string like "3", "37", or "3.7.3" into a tuple of ints. + + :return: A 2-tuple (version_info, error_msg), where `error_msg` is + non-None if and only if there was a parsing error. + """ + if not value: + # The empty string is the same as not providing a value. + return (None, None) + + parts = value.split('.') + if len(parts) > 3: + return ((), 'at most three version parts are allowed') + + if len(parts) == 1: + # Then we are in the case of "3" or "37". + value = parts[0] + if len(value) > 1: + parts = [value[0], value[1:]] + + try: + version_info = tuple(int(part) for part in parts) + except ValueError: + return ((), 'each version part must be an integer') + + return (version_info, None) + + +def _handle_python_version(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Handle a provided --python-version value. + """ + version_info, error_msg = _convert_python_version(value) + if error_msg is not None: + msg = ( + 'invalid --python-version value: {!r}: {}'.format( + value, error_msg, + ) + ) + raise_option_error(parser, option=option, msg=msg) + + parser.values.python_version = version_info python_version = partial( @@ -462,14 +540,17 @@ python_version = partial( '--python-version', dest='python_version', metavar='python_version', + action='callback', + callback=_handle_python_version, type='str', default=None, - help=("Only use wheels compatible with Python " - "interpreter version . If not specified, then the " - "current system interpreter minor version is used. A major " - "version (e.g. '2') can be specified to match all " - "minor revs of that major version. A minor version " - "(e.g. '34') can also be specified."), -) + help=dedent("""\ + The Python interpreter version to use for wheel and "Requires-Python" + compatibility checks. Defaults to a version derived from the running + interpreter. The version can be specified using up to three dot-separated + integers (e.g. "3" for 3.0.0, "3.7" for 3.7.0, or "3.7.3"). A major-minor + version can also be given as a string without dots (e.g. "37" for 3.7.0). + """), +) # type: Callable[..., Option] implementation = partial( @@ -483,7 +564,7 @@ implementation = partial( " or 'ip'. If not specified, then the current " "interpreter implementation is used. Use 'py' to force " "implementation-agnostic wheels."), -) +) # type: Callable[..., Option] abi = partial( @@ -498,10 +579,31 @@ abi = partial( "you will need to specify --implementation, " "--platform, and --python-version when using " "this option."), -) +) # type: Callable[..., Option] + + +def add_target_python_options(cmd_opts): + # type: (OptionGroup) -> None + cmd_opts.add_option(platform()) + cmd_opts.add_option(python_version()) + cmd_opts.add_option(implementation()) + cmd_opts.add_option(abi()) + + +def make_target_python(options): + # type: (Values) -> TargetPython + target_python = TargetPython( + platform=options.platform, + py_version_info=options.python_version, + abi=options.abi, + implementation=options.implementation, + ) + + return target_python def prefer_binary(): + # type: () -> Option return Option( "--prefer-binary", dest="prefer_binary", @@ -518,15 +620,45 @@ cache_dir = partial( default=USER_CACHE_DIR, metavar="dir", help="Store the cache data in ." -) +) # type: Callable[..., Option] + + +def _handle_no_cache_dir(option, opt, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Process a value provided for the --no-cache-dir option. + + This is an optparse.Option callback for the --no-cache-dir option. + """ + # The value argument will be None if --no-cache-dir is passed via the + # command-line, since the option doesn't accept arguments. However, + # the value can be non-None if the option is triggered e.g. by an + # environment variable, like PIP_NO_CACHE_DIR=true. + if value is not None: + # Then parse the string value to get argument error-checking. + try: + strtobool(value) + except ValueError as exc: + raise_option_error(parser, option=option, msg=str(exc)) + + # Originally, setting PIP_NO_CACHE_DIR to a value that strtobool() + # converted to 0 (like "false" or "no") caused cache_dir to be disabled + # rather than enabled (logic would say the latter). Thus, we disable + # the cache directory not just on values that parse to True, but (for + # backwards compatibility reasons) also on values that parse to False. + # In other words, always set it to False if the option is provided in + # some (valid) form. + parser.values.cache_dir = False + no_cache = partial( Option, "--no-cache-dir", dest="cache_dir", - action="store_false", + action="callback", + callback=_handle_no_cache_dir, help="Disable the cache.", -) +) # type: Callable[..., Option] no_deps = partial( Option, @@ -535,7 +667,7 @@ no_deps = partial( action='store_true', default=False, help="Don't install package dependencies.", -) # type: Any +) # type: Callable[..., Option] build_dir = partial( Option, @@ -547,7 +679,7 @@ build_dir = partial( 'The location of temporary directories can be controlled by setting ' 'the TMPDIR environment variable (TEMP on Windows) appropriately. ' 'When passed, build directories are not cleaned in case of failures.' -) # type: Any +) # type: Callable[..., Option] ignore_requires_python = partial( Option, @@ -555,7 +687,7 @@ ignore_requires_python = partial( dest='ignore_requires_python', action='store_true', help='Ignore the Requires-Python information.' -) # type: Any +) # type: Callable[..., Option] no_build_isolation = partial( Option, @@ -566,6 +698,51 @@ no_build_isolation = partial( help='Disable isolation when building a modern source distribution. ' 'Build dependencies specified by PEP 518 must be already installed ' 'if this option is used.' +) # type: Callable[..., Option] + + +def _handle_no_use_pep517(option, opt, value, parser): + # type: (Option, str, str, OptionParser) -> None + """ + Process a value provided for the --no-use-pep517 option. + + This is an optparse.Option callback for the no_use_pep517 option. + """ + # Since --no-use-pep517 doesn't accept arguments, the value argument + # will be None if --no-use-pep517 is passed via the command-line. + # However, the value can be non-None if the option is triggered e.g. + # by an environment variable, for example "PIP_NO_USE_PEP517=true". + if value is not None: + msg = """A value was passed for --no-use-pep517, + probably using either the PIP_NO_USE_PEP517 environment variable + or the "no-use-pep517" config file option. Use an appropriate value + of the PIP_USE_PEP517 environment variable or the "use-pep517" + config file option instead. + """ + raise_option_error(parser, option=option, msg=msg) + + # Otherwise, --no-use-pep517 was passed via the command-line. + parser.values.use_pep517 = False + + +use_pep517 = partial( + Option, + '--use-pep517', + dest='use_pep517', + action='store_true', + default=None, + help='Use PEP 517 for building source distributions ' + '(use --no-use-pep517 to force legacy behaviour).' +) # type: Any + +no_use_pep517 = partial( + Option, + '--no-use-pep517', + dest='use_pep517', + action='callback', + callback=_handle_no_use_pep517, + default=None, + help=SUPPRESS_HELP ) # type: Any install_options = partial( @@ -579,7 +756,7 @@ install_options = partial( "bin\"). Use multiple --install-option options to pass multiple " "options to setup.py install. If you are using an option with a " "directory path, be sure to use absolute path.", -) # type: Any +) # type: Callable[..., Option] global_options = partial( Option, @@ -589,7 +766,7 @@ global_options = partial( metavar='options', help="Extra global options to be supplied to the setup.py " "call before the install command.", -) # type: Any +) # type: Callable[..., Option] no_clean = partial( Option, @@ -597,7 +774,7 @@ no_clean = partial( action='store_true', default=False, help="Don't clean up build directories." -) # type: Any +) # type: Callable[..., Option] pre = partial( Option, @@ -606,7 +783,7 @@ pre = partial( default=False, help="Include pre-release and development versions. By default, " "pip only finds stable versions.", -) # type: Any +) # type: Callable[..., Option] disable_pip_version_check = partial( Option, @@ -616,7 +793,7 @@ disable_pip_version_check = partial( default=False, help="Don't periodically check PyPI to determine whether a new version " "of pip is available for download. Implied with --no-index.", -) # type: Any +) # type: Callable[..., Option] # Deprecated, Remove later @@ -626,10 +803,11 @@ always_unzip = partial( dest='always_unzip', action='store_true', help=SUPPRESS_HELP, -) # type: Any +) # type: Callable[..., Option] -def _merge_hash(option, opt_str, value, parser): +def _handle_merge_hash(option, opt_str, value, parser): + # type: (Option, str, str, OptionParser) -> None """Given a value spelled "algo:digest", append the digest to a list pointed to in a dict by the algo name.""" if not parser.values.hashes: @@ -653,11 +831,11 @@ hash = partial( # __dict__ copying in process_line(). dest='hashes', action='callback', - callback=_merge_hash, + callback=_handle_merge_hash, type='string', help="Verify that the package's archive matches this " 'hash before installing. Example: --hash=sha256:abcdef...', -) # type: Any +) # type: Callable[..., Option] require_hashes = partial( @@ -669,7 +847,25 @@ require_hashes = partial( help='Require a hash to check each requirement against, for ' 'repeatable installs. This option is implied when any package in a ' 'requirements file has a --hash option.', -) # type: Any +) # type: Callable[..., Option] + + +list_path = partial( + Option, + '--path', + dest='path', + action='append', + help='Restrict to the specified installation path for listing ' + 'packages (can be used multiple times).' +) # type: Callable[..., Option] + + +def check_list_path_option(options): + # type: (Values) -> None + if options.path and (options.user or options.local): + raise CommandError( + "Cannot combine '--path' with '--user' or '--local'" + ) ########## @@ -700,7 +896,7 @@ general_group = { disable_pip_version_check, no_color, ] -} +} # type: Dict[str, Any] index_group = { 'name': 'Package Index Options', @@ -709,6 +905,5 @@ index_group = { extra_index_url, no_index, find_links, - process_dependency_links, ] -} +} # type: Dict[str, Any] diff --git a/pipenv/patched/notpip/_internal/cli/command_context.py b/pipenv/patched/notpip/_internal/cli/command_context.py new file mode 100644 index 00000000..d529fb67 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/command_context.py @@ -0,0 +1,29 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from contextlib import contextmanager + +from pipenv.patched.notpip._vendor.contextlib2 import ExitStack + + +class CommandContextMixIn(object): + def __init__(self): + super(CommandContextMixIn, self).__init__() + self._in_main_context = False + self._main_context = ExitStack() + + @contextmanager + def main_context(self): + assert not self._in_main_context + + self._in_main_context = True + try: + with self._main_context: + yield + finally: + self._in_main_context = False + + def enter_context(self, context_provider): + assert self._in_main_context + + return self._main_context.enter_context(context_provider) diff --git a/pipenv/patched/notpip/_internal/cli/main_parser.py b/pipenv/patched/notpip/_internal/cli/main_parser.py index abe2f69e..769a7281 100644 --- a/pipenv/patched/notpip/_internal/cli/main_parser.py +++ b/pipenv/patched/notpip/_internal/cli/main_parser.py @@ -4,21 +4,25 @@ import os import sys -from pipenv.patched.notpip import __version__ from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.cli.parser import ( - ConfigOptionParser, UpdatingDefaultsHelpFormatter, -) -from pipenv.patched.notpip._internal.commands import ( - commands_dict, get_similar_commands, get_summaries, + ConfigOptionParser, + UpdatingDefaultsHelpFormatter, ) +from pipenv.patched.notpip._internal.commands import commands_dict, get_similar_commands from pipenv.patched.notpip._internal.exceptions import CommandError -from pipenv.patched.notpip._internal.utils.misc import get_prog +from pipenv.patched.notpip._internal.utils.misc import get_pip_version, get_prog +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple, List + __all__ = ["create_main_parser", "parse_command"] def create_main_parser(): + # type: () -> ConfigOptionParser """Creates and returns the main parser for pip's CLI """ @@ -33,28 +37,27 @@ def create_main_parser(): parser = ConfigOptionParser(**parser_kw) parser.disable_interspersed_args() - pip_pkg_dir = os.path.abspath(os.path.join( - os.path.dirname(__file__), "..", "..", - )) - parser.version = 'pip %s from %s (python %s)' % ( - __version__, pip_pkg_dir, sys.version[:3], - ) + parser.version = get_pip_version() # add the general options gen_opts = cmdoptions.make_option_group(cmdoptions.general_group, parser) parser.add_option_group(gen_opts) - parser.main = True # so the help formatter knows + # so the help formatter knows + parser.main = True # type: ignore # create command listing for description - command_summaries = get_summaries() - description = [''] + ['%-27s %s' % (i, j) for i, j in command_summaries] + description = [''] + [ + '%-27s %s' % (name, command_info.summary) + for name, command_info in commands_dict.items() + ] parser.description = '\n'.join(description) return parser def parse_command(args): + # type: (List[str]) -> Tuple[str, List[str]] parser = create_main_parser() # Note: parser calls disable_interspersed_args(), so the result of this @@ -68,7 +71,7 @@ def parse_command(args): # --version if general_options.version: - sys.stdout.write(parser.version) + sys.stdout.write(parser.version) # type: ignore sys.stdout.write(os.linesep) sys.exit() diff --git a/pipenv/patched/notpip/_internal/cli/parser.py b/pipenv/patched/notpip/_internal/cli/parser.py index 2d2c8f4d..3d5abd46 100644 --- a/pipenv/patched/notpip/_internal/cli/parser.py +++ b/pipenv/patched/notpip/_internal/cli/parser.py @@ -1,4 +1,8 @@ """Base option parser setup""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging diff --git a/pipenv/patched/notpip/_internal/cli/req_command.py b/pipenv/patched/notpip/_internal/cli/req_command.py new file mode 100644 index 00000000..ff76aeb8 --- /dev/null +++ b/pipenv/patched/notpip/_internal/cli/req_command.py @@ -0,0 +1,304 @@ +"""Contains the Command base classes that depend on PipSession. + +The classes in this module are in a separate module so the commands not +needing download / PackageFinder capability don't unnecessarily import the +PackageFinder machinery and all its vendored dependencies, etc. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import os +from functools import partial + +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.command_context import CommandContextMixIn +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.legacy_resolve import Resolver +from pipenv.patched.notpip._internal.models.selection_prefs import SelectionPreferences +from pipenv.patched.notpip._internal.network.session import PipSession +from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer +from pipenv.patched.notpip._internal.req.constructors import ( + install_req_from_editable, + install_req_from_line, + install_req_from_req_string, +) +from pipenv.patched.notpip._internal.req.req_file import parse_requirements +from pipenv.patched.notpip._internal.self_outdated_check import ( + make_link_collector, + pip_self_version_check, +) +from pipenv.patched.notpip._internal.utils.misc import normalize_path +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import List, Optional, Tuple + from pipenv.patched.notpip._internal.cache import WheelCache + from pipenv.patched.notpip._internal.models.target_python import TargetPython + from pipenv.patched.notpip._internal.req.req_set import RequirementSet + from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker + from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory + + +class SessionCommandMixin(CommandContextMixIn): + + """ + A class mixin for command classes needing _build_session(). + """ + def __init__(self): + super(SessionCommandMixin, self).__init__() + self._session = None # Optional[PipSession] + + @classmethod + def _get_index_urls(cls, options): + """Return a list of index urls from user-provided options.""" + index_urls = [] + if not getattr(options, "no_index", False): + url = getattr(options, "index_url", None) + if url: + index_urls.append(url) + urls = getattr(options, "extra_index_urls", None) + if urls: + index_urls.extend(urls) + # Return None rather than an empty list + return index_urls or None + + def get_default_session(self, options): + # type: (Values) -> PipSession + """Get a default-managed session.""" + if self._session is None: + self._session = self.enter_context(self._build_session(options)) + return self._session + + def _build_session(self, options, retries=None, timeout=None): + # type: (Values, Optional[int], Optional[int]) -> PipSession + session = PipSession( + cache=( + normalize_path(os.path.join(options.cache_dir, "http")) + if options.cache_dir else None + ), + retries=retries if retries is not None else options.retries, + trusted_hosts=options.trusted_hosts, + index_urls=self._get_index_urls(options), + ) + + # Handle custom ca-bundles from the user + if options.cert: + session.verify = options.cert + + # Handle SSL client certificate + if options.client_cert: + session.cert = options.client_cert + + # Handle timeouts + if options.timeout or timeout: + session.timeout = ( + timeout if timeout is not None else options.timeout + ) + + # Handle configured proxies + if options.proxy: + session.proxies = { + "http": options.proxy, + "https": options.proxy, + } + + # Determine if we can prompt the user for authentication or not + session.auth.prompting = not options.no_input + + return session + + +class IndexGroupCommand(Command, SessionCommandMixin): + + """ + Abstract base class for commands with the index_group options. + + This also corresponds to the commands that permit the pip version check. + """ + + def handle_pip_version_check(self, options): + # type: (Values) -> None + """ + Do the pip version check if not disabled. + + This overrides the default behavior of not doing the check. + """ + # Make sure the index_group options are present. + assert hasattr(options, 'no_index') + + if options.disable_pip_version_check or options.no_index: + return + + # Otherwise, check if we're using the latest version of pip available. + session = self._build_session( + options, + retries=0, + timeout=min(5, options.timeout) + ) + with session: + pip_self_version_check(session, options) + + +class RequirementCommand(IndexGroupCommand): + + @staticmethod + def make_requirement_preparer( + temp_build_dir, # type: TempDirectory + options, # type: Values + req_tracker, # type: RequirementTracker + download_dir=None, # type: str + wheel_download_dir=None, # type: str + ): + # type: (...) -> RequirementPreparer + """ + Create a RequirementPreparer instance for the given parameters. + """ + temp_build_dir_path = temp_build_dir.path + assert temp_build_dir_path is not None + return RequirementPreparer( + build_dir=temp_build_dir_path, + src_dir=options.src_dir, + download_dir=download_dir, + wheel_download_dir=wheel_download_dir, + progress_bar=options.progress_bar, + build_isolation=options.build_isolation, + req_tracker=req_tracker, + ) + + @staticmethod + def make_resolver( + preparer, # type: RequirementPreparer + session, # type: PipSession + finder, # type: PackageFinder + options, # type: Values + wheel_cache=None, # type: Optional[WheelCache] + use_user_site=False, # type: bool + ignore_installed=True, # type: bool + ignore_requires_python=False, # type: bool + force_reinstall=False, # type: bool + upgrade_strategy="to-satisfy-only", # type: str + use_pep517=None, # type: Optional[bool] + py_version_info=None # type: Optional[Tuple[int, ...]] + ): + # type: (...) -> Resolver + """ + Create a Resolver instance for the given parameters. + """ + make_install_req = partial( + install_req_from_req_string, + isolated=options.isolated_mode, + wheel_cache=wheel_cache, + use_pep517=use_pep517, + ) + return Resolver( + preparer=preparer, + session=session, + finder=finder, + make_install_req=make_install_req, + use_user_site=use_user_site, + ignore_dependencies=options.ignore_dependencies, + ignore_installed=ignore_installed, + ignore_requires_python=ignore_requires_python, + force_reinstall=force_reinstall, + upgrade_strategy=upgrade_strategy, + py_version_info=py_version_info + ) + + def populate_requirement_set( + self, + requirement_set, # type: RequirementSet + args, # type: List[str] + options, # type: Values + finder, # type: PackageFinder + session, # type: PipSession + wheel_cache, # type: Optional[WheelCache] + ): + # type: (...) -> None + """ + Marshal cmd line args into a requirement set. + """ + # NOTE: As a side-effect, options.require_hashes and + # requirement_set.require_hashes may be updated + + for filename in options.constraints: + for req_to_add in parse_requirements( + filename, + constraint=True, finder=finder, options=options, + session=session, wheel_cache=wheel_cache): + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + for req in args: + req_to_add = install_req_from_line( + req, None, isolated=options.isolated_mode, + use_pep517=options.use_pep517, + wheel_cache=wheel_cache + ) + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + for req in options.editables: + req_to_add = install_req_from_editable( + req, + isolated=options.isolated_mode, + use_pep517=options.use_pep517, + wheel_cache=wheel_cache + ) + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + + for filename in options.requirements: + for req_to_add in parse_requirements( + filename, + finder=finder, options=options, session=session, + wheel_cache=wheel_cache, + use_pep517=options.use_pep517): + req_to_add.is_direct = True + requirement_set.add_requirement(req_to_add) + # If --require-hashes was a line in a requirements file, tell + # RequirementSet about it: + requirement_set.require_hashes = options.require_hashes + + if not (args or options.editables or options.requirements): + opts = {'name': self.name} + if options.find_links: + raise CommandError( + 'You must give at least one requirement to %(name)s ' + '(maybe you meant "pip %(name)s %(links)s"?)' % + dict(opts, links=' '.join(options.find_links))) + else: + raise CommandError( + 'You must give at least one requirement to %(name)s ' + '(see "pip help %(name)s")' % opts) + + def _build_package_finder( + self, + options, # type: Values + session, # type: PipSession + target_python=None, # type: Optional[TargetPython] + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> PackageFinder + """ + Create a package finder appropriate to this requirement command. + + :param ignore_requires_python: Whether to ignore incompatible + "Requires-Python" values in links. Defaults to False. + """ + link_collector = make_link_collector(session, options=options) + selection_prefs = SelectionPreferences( + allow_yanked=True, + format_control=options.format_control, + allow_all_prereleases=options.pre, + prefer_binary=options.prefer_binary, + ignore_requires_python=ignore_requires_python, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + target_python=target_python, + ) diff --git a/pipenv/patched/notpip/_internal/collector.py b/pipenv/patched/notpip/_internal/collector.py new file mode 100644 index 00000000..1469cb7c --- /dev/null +++ b/pipenv/patched/notpip/_internal/collector.py @@ -0,0 +1,548 @@ +""" +The main purpose of this module is to expose LinkCollector.collect_links(). +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import cgi +import itertools +import logging +import mimetypes +import os +from collections import OrderedDict + +from pipenv.patched.notpip._vendor import html5lib, requests +from pipenv.patched.notpip._vendor.distlib.compat import unescape +from pipenv.patched.notpip._vendor.requests.exceptions import HTTPError, RetryError, SSLError +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request + +from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pipenv.patched.notpip._internal.utils.misc import redact_auth_from_url +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url, url_to_path +from pipenv.patched.notpip._internal.vcs import is_url, vcs + +if MYPY_CHECK_RUNNING: + from typing import ( + Callable, Dict, Iterable, List, MutableMapping, Optional, Sequence, + Tuple, Union, + ) + import xml.etree.ElementTree + + from pipenv.patched.notpip._vendor.requests import Response + + from pipenv.patched.notpip._internal.models.search_scope import SearchScope + from pipenv.patched.notpip._internal.network.session import PipSession + + HTMLElement = xml.etree.ElementTree.Element + ResponseHeaders = MutableMapping[str, str] + + +logger = logging.getLogger(__name__) + + +def _match_vcs_scheme(url): + # type: (str) -> Optional[str] + """Look for VCS schemes in the URL. + + Returns the matched VCS scheme, or None if there's no match. + """ + for scheme in vcs.schemes: + if url.lower().startswith(scheme) and url[len(scheme)] in '+:': + return scheme + return None + + +def _is_url_like_archive(url): + # type: (str) -> bool + """Return whether the URL looks like an archive. + """ + filename = Link(url).filename + for bad_ext in ARCHIVE_EXTENSIONS: + if filename.endswith(bad_ext): + return True + return False + + +class _NotHTML(Exception): + def __init__(self, content_type, request_desc): + # type: (str, str) -> None + super(_NotHTML, self).__init__(content_type, request_desc) + self.content_type = content_type + self.request_desc = request_desc + + +def _ensure_html_header(response): + # type: (Response) -> None + """Check the Content-Type header to ensure the response contains HTML. + + Raises `_NotHTML` if the content type is not text/html. + """ + content_type = response.headers.get("Content-Type", "") + if not content_type.lower().startswith("text/html"): + raise _NotHTML(content_type, response.request.method) + + +class _NotHTTP(Exception): + pass + + +def _ensure_html_response(url, session): + # type: (str, PipSession) -> None + """Send a HEAD request to the URL, and ensure the response contains HTML. + + Raises `_NotHTTP` if the URL is not available for a HEAD request, or + `_NotHTML` if the content type is not text/html. + """ + scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) + if scheme not in {'http', 'https'}: + raise _NotHTTP() + + resp = session.head(url, allow_redirects=True) + resp.raise_for_status() + + _ensure_html_header(resp) + + +def _get_html_response(url, session): + # type: (str, PipSession) -> Response + """Access an HTML page with GET, and return the response. + + This consists of three parts: + + 1. If the URL looks suspiciously like an archive, send a HEAD first to + check the Content-Type is HTML, to avoid downloading a large file. + Raise `_NotHTTP` if the content type cannot be determined, or + `_NotHTML` if it is not HTML. + 2. Actually perform the request. Raise HTTP exceptions on network failures. + 3. Check the Content-Type header to make sure we got HTML, and raise + `_NotHTML` otherwise. + """ + if _is_url_like_archive(url): + _ensure_html_response(url, session=session) + + logger.debug('Getting page %s', redact_auth_from_url(url)) + + resp = session.get( + url, + headers={ + "Accept": "text/html", + # We don't want to blindly returned cached data for + # /simple/, because authors generally expecting that + # twine upload && pip install will function, but if + # they've done a pip install in the last ~10 minutes + # it won't. Thus by setting this to zero we will not + # blindly use any cached data, however the benefit of + # using max-age=0 instead of no-cache, is that we will + # still support conditional requests, so we will still + # minimize traffic sent in cases where the page hasn't + # changed at all, we will just always incur the round + # trip for the conditional GET now instead of only + # once per 10 minutes. + # For more information, please see pypa/pip#5670. + "Cache-Control": "max-age=0", + }, + ) + resp.raise_for_status() + + # The check for archives above only works if the url ends with + # something that looks like an archive. However that is not a + # requirement of an url. Unless we issue a HEAD request on every + # url we cannot know ahead of time for sure if something is HTML + # or not. However we can check after we've downloaded it. + _ensure_html_header(resp) + + return resp + + +def _get_encoding_from_headers(headers): + # type: (ResponseHeaders) -> Optional[str] + """Determine if we have any encoding information in our headers. + """ + if headers and "Content-Type" in headers: + content_type, params = cgi.parse_header(headers["Content-Type"]) + if "charset" in params: + return params['charset'] + return None + + +def _determine_base_url(document, page_url): + # type: (HTMLElement, str) -> str + """Determine the HTML document's base URL. + + This looks for a ```` tag in the HTML document. If present, its href + attribute denotes the base URL of anchor tags in the document. If there is + no such tag (or if it does not have a valid href attribute), the HTML + file's URL is used as the base URL. + + :param document: An HTML document representation. The current + implementation expects the result of ``html5lib.parse()``. + :param page_url: The URL of the HTML document. + """ + for base in document.findall(".//base"): + href = base.get("href") + if href is not None: + return href + return page_url + + +def _clean_link(url): + # type: (str) -> str + """Makes sure a link is fully encoded. That is, if a ' ' shows up in + the link, it will be rewritten to %20 (while not over-quoting + % or other characters).""" + # Split the URL into parts according to the general structure + # `scheme://netloc/path;parameters?query#fragment`. Note that the + # `netloc` can be empty and the URI will then refer to a local + # filesystem path. + result = urllib_parse.urlparse(url) + # In both cases below we unquote prior to quoting to make sure + # nothing is double quoted. + if result.netloc == "": + # On Windows the path part might contain a drive letter which + # should not be quoted. On Linux where drive letters do not + # exist, the colon should be quoted. We rely on urllib.request + # to do the right thing here. + path = urllib_request.pathname2url( + urllib_request.url2pathname(result.path)) + else: + # In addition to the `/` character we protect `@` so that + # revision strings in VCS URLs are properly parsed. + path = urllib_parse.quote(urllib_parse.unquote(result.path), safe="/@") + return urllib_parse.urlunparse(result._replace(path=path)) + + +def _create_link_from_element( + anchor, # type: HTMLElement + page_url, # type: str + base_url, # type: str +): + # type: (...) -> Optional[Link] + """ + Convert an anchor element in a simple repository page to a Link. + """ + href = anchor.get("href") + if not href: + return None + + url = _clean_link(urllib_parse.urljoin(base_url, href)) + pyrequire = anchor.get('data-requires-python') + pyrequire = unescape(pyrequire) if pyrequire else None + + yanked_reason = anchor.get('data-yanked') + if yanked_reason: + # This is a unicode string in Python 2 (and 3). + yanked_reason = unescape(yanked_reason) + + link = Link( + url, + comes_from=page_url, + requires_python=pyrequire, + yanked_reason=yanked_reason, + ) + + return link + + +def parse_links(page): + # type: (HTMLPage) -> Iterable[Link] + """ + Parse an HTML document, and yield its anchor elements as Link objects. + """ + document = html5lib.parse( + page.content, + transport_encoding=page.encoding, + namespaceHTMLElements=False, + ) + + url = page.url + base_url = _determine_base_url(document, url) + for anchor in document.findall(".//a"): + link = _create_link_from_element( + anchor, + page_url=url, + base_url=base_url, + ) + if link is None: + continue + yield link + + +class HTMLPage(object): + """Represents one page, along with its URL""" + + def __init__( + self, + content, # type: bytes + encoding, # type: Optional[str] + url, # type: str + ): + # type: (...) -> None + """ + :param encoding: the encoding to decode the given content. + :param url: the URL from which the HTML was downloaded. + """ + self.content = content + self.encoding = encoding + self.url = url + + def __str__(self): + return redact_auth_from_url(self.url) + + +def _handle_get_page_fail( + link, # type: Link + reason, # type: Union[str, Exception] + meth=None # type: Optional[Callable[..., None]] +): + # type: (...) -> None + if meth is None: + meth = logger.debug + meth("Could not fetch URL %s: %s - skipping", link, reason) + + +def _make_html_page(response): + # type: (Response) -> HTMLPage + encoding = _get_encoding_from_headers(response.headers) + return HTMLPage(response.content, encoding=encoding, url=response.url) + + +def _get_html_page(link, session=None): + # type: (Link, Optional[PipSession]) -> Optional[HTMLPage] + if session is None: + raise TypeError( + "_get_html_page() missing 1 required keyword argument: 'session'" + ) + + url = link.url.split('#', 1)[0] + + # Check for VCS schemes that do not support lookup as web pages. + vcs_scheme = _match_vcs_scheme(url) + if vcs_scheme: + logger.debug('Cannot look at %s URL %s', vcs_scheme, link) + return None + + # Tack index.html onto file:// URLs that point to directories + scheme, _, path, _, _, _ = urllib_parse.urlparse(url) + if (scheme == 'file' and os.path.isdir(urllib_request.url2pathname(path))): + # add trailing slash if not present so urljoin doesn't trim + # final segment + if not url.endswith('/'): + url += '/' + url = urllib_parse.urljoin(url, 'index.html') + logger.debug(' file: URL is directory, getting %s', url) + + try: + resp = _get_html_response(url, session=session) + except _NotHTTP: + logger.debug( + 'Skipping page %s because it looks like an archive, and cannot ' + 'be checked by HEAD.', link, + ) + except _NotHTML as exc: + logger.debug( + 'Skipping page %s because the %s request got Content-Type: %s', + link, exc.request_desc, exc.content_type, + ) + except HTTPError as exc: + _handle_get_page_fail(link, exc) + except RetryError as exc: + _handle_get_page_fail(link, exc) + except SSLError as exc: + reason = "There was a problem confirming the ssl certificate: " + reason += str(exc) + _handle_get_page_fail(link, reason, meth=logger.info) + except requests.ConnectionError as exc: + _handle_get_page_fail(link, "connection error: %s" % exc) + except requests.Timeout: + _handle_get_page_fail(link, "timed out") + else: + return _make_html_page(resp) + return None + + +def _remove_duplicate_links(links): + # type: (Iterable[Link]) -> List[Link] + """ + Return a list of links, with duplicates removed and ordering preserved. + """ + # We preserve the ordering when removing duplicates because we can. + return list(OrderedDict.fromkeys(links)) + + +def group_locations(locations, expand_dir=False): + # type: (Sequence[str], bool) -> Tuple[List[str], List[str]] + """ + Divide a list of locations into two groups: "files" (archives) and "urls." + + :return: A pair of lists (files, urls). + """ + files = [] + urls = [] + + # puts the url for the given file path into the appropriate list + def sort_path(path): + url = path_to_url(path) + if mimetypes.guess_type(url, strict=False)[0] == 'text/html': + urls.append(url) + else: + files.append(url) + + for url in locations: + + is_local_path = os.path.exists(url) + is_file_url = url.startswith('file:') + + if is_local_path or is_file_url: + if is_local_path: + path = url + else: + path = url_to_path(url) + if os.path.isdir(path): + if expand_dir: + path = os.path.realpath(path) + for item in os.listdir(path): + sort_path(os.path.join(path, item)) + elif is_file_url: + urls.append(url) + else: + logger.warning( + "Path '{0}' is ignored: " + "it is a directory.".format(path), + ) + elif os.path.isfile(path): + sort_path(path) + else: + logger.warning( + "Url '%s' is ignored: it is neither a file " + "nor a directory.", url, + ) + elif is_url(url): + # Only add url with clear scheme + urls.append(url) + else: + logger.warning( + "Url '%s' is ignored. It is either a non-existing " + "path or lacks a specific scheme.", url, + ) + + return files, urls + + +class CollectedLinks(object): + + """ + Encapsulates all the Link objects collected by a call to + LinkCollector.collect_links(), stored separately as-- + + (1) links from the configured file locations, + (2) links from the configured find_links, and + (3) a dict mapping HTML page url to links from that page. + """ + + def __init__( + self, + files, # type: List[Link] + find_links, # type: List[Link] + pages, # type: Dict[str, List[Link]] + ): + # type: (...) -> None + """ + :param files: Links from file locations. + :param find_links: Links from find_links. + :param pages: A dict mapping HTML page url to links from that page. + """ + self.files = files + self.find_links = find_links + self.pages = pages + + +class LinkCollector(object): + + """ + Responsible for collecting Link objects from all configured locations, + making network requests as needed. + + The class's main method is its collect_links() method. + """ + + def __init__( + self, + session, # type: PipSession + search_scope, # type: SearchScope + ): + # type: (...) -> None + self.search_scope = search_scope + self.session = session + + @property + def find_links(self): + # type: () -> List[str] + return self.search_scope.find_links + + def _get_pages(self, locations): + # type: (Iterable[Link]) -> Iterable[HTMLPage] + """ + Yields (page, page_url) from the given locations, skipping + locations that have errors. + """ + for location in locations: + page = _get_html_page(location, session=self.session) + if page is None: + continue + + yield page + + def collect_links(self, project_name): + # type: (str) -> CollectedLinks + """Find all available links for the given project name. + + :return: All the Link objects (unfiltered), as a CollectedLinks object. + """ + search_scope = self.search_scope + index_locations = search_scope.get_index_urls_locations(project_name) + index_file_loc, index_url_loc = group_locations(index_locations) + fl_file_loc, fl_url_loc = group_locations( + self.find_links, expand_dir=True, + ) + + file_links = [ + Link(url) for url in itertools.chain(index_file_loc, fl_file_loc) + ] + + # We trust every directly linked archive in find_links + find_link_links = [Link(url, '-f') for url in self.find_links] + + # We trust every url that the user has given us whether it was given + # via --index-url or --find-links. + # We want to filter out anything that does not have a secure origin. + url_locations = [ + link for link in itertools.chain( + (Link(url) for url in index_url_loc), + (Link(url) for url in fl_url_loc), + ) + if self.session.is_secure_origin(link) + ] + + url_locations = _remove_duplicate_links(url_locations) + lines = [ + '{} location(s) to search for versions of {}:'.format( + len(url_locations), project_name, + ), + ] + for link in url_locations: + lines.append('* {}'.format(link)) + logger.debug('\n'.join(lines)) + + pages_links = {} + for page in self._get_pages(url_locations): + pages_links[page.url] = list(parse_links(page)) + + return CollectedLinks( + files=file_links, + find_links=find_link_links, + pages=pages_links, + ) diff --git a/pipenv/patched/notpip/_internal/commands/__init__.py b/pipenv/patched/notpip/_internal/commands/__init__.py index a403c6f9..ca155a94 100644 --- a/pipenv/patched/notpip/_internal/commands/__init__.py +++ b/pipenv/patched/notpip/_internal/commands/__init__.py @@ -1,57 +1,103 @@ """ Package containing all pip commands """ + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import -from pipenv.patched.notpip._internal.commands.completion import CompletionCommand -from pipenv.patched.notpip._internal.commands.configuration import ConfigurationCommand -from pipenv.patched.notpip._internal.commands.download import DownloadCommand -from pipenv.patched.notpip._internal.commands.freeze import FreezeCommand -from pipenv.patched.notpip._internal.commands.hash import HashCommand -from pipenv.patched.notpip._internal.commands.help import HelpCommand -from pipenv.patched.notpip._internal.commands.list import ListCommand -from pipenv.patched.notpip._internal.commands.check import CheckCommand -from pipenv.patched.notpip._internal.commands.search import SearchCommand -from pipenv.patched.notpip._internal.commands.show import ShowCommand -from pipenv.patched.notpip._internal.commands.install import InstallCommand -from pipenv.patched.notpip._internal.commands.uninstall import UninstallCommand -from pipenv.patched.notpip._internal.commands.wheel import WheelCommand +import importlib +from collections import OrderedDict, namedtuple from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import List, Type # noqa: F401 - from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 - -commands_order = [ - InstallCommand, - DownloadCommand, - UninstallCommand, - FreezeCommand, - ListCommand, - ShowCommand, - CheckCommand, - ConfigurationCommand, - SearchCommand, - WheelCommand, - HashCommand, - CompletionCommand, - HelpCommand, -] # type: List[Type[Command]] - -commands_dict = {c.name: c for c in commands_order} + from typing import Any + from pipenv.patched.notpip._internal.cli.base_command import Command -def get_summaries(ordered=True): - """Yields sorted (command name, command summary) tuples.""" +CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary') - if ordered: - cmditems = _sort_commands(commands_dict, commands_order) - else: - cmditems = commands_dict.items() +# The ordering matters for help display. +# Also, even though the module path starts with the same +# "pipenv.patched.notpip._internal.commands" prefix in each case, we include the full path +# because it makes testing easier (specifically when modifying commands_dict +# in test setup / teardown by adding info for a FakeCommand class defined +# in a test-related module). +# Finally, we need to pass an iterable of pairs here rather than a dict +# so that the ordering won't be lost when using Python 2.7. +commands_dict = OrderedDict([ + ('install', CommandInfo( + 'pipenv.patched.notpip._internal.commands.install', 'InstallCommand', + 'Install packages.', + )), + ('download', CommandInfo( + 'pipenv.patched.notpip._internal.commands.download', 'DownloadCommand', + 'Download packages.', + )), + ('uninstall', CommandInfo( + 'pipenv.patched.notpip._internal.commands.uninstall', 'UninstallCommand', + 'Uninstall packages.', + )), + ('freeze', CommandInfo( + 'pipenv.patched.notpip._internal.commands.freeze', 'FreezeCommand', + 'Output installed packages in requirements format.', + )), + ('list', CommandInfo( + 'pipenv.patched.notpip._internal.commands.list', 'ListCommand', + 'List installed packages.', + )), + ('show', CommandInfo( + 'pipenv.patched.notpip._internal.commands.show', 'ShowCommand', + 'Show information about installed packages.', + )), + ('check', CommandInfo( + 'pipenv.patched.notpip._internal.commands.check', 'CheckCommand', + 'Verify installed packages have compatible dependencies.', + )), + ('config', CommandInfo( + 'pipenv.patched.notpip._internal.commands.configuration', 'ConfigurationCommand', + 'Manage local and global configuration.', + )), + ('search', CommandInfo( + 'pipenv.patched.notpip._internal.commands.search', 'SearchCommand', + 'Search PyPI for packages.', + )), + ('wheel', CommandInfo( + 'pipenv.patched.notpip._internal.commands.wheel', 'WheelCommand', + 'Build wheels from your requirements.', + )), + ('hash', CommandInfo( + 'pipenv.patched.notpip._internal.commands.hash', 'HashCommand', + 'Compute hashes of package archives.', + )), + ('completion', CommandInfo( + 'pipenv.patched.notpip._internal.commands.completion', 'CompletionCommand', + 'A helper command used for command completion.', + )), + ('debug', CommandInfo( + 'pipenv.patched.notpip._internal.commands.debug', 'DebugCommand', + 'Show information useful for debugging.', + )), + ('help', CommandInfo( + 'pipenv.patched.notpip._internal.commands.help', 'HelpCommand', + 'Show help for commands.', + )), +]) # type: OrderedDict[str, CommandInfo] - for name, command_class in cmditems: - yield (name, command_class.summary) + +def create_command(name, **kwargs): + # type: (str, **Any) -> Command + """ + Create an instance of the Command class with the given name. + """ + module_path, class_name, summary = commands_dict[name] + module = importlib.import_module(module_path) + command_class = getattr(module, class_name) + command = command_class(name=name, summary=summary, **kwargs) + + return command def get_similar_commands(name): @@ -66,14 +112,3 @@ def get_similar_commands(name): return close_commands[0] else: return False - - -def _sort_commands(cmddict, order): - def keyfn(key): - try: - return order.index(key[1]) - except ValueError: - # unordered items should come last - return 0xff - - return sorted(cmddict.items(), key=keyfn) diff --git a/pipenv/patched/notpip/_internal/commands/check.py b/pipenv/patched/notpip/_internal/commands/check.py index adf4f5e7..8e1db559 100644 --- a/pipenv/patched/notpip/_internal/commands/check.py +++ b/pipenv/patched/notpip/_internal/commands/check.py @@ -1,28 +1,32 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + import logging from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.operations.check import ( - check_package_set, create_package_set_from_installed, + check_package_set, + create_package_set_from_installed, ) +from pipenv.patched.notpip._internal.utils.misc import write_output logger = logging.getLogger(__name__) class CheckCommand(Command): """Verify installed packages have compatible dependencies.""" - name = 'check' + usage = """ %prog [options]""" - summary = 'Verify installed packages have compatible dependencies.' def run(self, options, args): - package_set = create_package_set_from_installed() + package_set, parsing_probs = create_package_set_from_installed() missing, conflicting = check_package_set(package_set) for project_name in missing: version = package_set[project_name].version for dependency in missing[project_name]: - logger.info( + write_output( "%s %s requires %s, which is not installed.", project_name, version, dependency[0], ) @@ -30,12 +34,12 @@ class CheckCommand(Command): for project_name in conflicting: version = package_set[project_name].version for dep_name, dep_version, req in conflicting[project_name]: - logger.info( + write_output( "%s %s has requirement %s, but you have %s %s.", project_name, version, req, dep_name, dep_version, ) - if missing or conflicting: + if missing or conflicting or parsing_probs: return 1 else: - logger.info("No broken requirements found.") + write_output("No broken requirements found.") diff --git a/pipenv/patched/notpip/_internal/commands/completion.py b/pipenv/patched/notpip/_internal/commands/completion.py index cb8a11a7..4cccfbf2 100644 --- a/pipenv/patched/notpip/_internal/commands/completion.py +++ b/pipenv/patched/notpip/_internal/commands/completion.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import sys @@ -16,7 +19,7 @@ COMPLETION_SCRIPTS = { { COMPREPLY=( $( COMP_WORDS="${COMP_WORDS[*]}" \\ COMP_CWORD=$COMP_CWORD \\ - PIP_AUTO_COMPLETE=1 $1 ) ) + PIP_AUTO_COMPLETE=1 $1 2>/dev/null ) ) } complete -o default -F _pip_completion %(prog)s """, @@ -27,7 +30,7 @@ COMPLETION_SCRIPTS = { read -cn cword reply=( $( COMP_WORDS="$words[*]" \\ COMP_CWORD=$(( cword-1 )) \\ - PIP_AUTO_COMPLETE=1 $words[1] ) ) + PIP_AUTO_COMPLETE=1 $words[1] 2>/dev/null )) } compctl -K _pip_completion %(prog)s """, @@ -47,8 +50,7 @@ COMPLETION_SCRIPTS = { class CompletionCommand(Command): """A helper command to be used for command completion.""" - name = 'completion' - summary = 'A helper command used for command completion.' + ignore_require_venv = True def __init__(self, *args, **kw): diff --git a/pipenv/patched/notpip/_internal/commands/configuration.py b/pipenv/patched/notpip/_internal/commands/configuration.py index 6c1dbdfd..1573fb18 100644 --- a/pipenv/patched/notpip/_internal/commands/configuration.py +++ b/pipenv/patched/notpip/_internal/commands/configuration.py @@ -1,13 +1,19 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + import logging import os import subprocess from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS -from pipenv.patched.notpip._internal.configuration import Configuration, kinds +from pipenv.patched.notpip._internal.configuration import ( + Configuration, + get_configuration_files, + kinds, +) from pipenv.patched.notpip._internal.exceptions import PipError -from pipenv.patched.notpip._internal.locations import venv_config_file -from pipenv.patched.notpip._internal.utils.misc import get_prog +from pipenv.patched.notpip._internal.utils.misc import get_prog, write_output logger = logging.getLogger(__name__) @@ -23,13 +29,13 @@ class ConfigurationCommand(Command): set: Set the name=value unset: Unset the value associated with name - If none of --user, --global and --venv are passed, a virtual + If none of --user, --global and --site are passed, a virtual environment configuration file is used if one is active and the file exists. Otherwise, all modifications happen on the to the user file by default. """ - name = 'config' + ignore_require_venv = True usage = """ %prog [] list %prog [] [--editor ] edit @@ -39,8 +45,6 @@ class ConfigurationCommand(Command): %prog [] unset name """ - summary = "Manage local and global configuration." - def __init__(self, *args, **kwargs): super(ConfigurationCommand, self).__init__(*args, **kwargs) @@ -74,11 +78,11 @@ class ConfigurationCommand(Command): ) self.cmd_opts.add_option( - '--venv', - dest='venv_file', + '--site', + dest='site_file', action='store_true', default=False, - help='Use the virtualenv configuration file only' + help='Use the current environment configuration file only' ) self.parser.insert_option_group(0, self.cmd_opts) @@ -127,40 +131,42 @@ class ConfigurationCommand(Command): return SUCCESS def _determine_file(self, options, need_value): - file_options = { - kinds.USER: options.user_file, - kinds.GLOBAL: options.global_file, - kinds.VENV: options.venv_file - } + file_options = [key for key, value in ( + (kinds.USER, options.user_file), + (kinds.GLOBAL, options.global_file), + (kinds.SITE, options.site_file), + ) if value] - if sum(file_options.values()) == 0: + if not file_options: if not need_value: return None - # Default to user, unless there's a virtualenv file. - elif os.path.exists(venv_config_file): - return kinds.VENV + # Default to user, unless there's a site file. + elif any( + os.path.exists(site_config_file) + for site_config_file in get_configuration_files()[kinds.SITE] + ): + return kinds.SITE else: return kinds.USER - elif sum(file_options.values()) == 1: - # There's probably a better expression for this. - return [key for key in file_options if file_options[key]][0] + elif len(file_options) == 1: + return file_options[0] raise PipError( "Need exactly one file to operate upon " - "(--user, --venv, --global) to perform." + "(--user, --site, --global) to perform." ) def list_values(self, options, args): self._get_n_args(args, "list", n=0) for key, value in sorted(self.configuration.items()): - logger.info("%s=%r", key, value) + write_output("%s=%r", key, value) def get_name(self, options, args): key = self._get_n_args(args, "get [name]", n=1) value = self.configuration.get_value(key) - logger.info("%s", value) + write_output("%s", value) def set_name_value(self, options, args): key, value = self._get_n_args(args, "set [name] [value]", n=2) diff --git a/pipenv/patched/notpip/_internal/commands/debug.py b/pipenv/patched/notpip/_internal/commands/debug.py new file mode 100644 index 00000000..f83e7573 --- /dev/null +++ b/pipenv/patched/notpip/_internal/commands/debug.py @@ -0,0 +1,115 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import locale +import logging +import sys + +from pipenv.patched.notpip._internal.cli import cmdoptions +from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.cmdoptions import make_target_python +from pipenv.patched.notpip._internal.cli.status_codes import SUCCESS +from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.misc import get_pip_version +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.wheel import format_tag + +if MYPY_CHECK_RUNNING: + from typing import Any, List + from optparse import Values + +logger = logging.getLogger(__name__) + + +def show_value(name, value): + # type: (str, str) -> None + logger.info('{}: {}'.format(name, value)) + + +def show_sys_implementation(): + # type: () -> None + logger.info('sys.implementation:') + if hasattr(sys, 'implementation'): + implementation = sys.implementation # type: ignore + implementation_name = implementation.name + else: + implementation_name = '' + + with indent_log(): + show_value('name', implementation_name) + + +def show_tags(options): + # type: (Values) -> None + tag_limit = 10 + + target_python = make_target_python(options) + tags = target_python.get_tags() + + # Display the target options that were explicitly provided. + formatted_target = target_python.format_given() + suffix = '' + if formatted_target: + suffix = ' (target: {})'.format(formatted_target) + + msg = 'Compatible tags: {}{}'.format(len(tags), suffix) + logger.info(msg) + + if options.verbose < 1 and len(tags) > tag_limit: + tags_limited = True + tags = tags[:tag_limit] + else: + tags_limited = False + + with indent_log(): + for tag in tags: + logger.info(format_tag(tag)) + + if tags_limited: + msg = ( + '...\n' + '[First {tag_limit} tags shown. Pass --verbose to show all.]' + ).format(tag_limit=tag_limit) + logger.info(msg) + + +class DebugCommand(Command): + """ + Display debug information. + """ + + usage = """ + %prog """ + ignore_require_venv = True + + def __init__(self, *args, **kw): + super(DebugCommand, self).__init__(*args, **kw) + + cmd_opts = self.cmd_opts + cmdoptions.add_target_python_options(cmd_opts) + self.parser.insert_option_group(0, cmd_opts) + + def run(self, options, args): + # type: (Values, List[Any]) -> int + logger.warning( + "This command is only meant for debugging. " + "Do not use this with automation for parsing and getting these " + "details, since the output and options of this command may " + "change without notice." + ) + show_value('pip version', get_pip_version()) + show_value('sys.version', sys.version) + show_value('sys.executable', sys.executable) + show_value('sys.getdefaultencoding', sys.getdefaultencoding()) + show_value('sys.getfilesystemencoding', sys.getfilesystemencoding()) + show_value( + 'locale.getpreferredencoding', locale.getpreferredencoding(), + ) + show_value('sys.platform', sys.platform) + show_sys_implementation() + + show_tags(options) + + return SUCCESS diff --git a/pipenv/patched/notpip/_internal/commands/download.py b/pipenv/patched/notpip/_internal/commands/download.py index e5d87121..a56f0983 100644 --- a/pipenv/patched/notpip/_internal/commands/download.py +++ b/pipenv/patched/notpip/_internal/commands/download.py @@ -1,16 +1,18 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging import os from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand -from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer +from pipenv.patched.notpip._internal.cli.cmdoptions import make_target_python +from pipenv.patched.notpip._internal.cli.req_command import RequirementCommand from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker -from pipenv.patched.notpip._internal.resolve import Resolver from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner -from pipenv.patched.notpip._internal.utils.misc import ensure_dir, normalize_path +from pipenv.patched.notpip._internal.utils.misc import ensure_dir, normalize_path, write_output from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory logger = logging.getLogger(__name__) @@ -28,7 +30,6 @@ class DownloadCommand(RequirementCommand): pip also supports downloading from "requirements files", which provide an easy way to specify a whole environment to be downloaded. """ - name = 'download' usage = """ %prog [options] [package-index-options] ... @@ -37,8 +38,6 @@ class DownloadCommand(RequirementCommand): %prog [options] ... %prog [options] ...""" - summary = 'Download packages.' - def __init__(self, *args, **kw): super(DownloadCommand, self).__init__(*args, **kw) @@ -58,6 +57,8 @@ class DownloadCommand(RequirementCommand): cmd_opts.add_option(cmdoptions.require_hashes()) cmd_opts.add_option(cmdoptions.progress_bar()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option( '-d', '--dest', '--destination-dir', '--destination-directory', @@ -67,10 +68,7 @@ class DownloadCommand(RequirementCommand): help=("Download packages into ."), ) - cmd_opts.add_option(cmdoptions.platform()) - cmd_opts.add_option(cmdoptions.python_version()) - cmd_opts.add_option(cmdoptions.implementation()) - cmd_opts.add_option(cmdoptions.abi()) + cmdoptions.add_target_python_options(cmd_opts) index_opts = cmdoptions.make_option_group( cmdoptions.index_group, @@ -86,11 +84,6 @@ class DownloadCommand(RequirementCommand): # of the RequirementSet code require that property. options.editables = [] - if options.python_version: - python_versions = [options.python_version] - else: - python_versions = None - cmdoptions.check_dist_restriction(options) options.src_dir = os.path.abspath(options.src_dir) @@ -98,77 +91,66 @@ class DownloadCommand(RequirementCommand): ensure_dir(options.download_dir) - with self._build_session(options) as session: - finder = self._build_package_finder( - options=options, - session=session, - platform=options.platform, - python_versions=python_versions, - abi=options.abi, - implementation=options.implementation, + session = self.get_default_session(options) + + target_python = make_target_python(options) + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ) + build_delete = (not (options.no_clean or options.build_dir)) + if options.cache_dir and not check_path_owner(options.cache_dir): + logger.warning( + "The directory '%s' or its parent directory is not owned " + "by the current user and caching wheels has been " + "disabled. check the permissions and owner of that " + "directory. If executing pip with sudo, you may want " + "sudo's -H flag.", + options.cache_dir, ) - build_delete = (not (options.no_clean or options.build_dir)) - if options.cache_dir and not check_path_owner(options.cache_dir): - logger.warning( - "The directory '%s' or its parent directory is not owned " - "by the current user and caching wheels has been " - "disabled. check the permissions and owner of that " - "directory. If executing pip with sudo, you may want " - "sudo's -H flag.", - options.cache_dir, - ) - options.cache_dir = None + options.cache_dir = None - with RequirementTracker() as req_tracker, TempDirectory( - options.build_dir, delete=build_delete, kind="download" - ) as directory: + with RequirementTracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="download" + ) as directory: - requirement_set = RequirementSet( - require_hashes=options.require_hashes, - ) - self.populate_requirement_set( - requirement_set, - args, - options, - finder, - session, - self.name, - None - ) + requirement_set = RequirementSet( + require_hashes=options.require_hashes, + ) + self.populate_requirement_set( + requirement_set, + args, + options, + finder, + session, + None + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=options.download_dir, - wheel_download_dir=None, - progress_bar=options.progress_bar, - build_isolation=options.build_isolation, - req_tracker=req_tracker, - ) + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + download_dir=options.download_dir, + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=None, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=False, - ignore_installed=True, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + session=session, + options=options, + py_version_info=options.python_version, + ) + resolver.resolve(requirement_set) - downloaded = ' '.join([ - req.name for req in requirement_set.successfully_downloaded - ]) - if downloaded: - logger.info('Successfully downloaded %s', downloaded) + downloaded = ' '.join([ + req.name for req in requirement_set.successfully_downloaded + ]) + if downloaded: + write_output('Successfully downloaded %s', downloaded) - # Clean up - if not options.no_clean: - requirement_set.cleanup_files() + # Clean up + if not options.no_clean: + requirement_set.cleanup_files() return requirement_set diff --git a/pipenv/patched/notpip/_internal/commands/freeze.py b/pipenv/patched/notpip/_internal/commands/freeze.py index 343227ba..0c40522e 100644 --- a/pipenv/patched/notpip/_internal/commands/freeze.py +++ b/pipenv/patched/notpip/_internal/commands/freeze.py @@ -1,8 +1,12 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import sys from pipenv.patched.notpip._internal.cache import WheelCache +from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.models.format_control import FormatControl from pipenv.patched.notpip._internal.operations.freeze import freeze @@ -17,10 +21,9 @@ class FreezeCommand(Command): packages are listed in a case-insensitive sorted order. """ - name = 'freeze' + usage = """ %prog [options]""" - summary = 'Output installed packages in requirements format.' log_streams = ("ext://sys.stderr", "ext://sys.stderr") def __init__(self, *args, **kw): @@ -56,6 +59,7 @@ class FreezeCommand(Command): action='store_true', default=False, help='Only output packages installed in user-site.') + self.cmd_opts.add_option(cmdoptions.list_path()) self.cmd_opts.add_option( '--all', dest='freeze_all', @@ -77,11 +81,14 @@ class FreezeCommand(Command): if not options.freeze_all: skip.update(DEV_PKGS) + cmdoptions.check_list_path_option(options) + freeze_kwargs = dict( requirement=options.requirements, find_links=options.find_links, local_only=options.local, user_only=options.user, + paths=options.path, skip_regex=options.skip_requirements_regex, isolated=options.isolated_mode, wheel_cache=wheel_cache, diff --git a/pipenv/patched/notpip/_internal/commands/hash.py b/pipenv/patched/notpip/_internal/commands/hash.py index 183f11ae..eb8ce6e3 100644 --- a/pipenv/patched/notpip/_internal/commands/hash.py +++ b/pipenv/patched/notpip/_internal/commands/hash.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import hashlib @@ -7,7 +10,7 @@ import sys from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.cli.status_codes import ERROR from pipenv.patched.notpip._internal.utils.hashes import FAVORITE_HASH, STRONG_HASHES -from pipenv.patched.notpip._internal.utils.misc import read_chunks +from pipenv.patched.notpip._internal.utils.misc import read_chunks, write_output logger = logging.getLogger(__name__) @@ -18,11 +21,9 @@ class HashCommand(Command): These can be used with --hash in a requirements file to do repeatable installs. - """ - name = 'hash' + usage = '%prog [options] ...' - summary = 'Compute hashes of package archives.' ignore_require_venv = True def __init__(self, *args, **kw): @@ -44,8 +45,8 @@ class HashCommand(Command): algorithm = options.algorithm for path in args: - logger.info('%s:\n--hash=%s:%s', - path, algorithm, _hash_of_file(path, algorithm)) + write_output('%s:\n--hash=%s:%s', + path, algorithm, _hash_of_file(path, algorithm)) def _hash_of_file(path, algorithm): diff --git a/pipenv/patched/notpip/_internal/commands/help.py b/pipenv/patched/notpip/_internal/commands/help.py index f2c61965..73fd016b 100644 --- a/pipenv/patched/notpip/_internal/commands/help.py +++ b/pipenv/patched/notpip/_internal/commands/help.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import from pipenv.patched.notpip._internal.cli.base_command import Command @@ -7,14 +10,15 @@ from pipenv.patched.notpip._internal.exceptions import CommandError class HelpCommand(Command): """Show help for commands""" - name = 'help' + usage = """ %prog """ - summary = 'Show help for commands.' ignore_require_venv = True def run(self, options, args): - from pipenv.patched.notpip._internal.commands import commands_dict, get_similar_commands + from pipenv.patched.notpip._internal.commands import ( + commands_dict, create_command, get_similar_commands, + ) try: # 'pip help' with no args is handled by pip.__init__.parseopt() @@ -31,7 +35,7 @@ class HelpCommand(Command): raise CommandError(' - '.join(msg)) - command = commands_dict[cmd_name]() + command = create_command(cmd_name) command.parser.print_help() return SUCCESS diff --git a/pipenv/patched/notpip/_internal/commands/install.py b/pipenv/patched/notpip/_internal/commands/install.py index ddcb4759..76ae4d6d 100644 --- a/pipenv/patched/notpip/_internal/commands/install.py +++ b/pipenv/patched/notpip/_internal/commands/install.py @@ -1,3 +1,10 @@ +# The following comment should be removed at some point in the future. +# It's included for now because without it InstallCommand.run() has a +# couple errors where we have to know req.name is str rather than +# Optional[str] for the InstallRequirement req. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import errno @@ -8,37 +15,101 @@ import shutil from optparse import SUPPRESS_HELP from pipenv.patched.notpip._vendor import pkg_resources +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._internal.cache import WheelCache from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand -from pipenv.patched.notpip._internal.cli.status_codes import ERROR +from pipenv.patched.notpip._internal.cli.cmdoptions import make_target_python +from pipenv.patched.notpip._internal.cli.req_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS from pipenv.patched.notpip._internal.exceptions import ( - CommandError, InstallationError, PreviousBuildDirError, + CommandError, + InstallationError, + PreviousBuildDirError, ) -from pipenv.patched.notpip._internal.locations import distutils_scheme, virtualenv_no_global +from pipenv.patched.notpip._internal.locations import distutils_scheme from pipenv.patched.notpip._internal.operations.check import check_install_conflicts -from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet, install_given_reqs from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker -from pipenv.patched.notpip._internal.resolve import Resolver from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner from pipenv.patched.notpip._internal.utils.misc import ( - ensure_dir, get_installed_version, + ensure_dir, + get_installed_version, protect_pip_from_modification_on_windows, + write_output, ) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.virtualenv import virtualenv_no_global from pipenv.patched.notpip._internal.wheel import WheelBuilder -try: - import wheel -except ImportError: - wheel = None +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Any, List, Optional + + from pipenv.patched.notpip._internal.models.format_control import FormatControl + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from pipenv.patched.notpip._internal.wheel import BinaryAllowedPredicate logger = logging.getLogger(__name__) +def is_wheel_installed(): + """ + Return whether the wheel package is installed. + """ + try: + import wheel # noqa: F401 + except ImportError: + return False + + return True + + +def build_wheels( + builder, # type: WheelBuilder + pep517_requirements, # type: List[InstallRequirement] + legacy_requirements, # type: List[InstallRequirement] +): + # type: (...) -> List[InstallRequirement] + """ + Build wheels for requirements, depending on whether wheel is installed. + """ + # We don't build wheels for legacy requirements if wheel is not installed. + should_build_legacy = is_wheel_installed() + + # Always build PEP 517 requirements + build_failures = builder.build( + pep517_requirements, + should_unpack=True, + ) + + if should_build_legacy: + # We don't care about failures building legacy + # requirements, as we'll fall through to a direct + # install for those. + builder.build( + legacy_requirements, + should_unpack=True, + ) + + return build_failures + + +def get_check_binary_allowed(format_control): + # type: (FormatControl) -> BinaryAllowedPredicate + def check_binary_allowed(req): + # type: (InstallRequirement) -> bool + if req.use_pep517: + return True + canonical_name = canonicalize_name(req.name) + allowed_formats = format_control.get_allowed_formats(canonical_name) + return "binary" in allowed_formats + + return check_binary_allowed + + class InstallCommand(RequirementCommand): """ Install packages from: @@ -51,7 +122,6 @@ class InstallCommand(RequirementCommand): pip also supports installing from "requirements files", which provide an easy way to specify a whole environment to be installed. """ - name = 'install' usage = """ %prog [options] [package-index-options] ... @@ -60,8 +130,6 @@ class InstallCommand(RequirementCommand): %prog [options] [-e] ... %prog [options] ...""" - summary = 'Install packages.' - def __init__(self, *args, **kw): super(InstallCommand, self).__init__(*args, **kw) @@ -83,10 +151,7 @@ class InstallCommand(RequirementCommand): '. Use --upgrade to replace existing packages in ' 'with new versions.' ) - cmd_opts.add_option(cmdoptions.platform()) - cmd_opts.add_option(cmdoptions.python_version()) - cmd_opts.add_option(cmdoptions.implementation()) - cmd_opts.add_option(cmdoptions.abi()) + cmdoptions.add_target_python_options(cmd_opts) cmd_opts.add_option( '--user', @@ -154,10 +219,16 @@ class InstallCommand(RequirementCommand): '-I', '--ignore-installed', dest='ignore_installed', action='store_true', - help='Ignore the installed packages (reinstalling instead).') + help='Ignore the installed packages, overwriting them. ' + 'This can break your system if the existing package ' + 'is of a different version or was installed ' + 'with a different package manager!' + ) cmd_opts.add_option(cmdoptions.ignore_requires_python()) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.install_options()) cmd_opts.add_option(cmdoptions.global_options()) @@ -208,6 +279,7 @@ class InstallCommand(RequirementCommand): self.parser.insert_option_group(0, cmd_opts) def run(self, options, args): + # type: (Values, List[Any]) -> int cmdoptions.check_install_build_global(options) upgrade_strategy = "to-satisfy-only" if options.upgrade: @@ -218,11 +290,6 @@ class InstallCommand(RequirementCommand): cmdoptions.check_dist_restriction(options, check_target=True) - if options.python_version: - python_versions = [options.python_version] - else: - python_versions = None - options.src_dir = os.path.abspath(options.src_dir) install_options = options.install_options or [] if options.use_user_site: @@ -239,7 +306,8 @@ class InstallCommand(RequirementCommand): install_options.append('--user') install_options.append('--prefix=') - target_temp_dir = TempDirectory(kind="target") + target_temp_dir = None # type: Optional[TempDirectory] + target_temp_dir_path = None # type: Optional[str] if options.target_dir: options.ignore_installed = True options.target_dir = os.path.abspath(options.target_dir) @@ -251,169 +319,193 @@ class InstallCommand(RequirementCommand): ) # Create a target directory for using with the target option - target_temp_dir.create() - install_options.append('--home=' + target_temp_dir.path) + target_temp_dir = TempDirectory(kind="target") + target_temp_dir_path = target_temp_dir.path + install_options.append('--home=' + target_temp_dir_path) global_options = options.global_options or [] - with self._build_session(options) as session: - finder = self._build_package_finder( - options=options, - session=session, - platform=options.platform, - python_versions=python_versions, - abi=options.abi, - implementation=options.implementation, + session = self.get_default_session(options) + + target_python = make_target_python(options) + finder = self._build_package_finder( + options=options, + session=session, + target_python=target_python, + ignore_requires_python=options.ignore_requires_python, + ) + build_delete = (not (options.no_clean or options.build_dir)) + wheel_cache = WheelCache(options.cache_dir, options.format_control) + + if options.cache_dir and not check_path_owner(options.cache_dir): + logger.warning( + "The directory '%s' or its parent directory is not owned " + "by the current user and caching wheels has been " + "disabled. check the permissions and owner of that " + "directory. If executing pip with sudo, you may want " + "sudo's -H flag.", + options.cache_dir, ) - build_delete = (not (options.no_clean or options.build_dir)) - wheel_cache = WheelCache(options.cache_dir, options.format_control) + options.cache_dir = None - if options.cache_dir and not check_path_owner(options.cache_dir): - logger.warning( - "The directory '%s' or its parent directory is not owned " - "by the current user and caching wheels has been " - "disabled. check the permissions and owner of that " - "directory. If executing pip with sudo, you may want " - "sudo's -H flag.", - options.cache_dir, - ) - options.cache_dir = None + with RequirementTracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="install" + ) as directory: + requirement_set = RequirementSet( + require_hashes=options.require_hashes, + check_supported_wheels=not options.target_dir, + ) - with RequirementTracker() as req_tracker, TempDirectory( - options.build_dir, delete=build_delete, kind="install" - ) as directory: - requirement_set = RequirementSet( - require_hashes=options.require_hashes, - check_supported_wheels=not options.target_dir, + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + wheel_cache ) + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + ) + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + session=session, + options=options, + wheel_cache=wheel_cache, + use_user_site=options.use_user_site, + ignore_installed=options.ignore_installed, + ignore_requires_python=options.ignore_requires_python, + force_reinstall=options.force_reinstall, + upgrade_strategy=upgrade_strategy, + use_pep517=options.use_pep517, + ) + resolver.resolve(requirement_set) try: - self.populate_requirement_set( - requirement_set, args, options, finder, session, - self.name, wheel_cache - ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=None, - progress_bar=options.progress_bar, - build_isolation=options.build_isolation, - req_tracker=req_tracker, - ) + pip_req = requirement_set.get_requirement("pip") + except KeyError: + modifying_pip = None + else: + # If we're not replacing an already installed pip, + # we're not modifying it. + modifying_pip = getattr(pip_req, "satisfied_by", None) is None + protect_pip_from_modification_on_windows( + modifying_pip=modifying_pip + ) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=options.use_user_site, - upgrade_strategy=upgrade_strategy, - force_reinstall=options.force_reinstall, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=options.ignore_installed, - isolated=options.isolated_mode, - ) - resolver.resolve(requirement_set) + check_binary_allowed = get_check_binary_allowed( + finder.format_control + ) + # Consider legacy and PEP517-using requirements separately + legacy_requirements = [] + pep517_requirements = [] + for req in requirement_set.requirements.values(): + if req.use_pep517: + pep517_requirements.append(req) + else: + legacy_requirements.append(req) - protect_pip_from_modification_on_windows( - modifying_pip=requirement_set.has_requirement("pip") - ) + wheel_builder = WheelBuilder( + preparer, wheel_cache, + build_options=[], global_options=[], + check_binary_allowed=check_binary_allowed, + ) - # If caching is disabled or wheel is not installed don't - # try to build wheels. - if wheel and options.cache_dir: - # build wheels before install. - wb = WheelBuilder( - finder, preparer, wheel_cache, - build_options=[], global_options=[], + build_failures = build_wheels( + builder=wheel_builder, + pep517_requirements=pep517_requirements, + legacy_requirements=legacy_requirements, + ) + + # If we're using PEP 517, we cannot do a direct install + # so we fail here. + if build_failures: + raise InstallationError( + "Could not build wheels for {} which use" + " PEP 517 and cannot be installed directly".format( + ", ".join(r.name for r in build_failures))) + + to_install = resolver.get_installation_order( + requirement_set + ) + + # Consistency Checking of the package set we're installing. + should_warn_about_conflicts = ( + not options.ignore_dependencies and + options.warn_about_conflicts + ) + if should_warn_about_conflicts: + self._warn_about_conflicts(to_install) + + # Don't warn about script install locations if + # --target has been specified + warn_script_location = options.warn_script_location + if options.target_dir: + warn_script_location = False + + installed = install_given_reqs( + to_install, + install_options, + global_options, + root=options.root_path, + home=target_temp_dir_path, + prefix=options.prefix_path, + pycompile=options.compile, + warn_script_location=warn_script_location, + use_user_site=options.use_user_site, + ) + + lib_locations = get_lib_location_guesses( + user=options.use_user_site, + home=target_temp_dir_path, + root=options.root_path, + prefix=options.prefix_path, + isolated=options.isolated_mode, + ) + working_set = pkg_resources.WorkingSet(lib_locations) + + reqs = sorted(installed, key=operator.attrgetter('name')) + items = [] + for req in reqs: + item = req.name + try: + installed_version = get_installed_version( + req.name, working_set=working_set ) - # Ignore the result: a failed wheel will be - # installed from the sdist/vcs whatever. - wb.build( - requirement_set.requirements.values(), - session=session, autobuilding=True - ) - - to_install = resolver.get_installation_order( - requirement_set + if installed_version: + item += '-' + installed_version + except Exception: + pass + items.append(item) + installed_desc = ' '.join(items) + if installed_desc: + write_output( + 'Successfully installed %s', installed_desc, ) + except EnvironmentError as error: + show_traceback = (self.verbosity >= 1) - # Consistency Checking of the package set we're installing. - should_warn_about_conflicts = ( - not options.ignore_dependencies and - options.warn_about_conflicts - ) - if should_warn_about_conflicts: - self._warn_about_conflicts(to_install) + message = create_env_error_message( + error, show_traceback, options.use_user_site, + ) + logger.error(message, exc_info=show_traceback) - # Don't warn about script install locations if - # --target has been specified - warn_script_location = options.warn_script_location - if options.target_dir: - warn_script_location = False - - installed = install_given_reqs( - to_install, - install_options, - global_options, - root=options.root_path, - home=target_temp_dir.path, - prefix=options.prefix_path, - pycompile=options.compile, - warn_script_location=warn_script_location, - use_user_site=options.use_user_site, - ) - - lib_locations = get_lib_location_guesses( - user=options.use_user_site, - home=target_temp_dir.path, - root=options.root_path, - prefix=options.prefix_path, - isolated=options.isolated_mode, - ) - working_set = pkg_resources.WorkingSet(lib_locations) - - reqs = sorted(installed, key=operator.attrgetter('name')) - items = [] - for req in reqs: - item = req.name - try: - installed_version = get_installed_version( - req.name, working_set=working_set - ) - if installed_version: - item += '-' + installed_version - except Exception: - pass - items.append(item) - installed = ' '.join(items) - if installed: - logger.info('Successfully installed %s', installed) - except EnvironmentError as error: - show_traceback = (self.verbosity >= 1) - - message = create_env_error_message( - error, show_traceback, options.use_user_site, - ) - logger.error(message, exc_info=show_traceback) - - return ERROR - except PreviousBuildDirError: - options.no_clean = True - raise - finally: - # Clean up - if not options.no_clean: - requirement_set.cleanup_files() - wheel_cache.cleanup() + return ERROR + except PreviousBuildDirError: + options.no_clean = True + raise + finally: + # Clean up + if not options.no_clean: + requirement_set.cleanup_files() + wheel_cache.cleanup() if options.target_dir: self._handle_target_dir( options.target_dir, target_temp_dir, options.upgrade ) - return requirement_set + + return SUCCESS def _handle_target_dir(self, target_dir, target_temp_dir, upgrade): ensure_dir(target_dir) @@ -472,7 +564,11 @@ class InstallCommand(RequirementCommand): ) def _warn_about_conflicts(self, to_install): - package_set, _dep_info = check_install_conflicts(to_install) + try: + package_set, _dep_info = check_install_conflicts(to_install) + except Exception: + logger.error("Error checking for conflicts.", exc_info=True) + return missing, conflicting = _dep_info # NOTE: There is some duplication here from pipenv.patched.notpip check diff --git a/pipenv/patched/notpip/_internal/commands/list.py b/pipenv/patched/notpip/_internal/commands/list.py index 577c0b5f..b61b4c8c 100644 --- a/pipenv/patched/notpip/_internal/commands/list.py +++ b/pipenv/patched/notpip/_internal/commands/list.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import json @@ -7,27 +10,30 @@ from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import zip_longest from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.req_command import IndexGroupCommand from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.models.selection_prefs import SelectionPreferences +from pipenv.patched.notpip._internal.self_outdated_check import make_link_collector from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, + dist_is_editable, + get_installed_distributions, + write_output, ) from pipenv.patched.notpip._internal.utils.packaging import get_installer logger = logging.getLogger(__name__) -class ListCommand(Command): +class ListCommand(IndexGroupCommand): """ List installed packages, including editables. Packages are listed in a case-insensitive sorted order. """ - name = 'list' + usage = """ %prog [options]""" - summary = 'List installed packages.' def __init__(self, *args, **kw): super(ListCommand, self).__init__(*args, **kw) @@ -62,7 +68,7 @@ class ListCommand(Command): action='store_true', default=False, help='Only output packages installed in user-site.') - + cmd_opts.add_option(cmdoptions.list_path()) cmd_opts.add_option( '--pre', action='store_true', @@ -109,17 +115,21 @@ class ListCommand(Command): self.parser.insert_option_group(0, index_opts) self.parser.insert_option_group(0, cmd_opts) - def _build_package_finder(self, options, index_urls, session): + def _build_package_finder(self, options, session): """ Create a package finder appropriate to this list command. """ - return PackageFinder( - find_links=options.find_links, - index_urls=index_urls, + link_collector = make_link_collector(session, options=options) + + # Pass allow_yanked=False to ignore yanked versions. + selection_prefs = SelectionPreferences( + allow_yanked=False, allow_all_prereleases=options.pre, - trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, - session=session, + ) + + return PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, ) def run(self, options, args): @@ -127,21 +137,28 @@ class ListCommand(Command): raise CommandError( "Options --outdated and --uptodate cannot be combined.") + cmdoptions.check_list_path_option(options) + packages = get_installed_distributions( local_only=options.local, user_only=options.user, editables_only=options.editable, include_editables=options.include_editable, + paths=options.path, ) + # get_not_required must be called firstly in order to find and + # filter out all dependencies correctly. Otherwise a package + # can't be identified as requirement because some parent packages + # could be filtered out before. + if options.not_required: + packages = self.get_not_required(packages, options) + if options.outdated: packages = self.get_outdated(packages, options) elif options.uptodate: packages = self.get_uptodate(packages, options) - if options.not_required: - packages = self.get_not_required(packages, options) - self.output_package_listing(packages, options) def get_outdated(self, packages, options): @@ -163,21 +180,8 @@ class ListCommand(Command): return {pkg for pkg in packages if pkg.key not in dep_keys} def iter_packages_latest_infos(self, packages, options): - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) - index_urls = [] - - dependency_links = [] - for dist in packages: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt'), - ) - with self._build_session(options) as session: - finder = self._build_package_finder(options, index_urls, session) - finder.add_dependency_links(dependency_links) + finder = self._build_package_finder(options, session) for dist in packages: typ = 'unknown' @@ -187,12 +191,15 @@ class ListCommand(Command): all_candidates = [candidate for candidate in all_candidates if not candidate.version.is_prerelease] - if not all_candidates: + evaluator = finder.make_candidate_evaluator( + project_name=dist.project_name, + ) + best_candidate = evaluator.sort_best_candidate(all_candidates) + if best_candidate is None: continue - best_candidate = max(all_candidates, - key=finder._candidate_sort_key) + remote_version = best_candidate.version - if best_candidate.location.is_wheel: + if best_candidate.link.is_wheel: typ = 'wheel' else: typ = 'sdist' @@ -212,12 +219,12 @@ class ListCommand(Command): elif options.list_format == 'freeze': for dist in packages: if options.verbose >= 1: - logger.info("%s==%s (%s)", dist.project_name, - dist.version, dist.location) + write_output("%s==%s (%s)", dist.project_name, + dist.version, dist.location) else: - logger.info("%s==%s", dist.project_name, dist.version) + write_output("%s==%s", dist.project_name, dist.version) elif options.list_format == 'json': - logger.info(format_for_json(packages, options)) + write_output(format_for_json(packages, options)) def output_package_listing_columns(self, data, header): # insert the header first: we need to know the size of column names @@ -231,7 +238,7 @@ class ListCommand(Command): pkg_strings.insert(1, " ".join(map(lambda x: '-' * x, sizes))) for val in pkg_strings: - logger.info(val) + write_output(val) def tabulate(vals): diff --git a/pipenv/patched/notpip/_internal/commands/search.py b/pipenv/patched/notpip/_internal/commands/search.py index 986208f9..0da724d9 100644 --- a/pipenv/patched/notpip/_internal/commands/search.py +++ b/pipenv/patched/notpip/_internal/commands/search.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -12,22 +15,23 @@ from pipenv.patched.notpip._vendor.packaging.version import parse as parse_versi from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.req_command import SessionCommandMixin from pipenv.patched.notpip._internal.cli.status_codes import NO_MATCHES_FOUND, SUCCESS -from pipenv.patched.notpip._internal.download import PipXmlrpcTransport from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.network.xmlrpc import PipXmlrpcTransport from pipenv.patched.notpip._internal.utils.compat import get_terminal_size from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.misc import write_output logger = logging.getLogger(__name__) -class SearchCommand(Command): +class SearchCommand(Command, SessionCommandMixin): """Search for PyPI packages whose name or summary contains .""" - name = 'search' + usage = """ %prog [options] """ - summary = 'Search PyPI for packages.' ignore_require_venv = True def __init__(self, *args, **kw): @@ -59,11 +63,13 @@ class SearchCommand(Command): def search(self, query, options): index_url = options.index - with self._build_session(options) as session: - transport = PipXmlrpcTransport(index_url, session) - pypi = xmlrpc_client.ServerProxy(index_url, transport) - hits = pypi.search({'name': query, 'summary': query}, 'or') - return hits + + session = self.get_default_session(options) + + transport = PipXmlrpcTransport(index_url, session) + pypi = xmlrpc_client.ServerProxy(index_url, transport) + hits = pypi.search({'name': query, 'summary': query}, 'or') + return hits def transform_hits(hits): @@ -118,15 +124,19 @@ def print_results(hits, name_column_width=None, terminal_width=None): line = '%-*s - %s' % (name_column_width, '%s (%s)' % (name, latest), summary) try: - logger.info(line) + write_output(line) if name in installed_packages: dist = pkg_resources.get_distribution(name) with indent_log(): if dist.version == latest: - logger.info('INSTALLED: %s (latest)', dist.version) + write_output('INSTALLED: %s (latest)', dist.version) else: - logger.info('INSTALLED: %s', dist.version) - logger.info('LATEST: %s', latest) + write_output('INSTALLED: %s', dist.version) + if parse_version(latest).pre: + write_output('LATEST: %s (pre-release; install' + ' with "pip install --pre")', latest) + else: + write_output('LATEST: %s', latest) except UnicodeEncodeError: pass diff --git a/pipenv/patched/notpip/_internal/commands/show.py b/pipenv/patched/notpip/_internal/commands/show.py index 3fd24482..2bffb612 100644 --- a/pipenv/patched/notpip/_internal/commands/show.py +++ b/pipenv/patched/notpip/_internal/commands/show.py @@ -1,14 +1,18 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging import os -from email.parser import FeedParser # type: ignore +from email.parser import FeedParser from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._internal.cli.base_command import Command from pipenv.patched.notpip._internal.cli.status_codes import ERROR, SUCCESS +from pipenv.patched.notpip._internal.utils.misc import write_output logger = logging.getLogger(__name__) @@ -19,10 +23,9 @@ class ShowCommand(Command): The output is in RFC-compliant mail header format. """ - name = 'show' + usage = """ %prog [options] ...""" - summary = 'Show information about installed packages.' ignore_require_venv = True def __init__(self, *args, **kw): @@ -61,6 +64,20 @@ def search_packages_info(query): installed[canonicalize_name(p.project_name)] = p query_names = [canonicalize_name(name) for name in query] + missing = sorted( + [name for name, pkg in zip(query, query_names) if pkg not in installed] + ) + if missing: + logger.warning('Package(s) not found: %s', ', '.join(missing)) + + def get_requiring_packages(package_name): + canonical_name = canonicalize_name(package_name) + return [ + pkg.project_name for pkg in pkg_resources.working_set + if canonical_name in + [canonicalize_name(required.name) for required in + pkg.requires()] + ] for dist in [installed[pkg] for pkg in query_names if pkg in installed]: package = { @@ -68,6 +85,7 @@ def search_packages_info(query): 'version': dist.version, 'location': dist.location, 'requires': [dep.project_name for dep in dist.requires()], + 'required_by': get_requiring_packages(dist.project_name) } file_list = None metadata = None @@ -130,39 +148,33 @@ def print_results(distributions, list_files=False, verbose=False): for i, dist in enumerate(distributions): results_printed = True if i > 0: - logger.info("---") + write_output("---") - name = dist.get('name', '') - required_by = [ - pkg.project_name for pkg in pkg_resources.working_set - if name in [required.name for required in pkg.requires()] - ] - - logger.info("Name: %s", name) - logger.info("Version: %s", dist.get('version', '')) - logger.info("Summary: %s", dist.get('summary', '')) - logger.info("Home-page: %s", dist.get('home-page', '')) - logger.info("Author: %s", dist.get('author', '')) - logger.info("Author-email: %s", dist.get('author-email', '')) - logger.info("License: %s", dist.get('license', '')) - logger.info("Location: %s", dist.get('location', '')) - logger.info("Requires: %s", ', '.join(dist.get('requires', []))) - logger.info("Required-by: %s", ', '.join(required_by)) + write_output("Name: %s", dist.get('name', '')) + write_output("Version: %s", dist.get('version', '')) + write_output("Summary: %s", dist.get('summary', '')) + write_output("Home-page: %s", dist.get('home-page', '')) + write_output("Author: %s", dist.get('author', '')) + write_output("Author-email: %s", dist.get('author-email', '')) + write_output("License: %s", dist.get('license', '')) + write_output("Location: %s", dist.get('location', '')) + write_output("Requires: %s", ', '.join(dist.get('requires', []))) + write_output("Required-by: %s", ', '.join(dist.get('required_by', []))) if verbose: - logger.info("Metadata-Version: %s", - dist.get('metadata-version', '')) - logger.info("Installer: %s", dist.get('installer', '')) - logger.info("Classifiers:") + write_output("Metadata-Version: %s", + dist.get('metadata-version', '')) + write_output("Installer: %s", dist.get('installer', '')) + write_output("Classifiers:") for classifier in dist.get('classifiers', []): - logger.info(" %s", classifier) - logger.info("Entry-points:") + write_output(" %s", classifier) + write_output("Entry-points:") for entry in dist.get('entry_points', []): - logger.info(" %s", entry.strip()) + write_output(" %s", entry.strip()) if list_files: - logger.info("Files:") + write_output("Files:") for line in dist.get('files', []): - logger.info(" %s", line.strip()) + write_output(" %s", line.strip()) if "files" not in dist: - logger.info("Cannot locate installed-files.txt") + write_output("Cannot locate installed-files.txt") return results_printed diff --git a/pipenv/patched/notpip/_internal/commands/uninstall.py b/pipenv/patched/notpip/_internal/commands/uninstall.py index cf6a511c..0f92baeb 100644 --- a/pipenv/patched/notpip/_internal/commands/uninstall.py +++ b/pipenv/patched/notpip/_internal/commands/uninstall.py @@ -1,15 +1,19 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._internal.cli.base_command import Command +from pipenv.patched.notpip._internal.cli.req_command import SessionCommandMixin from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.req import parse_requirements from pipenv.patched.notpip._internal.req.constructors import install_req_from_line from pipenv.patched.notpip._internal.utils.misc import protect_pip_from_modification_on_windows -class UninstallCommand(Command): +class UninstallCommand(Command, SessionCommandMixin): """ Uninstall packages. @@ -19,11 +23,10 @@ class UninstallCommand(Command): leave behind no metadata to determine what files were installed. - Script wrappers installed by ``python setup.py develop``. """ - name = 'uninstall' + usage = """ %prog [options] ... %prog [options] -r ...""" - summary = 'Uninstall packages.' def __init__(self, *args, **kw): super(UninstallCommand, self).__init__(*args, **kw) @@ -45,34 +48,35 @@ class UninstallCommand(Command): self.parser.insert_option_group(0, self.cmd_opts) def run(self, options, args): - with self._build_session(options) as session: - reqs_to_uninstall = {} - for name in args: - req = install_req_from_line( - name, isolated=options.isolated_mode, - ) + session = self.get_default_session(options) + + reqs_to_uninstall = {} + for name in args: + req = install_req_from_line( + name, isolated=options.isolated_mode, + ) + if req.name: + reqs_to_uninstall[canonicalize_name(req.name)] = req + for filename in options.requirements: + for req in parse_requirements( + filename, + options=options, + session=session): if req.name: reqs_to_uninstall[canonicalize_name(req.name)] = req - for filename in options.requirements: - for req in parse_requirements( - filename, - options=options, - session=session): - if req.name: - reqs_to_uninstall[canonicalize_name(req.name)] = req - if not reqs_to_uninstall: - raise InstallationError( - 'You must give at least one requirement to %(name)s (see ' - '"pip help %(name)s")' % dict(name=self.name) - ) - - protect_pip_from_modification_on_windows( - modifying_pip="pip" in reqs_to_uninstall + if not reqs_to_uninstall: + raise InstallationError( + 'You must give at least one requirement to %(name)s (see ' + '"pip help %(name)s")' % dict(name=self.name) ) - for req in reqs_to_uninstall.values(): - uninstall_pathset = req.uninstall( - auto_confirm=options.yes, verbose=self.verbosity > 0, - ) - if uninstall_pathset: - uninstall_pathset.commit() + protect_pip_from_modification_on_windows( + modifying_pip="pip" in reqs_to_uninstall + ) + + for req in reqs_to_uninstall.values(): + uninstall_pathset = req.uninstall( + auto_confirm=options.yes, verbose=self.verbosity > 0, + ) + if uninstall_pathset: + uninstall_pathset.commit() diff --git a/pipenv/patched/notpip/_internal/commands/wheel.py b/pipenv/patched/notpip/_internal/commands/wheel.py index 08d695ab..8d963b4a 100644 --- a/pipenv/patched/notpip/_internal/commands/wheel.py +++ b/pipenv/patched/notpip/_internal/commands/wheel.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -6,15 +10,19 @@ import os from pipenv.patched.notpip._internal.cache import WheelCache from pipenv.patched.notpip._internal.cli import cmdoptions -from pipenv.patched.notpip._internal.cli.base_command import RequirementCommand +from pipenv.patched.notpip._internal.cli.req_command import RequirementCommand from pipenv.patched.notpip._internal.exceptions import CommandError, PreviousBuildDirError -from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer from pipenv.patched.notpip._internal.req import RequirementSet from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker -from pipenv.patched.notpip._internal.resolve import Resolver from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import WheelBuilder +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Any, List + + logger = logging.getLogger(__name__) @@ -33,7 +41,6 @@ class WheelCommand(RequirementCommand): """ - name = 'wheel' usage = """ %prog [options] ... %prog [options] -r ... @@ -41,8 +48,6 @@ class WheelCommand(RequirementCommand): %prog [options] [-e] ... %prog [options] ...""" - summary = 'Build wheels from your requirements.' - def __init__(self, *args, **kw): super(WheelCommand, self).__init__(*args, **kw) @@ -67,6 +72,8 @@ class WheelCommand(RequirementCommand): help="Extra arguments to be supplied to 'setup.py bdist_wheel'.", ) cmd_opts.add_option(cmdoptions.no_build_isolation()) + cmd_opts.add_option(cmdoptions.use_pep517()) + cmd_opts.add_option(cmdoptions.no_use_pep517()) cmd_opts.add_option(cmdoptions.constraints()) cmd_opts.add_option(cmdoptions.editable()) cmd_opts.add_option(cmdoptions.requirements()) @@ -104,80 +111,70 @@ class WheelCommand(RequirementCommand): self.parser.insert_option_group(0, cmd_opts) def run(self, options, args): + # type: (Values, List[Any]) -> None cmdoptions.check_install_build_global(options) - index_urls = [options.index_url] + options.extra_index_urls - if options.no_index: - logger.debug('Ignoring indexes: %s', ','.join(index_urls)) - index_urls = [] - if options.build_dir: options.build_dir = os.path.abspath(options.build_dir) options.src_dir = os.path.abspath(options.src_dir) - with self._build_session(options) as session: - finder = self._build_package_finder(options, session) - build_delete = (not (options.no_clean or options.build_dir)) - wheel_cache = WheelCache(options.cache_dir, options.format_control) + session = self.get_default_session(options) - with RequirementTracker() as req_tracker, TempDirectory( - options.build_dir, delete=build_delete, kind="wheel" - ) as directory: + finder = self._build_package_finder(options, session) + build_delete = (not (options.no_clean or options.build_dir)) + wheel_cache = WheelCache(options.cache_dir, options.format_control) - requirement_set = RequirementSet( - require_hashes=options.require_hashes, + with RequirementTracker() as req_tracker, TempDirectory( + options.build_dir, delete=build_delete, kind="wheel" + ) as directory: + + requirement_set = RequirementSet( + require_hashes=options.require_hashes, + ) + + try: + self.populate_requirement_set( + requirement_set, args, options, finder, session, + wheel_cache ) - try: - self.populate_requirement_set( - requirement_set, args, options, finder, session, - self.name, wheel_cache - ) + preparer = self.make_requirement_preparer( + temp_build_dir=directory, + options=options, + req_tracker=req_tracker, + wheel_download_dir=options.wheel_dir, + ) - preparer = RequirementPreparer( - build_dir=directory.path, - src_dir=options.src_dir, - download_dir=None, - wheel_download_dir=options.wheel_dir, - progress_bar=options.progress_bar, - build_isolation=options.build_isolation, - req_tracker=req_tracker, - ) + resolver = self.make_resolver( + preparer=preparer, + finder=finder, + session=session, + options=options, + wheel_cache=wheel_cache, + ignore_requires_python=options.ignore_requires_python, + use_pep517=options.use_pep517, + ) + resolver.resolve(requirement_set) - resolver = Resolver( - preparer=preparer, - finder=finder, - session=session, - wheel_cache=wheel_cache, - use_user_site=False, - upgrade_strategy="to-satisfy-only", - force_reinstall=False, - ignore_dependencies=options.ignore_dependencies, - ignore_requires_python=options.ignore_requires_python, - ignore_installed=True, - isolated=options.isolated_mode, + # build wheels + wb = WheelBuilder( + preparer, wheel_cache, + build_options=options.build_options or [], + global_options=options.global_options or [], + no_clean=options.no_clean, + ) + build_failures = wb.build( + requirement_set.requirements.values(), + ) + if len(build_failures) != 0: + raise CommandError( + "Failed to build one or more wheels" ) - resolver.resolve(requirement_set) - - # build wheels - wb = WheelBuilder( - finder, preparer, wheel_cache, - build_options=options.build_options or [], - global_options=options.global_options or [], - no_clean=options.no_clean, - ) - wheels_built_successfully = wb.build( - requirement_set.requirements.values(), session=session, - ) - if not wheels_built_successfully: - raise CommandError( - "Failed to build one or more wheels" - ) - except PreviousBuildDirError: - options.no_clean = True - raise - finally: - if not options.no_clean: - requirement_set.cleanup_files() - wheel_cache.cleanup() + except PreviousBuildDirError: + options.no_clean = True + raise + finally: + if not options.no_clean: + requirement_set.cleanup_files() + wheel_cache.cleanup() diff --git a/pipenv/patched/notpip/_internal/configuration.py b/pipenv/patched/notpip/_internal/configuration.py index 3c02c955..101934eb 100644 --- a/pipenv/patched/notpip/_internal/configuration.py +++ b/pipenv/patched/notpip/_internal/configuration.py @@ -11,25 +11,28 @@ Some terminology: A single word describing where the configuration key-value pair came from """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + import locale import logging import os +import sys -from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.six.moves import configparser from pipenv.patched.notpip._internal.exceptions import ( - ConfigurationError, ConfigurationFileCouldNotBeLoaded, -) -from pipenv.patched.notpip._internal.locations import ( - legacy_config_file, new_config_file, running_under_virtualenv, - site_config_files, venv_config_file, + ConfigurationError, + ConfigurationFileCouldNotBeLoaded, ) +from pipenv.patched.notpip._internal.utils import appdirs +from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser from pipenv.patched.notpip._internal.utils.misc import ensure_dir, enum from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ( # noqa: F401 + from typing import ( Any, Dict, Iterable, List, NewType, Optional, Tuple ) @@ -52,6 +55,12 @@ def _normalize_name(name): def _disassemble_key(name): # type: (str) -> List[str] + if "." not in name: + error_message = ( + "Key does not contain dot separated section and key. " + "Perhaps you wanted to use 'global.{}' instead?" + ).format(name) + raise ConfigurationError(error_message) return name.split(".", 1) @@ -59,12 +68,37 @@ def _disassemble_key(name): kinds = enum( USER="user", # User Specific GLOBAL="global", # System Wide - VENV="venv", # Virtual Environment Specific + SITE="site", # [Virtual] Environment Specific ENV="env", # from PIP_CONFIG_FILE ENV_VAR="env-var", # from Environment Variables ) +CONFIG_BASENAME = 'pip.ini' if WINDOWS else 'pip.conf' + + +def get_configuration_files(): + global_config_files = [ + os.path.join(path, CONFIG_BASENAME) + for path in appdirs.site_config_dirs('pip') + ] + + site_config_file = os.path.join(sys.prefix, CONFIG_BASENAME) + legacy_config_file = os.path.join( + expanduser('~'), + 'pip' if WINDOWS else '.pip', + CONFIG_BASENAME, + ) + new_config_file = os.path.join( + appdirs.user_config_dir("pip"), CONFIG_BASENAME + ) + return { + kinds.GLOBAL: global_config_files, + kinds.SITE: [site_config_file], + kinds.USER: [legacy_config_file, new_config_file], + } + + class Configuration(object): """Handles management of configuration. @@ -83,7 +117,7 @@ class Configuration(object): # type: (bool, Kind) -> None super(Configuration, self).__init__() - _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.VENV, None] + _valid_load_only = [kinds.USER, kinds.GLOBAL, kinds.SITE, None] if load_only not in _valid_load_only: raise ConfigurationError( "Got invalid value for load_only - should be one of {}".format( @@ -95,7 +129,7 @@ class Configuration(object): # The order here determines the override order. self._override_order = [ - kinds.GLOBAL, kinds.USER, kinds.VENV, kinds.ENV, kinds.ENV_VAR + kinds.GLOBAL, kinds.USER, kinds.SITE, kinds.ENV, kinds.ENV_VAR ] self._ignore_env_names = ["version", "help"] @@ -188,7 +222,7 @@ class Configuration(object): # name removed from parser, section may now be empty section_iter = iter(parser.items(section)) try: - val = six.next(section_iter) + val = next(section_iter) except StopIteration: val = None @@ -205,7 +239,7 @@ class Configuration(object): def save(self): # type: () -> None - """Save the currentin-memory state. + """Save the current in-memory state. """ self._ensure_have_load_only() @@ -216,7 +250,7 @@ class Configuration(object): ensure_dir(os.path.dirname(fname)) with open(fname, "w") as f: - parser.write(f) # type: ignore + parser.write(f) # # Private routines @@ -351,8 +385,10 @@ class Configuration(object): else: yield kinds.ENV, [] + config_files = get_configuration_files() + # at the base we have any global configuration - yield kinds.GLOBAL, list(site_config_files) + yield kinds.GLOBAL, config_files[kinds.GLOBAL] # per-user configuration next should_load_user_config = not self.isolated and not ( @@ -360,11 +396,10 @@ class Configuration(object): ) if should_load_user_config: # The legacy config file is overridden by the new config file - yield kinds.USER, [legacy_config_file, new_config_file] + yield kinds.USER, config_files[kinds.USER] # finally virtualenv configuration first trumping others - if running_under_virtualenv(): - yield kinds.VENV, [venv_config_file] + yield kinds.SITE, config_files[kinds.SITE] def _get_parser_to_modify(self): # type: () -> Tuple[str, RawConfigParser] diff --git a/pipenv/patched/notpip/_internal/distributions/__init__.py b/pipenv/patched/notpip/_internal/distributions/__init__.py new file mode 100644 index 00000000..9eb3d7d6 --- /dev/null +++ b/pipenv/patched/notpip/_internal/distributions/__init__.py @@ -0,0 +1,24 @@ +from pipenv.patched.notpip._internal.distributions.source.legacy import SourceDistribution +from pipenv.patched.notpip._internal.distributions.wheel import WheelDistribution +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pipenv.patched.notpip._internal.distributions.base import AbstractDistribution + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + + +def make_distribution_for_install_requirement(install_req): + # type: (InstallRequirement) -> AbstractDistribution + """Returns a Distribution for the given InstallRequirement + """ + # Editable requirements will always be source distributions. They use the + # legacy logic until we create a modern standard for them. + if install_req.editable: + return SourceDistribution(install_req) + + # If it's a wheel, it's a WheelDistribution + if install_req.is_wheel: + return WheelDistribution(install_req) + + # Otherwise, a SourceDistribution + return SourceDistribution(install_req) diff --git a/pipenv/patched/notpip/_internal/distributions/base.py b/pipenv/patched/notpip/_internal/distributions/base.py new file mode 100644 index 00000000..b479ff83 --- /dev/null +++ b/pipenv/patched/notpip/_internal/distributions/base.py @@ -0,0 +1,36 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import abc + +from pipenv.patched.notpip._vendor.six import add_metaclass + + +@add_metaclass(abc.ABCMeta) +class AbstractDistribution(object): + """A base class for handling installable artifacts. + + The requirements for anything installable are as follows: + + - we must be able to determine the requirement name + (or we can't correctly handle the non-upgrade case). + + - for packages with setup requirements, we must also be able + to determine their requirements without installing additional + packages (for the same reason as run-time dependencies) + + - we must be able to create a Distribution object exposing the + above metadata. + """ + + def __init__(self, req): + super(AbstractDistribution, self).__init__() + self.req = req + + @abc.abstractmethod + def get_pkg_resources_distribution(self): + raise NotImplementedError() + + @abc.abstractmethod + def prepare_distribution_metadata(self, finder, build_isolation): + raise NotImplementedError() diff --git a/pipenv/patched/notpip/_internal/distributions/installed.py b/pipenv/patched/notpip/_internal/distributions/installed.py new file mode 100644 index 00000000..78f29d52 --- /dev/null +++ b/pipenv/patched/notpip/_internal/distributions/installed.py @@ -0,0 +1,18 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from pipenv.patched.notpip._internal.distributions.base import AbstractDistribution + + +class InstalledDistribution(AbstractDistribution): + """Represents an installed package. + + This does not need any preparation as the required information has already + been computed. + """ + + def get_pkg_resources_distribution(self): + return self.req.satisfied_by + + def prepare_distribution_metadata(self, finder, build_isolation): + pass diff --git a/pipenv/patched/notpip/_internal/distributions/source/__init__.py b/pipenv/patched/notpip/_internal/distributions/source/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/patched/notpip/_internal/distributions/source/legacy.py b/pipenv/patched/notpip/_internal/distributions/source/legacy.py new file mode 100644 index 00000000..0e700d2a --- /dev/null +++ b/pipenv/patched/notpip/_internal/distributions/source/legacy.py @@ -0,0 +1,98 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pipenv.patched.notpip._internal.build_env import BuildEnvironment +from pipenv.patched.notpip._internal.distributions.base import AbstractDistribution +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.subprocess import runner_with_spinner_message + +logger = logging.getLogger(__name__) + + +class SourceDistribution(AbstractDistribution): + """Represents a source distribution. + + The preparation step for these needs metadata for the packages to be + generated, either using PEP 517 or using the legacy `setup.py egg_info`. + + NOTE from @pradyunsg (14 June 2019) + I expect SourceDistribution class will need to be split into + `legacy_source` (setup.py based) and `source` (PEP 517 based) when we start + bringing logic for preparation out of InstallRequirement into this class. + """ + + def get_pkg_resources_distribution(self): + return self.req.get_dist() + + def prepare_distribution_metadata(self, finder, build_isolation): + # Prepare for building. We need to: + # 1. Load pyproject.toml (if it exists) + # 2. Set up the build environment + + self.req.load_pyproject_toml() + should_isolate = self.req.use_pep517 and build_isolation + if should_isolate: + self._setup_isolation(finder) + + self.req.prepare_metadata() + self.req.assert_source_matches_version() + + def _setup_isolation(self, finder): + def _raise_conflicts(conflicting_with, conflicting_reqs): + format_string = ( + "Some build dependencies for {requirement} " + "conflict with {conflicting_with}: {description}." + ) + error_message = format_string.format( + requirement=self.req, + conflicting_with=conflicting_with, + description=', '.join( + '%s is incompatible with %s' % (installed, wanted) + for installed, wanted in sorted(conflicting) + ) + ) + raise InstallationError(error_message) + + # Isolate in a BuildEnvironment and install the build-time + # requirements. + self.req.build_env = BuildEnvironment() + self.req.build_env.install_requirements( + finder, self.req.pyproject_requires, 'overlay', + "Installing build dependencies" + ) + conflicting, missing = self.req.build_env.check_requirements( + self.req.requirements_to_check + ) + if conflicting: + _raise_conflicts("PEP 517/518 supported requirements", + conflicting) + if missing: + logger.warning( + "Missing build requirements in pyproject.toml for %s.", + self.req, + ) + logger.warning( + "The project does not specify a build backend, and " + "pip cannot fall back to setuptools without %s.", + " and ".join(map(repr, sorted(missing))) + ) + # Install any extra build dependencies that the backend requests. + # This must be done in a second pass, as the pyproject.toml + # dependencies must be installed before we can call the backend. + with self.req.build_env: + runner = runner_with_spinner_message( + "Getting requirements to build wheel" + ) + backend = self.req.pep517_backend + with backend.subprocess_runner(runner): + reqs = backend.get_requires_for_build_wheel() + + conflicting, missing = self.req.build_env.check_requirements(reqs) + if conflicting: + _raise_conflicts("the backend dependencies", conflicting) + self.req.build_env.install_requirements( + finder, missing, 'normal', + "Installing backend dependencies" + ) diff --git a/pipenv/patched/notpip/_internal/distributions/wheel.py b/pipenv/patched/notpip/_internal/distributions/wheel.py new file mode 100644 index 00000000..23e73ee7 --- /dev/null +++ b/pipenv/patched/notpip/_internal/distributions/wheel.py @@ -0,0 +1,20 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from pipenv.patched.notpip._vendor import pkg_resources + +from pipenv.patched.notpip._internal.distributions.base import AbstractDistribution + + +class WheelDistribution(AbstractDistribution): + """Represents a wheel distribution. + + This does not need any preparation as wheels can be directly unpacked. + """ + + def get_pkg_resources_distribution(self): + return list(pkg_resources.find_distributions( + self.req.source_dir))[0] + + def prepare_distribution_metadata(self, finder, build_isolation): + pass diff --git a/pipenv/patched/notpip/_internal/download.py b/pipenv/patched/notpip/_internal/download.py index 8f4c38f5..b8d12e17 100644 --- a/pipenv/patched/notpip/_internal/download.py +++ b/pipenv/patched/notpip/_internal/download.py @@ -1,403 +1,89 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import cgi -import email.utils -import getpass -import json import logging import mimetypes import os -import platform import re import shutil import sys -from pipenv.patched.notpip._vendor import requests, six, urllib3 -from pipenv.patched.notpip._vendor.cachecontrol import CacheControlAdapter -from pipenv.patched.notpip._vendor.cachecontrol.caches import FileCache -from pipenv.patched.notpip._vendor.lockfile import LockError -from pipenv.patched.notpip._vendor.requests.adapters import BaseAdapter, HTTPAdapter -from pipenv.patched.notpip._vendor.requests.auth import AuthBase, HTTPBasicAuth +from pipenv.patched.notpip._vendor import requests from pipenv.patched.notpip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -from pipenv.patched.notpip._vendor.requests.sessions import Session -from pipenv.patched.notpip._vendor.requests.structures import CaseInsensitiveDict -from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth -# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is -# why we ignore the type on this import -from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore +from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote -from pipenv.patched.notpip._vendor.urllib3.util import IS_PYOPENSSL -import pipenv.patched.notpip from pipenv.patched.notpip._internal.exceptions import HashMismatch, InstallationError -from pipenv.patched.notpip._internal.locations import write_delete_marker_file from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.network.session import PipSession from pipenv.patched.notpip._internal.utils.encoding import auto_decode -from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner -from pipenv.patched.notpip._internal.utils.glibc import libc_ver -from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.filesystem import copy2_fixed from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, ask_path_exists, backup_dir, call_subprocess, consume, - display_path, format_size, get_installed_version, rmtree, splitext, - unpack_file, + ask_path_exists, + backup_dir, + consume, + display_path, + format_size, + hide_url, + path_to_display, + rmtree, + splitext, ) -from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import DownloadProgressProvider +from pipenv.patched.notpip._internal.utils.unpacking import unpack_file +from pipenv.patched.notpip._internal.utils.urls import get_url_scheme from pipenv.patched.notpip._internal.vcs import vcs -try: - import ssl # noqa -except ImportError: - ssl = None +if MYPY_CHECK_RUNNING: + from typing import ( + IO, Callable, List, Optional, Text, Tuple, + ) + + from mypy_extensions import TypedDict + + from pipenv.patched.notpip._internal.models.link import Link + from pipenv.patched.notpip._internal.utils.hashes import Hashes + from pipenv.patched.notpip._internal.vcs.versioncontrol import VersionControl + + if PY2: + CopytreeKwargs = TypedDict( + 'CopytreeKwargs', + { + 'ignore': Callable[[str, List[str]], List[str]], + 'symlinks': bool, + }, + total=False, + ) + else: + CopytreeKwargs = TypedDict( + 'CopytreeKwargs', + { + 'copy_function': Callable[[str, str], None], + 'ignore': Callable[[str, List[str]], List[str]], + 'ignore_dangling_symlinks': bool, + 'symlinks': bool, + }, + total=False, + ) -HAS_TLS = (ssl is not None) or IS_PYOPENSSL __all__ = ['get_file_content', - 'is_url', 'url_to_path', 'path_to_url', - 'is_archive_file', 'unpack_vcs_link', - 'unpack_file_url', 'is_vcs_url', 'is_file_url', - 'unpack_http_url', 'unpack_url'] + 'unpack_vcs_link', + 'unpack_file_url', + 'unpack_http_url', 'unpack_url', + 'parse_content_disposition', 'sanitize_content_filename'] logger = logging.getLogger(__name__) -def user_agent(): - """ - Return a string representing the user agent. - """ - data = { - "installer": {"name": "pip", "version": pipenv.patched.notpip.__version__}, - "python": platform.python_version(), - "implementation": { - "name": platform.python_implementation(), - }, - } - - if data["implementation"]["name"] == 'CPython': - data["implementation"]["version"] = platform.python_version() - elif data["implementation"]["name"] == 'PyPy': - if sys.pypy_version_info.releaselevel == 'final': - pypy_version_info = sys.pypy_version_info[:3] - else: - pypy_version_info = sys.pypy_version_info - data["implementation"]["version"] = ".".join( - [str(x) for x in pypy_version_info] - ) - elif data["implementation"]["name"] == 'Jython': - # Complete Guess - data["implementation"]["version"] = platform.python_version() - elif data["implementation"]["name"] == 'IronPython': - # Complete Guess - data["implementation"]["version"] = platform.python_version() - - if sys.platform.startswith("linux"): - from pipenv.patched.notpip._vendor import distro - distro_infos = dict(filter( - lambda x: x[1], - zip(["name", "version", "id"], distro.linux_distribution()), - )) - libc = dict(filter( - lambda x: x[1], - zip(["lib", "version"], libc_ver()), - )) - if libc: - distro_infos["libc"] = libc - if distro_infos: - data["distro"] = distro_infos - - if sys.platform.startswith("darwin") and platform.mac_ver()[0]: - data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} - - if platform.system(): - data.setdefault("system", {})["name"] = platform.system() - - if platform.release(): - data.setdefault("system", {})["release"] = platform.release() - - if platform.machine(): - data["cpu"] = platform.machine() - - if HAS_TLS: - data["openssl_version"] = ssl.OPENSSL_VERSION - - setuptools_version = get_installed_version("setuptools") - if setuptools_version is not None: - data["setuptools_version"] = setuptools_version - - return "{data[installer][name]}/{data[installer][version]} {json}".format( - data=data, - json=json.dumps(data, separators=(",", ":"), sort_keys=True), - ) - - -class MultiDomainBasicAuth(AuthBase): - - def __init__(self, prompting=True): - self.prompting = prompting - self.passwords = {} - - def __call__(self, req): - parsed = urllib_parse.urlparse(req.url) - - # Get the netloc without any embedded credentials - netloc = parsed.netloc.rsplit("@", 1)[-1] - - # Set the url of the request to the url without any credentials - req.url = urllib_parse.urlunparse(parsed[:1] + (netloc,) + parsed[2:]) - - # Use any stored credentials that we have for this netloc - username, password = self.passwords.get(netloc, (None, None)) - - # Extract credentials embedded in the url if we have none stored - if username is None: - username, password = self.parse_credentials(parsed.netloc) - - # Get creds from netrc if we still don't have them - if username is None and password is None: - netrc_auth = get_netrc_auth(req.url) - username, password = netrc_auth if netrc_auth else (None, None) - - if username or password: - # Store the username and password - self.passwords[netloc] = (username, password) - - # Send the basic auth with this request - req = HTTPBasicAuth(username or "", password or "")(req) - - # Attach a hook to handle 401 responses - req.register_hook("response", self.handle_401) - - return req - - def handle_401(self, resp, **kwargs): - # We only care about 401 responses, anything else we want to just - # pass through the actual response - if resp.status_code != 401: - return resp - - # We are not able to prompt the user so simply return the response - if not self.prompting: - return resp - - parsed = urllib_parse.urlparse(resp.url) - - # Prompt the user for a new username and password - username = six.moves.input("User for %s: " % parsed.netloc) - password = getpass.getpass("Password: ") - - # Store the new username and password to use for future requests - if username or password: - self.passwords[parsed.netloc] = (username, password) - - # Consume content and release the original connection to allow our new - # request to reuse the same one. - resp.content - resp.raw.release_conn() - - # Add our new username and password to the request - req = HTTPBasicAuth(username or "", password or "")(resp.request) - - # Send our new request - new_resp = resp.connection.send(req, **kwargs) - new_resp.history.append(resp) - - return new_resp - - def parse_credentials(self, netloc): - if "@" in netloc: - userinfo = netloc.rsplit("@", 1)[0] - if ":" in userinfo: - user, pwd = userinfo.split(":", 1) - return (urllib_unquote(user), urllib_unquote(pwd)) - return urllib_unquote(userinfo), None - return None, None - - -class LocalFSAdapter(BaseAdapter): - - def send(self, request, stream=None, timeout=None, verify=None, cert=None, - proxies=None): - pathname = url_to_path(request.url) - - resp = Response() - resp.status_code = 200 - resp.url = request.url - - try: - stats = os.stat(pathname) - except OSError as exc: - resp.status_code = 404 - resp.raw = exc - else: - modified = email.utils.formatdate(stats.st_mtime, usegmt=True) - content_type = mimetypes.guess_type(pathname)[0] or "text/plain" - resp.headers = CaseInsensitiveDict({ - "Content-Type": content_type, - "Content-Length": stats.st_size, - "Last-Modified": modified, - }) - - resp.raw = open(pathname, "rb") - resp.close = resp.raw.close - - return resp - - def close(self): - pass - - -class SafeFileCache(FileCache): - """ - A file based cache which is safe to use even when the target directory may - not be accessible or writable. - """ - - def __init__(self, *args, **kwargs): - super(SafeFileCache, self).__init__(*args, **kwargs) - - # Check to ensure that the directory containing our cache directory - # is owned by the user current executing pip. If it does not exist - # we will check the parent directory until we find one that does exist. - # If it is not owned by the user executing pip then we will disable - # the cache and log a warning. - if not check_path_owner(self.directory): - logger.warning( - "The directory '%s' or its parent directory is not owned by " - "the current user and the cache has been disabled. Please " - "check the permissions and owner of that directory. If " - "executing pip with sudo, you may want sudo's -H flag.", - self.directory, - ) - - # Set our directory to None to disable the Cache - self.directory = None - - def get(self, *args, **kwargs): - # If we don't have a directory, then the cache should be a no-op. - if self.directory is None: - return - - try: - return super(SafeFileCache, self).get(*args, **kwargs) - except (LockError, OSError, IOError): - # We intentionally silence this error, if we can't access the cache - # then we can just skip caching and process the request as if - # caching wasn't enabled. - pass - - def set(self, *args, **kwargs): - # If we don't have a directory, then the cache should be a no-op. - if self.directory is None: - return - - try: - return super(SafeFileCache, self).set(*args, **kwargs) - except (LockError, OSError, IOError): - # We intentionally silence this error, if we can't access the cache - # then we can just skip caching and process the request as if - # caching wasn't enabled. - pass - - def delete(self, *args, **kwargs): - # If we don't have a directory, then the cache should be a no-op. - if self.directory is None: - return - - try: - return super(SafeFileCache, self).delete(*args, **kwargs) - except (LockError, OSError, IOError): - # We intentionally silence this error, if we can't access the cache - # then we can just skip caching and process the request as if - # caching wasn't enabled. - pass - - -class InsecureHTTPAdapter(HTTPAdapter): - - def cert_verify(self, conn, url, verify, cert): - conn.cert_reqs = 'CERT_NONE' - conn.ca_certs = None - - -class PipSession(requests.Session): - - timeout = None - - def __init__(self, *args, **kwargs): - retries = kwargs.pop("retries", 0) - cache = kwargs.pop("cache", None) - insecure_hosts = kwargs.pop("insecure_hosts", []) - - super(PipSession, self).__init__(*args, **kwargs) - - # Attach our User Agent to the request - self.headers["User-Agent"] = user_agent() - - # Attach our Authentication handler to the session - self.auth = MultiDomainBasicAuth() - - # Create our urllib3.Retry instance which will allow us to customize - # how we handle retries. - retries = urllib3.Retry( - # Set the total number of retries that a particular request can - # have. - total=retries, - - # A 503 error from PyPI typically means that the Fastly -> Origin - # connection got interrupted in some way. A 503 error in general - # is typically considered a transient error so we'll go ahead and - # retry it. - # A 500 may indicate transient error in Amazon S3 - # A 520 or 527 - may indicate transient error in CloudFlare - status_forcelist=[500, 503, 520, 527], - - # Add a small amount of back off between failed requests in - # order to prevent hammering the service. - backoff_factor=0.25, - ) - - # We want to _only_ cache responses on securely fetched origins. We do - # this because we can't validate the response of an insecurely fetched - # origin, and we don't want someone to be able to poison the cache and - # require manual eviction from the cache to fix it. - if cache: - secure_adapter = CacheControlAdapter( - cache=SafeFileCache(cache, use_dir_lock=True), - max_retries=retries, - ) - else: - secure_adapter = HTTPAdapter(max_retries=retries) - - # Our Insecure HTTPAdapter disables HTTPS validation. It does not - # support caching (see above) so we'll use it for all http:// URLs as - # well as any https:// host that we've marked as ignoring TLS errors - # for. - insecure_adapter = InsecureHTTPAdapter(max_retries=retries) - - self.mount("https://", secure_adapter) - self.mount("http://", insecure_adapter) - - # Enable file:// urls - self.mount("file://", LocalFSAdapter()) - - # We want to use a non-validating adapter for any requests which are - # deemed insecure. - for host in insecure_hosts: - self.mount("https://{}/".format(host), insecure_adapter) - - def request(self, method, url, *args, **kwargs): - # Allow setting a default timeout on a session - kwargs.setdefault("timeout", self.timeout) - - # Dispatch the actual request - return super(PipSession, self).request(method, url, *args, **kwargs) - - def get_file_content(url, comes_from=None, session=None): + # type: (str, Optional[str], Optional[PipSession]) -> Tuple[str, Text] """Gets the content of a file; it may be a filename, file: URL, or http: URL. Returns (location, content). Content is unicode. @@ -410,29 +96,30 @@ def get_file_content(url, comes_from=None, session=None): "get_file_content() missing 1 required keyword argument: 'session'" ) - match = _scheme_re.search(url) - if match: - scheme = match.group(1).lower() - if (scheme == 'file' and comes_from and - comes_from.startswith('http')): + scheme = get_url_scheme(url) + + if scheme in ['http', 'https']: + # FIXME: catch some errors + resp = session.get(url) + resp.raise_for_status() + return resp.url, resp.text + + elif scheme == 'file': + if comes_from and comes_from.startswith('http'): raise InstallationError( 'Requirements file %s references URL %s, which is local' % (comes_from, url)) - if scheme == 'file': - path = url.split(':', 1)[1] - path = path.replace('\\', '/') - match = _url_slash_drive_re.match(path) - if match: - path = match.group(1) + ':' + path.split('|', 1)[1] - path = urllib_parse.unquote(path) - if path.startswith('/'): - path = '/' + path.lstrip('/') - url = path - else: - # FIXME: catch some errors - resp = session.get(url) - resp.raise_for_status() - return resp.url, resp.text + + path = url.split(':', 1)[1] + path = path.replace('\\', '/') + match = _url_slash_drive_re.match(path) + if match: + path = match.group(1) + ':' + path.split('|', 1)[1] + path = urllib_parse.unquote(path) + if path.startswith('/'): + path = '/' + path.lstrip('/') + url = path + try: with open(url, 'rb') as f: content = auto_decode(f.read()) @@ -443,89 +130,39 @@ def get_file_content(url, comes_from=None, session=None): return url, content -_scheme_re = re.compile(r'^(http|https|file):', re.I) _url_slash_drive_re = re.compile(r'/*([a-z])\|', re.I) -def is_url(name): - """Returns true if the name looks like a URL""" - if ':' not in name: - return False - scheme = name.split(':', 1)[0].lower() - return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes - - -def url_to_path(url): - """ - Convert a file: URL to a path. - """ - assert url.startswith('file:'), ( - "You can only turn file: urls into filenames (not %r)" % url) - - _, netloc, path, _, _ = urllib_parse.urlsplit(url) - - # if we have a UNC path, prepend UNC share notation - if netloc: - netloc = '\\\\' + netloc - - path = urllib_request.url2pathname(netloc + path) - return path - - -def path_to_url(path): - """ - Convert a path to a file: URL. The path will be made absolute and have - quoted path parts. - """ - path = os.path.normpath(os.path.abspath(path)) - url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) - return url - - -def is_archive_file(name): - """Return True if `name` is a considered as an archive file.""" - ext = splitext(name)[1].lower() - if ext in ARCHIVE_EXTENSIONS: - return True - return False - - def unpack_vcs_link(link, location): + # type: (Link, str) -> None vcs_backend = _get_used_vcs_backend(link) - vcs_backend.unpack(location) + assert vcs_backend is not None + vcs_backend.unpack(location, url=hide_url(link.url)) def _get_used_vcs_backend(link): - for backend in vcs.backends: - if link.scheme in backend.schemes: - vcs_backend = backend(link.url) - return vcs_backend - - -def is_vcs_url(link): - return bool(_get_used_vcs_backend(link)) - - -def is_file_url(link): - return link.url.lower().startswith('file:') - - -def is_dir_url(link): - """Return whether a file:// Link points to a directory. - - ``link`` must not have any other scheme but file://. Call is_file_url() - first. - + # type: (Link) -> Optional[VersionControl] """ - link_path = url_to_path(link.url_without_fragment) - return os.path.isdir(link_path) + Return a VersionControl object or None. + """ + for vcs_backend in vcs.backends: + if link.scheme in vcs_backend.schemes: + return vcs_backend + return None def _progress_indicator(iterable, *args, **kwargs): return iterable -def _download_url(resp, link, content_file, hashes, progress_bar): +def _download_url( + resp, # type: Response + link, # type: Link + content_file, # type: IO + hashes, # type: Optional[Hashes] + progress_bar # type: str +): + # type: (...) -> None try: total_length = int(resp.headers['content-length']) except (ValueError, KeyError, TypeError): @@ -606,8 +243,6 @@ def _download_url(resp, link, content_file, hashes, progress_bar): else: logger.info("Downloading %s", url) - logger.debug('Downloading from URL %s', link) - downloaded_chunks = written_chunks( progress_indicator( resp_read(CONTENT_CHUNK_SIZE), @@ -647,8 +282,15 @@ def _copy_file(filename, location, link): logger.info('Saved %s', display_path(download_location)) -def unpack_http_url(link, location, download_dir=None, - session=None, hashes=None, progress_bar="on"): +def unpack_http_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None if session is None: raise TypeError( "unpack_http_url() missing 1 required keyword argument: 'session'" @@ -675,7 +317,7 @@ def unpack_http_url(link, location, download_dir=None, # unpack the archive to the build dir location. even when only # downloading archives, they have to be unpacked to parse dependencies - unpack_file(from_path, location, content_type, link) + unpack_file(from_path, location, content_type) # a download dir is specified; let's copy the archive there if download_dir and not already_downloaded_path: @@ -685,19 +327,64 @@ def unpack_http_url(link, location, download_dir=None, os.unlink(from_path) -def unpack_file_url(link, location, download_dir=None, hashes=None): +def _copy2_ignoring_special_files(src, dest): + # type: (str, str) -> None + """Copying special files is not supported, but as a convenience to users + we skip errors copying them. This supports tools that may create e.g. + socket files in the project source directory. + """ + try: + copy2_fixed(src, dest) + except shutil.SpecialFileError as e: + # SpecialFileError may be raised due to either the source or + # destination. If the destination was the cause then we would actually + # care, but since the destination directory is deleted prior to + # copy we ignore all of them assuming it is caused by the source. + logger.warning( + "Ignoring special file error '%s' encountered copying %s to %s.", + str(e), + path_to_display(src), + path_to_display(dest), + ) + + +def _copy_source_tree(source, target): + # type: (str, str) -> None + def ignore(d, names): + # Pulling in those directories can potentially be very slow, + # exclude the following directories if they appear in the top + # level dir (and only it). + # See discussion at https://github.com/pypa/pip/pull/6770 + return ['.tox', '.nox'] if d == source else [] + + kwargs = dict(ignore=ignore, symlinks=True) # type: CopytreeKwargs + + if not PY2: + # Python 2 does not support copy_function, so we only ignore + # errors on special file copy in Python 3. + kwargs['copy_function'] = _copy2_ignoring_special_files + + shutil.copytree(source, target, **kwargs) + + +def unpack_file_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + hashes=None # type: Optional[Hashes] +): + # type: (...) -> None """Unpack link into location. If download_dir is provided and link points to a file, make a copy of the link file inside download_dir. """ - link_path = url_to_path(link.url_without_fragment) - + link_path = link.file_path # If it's a url to a local directory - if is_dir_url(link): + if link.is_existing_dir(): if os.path.isdir(location): rmtree(location) - shutil.copytree(link_path, location, symlinks=True) + _copy_source_tree(link_path, location) if download_dir: logger.info('Link is a directory, ignoring download_dir') return @@ -726,81 +413,22 @@ def unpack_file_url(link, location, download_dir=None, hashes=None): # unpack the archive to the build dir location. even when only downloading # archives, they have to be unpacked to parse dependencies - unpack_file(from_path, location, content_type, link) + unpack_file(from_path, location, content_type) # a download dir is specified and not already downloaded if download_dir and not already_downloaded_path: _copy_file(from_path, download_dir, link) -def _copy_dist_from_dir(link_path, location): - """Copy distribution files in `link_path` to `location`. - - Invoked when user requests to install a local directory. E.g.: - - pip install . - pip install ~/dev/git-repos/python-prompt-toolkit - - """ - - # Note: This is currently VERY SLOW if you have a lot of data in the - # directory, because it copies everything with `shutil.copytree`. - # What it should really do is build an sdist and install that. - # See https://github.com/pypa/pip/issues/2195 - - if os.path.isdir(location): - rmtree(location) - - # build an sdist - setup_py = 'setup.py' - sdist_args = [sys.executable] - sdist_args.append('-c') - sdist_args.append(SETUPTOOLS_SHIM % setup_py) - sdist_args.append('sdist') - sdist_args += ['--dist-dir', location] - logger.info('Running setup.py sdist for %s', link_path) - - with indent_log(): - call_subprocess(sdist_args, cwd=link_path, show_stdout=False) - - # unpack sdist into `location` - sdist = os.path.join(location, os.listdir(location)[0]) - logger.info('Unpacking sdist %s into %s', sdist, location) - unpack_file(sdist, location, content_type=None, link=None) - - -class PipXmlrpcTransport(xmlrpc_client.Transport): - """Provide a `xmlrpclib.Transport` implementation via a `PipSession` - object. - """ - - def __init__(self, index_url, session, use_datetime=False): - xmlrpc_client.Transport.__init__(self, use_datetime) - index_parts = urllib_parse.urlparse(index_url) - self._scheme = index_parts.scheme - self._session = session - - def request(self, host, handler, request_body, verbose=False): - parts = (self._scheme, host, handler, None, None, None) - url = urllib_parse.urlunparse(parts) - try: - headers = {'Content-Type': 'text/xml'} - response = self._session.post(url, data=request_body, - headers=headers, stream=True) - response.raise_for_status() - self.verbose = verbose - return self.parse_response(response.raw) - except requests.HTTPError as exc: - logger.critical( - "HTTP error %s while getting %s", - exc.response.status_code, url, - ) - raise - - -def unpack_url(link, location, download_dir=None, - only_download=False, session=None, hashes=None, - progress_bar="on"): +def unpack_url( + link, # type: Link + location, # type: str + download_dir=None, # type: Optional[str] + session=None, # type: Optional[PipSession] + hashes=None, # type: Optional[Hashes] + progress_bar="on" # type: str +): + # type: (...) -> None """Unpack link. If link is a VCS link: if only_download, export into download_dir and ignore location @@ -816,11 +444,11 @@ def unpack_url(link, location, download_dir=None, would ordinarily raise HashUnsupported) are allowed. """ # non-editable vcs urls - if is_vcs_url(link): + if link.is_vcs: unpack_vcs_link(link, location) # file urls - elif is_file_url(link): + elif link.is_file: unpack_file_url(link, location, download_dir, hashes=hashes) # http urls @@ -836,11 +464,39 @@ def unpack_url(link, location, download_dir=None, hashes=hashes, progress_bar=progress_bar ) - if only_download: - write_delete_marker_file(location) -def _download_http_url(link, session, temp_dir, hashes, progress_bar): +def sanitize_content_filename(filename): + # type: (str) -> str + """ + Sanitize the "filename" value from a Content-Disposition header. + """ + return os.path.basename(filename) + + +def parse_content_disposition(content_disposition, default_filename): + # type: (str, str) -> str + """ + Parse the "filename" value from a Content-Disposition header, and + return the default filename if the result is empty. + """ + _type, params = cgi.parse_header(content_disposition) + filename = params.get('filename') + if filename: + # We need to sanitize the filename to prevent directory traversal + # in case the filename contains ".." path parts. + filename = sanitize_content_filename(filename) + return filename or default_filename + + +def _download_http_url( + link, # type: Link + session, # type: PipSession + temp_dir, # type: str + hashes, # type: Optional[Hashes] + progress_bar # type: str +): + # type: (...) -> Tuple[str, str] """Download link url into temp_dir using provided session""" target_url = link.url.split('#', 1)[0] try: @@ -880,11 +536,8 @@ def _download_http_url(link, session, temp_dir, hashes, progress_bar): # Have a look at the Content-Disposition header for a better guess content_disposition = resp.headers.get('content-disposition') if content_disposition: - type, params = cgi.parse_header(content_disposition) - # We use ``or`` here because we don't want to use an "empty" value - # from the filename param. - filename = params.get('filename') or filename - ext = splitext(filename)[1] + filename = parse_content_disposition(content_disposition, filename) + ext = splitext(filename)[1] # type: Optional[str] if not ext: ext = mimetypes.guess_extension(content_type) if ext: @@ -900,23 +553,26 @@ def _download_http_url(link, session, temp_dir, hashes, progress_bar): def _check_download_dir(link, download_dir, hashes): + # type: (Link, str, Optional[Hashes]) -> Optional[str] """ Check download_dir for previously downloaded file with correct hash If a correct file is found return its path else None """ download_path = os.path.join(download_dir, link.filename) - if os.path.exists(download_path): - # If already downloaded, does its hash match? - logger.info('File was already downloaded %s', download_path) - if hashes: - try: - hashes.check_against_path(download_path) - except HashMismatch: - logger.warning( - 'Previously-downloaded file %s has bad hash. ' - 'Re-downloading.', - download_path - ) - os.unlink(download_path) - return None - return download_path - return None + + if not os.path.exists(download_path): + return None + + # If already downloaded, does its hash match? + logger.info('File was already downloaded %s', download_path) + if hashes: + try: + hashes.check_against_path(download_path) + except HashMismatch: + logger.warning( + 'Previously-downloaded file %s has bad hash. ' + 'Re-downloading.', + download_path + ) + os.unlink(download_path) + return None + return download_path diff --git a/pipenv/patched/notpip/_internal/exceptions.py b/pipenv/patched/notpip/_internal/exceptions.py index 2eadcf28..368b433b 100644 --- a/pipenv/patched/notpip/_internal/exceptions.py +++ b/pipenv/patched/notpip/_internal/exceptions.py @@ -1,10 +1,21 @@ """Exceptions used throughout package""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import from itertools import chain, groupby, repeat from pipenv.patched.notpip._vendor.six import iteritems +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + from pipenv.patched.notpip._vendor.pkg_resources import Distribution + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + class PipError(Exception): """Base pip exception""" @@ -22,6 +33,36 @@ class UninstallationError(PipError): """General exception during uninstallation""" +class NoneMetadataError(PipError): + """ + Raised when accessing "METADATA" or "PKG-INFO" metadata for a + pip._vendor.pkg_resources.Distribution object and + `dist.has_metadata('METADATA')` returns True but + `dist.get_metadata('METADATA')` returns None (and similarly for + "PKG-INFO"). + """ + + def __init__(self, dist, metadata_name): + # type: (Distribution, str) -> None + """ + :param dist: A Distribution object. + :param metadata_name: The name of the metadata being accessed + (can be "METADATA" or "PKG-INFO"). + """ + self.dist = dist + self.metadata_name = metadata_name + + def __str__(self): + # type: () -> str + # Use `dist` in the error message because its stringification + # includes more information, like the version and location. + return ( + 'None {} metadata found for distribution: {}'.format( + self.metadata_name, self.dist, + ) + ) + + class DistributionNotFound(InstallationError): """Raised when a distribution cannot be found to satisfy a requirement""" @@ -96,7 +137,7 @@ class HashError(InstallationError): typically available earlier. """ - req = None + req = None # type: Optional[InstallRequirement] head = '' def body(self): @@ -240,7 +281,6 @@ class HashMismatch(HashError): for e in expecteds) lines.append(' Got %s\n' % self.gots[hash_name].hexdigest()) - prefix = ' or' return '\n'.join(lines) diff --git a/pipenv/patched/notpip/_internal/index.py b/pipenv/patched/notpip/_internal/index.py index b4b02373..0f212115 100644 --- a/pipenv/patched/notpip/_internal/index.py +++ b/pipenv/patched/notpip/_internal/index.py @@ -1,178 +1,601 @@ """Routines related to PyPI, indexes""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import -import cgi -import itertools import logging -import mimetypes -import os -import posixpath import re -import sys -from collections import namedtuple -from pipenv.patched.notpip._vendor import html5lib, requests, six -from pipenv.patched.notpip._vendor.distlib.compat import unescape from pipenv.patched.notpip._vendor.packaging import specifiers from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version -from pipenv.patched.notpip._vendor.requests.exceptions import SSLError -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request -from pipenv.patched.notpip._internal.download import HAS_TLS, is_url, path_to_url, url_to_path from pipenv.patched.notpip._internal.exceptions import ( - BestVersionAlreadyInstalled, DistributionNotFound, InvalidWheelFilename, + BestVersionAlreadyInstalled, + DistributionNotFound, + InvalidWheelFilename, UnsupportedWheel, ) from pipenv.patched.notpip._internal.models.candidate import InstallationCandidate from pipenv.patched.notpip._internal.models.format_control import FormatControl -from pipenv.patched.notpip._internal.models.index import PyPI from pipenv.patched.notpip._internal.models.link import Link -from pipenv.patched.notpip._internal.pep425tags import get_supported -from pipenv.patched.notpip._internal.utils.compat import ipaddress -from pipenv.patched.notpip._internal.utils.deprecation import deprecated +from pipenv.patched.notpip._internal.models.selection_prefs import SelectionPreferences +from pipenv.patched.notpip._internal.models.target_python import TargetPython +from pipenv.patched.notpip._internal.utils.filetypes import WHEEL_EXTENSION from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import ( - ARCHIVE_EXTENSIONS, SUPPORTED_EXTENSIONS, normalize_path, - remove_auth_from_url, -) +from pipenv.patched.notpip._internal.utils.misc import build_netloc from pipenv.patched.notpip._internal.utils.packaging import check_requires_python -from pipenv.patched.notpip._internal.wheel import Wheel, wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.unpacking import SUPPORTED_EXTENSIONS +from pipenv.patched.notpip._internal.utils.urls import url_to_path +from pipenv.patched.notpip._internal.wheel import Wheel -__all__ = ['FormatControl', 'PackageFinder'] +if MYPY_CHECK_RUNNING: + from typing import ( + FrozenSet, Iterable, List, Optional, Set, Text, Tuple, Union, + ) + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion + from pipenv.patched.notpip._internal.collector import LinkCollector + from pipenv.patched.notpip._internal.models.search_scope import SearchScope + from pipenv.patched.notpip._internal.req import InstallRequirement + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag + from pipenv.patched.notpip._internal.utils.hashes import Hashes + + BuildTag = Union[Tuple[()], Tuple[int, str]] + CandidateSortingKey = ( + Tuple[int, int, int, _BaseVersion, BuildTag, Optional[int]] + ) -SECURE_ORIGINS = [ - # protocol, hostname, port - # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC) - ("https", "*", "*"), - ("*", "localhost", "*"), - ("*", "127.0.0.0/8", "*"), - ("*", "::1/128", "*"), - ("file", "*", None), - # ssh is always secure. - ("ssh", "*", "*"), -] +__all__ = ['FormatControl', 'BestCandidateResult', 'PackageFinder'] logger = logging.getLogger(__name__) -def _get_content_type(url, session): - """Get the Content-Type of the given url, using a HEAD request""" - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(url) - if scheme not in {'http', 'https'}: - # FIXME: some warning or something? - # assertion error? - return '' +def _check_link_requires_python( + link, # type: Link + version_info, # type: Tuple[int, int, int] + ignore_requires_python=False, # type: bool +): + # type: (...) -> bool + """ + Return whether the given Python version is compatible with a link's + "Requires-Python" value. - resp = session.head(url, allow_redirects=True) - resp.raise_for_status() + :param version_info: A 3-tuple of ints representing the Python + major-minor-micro version to check. + :param ignore_requires_python: Whether to ignore the "Requires-Python" + value if the given Python version isn't compatible. + """ + try: + is_compatible = check_requires_python( + link.requires_python, version_info=version_info, + ) + except specifiers.InvalidSpecifier: + logger.debug( + "Ignoring invalid Requires-Python (%r) for link: %s", + link.requires_python, link, + ) + else: + if not is_compatible: + version = '.'.join(map(str, version_info)) + if not ignore_requires_python: + logger.debug( + 'Link requires a different Python (%s not in: %r): %s', + version, link.requires_python, link, + ) + return False - return resp.headers.get("Content-Type", "") + logger.debug( + 'Ignoring failed Requires-Python check (%s not in: %r) ' + 'for link: %s', + version, link.requires_python, link, + ) + + return True -def _handle_get_page_fail(link, reason, url, meth=None): - if meth is None: - meth = logger.debug - meth("Could not fetch URL %s: %s - skipping", link, reason) +class LinkEvaluator(object): + + """ + Responsible for evaluating links for a particular project. + """ + + _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') + + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + def __init__( + self, + project_name, # type: str + canonical_name, # type: str + formats, # type: FrozenSet + target_python, # type: TargetPython + allow_yanked, # type: bool + ignore_requires_python=None, # type: Optional[bool] + ignore_compatibility=None, # type: Optional[bool] + ): + # type: (...) -> None + """ + :param project_name: The user supplied package name. + :param canonical_name: The canonical package name. + :param formats: The formats allowed for this package. Should be a set + with 'binary' or 'source' or both in it. + :param target_python: The target Python interpreter to use when + evaluating link compatibility. This is used, for example, to + check wheel compatibility, as well as when checking the Python + version, e.g. the Python version embedded in a link filename + (or egg fragment) and against an HTML link's optional PEP 503 + "data-requires-python" attribute. + :param allow_yanked: Whether files marked as yanked (in the sense + of PEP 592) are permitted to be candidates for install. + :param ignore_requires_python: Whether to ignore incompatible + PEP 503 "data-requires-python" values in HTML links. Defaults + to False. + :param Optional[bool] ignore_compatibility: Whether to ignore + compatibility of python versions and allow all versions of packages. + """ + if ignore_requires_python is None: + ignore_requires_python = False + if ignore_compatibility is None: + ignore_compatibility = True + + self._allow_yanked = allow_yanked + self._canonical_name = canonical_name + self._ignore_requires_python = ignore_requires_python + self._formats = formats + self._target_python = target_python + self._ignore_compatibility = ignore_compatibility + + self.project_name = project_name + + def evaluate_link(self, link): + # type: (Link) -> Tuple[bool, Optional[Text]] + """ + Determine whether a link is a candidate for installation. + + :return: A tuple (is_candidate, result), where `result` is (1) a + version string if `is_candidate` is True, and (2) if + `is_candidate` is False, an optional string to log the reason + the link fails to qualify. + """ + version = None + if link.is_yanked and not self._allow_yanked: + reason = link.yanked_reason or '' + # Mark this as a unicode string to prevent "UnicodeEncodeError: + # 'ascii' codec can't encode character" in Python 2 when + # the reason contains non-ascii characters. + return (False, u'yanked for reason: {}'.format(reason)) + + if link.egg_fragment: + egg_info = link.egg_fragment + ext = link.ext + else: + egg_info, ext = link.splitext() + if not ext: + return (False, 'not a file') + if ext not in SUPPORTED_EXTENSIONS: + return (False, 'unsupported archive format: %s' % ext) + if "binary" not in self._formats and ext == WHEEL_EXTENSION and not self._ignore_compatibility: + reason = 'No binaries permitted for %s' % self.project_name + return (False, reason) + if "macosx10" in link.path and ext == '.zip' and not self._ignore_compatibility: + return (False, 'macosx10 one') + if ext == WHEEL_EXTENSION: + try: + wheel = Wheel(link.filename) + except InvalidWheelFilename: + return (False, 'invalid wheel filename') + if canonicalize_name(wheel.name) != self._canonical_name: + reason = 'wrong project name (not %s)' % self.project_name + return (False, reason) + + supported_tags = self._target_python.get_tags() + if not wheel.supported(supported_tags) and not self._ignore_compatibility: + # Include the wheel's tags in the reason string to + # simplify troubleshooting compatibility issues. + file_tags = wheel.get_formatted_file_tags() + reason = ( + "none of the wheel's tags match: {}".format( + ', '.join(file_tags) + ) + ) + return (False, reason) + + version = wheel.version + + # This should be up by the self.ok_binary check, but see issue 2700. + if "source" not in self._formats and ext != WHEEL_EXTENSION: + return (False, 'No sources permitted for %s' % self.project_name) + + if not version: + version = _extract_version_from_fragment( + egg_info, self._canonical_name, + ) + if not version: + return ( + False, 'Missing project version for %s' % self.project_name, + ) + + match = self._py_version_re.search(version) + if match: + version = version[:match.start()] + py_version = match.group(1) + if py_version != self._target_python.py_version: + return (False, 'Python version is incorrect') + + supports_python = _check_link_requires_python( + link, version_info=self._target_python.py_version_info, + ignore_requires_python=self._ignore_requires_python, + ) + if not supports_python and not self._ignore_compatibility: + # Return None for the reason text to suppress calling + # _log_skipped_link(). + return (False, None) + + logger.debug('Found link %s, version: %s', link, version) + + return (True, version) -def _get_html_page(link, session=None): - if session is None: - raise TypeError( - "_get_html_page() missing 1 required keyword argument: 'session'" +def filter_unallowed_hashes( + candidates, # type: List[InstallationCandidate] + hashes, # type: Hashes + project_name, # type: str +): + # type: (...) -> List[InstallationCandidate] + """ + Filter out candidates whose hashes aren't allowed, and return a new + list of candidates. + + If at least one candidate has an allowed hash, then all candidates with + either an allowed hash or no hash specified are returned. Otherwise, + the given candidates are returned. + + Including the candidates with no hash specified when there is a match + allows a warning to be logged if there is a more preferred candidate + with no hash specified. Returning all candidates in the case of no + matches lets pip report the hash of the candidate that would otherwise + have been installed (e.g. permitting the user to more easily update + their requirements file with the desired hash). + """ + if not hashes: + logger.debug( + 'Given no hashes to check %s links for project %r: ' + 'discarding no candidates', + len(candidates), + project_name, + ) + # Make sure we're not returning back the given value. + return list(candidates) + + matches_or_no_digest = [] + # Collect the non-matches for logging purposes. + non_matches = [] + match_count = 0 + for candidate in candidates: + link = candidate.link + if not link.has_hash: + pass + elif link.is_hash_allowed(hashes=hashes): + match_count += 1 + else: + non_matches.append(candidate) + continue + + matches_or_no_digest.append(candidate) + + if match_count: + filtered = matches_or_no_digest + else: + # Make sure we're not returning back the given value. + filtered = list(candidates) + + if len(filtered) == len(candidates): + discard_message = 'discarding no candidates' + else: + discard_message = 'discarding {} non-matches:\n {}'.format( + len(non_matches), + '\n '.join(str(candidate.link) for candidate in non_matches) ) - url = link.url - url = url.split('#', 1)[0] + logger.debug( + 'Checked %s links for project %r against %s hashes ' + '(%s matches, %s no digest): %s', + len(candidates), + project_name, + hashes.digest_count, + match_count, + len(matches_or_no_digest) - match_count, + discard_message + ) - # Check for VCS schemes that do not support lookup as web pages. - from pipenv.patched.notpip._internal.vcs import VcsSupport - for scheme in VcsSupport.schemes: - if url.lower().startswith(scheme) and url[len(scheme)] in '+:': - logger.debug('Cannot look at %s URL %s', scheme, link) + return filtered + + +class CandidatePreferences(object): + + """ + Encapsulates some of the preferences for filtering and sorting + InstallationCandidate objects. + """ + + def __init__( + self, + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + ): + # type: (...) -> None + """ + :param allow_all_prereleases: Whether to allow all pre-releases. + """ + self.allow_all_prereleases = allow_all_prereleases + self.prefer_binary = prefer_binary + + +class BestCandidateResult(object): + """A collection of candidates, returned by `PackageFinder.find_best_candidate`. + + This class is only intended to be instantiated by CandidateEvaluator's + `compute_best_candidate()` method. + """ + + def __init__( + self, + candidates, # type: List[InstallationCandidate] + applicable_candidates, # type: List[InstallationCandidate] + best_candidate, # type: Optional[InstallationCandidate] + ): + # type: (...) -> None + """ + :param candidates: A sequence of all available candidates found. + :param applicable_candidates: The applicable candidates. + :param best_candidate: The most preferred candidate found, or None + if no applicable candidates were found. + """ + assert set(applicable_candidates) <= set(candidates) + + if best_candidate is None: + assert not applicable_candidates + else: + assert best_candidate in applicable_candidates + + self._applicable_candidates = applicable_candidates + self._candidates = candidates + + self.best_candidate = best_candidate + + def iter_all(self): + # type: () -> Iterable[InstallationCandidate] + """Iterate through all candidates. + """ + return iter(self._candidates) + + def iter_applicable(self): + # type: () -> Iterable[InstallationCandidate] + """Iterate through the applicable candidates. + """ + return iter(self._applicable_candidates) + + +class CandidateEvaluator(object): + + """ + Responsible for filtering and sorting candidates for installation based + on what tags are valid. + """ + + @classmethod + def create( + cls, + project_name, # type: str + target_python=None, # type: Optional[TargetPython] + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> CandidateEvaluator + """Create a CandidateEvaluator object. + + :param target_python: The target Python interpreter to use when + checking compatibility. If None (the default), a TargetPython + object will be constructed from the running Python. + :param specifier: An optional object implementing `filter` + (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable + versions. + :param hashes: An optional collection of allowed hashes. + """ + if target_python is None: + target_python = TargetPython() + if specifier is None: + specifier = specifiers.SpecifierSet() + + supported_tags = target_python.get_tags() + + return cls( + project_name=project_name, + supported_tags=supported_tags, + specifier=specifier, + prefer_binary=prefer_binary, + allow_all_prereleases=allow_all_prereleases, + hashes=hashes, + ) + + def __init__( + self, + project_name, # type: str + supported_tags, # type: List[Pep425Tag] + specifier, # type: specifiers.BaseSpecifier + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> None + """ + :param supported_tags: The PEP 425 tags supported by the target + Python in order of preference (most preferred first). + """ + self._allow_all_prereleases = allow_all_prereleases + self._hashes = hashes + self._prefer_binary = prefer_binary + self._project_name = project_name + self._specifier = specifier + self._supported_tags = supported_tags + + def get_applicable_candidates( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> List[InstallationCandidate] + """ + Return the applicable candidates from a list of candidates. + """ + # Using None infers from the specifier instead. + allow_prereleases = self._allow_all_prereleases or None + specifier = self._specifier + versions = { + str(v) for v in specifier.filter( + # We turn the version object into a str here because otherwise + # when we're debundled but setuptools isn't, Python will see + # packaging.version.Version and + # pkg_resources._vendor.packaging.version.Version as different + # types. This way we'll use a str as a common data interchange + # format. If we stop using the pkg_resources provided specifier + # and start using our own, we can drop the cast to str(). + (str(c.version) for c in candidates), + prereleases=allow_prereleases, + ) + } + + # Again, converting version to str to deal with debundling. + applicable_candidates = [ + c for c in candidates if str(c.version) in versions + ] + + return filter_unallowed_hashes( + candidates=applicable_candidates, + hashes=self._hashes, + project_name=self._project_name, + ) + + def _sort_key(self, candidate, ignore_compatibility=True): + # type: (InstallationCandidate, bool) -> CandidateSortingKey + """ + Function to pass as the `key` argument to a call to sorted() to sort + InstallationCandidates by preference. + + Returns a tuple such that tuples sorting as greater using Python's + default comparison operator are more preferred. + + The preference is as follows: + + First and foremost, candidates with allowed (matching) hashes are + always preferred over candidates without matching hashes. This is + because e.g. if the only candidate with an allowed hash is yanked, + we still want to use that candidate. + + Second, excepting hash considerations, candidates that have been + yanked (in the sense of PEP 592) are always less preferred than + candidates that haven't been yanked. Then: + + If not finding wheels, they are sorted by version only. + If finding wheels, then the sort order is by version, then: + 1. existing installs + 2. wheels ordered via Wheel.support_index_min(self._supported_tags) + 3. source archives + If prefer_binary was set, then all wheels are sorted above sources. + + Note: it was considered to embed this logic into the Link + comparison operators, but then different sdist links + with the same version, would have to be considered equal + """ + valid_tags = self._supported_tags + support_num = len(valid_tags) + build_tag = () # type: BuildTag + binary_preference = 0 + link = candidate.link + if link.is_wheel: + # can raise InvalidWheelFilename + wheel = Wheel(link.filename) + if not wheel.supported(valid_tags) and not ignore_compatibility: + raise UnsupportedWheel( + "%s is not a supported wheel for this platform. It " + "can't be sorted." % wheel.filename + ) + if self._prefer_binary: + binary_preference = 1 + tags = self.valid_tags if not ignore_compatibility else None + try: + pri = -(wheel.support_index_min(tags=tags)) + except TypeError: + pri = -(support_num) + if wheel.build_tag is not None: + match = re.match(r'^(\d+)(.*)$', wheel.build_tag) + build_tag_groups = match.groups() + build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) + else: # sdist + pri = -(support_num) + has_allowed_hash = int(link.is_hash_allowed(self._hashes)) + yank_value = -1 * int(link.is_yanked) # -1 for yanked. + return ( + has_allowed_hash, yank_value, binary_preference, candidate.version, + build_tag, pri, + ) + + def sort_best_candidate( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> Optional[InstallationCandidate] + """ + Return the best candidate per the instance's sort order, or None if + no candidate is acceptable. + """ + if not candidates: return None - try: - filename = link.filename - for bad_ext in ARCHIVE_EXTENSIONS: - if filename.endswith(bad_ext): - content_type = _get_content_type(url, session=session) - if content_type.lower().startswith('text/html'): - break - else: - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return + best_candidate = max(candidates, key=self._sort_key) - logger.debug('Getting page %s', url) + # Log a warning per PEP 592 if necessary before returning. + link = best_candidate.link + if link.is_yanked: + reason = link.yanked_reason or '' + msg = ( + # Mark this as a unicode string to prevent + # "UnicodeEncodeError: 'ascii' codec can't encode character" + # in Python 2 when the reason contains non-ascii characters. + u'The candidate selected for download or install is a ' + 'yanked version: {candidate}\n' + 'Reason for being yanked: {reason}' + ).format(candidate=best_candidate, reason=reason) + logger.warning(msg) - # Tack index.html onto file:// URLs that point to directories - (scheme, netloc, path, params, query, fragment) = \ - urllib_parse.urlparse(url) - if (scheme == 'file' and - os.path.isdir(urllib_request.url2pathname(path))): - # add trailing slash if not present so urljoin doesn't trim - # final segment - if not url.endswith('/'): - url += '/' - url = urllib_parse.urljoin(url, 'index.html') - logger.debug(' file: URL is directory, getting %s', url) + return best_candidate - resp = session.get( - url, - headers={ - "Accept": "text/html", - # We don't want to blindly returned cached data for - # /simple/, because authors generally expecting that - # twine upload && pip install will function, but if - # they've done a pip install in the last ~10 minutes - # it won't. Thus by setting this to zero we will not - # blindly use any cached data, however the benefit of - # using max-age=0 instead of no-cache, is that we will - # still support conditional requests, so we will still - # minimize traffic sent in cases where the page hasn't - # changed at all, we will just always incur the round - # trip for the conditional GET now instead of only - # once per 10 minutes. - # For more information, please see pypa/pip#5670. - "Cache-Control": "max-age=0", - }, + def compute_best_candidate( + self, + candidates, # type: List[InstallationCandidate] + ): + # type: (...) -> BestCandidateResult + """ + Compute and return a `BestCandidateResult` instance. + """ + applicable_candidates = self.get_applicable_candidates(candidates) + + best_candidate = self.sort_best_candidate(applicable_candidates) + + return BestCandidateResult( + candidates, + applicable_candidates=applicable_candidates, + best_candidate=best_candidate, ) - resp.raise_for_status() - - # The check for archives above only works if the url ends with - # something that looks like an archive. However that is not a - # requirement of an url. Unless we issue a HEAD request on every - # url we cannot know ahead of time for sure if something is HTML - # or not. However we can check after we've downloaded it. - content_type = resp.headers.get('Content-Type', 'unknown') - if not content_type.lower().startswith("text/html"): - logger.debug( - 'Skipping page %s because of Content-Type: %s', - link, - content_type, - ) - return - - inst = HTMLPage(resp.content, resp.url, resp.headers) - except requests.HTTPError as exc: - _handle_get_page_fail(link, exc, url) - except SSLError as exc: - reason = "There was a problem confirming the ssl certificate: " - reason += str(exc) - _handle_get_page_fail(link, reason, url, meth=logger.info) - except requests.ConnectionError as exc: - _handle_get_page_fail(link, "connection error: %s" % exc, url) - except requests.Timeout: - _handle_get_page_fail(link, "timed out", url) - else: - return inst class PackageFinder(object): @@ -182,124 +605,85 @@ class PackageFinder(object): packages, by reading pages and looking for appropriate links. """ - def __init__(self, find_links, index_urls, allow_all_prereleases=False, - trusted_hosts=None, process_dependency_links=False, - session=None, format_control=None, platform=None, - versions=None, abi=None, implementation=None, - prefer_binary=False): - """Create a PackageFinder. + def __init__( + self, + link_collector, # type: LinkCollector + target_python, # type: TargetPython + allow_yanked, # type: bool + format_control=None, # type: Optional[FormatControl] + candidate_prefs=None, # type: CandidatePreferences + ignore_requires_python=None, # type: Optional[bool] + ignore_compatibility=None, # type: Optional[bool] + ): + # type: (...) -> None + """ + This constructor is primarily meant to be used by the create() class + method and from tests. - :param format_control: A FormatControl object or None. Used to control + :param format_control: A FormatControl object, used to control the selection of source packages / binary packages when consulting the index and links. - :param platform: A string or None. If None, searches for packages - that are supported by the current system. Otherwise, will find - packages that can be built on the platform passed in. These - packages will only be downloaded for distribution: they will - not be built locally. - :param versions: A list of strings or None. This is passed directly - to pep425tags.py in the get_supported() method. - :param abi: A string or None. This is passed directly - to pep425tags.py in the get_supported() method. - :param implementation: A string or None. This is passed directly - to pep425tags.py in the get_supported() method. + :param candidate_prefs: Options to use when creating a + CandidateEvaluator object. """ - if session is None: - raise TypeError( - "PackageFinder() missing 1 required keyword argument: " - "'session'" - ) + if candidate_prefs is None: + candidate_prefs = CandidatePreferences() + if ignore_compatibility is None: + ignore_compatibility = False - # Build find_links. If an argument starts with ~, it may be - # a local file relative to a home directory. So try normalizing - # it and if it exists, use the normalized version. - # This is deliberately conservative - it might be fine just to - # blindly normalize anything starting with a ~... - self.find_links = [] - for link in find_links: - if link.startswith('~'): - new_link = normalize_path(link) - if os.path.exists(new_link): - link = new_link - self.find_links.append(link) + format_control = format_control or FormatControl(set(), set()) - self.index_urls = index_urls - self.dependency_links = [] + self._allow_yanked = allow_yanked + self._candidate_prefs = candidate_prefs + self._ignore_requires_python = ignore_requires_python + self._link_collector = link_collector + self._target_python = target_python + self._ignore_compatibility = ignore_compatibility - # These are boring links that have already been logged somehow: - self.logged_links = set() + self.format_control = format_control - self.format_control = format_control or FormatControl(set(), set()) - - # Domains that we won't emit warnings for when not using HTTPS - self.secure_origins = [ - ("*", host, "*") - for host in (trusted_hosts if trusted_hosts else []) - ] - - # Do we want to allow _all_ pre-releases? - self.allow_all_prereleases = allow_all_prereleases - - # Do we process dependency links? - self.process_dependency_links = process_dependency_links - - # The Session we'll use to make requests - self.session = session + # These are boring links that have already been logged somehow. + self._logged_links = set() # type: Set[Link] # Kenneth's Hack self.extra = None - # The valid tags to check potential found wheel candidates against - self.valid_tags = get_supported( - versions=versions, - platform=platform, - abi=abi, - impl=implementation, + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + @classmethod + def create( + cls, + link_collector, # type: LinkCollector + selection_prefs, # type: SelectionPreferences + target_python=None, # type: Optional[TargetPython] + ): + # type: (...) -> PackageFinder + """Create a PackageFinder. + + :param selection_prefs: The candidate selection preferences, as a + SelectionPreferences object. + :param target_python: The target Python interpreter to use when + checking compatibility. If None (the default), a TargetPython + object will be constructed from the running Python. + """ + if target_python is None: + target_python = TargetPython() + + candidate_prefs = CandidatePreferences( + prefer_binary=selection_prefs.prefer_binary, + allow_all_prereleases=selection_prefs.allow_all_prereleases, ) - # Do we prefer old, but valid, binary dist over new source dist - self.prefer_binary = prefer_binary - - # If we don't have TLS enabled, then WARN if anyplace we're looking - # relies on TLS. - if not HAS_TLS: - for link in itertools.chain(self.index_urls, self.find_links): - parsed = urllib_parse.urlparse(link) - if parsed.scheme == "https": - logger.warning( - "pip is configured with locations that require " - "TLS/SSL, however the ssl module in Python is not " - "available." - ) - break - - def get_formatted_locations(self): - lines = [] - if self.index_urls and self.index_urls != [PyPI.simple_url]: - lines.append( - "Looking in indexes: {}".format(", ".join( - remove_auth_from_url(url) for url in self.index_urls)) - ) - if self.find_links: - lines.append( - "Looking in links: {}".format(", ".join(self.find_links)) - ) - return "\n".join(lines) - - def add_dependency_links(self, links): - # FIXME: this shouldn't be global list this, it should only - # apply to requirements of the package that specifies the - # dependency_links value - # FIXME: also, we should track comes_from (i.e., use Link) - if self.process_dependency_links: - deprecated( - "Dependency Links processing has been deprecated and will be " - "removed in a future release.", - replacement="PEP 508 URL dependencies", - gone_in="18.2", - issue=4187, - ) - self.dependency_links.extend(links) + return cls( + candidate_prefs=candidate_prefs, + link_collector=link_collector, + target_python=target_python, + allow_yanked=selection_prefs.allow_yanked, + format_control=selection_prefs.format_control, + ignore_requires_python=selection_prefs.ignore_requires_python, + ) @staticmethod def get_extras_links(links): @@ -316,335 +700,242 @@ class PackageFinder(object): extras[link[1:-1]] = current_list else: current_list.append(link) - return extras + @property + def search_scope(self): + # type: () -> SearchScope + return self._link_collector.search_scope - @staticmethod - def _sort_locations(locations, expand_dir=False): - """ - Sort locations into "files" (archives) and "urls", and return - a pair of lists (files,urls) - """ - files = [] - urls = [] + @search_scope.setter + def search_scope(self, search_scope): + # type: (SearchScope) -> None + self._link_collector.search_scope = search_scope - # puts the url for the given file path into the appropriate list - def sort_path(path): - url = path_to_url(path) - if mimetypes.guess_type(url, strict=False)[0] == 'text/html': - urls.append(url) - else: - files.append(url) + @property + def find_links(self): + # type: () -> List[str] + return self._link_collector.find_links - for url in locations: + @property + def index_urls(self): + # type: () -> List[str] + return self.search_scope.index_urls - is_local_path = os.path.exists(url) - is_file_url = url.startswith('file:') + @property + def trusted_hosts(self): + # type: () -> Iterable[str] + for host_port in self._link_collector.session.pip_trusted_origins: + yield build_netloc(*host_port) - if is_local_path or is_file_url: - if is_local_path: - path = url - else: - path = url_to_path(url) - if os.path.isdir(path): - if expand_dir: - path = os.path.realpath(path) - for item in os.listdir(path): - sort_path(os.path.join(path, item)) - elif is_file_url: - urls.append(url) - elif os.path.isfile(path): - sort_path(path) - else: - logger.warning( - "Url '%s' is ignored: it is neither a file " - "nor a directory.", url, - ) - elif is_url(url): - # Only add url with clear scheme - urls.append(url) - else: - logger.warning( - "Url '%s' is ignored. It is either a non-existing " - "path or lacks a specific scheme.", url, - ) + @property + def allow_all_prereleases(self): + # type: () -> bool + return self._candidate_prefs.allow_all_prereleases - return files, urls - - def _candidate_sort_key(self, candidate, ignore_compatibility=True): - """ - Function used to generate link sort key for link tuples. - The greater the return value, the more preferred it is. - If not finding wheels, then sorted by version only. - If finding wheels, then the sort order is by version, then: - 1. existing installs - 2. wheels ordered via Wheel.support_index_min(self.valid_tags) - 3. source archives - If prefer_binary was set, then all wheels are sorted above sources. - Note: it was considered to embed this logic into the Link - comparison operators, but then different sdist links - with the same version, would have to be considered equal - """ - support_num = len(self.valid_tags) - build_tag = tuple() - binary_preference = 0 - if candidate.location.is_wheel: - # can raise InvalidWheelFilename - wheel = Wheel(candidate.location.filename) - if not wheel.supported(self.valid_tags) and not ignore_compatibility: - raise UnsupportedWheel( - "%s is not a supported wheel for this platform. It " - "can't be sorted." % wheel.filename - ) - if self.prefer_binary: - binary_preference = 1 - tags = self.valid_tags if not ignore_compatibility else None - try: - pri = -(wheel.support_index_min(tags=tags)) - except TypeError: - pri = -(support_num) - - if wheel.build_tag is not None: - match = re.match(r'^(\d+)(.*)$', wheel.build_tag) - build_tag_groups = match.groups() - build_tag = (int(build_tag_groups[0]), build_tag_groups[1]) - else: # sdist - pri = -(support_num) - return (binary_preference, candidate.version, build_tag, pri) - - def _validate_secure_origin(self, logger, location): - # Determine if this url used a secure transport mechanism - parsed = urllib_parse.urlparse(str(location)) - origin = (parsed.scheme, parsed.hostname, parsed.port) - - # The protocol to use to see if the protocol matches. - # Don't count the repository type as part of the protocol: in - # cases such as "git+ssh", only use "ssh". (I.e., Only verify against - # the last scheme.) - protocol = origin[0].rsplit('+', 1)[-1] - - # Determine if our origin is a secure origin by looking through our - # hardcoded list of secure origins, as well as any additional ones - # configured on this PackageFinder instance. - for secure_origin in (SECURE_ORIGINS + self.secure_origins): - if protocol != secure_origin[0] and secure_origin[0] != "*": - continue - - try: - # We need to do this decode dance to ensure that we have a - # unicode object, even on Python 2.x. - addr = ipaddress.ip_address( - origin[1] - if ( - isinstance(origin[1], six.text_type) or - origin[1] is None - ) - else origin[1].decode("utf8") - ) - network = ipaddress.ip_network( - secure_origin[1] - if isinstance(secure_origin[1], six.text_type) - else secure_origin[1].decode("utf8") - ) - except ValueError: - # We don't have both a valid address or a valid network, so - # we'll check this origin against hostnames. - if (origin[1] and - origin[1].lower() != secure_origin[1].lower() and - secure_origin[1] != "*"): - continue - else: - # We have a valid address and network, so see if the address - # is contained within the network. - if addr not in network: - continue - - # Check to see if the port patches - if (origin[2] != secure_origin[2] and - secure_origin[2] != "*" and - secure_origin[2] is not None): - continue - - # If we've gotten here, then this origin matches the current - # secure origin and we should return True - return True - - # If we've gotten to this point, then the origin isn't secure and we - # will not accept it as a valid location to search. We will however - # log a warning that we are ignoring it. - logger.warning( - "The repository located at %s is not a trusted or secure host and " - "is being ignored. If this repository is available via HTTPS we " - "recommend you use HTTPS instead, otherwise you may silence " - "this warning and allow it anyway with '--trusted-host %s'.", - parsed.hostname, - parsed.hostname, - ) - - return False - - def _get_index_urls_locations(self, project_name): - """Returns the locations found via self.index_urls - - Checks the url_name on the main (first in the list) index and - use this url_name to produce all locations - """ - - def mkurl_pypi_url(url): - loc = posixpath.join( - url, - urllib_parse.quote(canonicalize_name(project_name))) - # For maximum compatibility with easy_install, ensure the path - # ends in a trailing slash. Although this isn't in the spec - # (and PyPI can handle it without the slash) some other index - # implementations might break if they relied on easy_install's - # behavior. - if not loc.endswith('/'): - loc = loc + '/' - return loc - - return [mkurl_pypi_url(url) for url in self.index_urls] - - def find_all_candidates(self, project_name): - """Find all available InstallationCandidate for project_name - - This checks index_urls, find_links and dependency_links. - All versions found are returned as an InstallationCandidate list. - - See _link_package_versions for details on which files are accepted - """ - index_locations = self._get_index_urls_locations(project_name) - index_file_loc, index_url_loc = self._sort_locations(index_locations) - fl_file_loc, fl_url_loc = self._sort_locations( - self.find_links, expand_dir=True, - ) - dep_file_loc, dep_url_loc = self._sort_locations(self.dependency_links) - - file_locations = (Link(url) for url in itertools.chain( - index_file_loc, fl_file_loc, dep_file_loc, - )) - - # We trust every url that the user has given us whether it was given - # via --index-url or --find-links - # We explicitly do not trust links that came from dependency_links - # We want to filter out any thing which does not have a secure origin. - url_locations = [ - link for link in itertools.chain( - (Link(url) for url in index_url_loc), - (Link(url) for url in fl_url_loc), - (Link(url) for url in dep_url_loc), - ) - if self._validate_secure_origin(logger, link) - ] - - logger.debug('%d location(s) to search for versions of %s:', - len(url_locations), project_name) - - for location in url_locations: - logger.debug('* %s', location) + def set_allow_all_prereleases(self): + # type: () -> None + self._candidate_prefs.allow_all_prereleases = True + def make_link_evaluator(self, project_name): + # type: (str) -> LinkEvaluator canonical_name = canonicalize_name(project_name) formats = self.format_control.get_allowed_formats(canonical_name) - search = Search(project_name, canonical_name, formats) - find_links_versions = self._package_versions( - # We trust every directly linked archive in find_links - (Link(url, '-f') for url in self.find_links), - search + + return LinkEvaluator( + project_name=project_name, + canonical_name=canonical_name, + formats=formats, + target_python=self._target_python, + allow_yanked=self._allow_yanked, + ignore_requires_python=self._ignore_requires_python, + ignore_compatibility=self._ignore_compatibility + ) + + def _sort_links(self, links): + # type: (Iterable[Link]) -> List[Link] + """ + Returns elements of links in order, non-egg links first, egg links + second, while eliminating duplicates + """ + eggs, no_eggs = [], [] + seen = set() # type: Set[Link] + for link in links: + if link not in seen: + seen.add(link) + if link.egg_fragment: + eggs.append(link) + else: + no_eggs.append(link) + return no_eggs + eggs + + def _log_skipped_link(self, link, reason): + # type: (Link, Text) -> None + if link not in self._logged_links: + # Mark this as a unicode string to prevent "UnicodeEncodeError: + # 'ascii' codec can't encode character" in Python 2 when + # the reason contains non-ascii characters. + # Also, put the link at the end so the reason is more visible + # and because the link string is usually very long. + logger.debug(u'Skipping link: %s: %s', reason, link) + self._logged_links.add(link) + + def get_install_candidate(self, link_evaluator, link): + # type: (LinkEvaluator, Link) -> Optional[InstallationCandidate] + """ + If the link is a candidate for install, convert it to an + InstallationCandidate and return it. Otherwise, return None. + """ + is_candidate, result = link_evaluator.evaluate_link(link) + if not is_candidate: + if result: + self._log_skipped_link(link, reason=result) + return None + + return InstallationCandidate( + project=link_evaluator.project_name, + link=link, + # Convert the Text result to str since InstallationCandidate + # accepts str. + version=str(result), + requires_python=getattr(link, "requires_python", None) + ) + + def evaluate_links(self, link_evaluator, links): + # type: (LinkEvaluator, Iterable[Link]) -> List[InstallationCandidate] + """ + Convert links that are candidates to InstallationCandidate objects. + """ + candidates = [] + for link in self._sort_links(links): + candidate = self.get_install_candidate(link_evaluator, link) + if candidate is not None: + candidates.append(candidate) + + return candidates + + def find_all_candidates(self, project_name): + # type: (str) -> List[InstallationCandidate] + """Find all available InstallationCandidate for project_name + + This checks index_urls and find_links. + All versions found are returned as an InstallationCandidate list. + + See LinkEvaluator.evaluate_link() for details on which files + are accepted. + """ + collected_links = self._link_collector.collect_links(project_name) + + link_evaluator = self.make_link_evaluator(project_name) + + find_links_versions = self.evaluate_links( + link_evaluator, + links=collected_links.find_links, ) page_versions = [] - for page in self._get_pages(url_locations, project_name): - try: - logger.debug('Analyzing links from page %s', page.url) - except AttributeError: - continue + for page_url, page_links in collected_links.pages.items(): + logger.debug('Analyzing links from page %s', page_url) with indent_log(): - page_versions.extend( - self._package_versions(page.iter_links(), search) + new_versions = self.evaluate_links( + link_evaluator, + links=page_links, ) + page_versions.extend(new_versions) - dependency_versions = self._package_versions( - (Link(url) for url in self.dependency_links), search + file_versions = self.evaluate_links( + link_evaluator, + links=collected_links.files, ) - if dependency_versions: - logger.debug( - 'dependency_links found: %s', - ', '.join([ - version.location.url for version in dependency_versions - ]) - ) - - file_versions = self._package_versions(file_locations, search) if file_versions: file_versions.sort(reverse=True) logger.debug( 'Local files found: %s', ', '.join([ - url_to_path(candidate.location.url) + url_to_path(candidate.link.url) for candidate in file_versions ]) ) # This is an intentional priority ordering - return ( - file_versions + find_links_versions + page_versions + - dependency_versions + return file_versions + find_links_versions + page_versions + + def make_candidate_evaluator( + self, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> CandidateEvaluator + """Create a CandidateEvaluator object to use. + """ + candidate_prefs = self._candidate_prefs + return CandidateEvaluator.create( + project_name=project_name, + target_python=self._target_python, + prefer_binary=candidate_prefs.prefer_binary, + allow_all_prereleases=candidate_prefs.allow_all_prereleases, + specifier=specifier, + hashes=hashes, ) - def find_requirement(self, req, upgrade, ignore_compatibility=False): + def find_best_candidate( + self, + project_name, # type: str + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Hashes] + ): + # type: (...) -> BestCandidateResult + """Find matches for the given project and specifier. + + :param specifier: An optional object implementing `filter` + (e.g. `packaging.specifiers.SpecifierSet`) to filter applicable + versions. + + :return: A `BestCandidateResult` instance. + """ + candidates = self.find_all_candidates(project_name) + candidate_evaluator = self.make_candidate_evaluator( + project_name=project_name, + specifier=specifier, + hashes=hashes, + ) + return candidate_evaluator.compute_best_candidate(candidates) + + def find_requirement(self, req, upgrade): + # type: (InstallRequirement, bool) -> Optional[Link] """Try to find a Link matching req Expects req, an InstallRequirement and upgrade, a boolean Returns a Link if found, Raises DistributionNotFound or BestVersionAlreadyInstalled otherwise """ - all_candidates = self.find_all_candidates(req.name) - - # Filter out anything which doesn't match our specifier - compatible_versions = set( - req.specifier.filter( - # We turn the version object into a str here because otherwise - # when we're debundled but setuptools isn't, Python will see - # packaging.version.Version and - # pkg_resources._vendor.packaging.version.Version as different - # types. This way we'll use a str as a common data interchange - # format. If we stop using the pkg_resources provided specifier - # and start using our own, we can drop the cast to str(). - [str(c.version) for c in all_candidates], - prereleases=( - self.allow_all_prereleases - if self.allow_all_prereleases else None - ), - ) + hashes = req.hashes(trust_internet=False) + best_candidate_result = self.find_best_candidate( + req.name, specifier=req.specifier, hashes=hashes, ) - applicable_candidates = [ - # Again, converting to str to deal with debundling. - c for c in all_candidates if str(c.version) in compatible_versions - ] - - if applicable_candidates: - best_candidate = max(applicable_candidates, - key=self._candidate_sort_key) - else: - best_candidate = None + best_candidate = best_candidate_result.best_candidate + installed_version = None # type: Optional[_BaseVersion] if req.satisfied_by is not None: installed_version = parse_version(req.satisfied_by.version) - else: - installed_version = None + + def _format_versions(cand_iter): + # This repeated parse_version and str() conversion is needed to + # handle different vendoring sources from pipenv.patched.notpip and pkg_resources. + # If we stop using the pkg_resources provided specifier and start + # using our own, we can drop the cast to str(). + return ", ".join(sorted( + {str(c.version) for c in cand_iter}, + key=parse_version, + )) or "none" if installed_version is None and best_candidate is None: logger.critical( 'Could not find a version that satisfies the requirement %s ' '(from versions: %s)', req, - ', '.join( - sorted( - {str(c.version) for c in all_candidates}, - key=parse_version, - ) - ) + _format_versions(best_candidate_result.iter_all()), ) raise DistributionNotFound( @@ -679,254 +970,59 @@ class PackageFinder(object): 'Installed version (%s) is most up-to-date (past versions: ' '%s)', installed_version, - ', '.join(sorted(compatible_versions, key=parse_version)) or - "none", + _format_versions(best_candidate_result.iter_applicable()), ) raise BestVersionAlreadyInstalled logger.debug( 'Using version %s (newest of versions: %s)', best_candidate.version, - ', '.join(sorted(compatible_versions, key=parse_version)) + _format_versions(best_candidate_result.iter_applicable()), ) - return best_candidate.location - - def _get_pages(self, locations, project_name): - """ - Yields (page, page_url) from the given locations, skipping - locations that have errors. - """ - seen = set() - for location in locations: - if location in seen: - continue - seen.add(location) - - try: - page = self._get_page(location) - except requests.HTTPError: - continue - if page is None: - continue - - yield page - - _py_version_re = re.compile(r'-py([123]\.?[0-9]?)$') - - def _sort_links(self, links): - """ - Returns elements of links in order, non-egg links first, egg links - second, while eliminating duplicates - """ - eggs, no_eggs = [], [] - seen = set() - for link in links: - if link not in seen: - seen.add(link) - if link.egg_fragment: - eggs.append(link) - else: - no_eggs.append(link) - return no_eggs + eggs - - def _package_versions(self, links, search): - result = [] - for link in self._sort_links(links): - v = self._link_package_versions(link, search) - if v is not None: - result.append(v) - return result - - def _log_skipped_link(self, link, reason): - if link not in self.logged_links: - logger.debug('Skipping link %s; %s', link, reason) - self.logged_links.add(link) - - def _link_package_versions(self, link, search, ignore_compatibility=True): - """Return an InstallationCandidate or None""" - version = None - if link.egg_fragment: - egg_info = link.egg_fragment - ext = link.ext - else: - egg_info, ext = link.splitext() - if not ext: - self._log_skipped_link(link, 'not a file') - return - if ext not in SUPPORTED_EXTENSIONS: - self._log_skipped_link( - link, 'unsupported archive format: %s' % ext, - ) - return - if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: - self._log_skipped_link( - link, 'No binaries permitted for %s' % search.supplied, - ) - return - if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: - self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: - try: - wheel = Wheel(link.filename) - except InvalidWheelFilename: - self._log_skipped_link(link, 'invalid wheel filename') - return - if canonicalize_name(wheel.name) != search.canonical: - self._log_skipped_link( - link, 'wrong project name (not %s)' % search.supplied) - return - - if not wheel.supported(self.valid_tags) and not ignore_compatibility: - self._log_skipped_link( - link, 'it is not compatible with this Python') - return - - version = wheel.version - - # This should be up by the search.ok_binary check, but see issue 2700. - if "source" not in search.formats and ext != wheel_ext: - self._log_skipped_link( - link, 'No sources permitted for %s' % search.supplied, - ) - return - - if not version: - version = egg_info_matches(egg_info, search.supplied, link) - if version is None: - self._log_skipped_link( - link, 'Missing project version for %s' % search.supplied) - return - - match = self._py_version_re.search(version) - if match: - version = version[:match.start()] - py_version = match.group(1) - if py_version != sys.version[:3]: - self._log_skipped_link( - link, 'Python version is incorrect') - return - try: - support_this_python = check_requires_python(link.requires_python) - except specifiers.InvalidSpecifier: - logger.debug("Package %s has an invalid Requires-Python entry: %s", - link.filename, link.requires_python) - support_this_python = True - - if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", - link, link.requires_python) - return - logger.debug('Found link %s, version: %s', link, version) - - return InstallationCandidate(search.supplied, version, link, link.requires_python) - - def _get_page(self, link): - return _get_html_page(link, session=self.session) + return best_candidate.link -def egg_info_matches( - egg_info, search_name, link, - _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): - """Pull the version part out of a string. +def _find_name_version_sep(fragment, canonical_name): + # type: (str, str) -> int + """Find the separator's index based on the package's canonical name. - :param egg_info: The string to parse. E.g. foo-2.1 - :param search_name: The name of the package this belongs to. None to - infer the name. Note that this cannot unambiguously parse strings - like foo-2-2 which might be foo, 2-2 or foo-2, 2. - :param link: The link the string came from, for logging on failure. + :param fragment: A + filename "fragment" (stem) or + egg fragment. + :param canonical_name: The package's canonical name. + + This function is needed since the canonicalized name does not necessarily + have the same length as the egg info's name part. An example:: + + >>> fragment = 'foo__bar-1.0' + >>> canonical_name = 'foo-bar' + >>> _find_name_version_sep(fragment, canonical_name) + 8 """ - match = _egg_info_re.search(egg_info) - if not match: - logger.debug('Could not parse version from link: %s', link) + # Project name and version must be separated by one single dash. Find all + # occurrences of dashes; if the string in front of it matches the canonical + # name, this is the one separating the name and version parts. + for i, c in enumerate(fragment): + if c != "-": + continue + if canonicalize_name(fragment[:i]) == canonical_name: + return i + raise ValueError("{} does not match {}".format(fragment, canonical_name)) + + +def _extract_version_from_fragment(fragment, canonical_name): + # type: (str, str) -> Optional[str] + """Parse the version string from a + filename + "fragment" (stem) or egg fragment. + + :param fragment: The string to parse. E.g. foo-2.1 + :param canonical_name: The canonicalized name of the package this + belongs to. + """ + try: + version_start = _find_name_version_sep(fragment, canonical_name) + 1 + except ValueError: return None - if search_name is None: - full_match = match.group(0) - return full_match.split('-', 1)[-1] - name = match.group(0).lower() - # To match the "safe" name that pkg_resources creates: - name = name.replace('_', '-') - # project name and version must be separated by a dash - look_for = search_name.lower() + "-" - if name.startswith(look_for): - return match.group(0)[len(look_for):] - else: + version = fragment[version_start:] + if not version: return None - - -def _determine_base_url(document, page_url): - """Determine the HTML document's base URL. - - This looks for a ```` tag in the HTML document. If present, its href - attribute denotes the base URL of anchor tags in the document. If there is - no such tag (or if it does not have a valid href attribute), the HTML - file's URL is used as the base URL. - - :param document: An HTML document representation. The current - implementation expects the result of ``html5lib.parse()``. - :param page_url: The URL of the HTML document. - """ - for base in document.findall(".//base"): - href = base.get("href") - if href is not None: - return href - return page_url - - -def _get_encoding_from_headers(headers): - """Determine if we have any encoding information in our headers. - """ - if headers and "Content-Type" in headers: - content_type, params = cgi.parse_header(headers["Content-Type"]) - if "charset" in params: - return params['charset'] - return None - - -_CLEAN_LINK_RE = re.compile(r'[^a-z0-9$&+,/:;=?@.#%_\\|-]', re.I) - - -def _clean_link(url): - """Makes sure a link is fully encoded. That is, if a ' ' shows up in - the link, it will be rewritten to %20 (while not over-quoting - % or other characters).""" - return _CLEAN_LINK_RE.sub(lambda match: '%%%2x' % ord(match.group(0)), url) - - -class HTMLPage(object): - """Represents one page, along with its URL""" - - def __init__(self, content, url, headers=None): - self.content = content - self.url = url - self.headers = headers - - def __str__(self): - return self.url - - def iter_links(self): - """Yields all links in the page""" - document = html5lib.parse( - self.content, - transport_encoding=_get_encoding_from_headers(self.headers), - namespaceHTMLElements=False, - ) - base_url = _determine_base_url(document, self.url) - for anchor in document.findall(".//a"): - if anchor.get("href"): - href = anchor.get("href") - url = _clean_link(urllib_parse.urljoin(base_url, href)) - pyrequire = anchor.get('data-requires-python') - pyrequire = unescape(pyrequire) if pyrequire else None - yield Link(url, self.url, requires_python=pyrequire) - - -Search = namedtuple('Search', 'supplied canonical formats') -"""Capture key aspects of a search. - -:attribute supplied: The user supplied package. -:attribute canonical: The canonical package name. -:attribute formats: The formats allowed for this package. Should be a set - with 'binary' or 'source' or both in it. -""" + return version diff --git a/pipenv/patched/notpip/_internal/resolve.py b/pipenv/patched/notpip/_internal/legacy_resolve.py similarity index 68% rename from pipenv/patched/notpip/_internal/resolve.py rename to pipenv/patched/notpip/_internal/legacy_resolve.py index b0d096f9..674efd09 100644 --- a/pipenv/patched/notpip/_internal/resolve.py +++ b/pipenv/patched/notpip/_internal/legacy_resolve.py @@ -10,22 +10,102 @@ for sub-dependencies a. "first found, wins" (where the order is breadth first) """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + import logging +import sys from collections import defaultdict from itertools import chain +from pipenv.patched.notpip._vendor.packaging import specifiers + from pipenv.patched.notpip._internal.exceptions import ( - BestVersionAlreadyInstalled, DistributionNotFound, HashError, HashErrors, + BestVersionAlreadyInstalled, + DistributionNotFound, + HashError, + HashErrors, UnsupportedPythonVersion, ) -from pipenv.patched.notpip._internal.req.constructors import install_req_from_req from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import dist_in_usersite, ensure_dir -from pipenv.patched.notpip._internal.utils.packaging import check_dist_requires_python +from pipenv.patched.notpip._internal.utils.misc import ( + dist_in_usersite, + ensure_dir, + normalize_version_info, +) +from pipenv.patched.notpip._internal.utils.packaging import ( + check_requires_python, + get_requires_python, +) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Callable, DefaultDict, List, Optional, Set, Tuple + from pipenv.patched.notpip._vendor import pkg_resources + + from pipenv.patched.notpip._internal.distributions import AbstractDistribution + from pipenv.patched.notpip._internal.network.session import PipSession + from pipenv.patched.notpip._internal.index import PackageFinder + from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from pipenv.patched.notpip._internal.req.req_set import RequirementSet + + InstallRequirementProvider = Callable[ + [str, InstallRequirement], InstallRequirement + ] logger = logging.getLogger(__name__) +def _check_dist_requires_python( + dist, # type: pkg_resources.Distribution + version_info, # type: Tuple[int, int, int] + ignore_requires_python=False, # type: bool +): + # type: (...) -> None + """ + Check whether the given Python version is compatible with a distribution's + "Requires-Python" value. + + :param version_info: A 3-tuple of ints representing the Python + major-minor-micro version to check. + :param ignore_requires_python: Whether to ignore the "Requires-Python" + value if the given Python version isn't compatible. + + :raises UnsupportedPythonVersion: When the given Python version isn't + compatible. + """ + requires_python = get_requires_python(dist) + try: + is_compatible = check_requires_python( + requires_python, version_info=version_info, + ) + except specifiers.InvalidSpecifier as exc: + logger.warning( + "Package %r has an invalid Requires-Python: %s", + dist.project_name, exc, + ) + return + + if is_compatible: + return + + version = '.'.join(map(str, version_info)) + if ignore_requires_python: + logger.debug( + 'Ignoring failed Requires-Python check for package %r: ' + '%s not in %r', + dist.project_name, version, requires_python, + ) + return + + raise UnsupportedPythonVersion( + 'Package {!r} requires a different Python: {} not in {!r}'.format( + dist.project_name, version, requires_python, + )) + + class Resolver(object): """Resolves which packages need to be installed/uninstalled to perform \ the requested operation without breaking the requirements of any package. @@ -33,37 +113,56 @@ class Resolver(object): _allowed_strategies = {"eager", "only-if-needed", "to-satisfy-only"} - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, - force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): + def __init__( + self, + preparer, # type: RequirementPreparer + session, # type: PipSession + finder, # type: PackageFinder + make_install_req, # type: InstallRequirementProvider + use_user_site, # type: bool + ignore_dependencies, # type: bool + ignore_installed, # type: bool + ignore_requires_python, # type: bool + force_reinstall, # type: bool + upgrade_strategy, # type: str + py_version_info=None, # type: Optional[Tuple[int, ...]] + ignore_compatibility=False, # type: bool + ): + # type: (...) -> None super(Resolver, self).__init__() assert upgrade_strategy in self._allowed_strategies + if py_version_info is None: + py_version_info = sys.version_info[:3] + else: + py_version_info = normalize_version_info(py_version_info) + + self._py_version_info = py_version_info + self.preparer = preparer self.finder = finder self.session = session - # NOTE: This would eventually be replaced with a cache that can give - # information about both sdist and wheels transparently. - self.wheel_cache = wheel_cache - - self.require_hashes = None # This is set in resolve + # This is set in resolve + self.require_hashes = None # type: Optional[bool] self.upgrade_strategy = upgrade_strategy self.force_reinstall = force_reinstall - self.isolated = isolated self.ignore_dependencies = ignore_dependencies self.ignore_installed = ignore_installed self.ignore_requires_python = ignore_requires_python - self.ignore_compatibility = ignore_compatibility self.use_user_site = use_user_site + self._make_install_req = make_install_req + self.ignore_compatibility = ignore_compatibility self.requires_python = None if self.ignore_compatibility: self.ignore_requires_python = True - self._discovered_dependencies = defaultdict(list) + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] def resolve(self, requirement_set): + # type: (RequirementSet) -> None """Resolve what operations need to be done As a side-effect of this method, the packages (and their dependencies) @@ -90,7 +189,8 @@ class Resolver(object): ) # Display where finder is looking for packages - locations = self.finder.get_formatted_locations() + search_scope = self.finder.search_scope + locations = search_scope.get_formatted_locations() if locations: logger.info(locations) @@ -98,7 +198,7 @@ class Resolver(object): # exceptions cannot be checked ahead of time, because # req.populate_link() needs to be called before we can make decisions # based on link type. - discovered_reqs = [] + discovered_reqs = [] # type: List[InstallRequirement] hash_errors = HashErrors() for req in chain(root_reqs, discovered_reqs): try: @@ -113,6 +213,7 @@ class Resolver(object): raise hash_errors def _is_upgrade_allowed(self, req): + # type: (InstallRequirement) -> bool if self.upgrade_strategy == "to-satisfy-only": return False elif self.upgrade_strategy == "eager": @@ -122,6 +223,7 @@ class Resolver(object): return req.is_direct def _set_req_to_reinstall(self, req): + # type: (InstallRequirement) -> None """ Set a requirement to be installed. """ @@ -131,8 +233,8 @@ class Resolver(object): req.conflicts_with = req.satisfied_by req.satisfied_by = None - # XXX: Stop passing requirement_set for options def _check_skip_installed(self, req_to_install): + # type: (InstallRequirement) -> Optional[str] """Check if req_to_install should be skipped. This will check if the req is installed, and whether we should upgrade @@ -185,6 +287,7 @@ class Resolver(object): return None def _get_abstract_dist_for(self, req): + # type: (InstallRequirement) -> AbstractDistribution """Takes a InstallRequirement and returns a single AbstractDist \ representing a prepared variant of the same. """ @@ -208,9 +311,11 @@ class Resolver(object): ) upgrade_allowed = self._is_upgrade_allowed(req) + + # We eagerly populate the link, since that's our "legacy" behavior. + req.populate_link(self.finder, upgrade_allowed, self.require_hashes) abstract_dist = self.preparer.prepare_linked_requirement( - req, self.session, self.finder, upgrade_allowed, - self.require_hashes + req, self.session, self.finder, self.require_hashes ) # NOTE @@ -241,7 +346,13 @@ class Resolver(object): return abstract_dist - def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): + def _resolve_one( + self, + requirement_set, # type: RequirementSet + req_to_install, # type: InstallRequirement + ignore_requires_python=False, # type: bool + ): + # type: (...) -> List[InstallRequirement] """Prepare a single requirements file. :return: A list of additional InstallRequirements to also install. @@ -260,29 +371,30 @@ class Resolver(object): abstract_dist = self._get_abstract_dist_for(req_to_install) # Parse and return dependencies - dist = abstract_dist.dist(self.finder) + dist = abstract_dist.get_pkg_resources_distribution() + # This will raise UnsupportedPythonVersion if the given Python + # version isn't compatible with the distribution's Requires-Python. + ignore_requires_python = ( + ignore_requires_python or self.ignore_requires_python or + self.ignore_compatibility + ) + _check_dist_requires_python( + dist, version_info=self._py_version_info, + ignore_requires_python=ignore_requires_python, + ) + # Patched in - lets get the python version on here then + # FIXME: Does this patch even work? it puts the python version + # on the resolver... why? try: - check_dist_requires_python(dist) - except UnsupportedPythonVersion as err: - if self.ignore_requires_python or self.ignore_compatibility: - logger.warning(err.args[0]) - else: - raise - - # A huge hack, by Kenneth Reitz. - try: - self.requires_python = check_dist_requires_python(dist, absorb=False) + self.requires_python = get_requires_python(dist) except TypeError: self.requires_python = None - - more_reqs = [] + more_reqs = [] # type: List[InstallRequirement] def add_req(subreq, extras_requested): - sub_install_req = install_req_from_req( + sub_install_req = self._make_install_req( str(subreq), req_to_install, - isolated=self.isolated, - wheel_cache=self.wheel_cache, ) parent_req_name = req_to_install.name to_scan_again, add_to_parent = requirement_set.add_requirement( @@ -331,19 +443,6 @@ class Resolver(object): for subreq in dist.requires(available_requested): add_req(subreq, extras_requested=available_requested) - # Hack for deep-resolving extras. - for available in available_requested: - if hasattr(dist, '_DistInfoDistribution__dep_map'): - for req in dist._DistInfoDistribution__dep_map[available]: - req = install_req_from_req( - str(req), - req_to_install, - isolated=self.isolated, - wheel_cache=self.wheel_cache, - ) - - more_reqs.append(req) - if not req_to_install.editable and not req_to_install.satisfied_by: # XXX: --no-install leads this to report 'Successfully # downloaded' for only non-editable reqs, even though we took @@ -353,6 +452,7 @@ class Resolver(object): return more_reqs def get_installation_order(self, req_set): + # type: (RequirementSet) -> List[InstallRequirement] """Create the installation order. The installation order is topological - requirements are installed @@ -363,7 +463,7 @@ class Resolver(object): # installs the user specified things in the order given, except when # dependencies must come earlier to achieve topological order. order = [] - ordered_reqs = set() + ordered_reqs = set() # type: Set[InstallRequirement] def schedule(req): if req.satisfied_by or req in ordered_reqs: diff --git a/pipenv/patched/notpip/_internal/locations.py b/pipenv/patched/notpip/_internal/locations.py index 3c7d5bd8..4bd3c87a 100644 --- a/pipenv/patched/notpip/_internal/locations.py +++ b/pipenv/patched/notpip/_internal/locations.py @@ -1,4 +1,9 @@ """Locations where we look for configs, install stuff, etc""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import os @@ -11,76 +16,49 @@ from distutils import sysconfig as distutils_sysconfig from distutils.command.install import SCHEME_KEYS # type: ignore from pipenv.patched.notpip._internal.utils import appdirs -from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.virtualenv import running_under_virtualenv + +if MYPY_CHECK_RUNNING: + from typing import Any, Union, Dict, List, Optional + # Application Directories USER_CACHE_DIR = appdirs.user_cache_dir("pip") -DELETE_MARKER_MESSAGE = '''\ -This file is placed here by pip to indicate the source was put -here by pip. - -Once this package is successfully installed this source code will be -deleted (unless you remove this file). -''' -PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' - - -def write_delete_marker_file(directory): +def get_major_minor_version(): + # type: () -> str """ - Write the pip delete marker file into this directory. + Return the major-minor version of the current Python as a string, e.g. + "3.7" or "3.10". """ - filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME) - with open(filepath, 'w') as marker_fp: - marker_fp.write(DELETE_MARKER_MESSAGE) + return '{}.{}'.format(*sys.version_info) -def running_under_virtualenv(): - """ - Return True if we're running inside a virtualenv, False otherwise. +def get_src_prefix(): + if running_under_virtualenv(): + src_prefix = os.path.join(sys.prefix, 'src') + else: + # FIXME: keep src in cwd for now (it is not a temporary folder) + try: + src_prefix = os.path.join(os.getcwd(), 'src') + except OSError: + # In case the current working directory has been renamed or deleted + sys.exit( + "The folder you are executing pip from can no longer be found." + ) - """ - if hasattr(sys, 'real_prefix'): - return True - elif sys.prefix != getattr(sys, "base_prefix", sys.prefix): - return True + # under macOS + virtualenv sys.prefix is not properly resolved + # it is something like /path/to/python/bin/.. + return os.path.abspath(src_prefix) - return False - - -def virtualenv_no_global(): - """ - Return True if in a venv and no system site packages. - """ - # this mirrors the logic in virtualenv.py for locating the - # no-global-site-packages.txt file - site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) - no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') - if running_under_virtualenv() and os.path.isfile(no_global_file): - return True - - -if running_under_virtualenv(): - src_prefix = os.path.join(sys.prefix, 'src') -else: - # FIXME: keep src in cwd for now (it is not a temporary folder) - try: - src_prefix = os.path.join(os.getcwd(), 'src') - except OSError: - # In case the current working directory has been renamed or deleted - sys.exit( - "The folder you are executing pip from can no longer be found." - ) - -# under macOS + virtualenv sys.prefix is not properly resolved -# it is something like /path/to/python/bin/.. -# Note: using realpath due to tmp dirs on OSX being symlinks -src_prefix = os.path.abspath(src_prefix) # FIXME doesn't account for venv linked to global site-packages -site_packages = sysconfig.get_path("purelib") +site_packages = sysconfig.get_path("purelib") # type: Optional[str] + # This is because of a bug in PyPy's sysconfig module, see # https://bitbucket.org/pypy/pypy/issues/2506/sysconfig-returns-incorrect-paths # for more information. @@ -92,7 +70,7 @@ try: user_site = site.getusersitepackages() except AttributeError: user_site = site.USER_SITE -user_dir = expanduser('~') + if WINDOWS: bin_py = os.path.join(sys.prefix, 'Scripts') bin_user = os.path.join(user_site, 'Scripts') @@ -100,41 +78,19 @@ if WINDOWS: if not os.path.exists(bin_py): bin_py = os.path.join(sys.prefix, 'bin') bin_user = os.path.join(user_site, 'bin') - - config_basename = 'pip.ini' - - legacy_storage_dir = os.path.join(user_dir, 'pip') - legacy_config_file = os.path.join( - legacy_storage_dir, - config_basename, - ) else: bin_py = os.path.join(sys.prefix, 'bin') bin_user = os.path.join(user_site, 'bin') - config_basename = 'pip.conf' - - legacy_storage_dir = os.path.join(user_dir, '.pip') - legacy_config_file = os.path.join( - legacy_storage_dir, - config_basename, - ) # Forcing to use /usr/local/bin for standard macOS framework installs # Also log to ~/Library/Logs/ for use with the Console.app log viewer if sys.platform[:6] == 'darwin' and sys.prefix[:16] == '/System/Library/': bin_py = '/usr/local/bin' -site_config_files = [ - os.path.join(path, config_basename) - for path in appdirs.site_config_dirs('pip') -] - -venv_config_file = os.path.join(sys.prefix, config_basename) -new_config_file = os.path.join(appdirs.user_config_dir("pip"), config_basename) - def distutils_scheme(dist_name, user=False, home=None, root=None, isolated=False, prefix=None): + # type:(str, bool, str, str, bool, str) -> dict """ Return a distutils install scheme """ @@ -146,18 +102,22 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, extra_dist_args = {"script_args": ["--no-user-cfg"]} else: extra_dist_args = {} - dist_args = {'name': dist_name} + dist_args = {'name': dist_name} # type: Dict[str, Union[str, List[str]]] dist_args.update(extra_dist_args) d = Distribution(dist_args) + # Ignoring, typeshed issue reported python/typeshed/issues/2567 d.parse_config_files() - i = d.get_command_obj('install', create=True) + # NOTE: Ignoring type since mypy can't find attributes on 'Command' + i = d.get_command_obj('install', create=True) # type: Any + assert i is not None # NOTE: setting user or home has the side-effect of creating the home dir # or user base for installations during finalize_options() # ideally, we'd prefer a scheme class that has no side-effects. assert not (user and prefix), "user={} prefix={}".format(user, prefix) + assert not (home and prefix), "home={} prefix={}".format(home, prefix) i.user = user or i.user - if user: + if user or home: i.prefix = "" i.prefix = prefix or i.prefix i.home = home or i.home @@ -171,7 +131,9 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, # platlib). Note, i.install_lib is *always* set after # finalize_options(); we only want to override here if the user # has explicitly requested it hence going back to the config - if 'install_lib' in d.get_option_dict('install'): + + # Ignoring, typeshed issue reported python/typeshed/issues/2567 + if 'install_lib' in d.get_option_dict('install'): # type: ignore scheme.update(dict(purelib=i.install_lib, platlib=i.install_lib)) if running_under_virtualenv(): @@ -179,7 +141,7 @@ def distutils_scheme(dist_name, user=False, home=None, root=None, sys.prefix, 'include', 'site', - 'python' + sys.version[:3], + 'python{}'.format(get_major_minor_version()), dist_name, ) diff --git a/pipenv/patched/notpip/_internal/main.py b/pipenv/patched/notpip/_internal/main.py new file mode 100644 index 00000000..ed712c42 --- /dev/null +++ b/pipenv/patched/notpip/_internal/main.py @@ -0,0 +1,47 @@ +"""Primary application entrypoint. +""" +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import locale +import logging +import os +import sys + +from pipenv.patched.notpip._internal.cli.autocompletion import autocomplete +from pipenv.patched.notpip._internal.cli.main_parser import parse_command +from pipenv.patched.notpip._internal.commands import create_command +from pipenv.patched.notpip._internal.exceptions import PipError +from pipenv.patched.notpip._internal.utils import deprecation + +logger = logging.getLogger(__name__) + + +def main(args=None): + if args is None: + args = sys.argv[1:] + + # Configure our deprecation warnings to be sent through loggers + deprecation.install_warning_logger() + + autocomplete() + + try: + cmd_name, cmd_args = parse_command(args) + except PipError as exc: + sys.stderr.write("ERROR: %s" % exc) + sys.stderr.write(os.linesep) + sys.exit(1) + + # Needed for locale.getpreferredencoding(False) to work + # in pip._internal.utils.encoding.auto_decode + try: + locale.setlocale(locale.LC_ALL, '') + except locale.Error as e: + # setlocale can apparently crash if locale are uninitialized + logger.debug("Ignoring error %s when setting locale", e) + command = create_command(cmd_name, isolated=("--isolated" in cmd_args)) + + return command.main(cmd_args) diff --git a/pipenv/patched/notpip/_internal/models/candidate.py b/pipenv/patched/notpip/_internal/models/candidate.py index 9627589e..937d872f 100644 --- a/pipenv/patched/notpip/_internal/models/candidate.py +++ b/pipenv/patched/notpip/_internal/models/candidate.py @@ -1,24 +1,40 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from pipenv.patched.notpip._vendor.packaging.version import _BaseVersion + from pipenv.patched.notpip._internal.models.link import Link + from typing import Any class InstallationCandidate(KeyBasedCompareMixin): """Represents a potential "candidate" for installation. """ - def __init__(self, project, version, location, requires_python=None): + def __init__(self, project, version, link, requires_python=None): + # type: (Any, str, Link, Any) -> None self.project = project - self.version = parse_version(version) - self.location = location + self.version = parse_version(version) # type: _BaseVersion + self.link = link self.requires_python = requires_python super(InstallationCandidate, self).__init__( - key=(self.project, self.version, self.location), + key=(self.project, self.version, self.link), defining_class=InstallationCandidate ) def __repr__(self): + # type: () -> str return "".format( - self.project, self.version, self.location, + self.project, self.version, self.link, + ) + + def __str__(self): + return '{!r} candidate (version {} at {})'.format( + self.project, self.version, self.link, ) diff --git a/pipenv/patched/notpip/_internal/models/format_control.py b/pipenv/patched/notpip/_internal/models/format_control.py index caad3cba..cbb57958 100644 --- a/pipenv/patched/notpip/_internal/models/format_control.py +++ b/pipenv/patched/notpip/_internal/models/format_control.py @@ -1,16 +1,29 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._internal.exceptions import CommandError +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Set, FrozenSet + class FormatControl(object): - """A helper class for controlling formats from which packages are installed. - If a field is falsy, it isn't set. If it is {':all:'}, it should match all - packages except those listed in the other field. Only one field can be set - to {':all:'} at a time. The rest of the time exact package name matches - are listed, with any given package only showing up in one field at a time. + """Helper for managing formats from which a package can be installed. """ + def __init__(self, no_binary=None, only_binary=None): - self.no_binary = set() if no_binary is None else no_binary - self.only_binary = set() if only_binary is None else only_binary + # type: (Optional[Set], Optional[Set]) -> None + if no_binary is None: + no_binary = set() + if only_binary is None: + only_binary = set() + + self.no_binary = no_binary + self.only_binary = only_binary def __eq__(self, other): return self.__dict__ == other.__dict__ @@ -27,6 +40,11 @@ class FormatControl(object): @staticmethod def handle_mutual_excludes(value, target, other): + # type: (str, Optional[Set], Optional[Set]) -> None + if value.startswith('-'): + raise CommandError( + "--no-binary / --only-binary option requires 1 argument." + ) new = value.split(',') while ':all:' in new: other.clear() @@ -45,6 +63,7 @@ class FormatControl(object): target.add(name) def get_allowed_formats(self, canonical_name): + # type: (str) -> FrozenSet result = {"binary", "source"} if canonical_name in self.only_binary: result.discard('source') @@ -57,6 +76,7 @@ class FormatControl(object): return frozenset(result) def disallow_binaries(self): + # type: () -> None self.handle_mutual_excludes( ':all:', self.no_binary, self.only_binary, ) diff --git a/pipenv/patched/notpip/_internal/models/index.py b/pipenv/patched/notpip/_internal/models/index.py index 0983fc9c..b2895dab 100644 --- a/pipenv/patched/notpip/_internal/models/index.py +++ b/pipenv/patched/notpip/_internal/models/index.py @@ -6,6 +6,7 @@ class PackageIndex(object): """ def __init__(self, url, file_storage_domain): + # type: (str, str) -> None super(PackageIndex, self).__init__() self.url = url self.netloc = urllib_parse.urlsplit(url).netloc @@ -18,6 +19,7 @@ class PackageIndex(object): self.file_storage_domain = file_storage_domain def _url_for_path(self, path): + # type: (str) -> str return urllib_parse.urljoin(self.url, path) diff --git a/pipenv/patched/notpip/_internal/models/link.py b/pipenv/patched/notpip/_internal/models/link.py index 686af1d0..688bd14f 100644 --- a/pipenv/patched/notpip/_internal/models/link.py +++ b/pipenv/patched/notpip/_internal/models/link.py @@ -1,42 +1,70 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import os import posixpath import re from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import splitext +from pipenv.patched.notpip._internal.utils.filetypes import WHEEL_EXTENSION +from pipenv.patched.notpip._internal.utils.misc import ( + redact_auth_from_url, + split_auth_from_netloc, + splitext, +) from pipenv.patched.notpip._internal.utils.models import KeyBasedCompareMixin -from pipenv.patched.notpip._internal.wheel import wheel_ext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url, url_to_path + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Tuple, Union + from pipenv.patched.notpip._internal.collector import HTMLPage + from pipenv.patched.notpip._internal.utils.hashes import Hashes class Link(KeyBasedCompareMixin): """Represents a parsed link from a Package Index's simple URL """ - def __init__(self, url, comes_from=None, requires_python=None): + def __init__( + self, + url, # type: str + comes_from=None, # type: Optional[Union[str, HTMLPage]] + requires_python=None, # type: Optional[str] + yanked_reason=None, # type: Optional[Text] + ): + # type: (...) -> None """ - url: - url of the resource pointed to (href of the link) - comes_from: - instance of HTMLPage where the link was found, or string. - requires_python: - String containing the `Requires-Python` metadata field, specified - in PEP 345. This may be specified by a data-requires-python - attribute in the HTML link tag, as described in PEP 503. + :param url: url of the resource pointed to (href of the link) + :param comes_from: instance of HTMLPage where the link was found, + or string. + :param requires_python: String containing the `Requires-Python` + metadata field, specified in PEP 345. This may be specified by + a data-requires-python attribute in the HTML link tag, as + described in PEP 503. + :param yanked_reason: the reason the file has been yanked, if the + file has been yanked, or None if the file hasn't been yanked. + This is the value of the "data-yanked" attribute, if present, in + a simple repository HTML link. If the file has been yanked but + no reason was provided, this should be the empty string. See + PEP 592 for more information and the specification. """ # url can be a UNC windows share if url.startswith('\\\\'): url = path_to_url(url) - self.url = url + self._parsed_url = urllib_parse.urlsplit(url) + # Store the url as a private attribute to prevent accidentally + # trying to set a new value. + self._url = url + self.comes_from = comes_from self.requires_python = requires_python if requires_python else None + self.yanked_reason = yanked_reason - super(Link, self).__init__( - key=(self.url), - defining_class=Link - ) + super(Link, self).__init__(key=url, defining_class=Link) def __str__(self): if self.requires_python: @@ -44,50 +72,78 @@ class Link(KeyBasedCompareMixin): else: rp = '' if self.comes_from: - return '%s (from %s)%s' % (self.url, self.comes_from, rp) + return '%s (from %s)%s' % (redact_auth_from_url(self._url), + self.comes_from, rp) else: - return str(self.url) + return redact_auth_from_url(str(self._url)) def __repr__(self): return '' % self + @property + def url(self): + # type: () -> str + return self._url + @property def filename(self): - _, netloc, path, _, _ = urllib_parse.urlsplit(self.url) - name = posixpath.basename(path.rstrip('/')) or netloc + # type: () -> str + path = self.path.rstrip('/') + name = posixpath.basename(path) + if not name: + # Make sure we don't leak auth information if the netloc + # includes a username and password. + netloc, user_pass = split_auth_from_netloc(self.netloc) + return netloc + name = urllib_parse.unquote(name) - assert name, ('URL %r produced no filename' % self.url) + assert name, ('URL %r produced no filename' % self._url) return name + @property + def file_path(self): + # type: () -> str + return url_to_path(self.url) + @property def scheme(self): - return urllib_parse.urlsplit(self.url)[0] + # type: () -> str + return self._parsed_url.scheme @property def netloc(self): - return urllib_parse.urlsplit(self.url)[1] + # type: () -> str + """ + This can contain auth information. + """ + return self._parsed_url.netloc @property def path(self): - return urllib_parse.unquote(urllib_parse.urlsplit(self.url)[2]) + # type: () -> str + return urllib_parse.unquote(self._parsed_url.path) def splitext(self): + # type: () -> Tuple[str, str] return splitext(posixpath.basename(self.path.rstrip('/'))) @property def ext(self): + # type: () -> str return self.splitext()[1] @property def url_without_fragment(self): - scheme, netloc, path, query, fragment = urllib_parse.urlsplit(self.url) + # type: () -> str + scheme, netloc, path, query, fragment = self._parsed_url return urllib_parse.urlunsplit((scheme, netloc, path, query, None)) _egg_fragment_re = re.compile(r'[#&]egg=([^&]*)') @property def egg_fragment(self): - match = self._egg_fragment_re.search(self.url) + # type: () -> Optional[str] + match = self._egg_fragment_re.search(self._url) if not match: return None return match.group(1) @@ -96,7 +152,8 @@ class Link(KeyBasedCompareMixin): @property def subdirectory_fragment(self): - match = self._subdirectory_fragment_re.search(self.url) + # type: () -> Optional[str] + match = self._subdirectory_fragment_re.search(self._url) if not match: return None return match.group(1) @@ -107,35 +164,64 @@ class Link(KeyBasedCompareMixin): @property def hash(self): - match = self._hash_re.search(self.url) + # type: () -> Optional[str] + match = self._hash_re.search(self._url) if match: return match.group(2) return None @property def hash_name(self): - match = self._hash_re.search(self.url) + # type: () -> Optional[str] + match = self._hash_re.search(self._url) if match: return match.group(1) return None @property def show_url(self): - return posixpath.basename(self.url.split('#', 1)[0].split('?', 1)[0]) + # type: () -> Optional[str] + return posixpath.basename(self._url.split('#', 1)[0].split('?', 1)[0]) + + @property + def is_file(self): + # type: () -> bool + return self.scheme == 'file' + + def is_existing_dir(self): + # type: () -> bool + return self.is_file and os.path.isdir(self.file_path) @property def is_wheel(self): - return self.ext == wheel_ext + # type: () -> bool + return self.ext == WHEEL_EXTENSION @property - def is_artifact(self): - """ - Determines if this points to an actual artifact (e.g. a tarball) or if - it points to an "abstract" thing like a path or a VCS location. - """ + def is_vcs(self): + # type: () -> bool from pipenv.patched.notpip._internal.vcs import vcs - if self.scheme in vcs.all_schemes: - return False + return self.scheme in vcs.all_schemes - return True + @property + def is_yanked(self): + # type: () -> bool + return self.yanked_reason is not None + + @property + def has_hash(self): + return self.hash_name is not None + + def is_hash_allowed(self, hashes): + # type: (Optional[Hashes]) -> bool + """ + Return True if the link has a hash and it is allowed. + """ + if hashes is None or not self.has_hash: + return False + # Assert non-None so mypy knows self.hash_name and self.hash are str. + assert self.hash_name is not None + assert self.hash is not None + + return hashes.is_hash_allowed(self.hash_name, hex_digest=self.hash) diff --git a/pipenv/patched/notpip/_internal/models/search_scope.py b/pipenv/patched/notpip/_internal/models/search_scope.py new file mode 100644 index 00000000..9e82ccb3 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/search_scope.py @@ -0,0 +1,116 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import itertools +import logging +import os +import posixpath + +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.models.index import PyPI +from pipenv.patched.notpip._internal.utils.compat import HAS_TLS +from pipenv.patched.notpip._internal.utils.misc import normalize_path, redact_auth_from_url +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List + + +logger = logging.getLogger(__name__) + + +class SearchScope(object): + + """ + Encapsulates the locations that pip is configured to search. + """ + + @classmethod + def create( + cls, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> SearchScope + """ + Create a SearchScope object after normalizing the `find_links`. + """ + # Build find_links. If an argument starts with ~, it may be + # a local file relative to a home directory. So try normalizing + # it and if it exists, use the normalized version. + # This is deliberately conservative - it might be fine just to + # blindly normalize anything starting with a ~... + built_find_links = [] # type: List[str] + for link in find_links: + if link.startswith('~'): + new_link = normalize_path(link) + if os.path.exists(new_link): + link = new_link + built_find_links.append(link) + + # If we don't have TLS enabled, then WARN if anyplace we're looking + # relies on TLS. + if not HAS_TLS: + for link in itertools.chain(index_urls, built_find_links): + parsed = urllib_parse.urlparse(link) + if parsed.scheme == 'https': + logger.warning( + 'pip is configured with locations that require ' + 'TLS/SSL, however the ssl module in Python is not ' + 'available.' + ) + break + + return cls( + find_links=built_find_links, + index_urls=index_urls, + ) + + def __init__( + self, + find_links, # type: List[str] + index_urls, # type: List[str] + ): + # type: (...) -> None + self.find_links = find_links + self.index_urls = index_urls + + def get_formatted_locations(self): + # type: () -> str + lines = [] + if self.index_urls and self.index_urls != [PyPI.simple_url]: + lines.append( + 'Looking in indexes: {}'.format(', '.join( + redact_auth_from_url(url) for url in self.index_urls)) + ) + if self.find_links: + lines.append( + 'Looking in links: {}'.format(', '.join( + redact_auth_from_url(url) for url in self.find_links)) + ) + return '\n'.join(lines) + + def get_index_urls_locations(self, project_name): + # type: (str) -> List[str] + """Returns the locations found via self.index_urls + + Checks the url_name on the main (first in the list) index and + use this url_name to produce all locations + """ + + def mkurl_pypi_url(url): + loc = posixpath.join( + url, + urllib_parse.quote(canonicalize_name(project_name))) + # For maximum compatibility with easy_install, ensure the path + # ends in a trailing slash. Although this isn't in the spec + # (and PyPI can handle it without the slash) some other index + # implementations might break if they relied on easy_install's + # behavior. + if not loc.endswith('/'): + loc = loc + '/' + return loc + + return [mkurl_pypi_url(url) for url in self.index_urls] diff --git a/pipenv/patched/notpip/_internal/models/selection_prefs.py b/pipenv/patched/notpip/_internal/models/selection_prefs.py new file mode 100644 index 00000000..256e903a --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/selection_prefs.py @@ -0,0 +1,47 @@ +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + from pipenv.patched.notpip._internal.models.format_control import FormatControl + + +class SelectionPreferences(object): + + """ + Encapsulates the candidate selection preferences for downloading + and installing files. + """ + + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps + # people when reading the code. + def __init__( + self, + allow_yanked, # type: bool + allow_all_prereleases=False, # type: bool + format_control=None, # type: Optional[FormatControl] + prefer_binary=False, # type: bool + ignore_requires_python=None, # type: Optional[bool] + ): + # type: (...) -> None + """Create a SelectionPreferences object. + + :param allow_yanked: Whether files marked as yanked (in the sense + of PEP 592) are permitted to be candidates for install. + :param format_control: A FormatControl object or None. Used to control + the selection of source packages / binary packages when consulting + the index and links. + :param prefer_binary: Whether to prefer an old, but valid, binary + dist over a new source dist. + :param ignore_requires_python: Whether to ignore incompatible + "Requires-Python" values in links. Defaults to False. + """ + if ignore_requires_python is None: + ignore_requires_python = False + + self.allow_yanked = allow_yanked + self.allow_all_prereleases = allow_all_prereleases + self.format_control = format_control + self.prefer_binary = prefer_binary + self.ignore_requires_python = ignore_requires_python diff --git a/pipenv/patched/notpip/_internal/models/target_python.py b/pipenv/patched/notpip/_internal/models/target_python.py new file mode 100644 index 00000000..c815b743 --- /dev/null +++ b/pipenv/patched/notpip/_internal/models/target_python.py @@ -0,0 +1,106 @@ +import sys + +from pipenv.patched.notpip._internal.pep425tags import get_supported, version_info_to_nodot +from pipenv.patched.notpip._internal.utils.misc import normalize_version_info +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Optional, Tuple + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag + + +class TargetPython(object): + + """ + Encapsulates the properties of a Python interpreter one is targeting + for a package install, download, etc. + """ + + def __init__( + self, + platform=None, # type: Optional[str] + py_version_info=None, # type: Optional[Tuple[int, ...]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + ): + # type: (...) -> None + """ + :param platform: A string or None. If None, searches for packages + that are supported by the current system. Otherwise, will find + packages that can be built on the platform passed in. These + packages will only be downloaded for distribution: they will + not be built locally. + :param py_version_info: An optional tuple of ints representing the + Python version information to use (e.g. `sys.version_info[:3]`). + This can have length 1, 2, or 3 when provided. + :param abi: A string or None. This is passed to pep425tags.py's + get_supported() function as is. + :param implementation: A string or None. This is passed to + pep425tags.py's get_supported() function as is. + """ + # Store the given py_version_info for when we call get_supported(). + self._given_py_version_info = py_version_info + + if py_version_info is None: + py_version_info = sys.version_info[:3] + else: + py_version_info = normalize_version_info(py_version_info) + + py_version = '.'.join(map(str, py_version_info[:2])) + + self.abi = abi + self.implementation = implementation + self.platform = platform + self.py_version = py_version + self.py_version_info = py_version_info + + # This is used to cache the return value of get_tags(). + self._valid_tags = None # type: Optional[List[Pep425Tag]] + + def format_given(self): + # type: () -> str + """ + Format the given, non-None attributes for display. + """ + display_version = None + if self._given_py_version_info is not None: + display_version = '.'.join( + str(part) for part in self._given_py_version_info + ) + + key_values = [ + ('platform', self.platform), + ('version_info', display_version), + ('abi', self.abi), + ('implementation', self.implementation), + ] + return ' '.join( + '{}={!r}'.format(key, value) for key, value in key_values + if value is not None + ) + + def get_tags(self): + # type: () -> List[Pep425Tag] + """ + Return the supported PEP 425 tags to check wheel candidates against. + + The tags are returned in order of preference (most preferred first). + """ + if self._valid_tags is None: + # Pass versions=None if no py_version_info was given since + # versions=None uses special default logic. + py_version_info = self._given_py_version_info + if py_version_info is None: + versions = None + else: + versions = [version_info_to_nodot(py_version_info)] + + tags = get_supported( + versions=versions, + platform=self.platform, + abi=self.abi, + impl=self.implementation, + ) + self._valid_tags = tags + + return self._valid_tags diff --git a/pipenv/patched/notpip/_internal/network/__init__.py b/pipenv/patched/notpip/_internal/network/__init__.py new file mode 100644 index 00000000..b51bde91 --- /dev/null +++ b/pipenv/patched/notpip/_internal/network/__init__.py @@ -0,0 +1,2 @@ +"""Contains purely network-related utilities. +""" diff --git a/pipenv/patched/notpip/_internal/network/auth.py b/pipenv/patched/notpip/_internal/network/auth.py new file mode 100644 index 00000000..943c48bc --- /dev/null +++ b/pipenv/patched/notpip/_internal/network/auth.py @@ -0,0 +1,298 @@ +"""Network Authentication Helpers + +Contains interface (MultiDomainBasicAuth) and associated glue code for +providing credentials in the context of network requests. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pipenv.patched.notpip._vendor.requests.auth import AuthBase, HTTPBasicAuth +from pipenv.patched.notpip._vendor.requests.utils import get_netrc_auth +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.utils.misc import ( + ask, + ask_input, + ask_password, + remove_auth_from_url, + split_auth_netloc_from_url, +) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from optparse import Values + from typing import Dict, Optional, Tuple + + from pipenv.patched.notpip._internal.vcs.versioncontrol import AuthInfo + + Credentials = Tuple[str, str, str] + +logger = logging.getLogger(__name__) + +try: + import keyring # noqa +except ImportError: + keyring = None +except Exception as exc: + logger.warning( + "Keyring is skipped due to an exception: %s", str(exc), + ) + keyring = None + + +def get_keyring_auth(url, username): + """Return the tuple auth for a given url from keyring.""" + if not url or not keyring: + return None + + try: + try: + get_credential = keyring.get_credential + except AttributeError: + pass + else: + logger.debug("Getting credentials from keyring for %s", url) + cred = get_credential(url, username) + if cred is not None: + return cred.username, cred.password + return None + + if username: + logger.debug("Getting password from keyring for %s", url) + password = keyring.get_password(url, username) + if password: + return username, password + + except Exception as exc: + logger.warning( + "Keyring is skipped due to an exception: %s", str(exc), + ) + + +class MultiDomainBasicAuth(AuthBase): + + def __init__(self, prompting=True, index_urls=None): + # type: (bool, Optional[Values]) -> None + self.prompting = prompting + self.index_urls = index_urls + self.passwords = {} # type: Dict[str, AuthInfo] + # When the user is prompted to enter credentials and keyring is + # available, we will offer to save them. If the user accepts, + # this value is set to the credentials they entered. After the + # request authenticates, the caller should call + # ``save_credentials`` to save these. + self._credentials_to_save = None # type: Optional[Credentials] + + def _get_index_url(self, url): + """Return the original index URL matching the requested URL. + + Cached or dynamically generated credentials may work against + the original index URL rather than just the netloc. + + The provided url should have had its username and password + removed already. If the original index url had credentials then + they will be included in the return value. + + Returns None if no matching index was found, or if --no-index + was specified by the user. + """ + if not url or not self.index_urls: + return None + + for u in self.index_urls: + prefix = remove_auth_from_url(u).rstrip("/") + "/" + if url.startswith(prefix): + return u + + def _get_new_credentials(self, original_url, allow_netrc=True, + allow_keyring=True): + """Find and return credentials for the specified URL.""" + # Split the credentials and netloc from the url. + url, netloc, url_user_password = split_auth_netloc_from_url( + original_url, + ) + + # Start with the credentials embedded in the url + username, password = url_user_password + if username is not None and password is not None: + logger.debug("Found credentials in url for %s", netloc) + return url_user_password + + # Find a matching index url for this request + index_url = self._get_index_url(url) + if index_url: + # Split the credentials from the url. + index_info = split_auth_netloc_from_url(index_url) + if index_info: + index_url, _, index_url_user_password = index_info + logger.debug("Found index url %s", index_url) + + # If an index URL was found, try its embedded credentials + if index_url and index_url_user_password[0] is not None: + username, password = index_url_user_password + if username is not None and password is not None: + logger.debug("Found credentials in index url for %s", netloc) + return index_url_user_password + + # Get creds from netrc if we still don't have them + if allow_netrc: + netrc_auth = get_netrc_auth(original_url) + if netrc_auth: + logger.debug("Found credentials in netrc for %s", netloc) + return netrc_auth + + # If we don't have a password and keyring is available, use it. + if allow_keyring: + # The index url is more specific than the netloc, so try it first + kr_auth = ( + get_keyring_auth(index_url, username) or + get_keyring_auth(netloc, username) + ) + if kr_auth: + logger.debug("Found credentials in keyring for %s", netloc) + return kr_auth + + return username, password + + def _get_url_and_credentials(self, original_url): + """Return the credentials to use for the provided URL. + + If allowed, netrc and keyring may be used to obtain the + correct credentials. + + Returns (url_without_credentials, username, password). Note + that even if the original URL contains credentials, this + function may return a different username and password. + """ + url, netloc, _ = split_auth_netloc_from_url(original_url) + + # Use any stored credentials that we have for this netloc + username, password = self.passwords.get(netloc, (None, None)) + + if username is None and password is None: + # No stored credentials. Acquire new credentials without prompting + # the user. (e.g. from netrc, keyring, or the URL itself) + username, password = self._get_new_credentials(original_url) + + if username is not None or password is not None: + # Convert the username and password if they're None, so that + # this netloc will show up as "cached" in the conditional above. + # Further, HTTPBasicAuth doesn't accept None, so it makes sense to + # cache the value that is going to be used. + username = username or "" + password = password or "" + + # Store any acquired credentials. + self.passwords[netloc] = (username, password) + + assert ( + # Credentials were found + (username is not None and password is not None) or + # Credentials were not found + (username is None and password is None) + ), "Could not load credentials from url: {}".format(original_url) + + return url, username, password + + def __call__(self, req): + # Get credentials for this request + url, username, password = self._get_url_and_credentials(req.url) + + # Set the url of the request to the url without any credentials + req.url = url + + if username is not None and password is not None: + # Send the basic auth with this request + req = HTTPBasicAuth(username, password)(req) + + # Attach a hook to handle 401 responses + req.register_hook("response", self.handle_401) + + return req + + # Factored out to allow for easy patching in tests + def _prompt_for_password(self, netloc): + username = ask_input("User for %s: " % netloc) + if not username: + return None, None + auth = get_keyring_auth(netloc, username) + if auth: + return auth[0], auth[1], False + password = ask_password("Password: ") + return username, password, True + + # Factored out to allow for easy patching in tests + def _should_save_password_to_keyring(self): + if not keyring: + return False + return ask("Save credentials to keyring [y/N]: ", ["y", "n"]) == "y" + + def handle_401(self, resp, **kwargs): + # We only care about 401 responses, anything else we want to just + # pass through the actual response + if resp.status_code != 401: + return resp + + # We are not able to prompt the user so simply return the response + if not self.prompting: + return resp + + parsed = urllib_parse.urlparse(resp.url) + + # Prompt the user for a new username and password + username, password, save = self._prompt_for_password(parsed.netloc) + + # Store the new username and password to use for future requests + self._credentials_to_save = None + if username is not None and password is not None: + self.passwords[parsed.netloc] = (username, password) + + # Prompt to save the password to keyring + if save and self._should_save_password_to_keyring(): + self._credentials_to_save = (parsed.netloc, username, password) + + # Consume content and release the original connection to allow our new + # request to reuse the same one. + resp.content + resp.raw.release_conn() + + # Add our new username and password to the request + req = HTTPBasicAuth(username or "", password or "")(resp.request) + req.register_hook("response", self.warn_on_401) + + # On successful request, save the credentials that were used to + # keyring. (Note that if the user responded "no" above, this member + # is not set and nothing will be saved.) + if self._credentials_to_save: + req.register_hook("response", self.save_credentials) + + # Send our new request + new_resp = resp.connection.send(req, **kwargs) + new_resp.history.append(resp) + + return new_resp + + def warn_on_401(self, resp, **kwargs): + """Response callback to warn about incorrect credentials.""" + if resp.status_code == 401: + logger.warning( + '401 Error, Credentials not correct for %s', resp.request.url, + ) + + def save_credentials(self, resp, **kwargs): + """Response callback to save credentials on success.""" + assert keyring is not None, "should never reach here without keyring" + if not keyring: + return + + creds = self._credentials_to_save + self._credentials_to_save = None + if creds and resp.status_code < 400: + try: + logger.info('Saving credentials to keyring') + keyring.set_password(*creds) + except Exception: + logger.exception('Failed to save credentials') diff --git a/pipenv/patched/notpip/_internal/network/cache.py b/pipenv/patched/notpip/_internal/network/cache.py new file mode 100644 index 00000000..9954009c --- /dev/null +++ b/pipenv/patched/notpip/_internal/network/cache.py @@ -0,0 +1,75 @@ +"""HTTP cache implementation. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import os +from contextlib import contextmanager + +from pipenv.patched.notpip._vendor.cachecontrol.cache import BaseCache +from pipenv.patched.notpip._vendor.cachecontrol.caches import FileCache + +from pipenv.patched.notpip._internal.utils.filesystem import adjacent_tmp_file, replace +from pipenv.patched.notpip._internal.utils.misc import ensure_dir +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + + +@contextmanager +def suppressed_cache_errors(): + """If we can't access the cache then we can just skip caching and process + requests as if caching wasn't enabled. + """ + try: + yield + except (OSError, IOError): + pass + + +class SafeFileCache(BaseCache): + """ + A file based cache which is safe to use even when the target directory may + not be accessible or writable. + """ + + def __init__(self, directory): + # type: (str) -> None + assert directory is not None, "Cache directory must not be None." + super(SafeFileCache, self).__init__() + self.directory = directory + + def _get_cache_path(self, name): + # type: (str) -> str + # From cachecontrol.caches.file_cache.FileCache._fn, brought into our + # class for backwards-compatibility and to avoid using a non-public + # method. + hashed = FileCache.encode(name) + parts = list(hashed[:5]) + [hashed] + return os.path.join(self.directory, *parts) + + def get(self, key): + # type: (str) -> Optional[bytes] + path = self._get_cache_path(key) + with suppressed_cache_errors(): + with open(path, 'rb') as f: + return f.read() + + def set(self, key, value): + # type: (str, bytes) -> None + path = self._get_cache_path(key) + with suppressed_cache_errors(): + ensure_dir(os.path.dirname(path)) + + with adjacent_tmp_file(path) as f: + f.write(value) + + replace(f.name, path) + + def delete(self, key): + # type: (str) -> None + path = self._get_cache_path(key) + with suppressed_cache_errors(): + os.remove(path) diff --git a/pipenv/patched/notpip/_internal/network/session.py b/pipenv/patched/notpip/_internal/network/session.py new file mode 100644 index 00000000..178c0457 --- /dev/null +++ b/pipenv/patched/notpip/_internal/network/session.py @@ -0,0 +1,426 @@ +"""PipSession and supporting code, containing all pip-specific +network request configuration and behavior. +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import email.utils +import json +import logging +import mimetypes +import os +import platform +import sys +import warnings + +from pipenv.patched.notpip._vendor import requests, six, urllib3 +from pipenv.patched.notpip._vendor.cachecontrol import CacheControlAdapter +from pipenv.patched.notpip._vendor.requests.adapters import BaseAdapter, HTTPAdapter +from pipenv.patched.notpip._vendor.requests.models import Response +from pipenv.patched.notpip._vendor.requests.structures import CaseInsensitiveDict +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.urllib3.exceptions import InsecureRequestWarning + +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.network.auth import MultiDomainBasicAuth +from pipenv.patched.notpip._internal.network.cache import SafeFileCache +# Import ssl from compat so the initial import occurs in only one place. +from pipenv.patched.notpip._internal.utils.compat import HAS_TLS, ipaddress, ssl +from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner +from pipenv.patched.notpip._internal.utils.glibc import libc_ver +from pipenv.patched.notpip._internal.utils.misc import ( + build_url_from_netloc, + get_installed_version, + parse_netloc, +) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import url_to_path + +if MYPY_CHECK_RUNNING: + from typing import ( + Iterator, List, Optional, Tuple, Union, + ) + + from pipenv.patched.notpip._internal.models.link import Link + + SecureOrigin = Tuple[str, str, Optional[Union[int, str]]] + + +logger = logging.getLogger(__name__) + + +# Ignore warning raised when using --trusted-host. +warnings.filterwarnings("ignore", category=InsecureRequestWarning) + + +SECURE_ORIGINS = [ + # protocol, hostname, port + # Taken from Chrome's list of secure origins (See: http://bit.ly/1qrySKC) + ("https", "*", "*"), + ("*", "localhost", "*"), + ("*", "127.0.0.0/8", "*"), + ("*", "::1/128", "*"), + ("file", "*", None), + # ssh is always secure. + ("ssh", "*", "*"), +] # type: List[SecureOrigin] + + +# These are environment variables present when running under various +# CI systems. For each variable, some CI systems that use the variable +# are indicated. The collection was chosen so that for each of a number +# of popular systems, at least one of the environment variables is used. +# This list is used to provide some indication of and lower bound for +# CI traffic to PyPI. Thus, it is okay if the list is not comprehensive. +# For more background, see: https://github.com/pypa/pip/issues/5499 +CI_ENVIRONMENT_VARIABLES = ( + # Azure Pipelines + 'BUILD_BUILDID', + # Jenkins + 'BUILD_ID', + # AppVeyor, CircleCI, Codeship, Gitlab CI, Shippable, Travis CI + 'CI', + # Explicit environment variable. + 'PIP_IS_CI', +) + + +def looks_like_ci(): + # type: () -> bool + """ + Return whether it looks like pip is running under CI. + """ + # We don't use the method of checking for a tty (e.g. using isatty()) + # because some CI systems mimic a tty (e.g. Travis CI). Thus that + # method doesn't provide definitive information in either direction. + return any(name in os.environ for name in CI_ENVIRONMENT_VARIABLES) + + +def user_agent(): + """ + Return a string representing the user agent. + """ + data = { + "installer": {"name": "pip", "version": __version__}, + "python": platform.python_version(), + "implementation": { + "name": platform.python_implementation(), + }, + } + + if data["implementation"]["name"] == 'CPython': + data["implementation"]["version"] = platform.python_version() + elif data["implementation"]["name"] == 'PyPy': + if sys.pypy_version_info.releaselevel == 'final': + pypy_version_info = sys.pypy_version_info[:3] + else: + pypy_version_info = sys.pypy_version_info + data["implementation"]["version"] = ".".join( + [str(x) for x in pypy_version_info] + ) + elif data["implementation"]["name"] == 'Jython': + # Complete Guess + data["implementation"]["version"] = platform.python_version() + elif data["implementation"]["name"] == 'IronPython': + # Complete Guess + data["implementation"]["version"] = platform.python_version() + + if sys.platform.startswith("linux"): + from pipenv.patched.notpip._vendor import distro + distro_infos = dict(filter( + lambda x: x[1], + zip(["name", "version", "id"], distro.linux_distribution()), + )) + libc = dict(filter( + lambda x: x[1], + zip(["lib", "version"], libc_ver()), + )) + if libc: + distro_infos["libc"] = libc + if distro_infos: + data["distro"] = distro_infos + + if sys.platform.startswith("darwin") and platform.mac_ver()[0]: + data["distro"] = {"name": "macOS", "version": platform.mac_ver()[0]} + + if platform.system(): + data.setdefault("system", {})["name"] = platform.system() + + if platform.release(): + data.setdefault("system", {})["release"] = platform.release() + + if platform.machine(): + data["cpu"] = platform.machine() + + if HAS_TLS: + data["openssl_version"] = ssl.OPENSSL_VERSION + + setuptools_version = get_installed_version("setuptools") + if setuptools_version is not None: + data["setuptools_version"] = setuptools_version + + # Use None rather than False so as not to give the impression that + # pip knows it is not being run under CI. Rather, it is a null or + # inconclusive result. Also, we include some value rather than no + # value to make it easier to know that the check has been run. + data["ci"] = True if looks_like_ci() else None + + user_data = os.environ.get("PIP_USER_AGENT_USER_DATA") + if user_data is not None: + data["user_data"] = user_data + + return "{data[installer][name]}/{data[installer][version]} {json}".format( + data=data, + json=json.dumps(data, separators=(",", ":"), sort_keys=True), + ) + + +class LocalFSAdapter(BaseAdapter): + + def send(self, request, stream=None, timeout=None, verify=None, cert=None, + proxies=None): + pathname = url_to_path(request.url) + + resp = Response() + resp.status_code = 200 + resp.url = request.url + + try: + stats = os.stat(pathname) + except OSError as exc: + resp.status_code = 404 + resp.raw = exc + else: + modified = email.utils.formatdate(stats.st_mtime, usegmt=True) + content_type = mimetypes.guess_type(pathname)[0] or "text/plain" + resp.headers = CaseInsensitiveDict({ + "Content-Type": content_type, + "Content-Length": stats.st_size, + "Last-Modified": modified, + }) + + resp.raw = open(pathname, "rb") + resp.close = resp.raw.close + + return resp + + def close(self): + pass + + +class InsecureHTTPAdapter(HTTPAdapter): + + def cert_verify(self, conn, url, verify, cert): + conn.cert_reqs = 'CERT_NONE' + conn.ca_certs = None + + +class PipSession(requests.Session): + + timeout = None # type: Optional[int] + + def __init__(self, *args, **kwargs): + """ + :param trusted_hosts: Domains not to emit warnings for when not using + HTTPS. + """ + retries = kwargs.pop("retries", 0) + cache = kwargs.pop("cache", None) + trusted_hosts = kwargs.pop("trusted_hosts", []) # type: List[str] + index_urls = kwargs.pop("index_urls", None) + + super(PipSession, self).__init__(*args, **kwargs) + + # Namespace the attribute with "pip_" just in case to prevent + # possible conflicts with the base class. + self.pip_trusted_origins = [] # type: List[Tuple[str, Optional[int]]] + + # Attach our User Agent to the request + self.headers["User-Agent"] = user_agent() + + # Attach our Authentication handler to the session + self.auth = MultiDomainBasicAuth(index_urls=index_urls) + + # Create our urllib3.Retry instance which will allow us to customize + # how we handle retries. + retries = urllib3.Retry( + # Set the total number of retries that a particular request can + # have. + total=retries, + + # A 503 error from PyPI typically means that the Fastly -> Origin + # connection got interrupted in some way. A 503 error in general + # is typically considered a transient error so we'll go ahead and + # retry it. + # A 500 may indicate transient error in Amazon S3 + # A 520 or 527 - may indicate transient error in CloudFlare + status_forcelist=[500, 503, 520, 527], + + # Add a small amount of back off between failed requests in + # order to prevent hammering the service. + backoff_factor=0.25, + ) + + # Check to ensure that the directory containing our cache directory + # is owned by the user current executing pip. If it does not exist + # we will check the parent directory until we find one that does exist. + if cache and not check_path_owner(cache): + logger.warning( + "The directory '%s' or its parent directory is not owned by " + "the current user and the cache has been disabled. Please " + "check the permissions and owner of that directory. If " + "executing pip with sudo, you may want sudo's -H flag.", + cache, + ) + cache = None + + # We want to _only_ cache responses on securely fetched origins. We do + # this because we can't validate the response of an insecurely fetched + # origin, and we don't want someone to be able to poison the cache and + # require manual eviction from the cache to fix it. + if cache: + secure_adapter = CacheControlAdapter( + cache=SafeFileCache(cache), + max_retries=retries, + ) + else: + secure_adapter = HTTPAdapter(max_retries=retries) + + # Our Insecure HTTPAdapter disables HTTPS validation. It does not + # support caching (see above) so we'll use it for all http:// URLs as + # well as any https:// host that we've marked as ignoring TLS errors + # for. + insecure_adapter = InsecureHTTPAdapter(max_retries=retries) + # Save this for later use in add_insecure_host(). + self._insecure_adapter = insecure_adapter + + self.mount("https://", secure_adapter) + self.mount("http://", insecure_adapter) + + # Enable file:// urls + self.mount("file://", LocalFSAdapter()) + + for host in trusted_hosts: + self.add_trusted_host(host, suppress_logging=True) + + def add_trusted_host(self, host, source=None, suppress_logging=False): + # type: (str, Optional[str], bool) -> None + """ + :param host: It is okay to provide a host that has previously been + added. + :param source: An optional source string, for logging where the host + string came from. + """ + if not suppress_logging: + msg = 'adding trusted host: {!r}'.format(host) + if source is not None: + msg += ' (from {})'.format(source) + logger.info(msg) + + host_port = parse_netloc(host) + if host_port not in self.pip_trusted_origins: + self.pip_trusted_origins.append(host_port) + + self.mount(build_url_from_netloc(host) + '/', self._insecure_adapter) + if not host_port[1]: + # Mount wildcard ports for the same host. + self.mount( + build_url_from_netloc(host) + ':', + self._insecure_adapter + ) + + def iter_secure_origins(self): + # type: () -> Iterator[SecureOrigin] + for secure_origin in SECURE_ORIGINS: + yield secure_origin + for host, port in self.pip_trusted_origins: + yield ('*', host, '*' if port is None else port) + + def is_secure_origin(self, location): + # type: (Link) -> bool + # Determine if this url used a secure transport mechanism + parsed = urllib_parse.urlparse(str(location)) + origin_protocol, origin_host, origin_port = ( + parsed.scheme, parsed.hostname, parsed.port, + ) + + # The protocol to use to see if the protocol matches. + # Don't count the repository type as part of the protocol: in + # cases such as "git+ssh", only use "ssh". (I.e., Only verify against + # the last scheme.) + origin_protocol = origin_protocol.rsplit('+', 1)[-1] + + # Determine if our origin is a secure origin by looking through our + # hardcoded list of secure origins, as well as any additional ones + # configured on this PackageFinder instance. + for secure_origin in self.iter_secure_origins(): + secure_protocol, secure_host, secure_port = secure_origin + if origin_protocol != secure_protocol and secure_protocol != "*": + continue + + try: + # We need to do this decode dance to ensure that we have a + # unicode object, even on Python 2.x. + addr = ipaddress.ip_address( + origin_host + if ( + isinstance(origin_host, six.text_type) or + origin_host is None + ) + else origin_host.decode("utf8") + ) + network = ipaddress.ip_network( + secure_host + if isinstance(secure_host, six.text_type) + # setting secure_host to proper Union[bytes, str] + # creates problems in other places + else secure_host.decode("utf8") # type: ignore + ) + except ValueError: + # We don't have both a valid address or a valid network, so + # we'll check this origin against hostnames. + if ( + origin_host and + origin_host.lower() != secure_host.lower() and + secure_host != "*" + ): + continue + else: + # We have a valid address and network, so see if the address + # is contained within the network. + if addr not in network: + continue + + # Check to see if the port matches. + if ( + origin_port != secure_port and + secure_port != "*" and + secure_port is not None + ): + continue + + # If we've gotten here, then this origin matches the current + # secure origin and we should return True + return True + + # If we've gotten to this point, then the origin isn't secure and we + # will not accept it as a valid location to search. We will however + # log a warning that we are ignoring it. + logger.warning( + "The repository located at %s is not a trusted or secure host and " + "is being ignored. If this repository is available via HTTPS we " + "recommend you use HTTPS instead, otherwise you may silence " + "this warning and allow it anyway with '--trusted-host %s'.", + origin_host, + origin_host, + ) + + return False + + def request(self, method, url, *args, **kwargs): + # Allow setting a default timeout on a session + kwargs.setdefault("timeout", self.timeout) + + # Dispatch the actual request + return super(PipSession, self).request(method, url, *args, **kwargs) diff --git a/pipenv/patched/notpip/_internal/network/xmlrpc.py b/pipenv/patched/notpip/_internal/network/xmlrpc.py new file mode 100644 index 00000000..a519d744 --- /dev/null +++ b/pipenv/patched/notpip/_internal/network/xmlrpc.py @@ -0,0 +1,44 @@ +"""xmlrpclib.Transport implementation +""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import logging + +from pipenv.patched.notpip._vendor import requests +# NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import +from pipenv.patched.notpip._vendor.six.moves import xmlrpc_client # type: ignore +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +logger = logging.getLogger(__name__) + + +class PipXmlrpcTransport(xmlrpc_client.Transport): + """Provide a `xmlrpclib.Transport` implementation via a `PipSession` + object. + """ + + def __init__(self, index_url, session, use_datetime=False): + xmlrpc_client.Transport.__init__(self, use_datetime) + index_parts = urllib_parse.urlparse(index_url) + self._scheme = index_parts.scheme + self._session = session + + def request(self, host, handler, request_body, verbose=False): + parts = (self._scheme, host, handler, None, None, None) + url = urllib_parse.urlunparse(parts) + try: + headers = {'Content-Type': 'text/xml'} + response = self._session.post(url, data=request_body, + headers=headers, stream=True) + response.raise_for_status() + self.verbose = verbose + return self.parse_response(response.raw) + except requests.HTTPError as exc: + logger.critical( + "HTTP error %s while getting %s", + exc.response.status_code, url, + ) + raise diff --git a/pipenv/patched/notpip/_internal/operations/check.py b/pipenv/patched/notpip/_internal/operations/check.py index 9c8ea08e..9f2fb187 100644 --- a/pipenv/patched/notpip/_internal/operations/check.py +++ b/pipenv/patched/notpip/_internal/operations/check.py @@ -1,18 +1,28 @@ """Validation of dependencies of packages """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +import logging from collections import namedtuple from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name +from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError -from pipenv.patched.notpip._internal.operations.prepare import make_abstract_dist +from pipenv.patched.notpip._internal.distributions import ( + make_distribution_for_install_requirement, +) from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +logger = logging.getLogger(__name__) + if MYPY_CHECK_RUNNING: - from pipenv.patched.notpip._internal.req.req_install import InstallRequirement # noqa: F401 - from typing import ( # noqa: F401 - Any, Callable, Dict, Iterator, Optional, Set, Tuple, List + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from typing import ( + Any, Callable, Dict, Optional, Set, Tuple, List ) # Shorthands @@ -28,7 +38,7 @@ PackageDetails = namedtuple('PackageDetails', ['version', 'requires']) def create_package_set_from_installed(**kwargs): - # type: (**Any) -> PackageSet + # type: (**Any) -> Tuple[PackageSet, bool] """Converts a list of distributions into a PackageSet. """ # Default to using all packages installed on the system @@ -36,10 +46,16 @@ def create_package_set_from_installed(**kwargs): kwargs = {"local_only": False, "skip": ()} package_set = {} + problems = False for dist in get_installed_distributions(**kwargs): name = canonicalize_name(dist.project_name) - package_set[name] = PackageDetails(dist.version, dist.requires()) - return package_set + try: + package_set[name] = PackageDetails(dist.version, dist.requires()) + except RequirementParseError as e: + # Don't crash on broken metadata + logging.warning("Error parsing requirements for %s: %s", name, e) + problems = True + return package_set, problems def check_package_set(package_set, should_ignore=None): @@ -53,8 +69,8 @@ def check_package_set(package_set, should_ignore=None): def should_ignore(name): return False - missing = dict() - conflicting = dict() + missing = {} + conflicting = {} for package_name in package_set: # Info about dependencies of package_name @@ -95,7 +111,7 @@ def check_install_conflicts(to_install): installing given requirements """ # Start from the current state - package_set = create_package_set_from_installed() + package_set, _ = create_package_set_from_installed() # Install packages would_be_installed = _simulate_installation_of(to_install, package_set) @@ -110,9 +126,6 @@ def check_install_conflicts(to_install): ) -# NOTE from @pradyunsg -# This required a minor update in dependency link handling logic over at -# operations.prepare.IsSDist.dist() to get it working def _simulate_installation_of(to_install, package_set): # type: (List[InstallRequirement], PackageSet) -> Set[str] """Computes the version of packages after installing to_install. @@ -123,7 +136,9 @@ def _simulate_installation_of(to_install, package_set): # Modify it as installing requirement_set would (assuming no errors) for inst_req in to_install: - dist = make_abstract_dist(inst_req).dist(finder=None) + abstract_dist = make_distribution_for_install_requirement(inst_req) + dist = abstract_dist.get_pkg_resources_distribution() + name = canonicalize_name(dist.key) package_set[name] = PackageDetails(dist.version, dist.requires()) diff --git a/pipenv/patched/notpip/_internal/operations/freeze.py b/pipenv/patched/notpip/_internal/operations/freeze.py index b18b98e4..0fe5399f 100644 --- a/pipenv/patched/notpip/_internal/operations/freeze.py +++ b/pipenv/patched/notpip/_internal/operations/freeze.py @@ -1,3 +1,7 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import collections @@ -5,61 +9,73 @@ import logging import os import re -from pipenv.patched.notpip._vendor import pkg_resources, six +from pipenv.patched.notpip._vendor import six from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError -from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.exceptions import BadCommand, InstallationError from pipenv.patched.notpip._internal.req.constructors import ( - install_req_from_editable, install_req_from_line, + install_req_from_editable, + install_req_from_line, ) from pipenv.patched.notpip._internal.req.req_file import COMMENT_RE -from pipenv.patched.notpip._internal.utils.deprecation import deprecated from pipenv.patched.notpip._internal.utils.misc import ( - dist_is_editable, get_installed_distributions, make_vcs_requirement_url, + dist_is_editable, + get_installed_distributions, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Iterator, Optional, List, Container, Set, Dict, Tuple, Iterable, Union + ) + from pipenv.patched.notpip._internal.cache import WheelCache + from pipenv.patched.notpip._vendor.pkg_resources import ( + Distribution, Requirement + ) + + RequirementInfo = Tuple[Optional[Union[str, Requirement]], bool, List[str]] + logger = logging.getLogger(__name__) def freeze( - requirement=None, - find_links=None, local_only=None, user_only=None, skip_regex=None, - isolated=False, - wheel_cache=None, - exclude_editable=False, - skip=()): + requirement=None, # type: Optional[List[str]] + find_links=None, # type: Optional[List[str]] + local_only=None, # type: Optional[bool] + user_only=None, # type: Optional[bool] + paths=None, # type: Optional[List[str]] + skip_regex=None, # type: Optional[str] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + exclude_editable=False, # type: bool + skip=() # type: Container[str] +): + # type: (...) -> Iterator[str] find_links = find_links or [] skip_match = None if skip_regex: skip_match = re.compile(skip_regex).search - dependency_links = [] - - for dist in pkg_resources.working_set: - if dist.has_metadata('dependency_links.txt'): - dependency_links.extend( - dist.get_metadata_lines('dependency_links.txt') - ) - for link in find_links: - if '#egg=' in link: - dependency_links.append(link) for link in find_links: yield '-f %s' % link - installations = {} + installations = {} # type: Dict[str, FrozenRequirement] for dist in get_installed_distributions(local_only=local_only, skip=(), - user_only=user_only): + user_only=user_only, + paths=paths): try: - req = FrozenRequirement.from_dist( - dist, - dependency_links - ) - except RequirementParseError: + req = FrozenRequirement.from_dist(dist) + except RequirementParseError as exc: + # We include dist rather than dist.project_name because the + # dist string includes more information, like the version and + # location. We also include the exception message to aid + # troubleshooting. logger.warning( - "Could not parse requirement: %s", - dist.project_name + 'Could not generate requirement for distribution %r: %s', + dist, exc ) continue if exclude_editable and req.editable: @@ -71,10 +87,10 @@ def freeze( # should only be emitted once, even if the same option is in multiple # requirements files, so we need to keep track of what has been emitted # so that we don't emit it again if it's seen again - emitted_options = set() + emitted_options = set() # type: Set[str] # keep track of which files a requirement is in so that we can # give an accurate warning if a requirement appears multiple times. - req_files = collections.defaultdict(list) + req_files = collections.defaultdict(list) # type: Dict[str, List[str]] for req_file_path in requirement: with open(req_file_path) as req_file: for line in req_file: @@ -128,10 +144,10 @@ def freeze( # but has been processed already if not req_files[line_req.name]: logger.warning( - "Requirement file [%s] contains %s, but that " - "package is not installed", + "Requirement file [%s] contains %s, but " + "package %r is not installed", req_file_path, - COMMENT_RE.sub('', line).strip(), + COMMENT_RE.sub('', line).strip(), line_req.name ) else: req_files[line_req.name].append(req_file_path) @@ -157,105 +173,84 @@ def freeze( yield str(installation).rstrip() +def get_requirement_info(dist): + # type: (Distribution) -> RequirementInfo + """ + Compute and return values (req, editable, comments) for use in + FrozenRequirement.from_dist(). + """ + if not dist_is_editable(dist): + return (None, False, []) + + location = os.path.normcase(os.path.abspath(dist.location)) + + from pipenv.patched.notpip._internal.vcs import vcs, RemoteNotFoundError + vcs_backend = vcs.get_backend_for_dir(location) + + if vcs_backend is None: + req = dist.as_requirement() + logger.debug( + 'No VCS found for editable requirement "%s" in: %r', req, + location, + ) + comments = [ + '# Editable install with no version control ({})'.format(req) + ] + return (location, True, comments) + + try: + req = vcs_backend.get_src_requirement(location, dist.project_name) + except RemoteNotFoundError: + req = dist.as_requirement() + comments = [ + '# Editable {} install with no remote ({})'.format( + type(vcs_backend).__name__, req, + ) + ] + return (location, True, comments) + + except BadCommand: + logger.warning( + 'cannot determine version of editable source in %s ' + '(%s command not found in path)', + location, + vcs_backend.name, + ) + return (None, True, []) + + except InstallationError as exc: + logger.warning( + "Error when trying to get requirement for VCS system %s, " + "falling back to uneditable format", exc + ) + else: + if req is not None: + return (req, True, []) + + logger.warning( + 'Could not determine repository location of %s', location + ) + comments = ['## !! Could not determine repository location'] + + return (None, False, comments) + + class FrozenRequirement(object): def __init__(self, name, req, editable, comments=()): + # type: (str, Union[str, Requirement], bool, Iterable[str]) -> None self.name = name self.req = req self.editable = editable self.comments = comments - _rev_re = re.compile(r'-r(\d+)$') - _date_re = re.compile(r'-(20\d\d\d\d\d\d)$') - @classmethod - def _init_args_from_dist(cls, dist, dependency_links): - """ - Compute and return arguments (req, editable, comments) to pass to - FrozenRequirement.__init__(). - - This method is for use in FrozenRequirement.from_dist(). - """ - location = os.path.normcase(os.path.abspath(dist.location)) - comments = [] - from pipenv.patched.notpip._internal.vcs import vcs, get_src_requirement - if dist_is_editable(dist) and vcs.get_backend_name(location): - editable = True - try: - req = get_src_requirement(dist, location) - except InstallationError as exc: - logger.warning( - "Error when trying to get requirement for VCS system %s, " - "falling back to uneditable format", exc - ) - req = None - if req is None: - logger.warning( - 'Could not determine repository location of %s', location - ) - comments.append( - '## !! Could not determine repository location' - ) - req = dist.as_requirement() - editable = False - else: - editable = False + def from_dist(cls, dist): + # type: (Distribution) -> FrozenRequirement + req, editable, comments = get_requirement_info(dist) + if req is None: req = dist.as_requirement() - specs = req.specs - assert len(specs) == 1 and specs[0][0] in ["==", "==="], \ - 'Expected 1 spec with == or ===; specs = %r; dist = %r' % \ - (specs, dist) - version = specs[0][1] - ver_match = cls._rev_re.search(version) - date_match = cls._date_re.search(version) - if ver_match or date_match: - svn_backend = vcs.get_backend('svn') - if svn_backend: - svn_location = svn_backend().get_location( - dist, - dependency_links, - ) - if not svn_location: - logger.warning( - 'Warning: cannot find svn location for %s', req, - ) - comments.append( - '## FIXME: could not find svn URL in dependency_links ' - 'for this package:' - ) - else: - deprecated( - "SVN editable detection based on dependency links " - "will be dropped in the future.", - replacement=None, - gone_in="18.2", - issue=4187, - ) - comments.append( - '# Installing as editable to satisfy requirement %s:' % - req - ) - if ver_match: - rev = ver_match.group(1) - else: - rev = '{%s}' % date_match.group(1) - editable = True - egg_name = cls.egg_name(dist) - req = make_vcs_requirement_url(svn_location, rev, egg_name) - return (req, editable, comments) - - @classmethod - def from_dist(cls, dist, dependency_links): - args = cls._init_args_from_dist(dist, dependency_links) - return cls(dist.project_name, *args) - - @staticmethod - def egg_name(dist): - name = dist.egg_name() - match = re.search(r'-py\d\.\d$', name) - if match: - name = name[:match.start()] - return name + return cls(dist.project_name, req, editable, comments=comments) def __str__(self): req = self.req diff --git a/pipenv/patched/notpip/_internal/operations/generate_metadata.py b/pipenv/patched/notpip/_internal/operations/generate_metadata.py new file mode 100644 index 00000000..dd30f553 --- /dev/null +++ b/pipenv/patched/notpip/_internal/operations/generate_metadata.py @@ -0,0 +1,136 @@ +"""Metadata generation logic for source distributions. +""" + +import logging +import os + +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.misc import ensure_dir +from pipenv.patched.notpip._internal.utils.setuptools_build import make_setuptools_shim_args +from pipenv.patched.notpip._internal.utils.subprocess import call_subprocess +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.vcs import vcs + +if MYPY_CHECK_RUNNING: + from typing import Callable, List + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + +logger = logging.getLogger(__name__) + + +def get_metadata_generator(install_req): + # type: (InstallRequirement) -> Callable[[InstallRequirement], str] + """Return a callable metadata generator for this InstallRequirement. + + A metadata generator takes an InstallRequirement (install_req) as an input, + generates metadata via the appropriate process for that install_req and + returns the generated metadata directory. + """ + if not install_req.use_pep517: + return _generate_metadata_legacy + + return _generate_metadata + + +def _find_egg_info(source_directory, is_editable): + # type: (str, bool) -> str + """Find an .egg-info in `source_directory`, based on `is_editable`. + """ + + def looks_like_virtual_env(path): + # type: (str) -> bool + return ( + os.path.lexists(os.path.join(path, 'bin', 'python')) or + os.path.exists(os.path.join(path, 'Scripts', 'Python.exe')) + ) + + def locate_editable_egg_info(base): + # type: (str) -> List[str] + candidates = [] # type: List[str] + for root, dirs, files in os.walk(base): + for dir_ in vcs.dirnames: + if dir_ in dirs: + dirs.remove(dir_) + # Iterate over a copy of ``dirs``, since mutating + # a list while iterating over it can cause trouble. + # (See https://github.com/pypa/pip/pull/462.) + for dir_ in list(dirs): + if looks_like_virtual_env(os.path.join(root, dir_)): + dirs.remove(dir_) + # Also don't search through tests + elif dir_ == 'test' or dir_ == 'tests': + dirs.remove(dir_) + candidates.extend(os.path.join(root, dir_) for dir_ in dirs) + return [f for f in candidates if f.endswith('.egg-info')] + + def depth_of_directory(dir_): + # type: (str) -> int + return ( + dir_.count(os.path.sep) + + (os.path.altsep and dir_.count(os.path.altsep) or 0) + ) + + base = source_directory + if is_editable: + filenames = locate_editable_egg_info(base) + else: + base = os.path.join(base, 'pip-egg-info') + filenames = os.listdir(base) + + if not filenames: + raise InstallationError( + "Files/directories not found in %s" % base + ) + + # If we have more than one match, we pick the toplevel one. This + # can easily be the case if there is a dist folder which contains + # an extracted tarball for testing purposes. + if len(filenames) > 1: + filenames.sort(key=depth_of_directory) + + return os.path.join(base, filenames[0]) + + +def _generate_metadata_legacy(install_req): + # type: (InstallRequirement) -> str + req_details_str = install_req.name or "from {}".format(install_req.link) + logger.debug( + 'Running setup.py (path:%s) egg_info for package %s', + install_req.setup_py_path, req_details_str, + ) + + # Compose arguments for subprocess call + base_cmd = make_setuptools_shim_args(install_req.setup_py_path) + if install_req.isolated: + base_cmd += ["--no-user-cfg"] + + # For non-editable installs, don't put the .egg-info files at the root, + # to avoid confusion due to the source code being considered an installed + # egg. + egg_base_option = [] # type: List[str] + if not install_req.editable: + egg_info_dir = os.path.join( + install_req.unpacked_source_directory, 'pip-egg-info', + ) + egg_base_option = ['--egg-base', egg_info_dir] + + # setuptools complains if the target directory does not exist. + ensure_dir(egg_info_dir) + + with install_req.build_env: + call_subprocess( + base_cmd + ["egg_info"] + egg_base_option, + cwd=install_req.unpacked_source_directory, + command_desc='python setup.py egg_info', + ) + + # Return the .egg-info directory. + return _find_egg_info( + install_req.unpacked_source_directory, + install_req.editable, + ) + + +def _generate_metadata(install_req): + # type: (InstallRequirement) -> str + return install_req.prepare_pep517_metadata() diff --git a/pipenv/patched/notpip/_internal/operations/prepare.py b/pipenv/patched/notpip/_internal/operations/prepare.py index d61270b4..6128f9b9 100644 --- a/pipenv/patched/notpip/_internal/operations/prepare.py +++ b/pipenv/patched/notpip/_internal/operations/prepare.py @@ -1,159 +1,80 @@ """Prepares a distribution for installation """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + import logging import os -from pipenv.patched.notpip._vendor import pkg_resources, requests +from pipenv.patched.notpip._vendor import requests -from pipenv.patched.notpip._internal.build_env import BuildEnvironment -from pipenv.patched.notpip._internal.download import ( - is_dir_url, is_file_url, is_vcs_url, unpack_url, url_to_path, +from pipenv.patched.notpip._internal.distributions import ( + make_distribution_for_install_requirement, ) +from pipenv.patched.notpip._internal.distributions.installed import InstalledDistribution +from pipenv.patched.notpip._internal.download import unpack_url from pipenv.patched.notpip._internal.exceptions import ( - DirectoryUrlHashUnsupported, HashUnpinned, InstallationError, - PreviousBuildDirError, VcsHashUnsupported, + DirectoryUrlHashUnsupported, + HashUnpinned, + InstallationError, + PreviousBuildDirError, + VcsHashUnsupported, ) from pipenv.patched.notpip._internal.utils.compat import expanduser from pipenv.patched.notpip._internal.utils.hashes import MissingHashes from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path, rmtree -from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.utils.marker_files import write_delete_marker_file +from pipenv.patched.notpip._internal.utils.misc import display_path, normalize_path +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional + + from pipenv.patched.notpip._internal.distributions import AbstractDistribution + from pipenv.patched.notpip._internal.index import PackageFinder + from pipenv.patched.notpip._internal.network.session import PipSession + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker logger = logging.getLogger(__name__) -def make_abstract_dist(req): - """Factory to make an abstract dist object. - - Preconditions: Either an editable req with a source_dir, or satisfied_by or - a wheel link, or a non-editable req with a source_dir. - - :return: A concrete DistAbstraction. +def _get_prepared_distribution(req, req_tracker, finder, build_isolation): + """Prepare a distribution for installation. """ - if req.editable: - return IsSDist(req) - elif req.link and req.link.is_wheel: - return IsWheel(req) - else: - return IsSDist(req) - - -class DistAbstraction(object): - """Abstracts out the wheel vs non-wheel Resolver.resolve() logic. - - The requirements for anything installable are as follows: - - we must be able to determine the requirement name - (or we can't correctly handle the non-upgrade case). - - we must be able to generate a list of run-time dependencies - without installing any additional packages (or we would - have to either burn time by doing temporary isolated installs - or alternatively violate pips 'don't start installing unless - all requirements are available' rule - neither of which are - desirable). - - for packages with setup requirements, we must also be able - to determine their requirements without installing additional - packages (for the same reason as run-time dependencies) - - we must be able to create a Distribution object exposing the - above metadata. - """ - - def __init__(self, req): - self.req = req - - def dist(self, finder): - """Return a setuptools Dist object.""" - raise NotImplementedError(self.dist) - - def prep_for_dist(self, finder, build_isolation): - """Ensure that we can get a Dist for this requirement.""" - raise NotImplementedError(self.dist) - - -class IsWheel(DistAbstraction): - - def dist(self, finder): - return list(pkg_resources.find_distributions( - self.req.source_dir))[0] - - def prep_for_dist(self, finder, build_isolation): - # FIXME:https://github.com/pypa/pip/issues/1112 - pass - - -class IsSDist(DistAbstraction): - - def dist(self, finder): - dist = self.req.get_dist() - # FIXME: shouldn't be globally added. - if finder and dist.has_metadata('dependency_links.txt'): - finder.add_dependency_links( - dist.get_metadata_lines('dependency_links.txt') - ) - return dist - - def prep_for_dist(self, finder, build_isolation): - # Prepare for building. We need to: - # 1. Load pyproject.toml (if it exists) - # 2. Set up the build environment - - self.req.load_pyproject_toml() - should_isolate = self.req.use_pep517 and build_isolation - - if should_isolate: - # Isolate in a BuildEnvironment and install the build-time - # requirements. - self.req.build_env = BuildEnvironment() - self.req.build_env.install_requirements( - finder, self.req.pyproject_requires, - "Installing build dependencies" - ) - missing = [] - if self.req.requirements_to_check: - check = self.req.requirements_to_check - missing = self.req.build_env.missing_requirements(check) - if missing: - logger.warning( - "Missing build requirements in pyproject.toml for %s.", - self.req, - ) - logger.warning( - "The project does not specify a build backend, and pip " - "cannot fall back to setuptools without %s.", - " and ".join(map(repr, sorted(missing))) - ) - - try: - self.req.run_egg_info() - except (OSError, TypeError): - self.req._correct_build_location() - self.req.run_egg_info() - self.req.assert_source_matches_version() - - -class Installed(DistAbstraction): - - def dist(self, finder): - return self.req.satisfied_by - - def prep_for_dist(self, finder, build_isolation): - pass + abstract_dist = make_distribution_for_install_requirement(req) + with req_tracker.track(req): + abstract_dist.prepare_distribution_metadata(finder, build_isolation) + return abstract_dist class RequirementPreparer(object): """Prepares a Requirement """ - def __init__(self, build_dir, download_dir, src_dir, wheel_download_dir, - progress_bar, build_isolation, req_tracker): + def __init__( + self, + build_dir, # type: str + download_dir, # type: Optional[str] + src_dir, # type: str + wheel_download_dir, # type: Optional[str] + progress_bar, # type: str + build_isolation, # type: bool + req_tracker # type: RequirementTracker + ): + # type: (...) -> None super(RequirementPreparer, self).__init__() self.src_dir = src_dir self.build_dir = build_dir self.req_tracker = req_tracker - # Where still packed archives should be written to. If None, they are + # Where still-packed archives should be written to. If None, they are # not saved, and are deleted immediately after unpacking. + if download_dir: + download_dir = expanduser(download_dir) self.download_dir = download_dir # Where still-packed .whl files should be written to. If None, they are @@ -175,28 +96,37 @@ class RequirementPreparer(object): @property def _download_should_save(self): - # TODO: Modify to reduce indentation needed - if self.download_dir: - self.download_dir = expanduser(self.download_dir) - if os.path.exists(self.download_dir): - return True - else: - logger.critical('Could not find download directory') - raise InstallationError( - "Could not find or access download directory '%s'" - % display_path(self.download_dir)) - return False + # type: () -> bool + if not self.download_dir: + return False - def prepare_linked_requirement(self, req, session, finder, - upgrade_allowed, require_hashes): + if os.path.exists(self.download_dir): + return True + + logger.critical('Could not find download directory') + raise InstallationError( + "Could not find or access download directory '%s'" + % display_path(self.download_dir)) + + def prepare_linked_requirement( + self, + req, # type: InstallRequirement + session, # type: PipSession + finder, # type: PackageFinder + require_hashes, # type: bool + ): + # type: (...) -> AbstractDistribution """Prepare a requirement that would be obtained from req.link """ + assert req.link + link = req.link + # TODO: Breakup into smaller functions - if req.link and req.link.scheme == 'file': - path = url_to_path(req.link.url) + if link.scheme == 'file': + path = link.file_path logger.info('Processing %s', display_path(path)) else: - logger.info('Collecting %s', req) + logger.info('Collecting %s', req.req or req) with indent_log(): # @@ if filesystem packages are not marked @@ -211,17 +141,6 @@ class RequirementPreparer(object): # package unpacked in `req.source_dir` if os.path.exists(os.path.join(req.source_dir, 'setup.py')): rmtree(req.source_dir) - req.populate_link(finder, upgrade_allowed, require_hashes) - - # We can't hit this spot and have populate_link return None. - # req.satisfied_by is None here (because we're - # guarded) and upgrade has no impact except when satisfied_by - # is not None. - # Then inside find_requirement existing_applicable -> False - # If no new versions are found, DistributionNotFound is raised, - # otherwise a result is guaranteed. - assert req.link - link = req.link # Now that we have the real link, we can tell what kind of # requirements we have and raise some more informative errors @@ -233,9 +152,9 @@ class RequirementPreparer(object): # we would report less-useful error messages for # unhashable requirements, complaining that there's no # hash provided. - if is_vcs_url(link): + if link.is_vcs: raise VcsHashUnsupported() - elif is_file_url(link) and is_dir_url(link): + elif link.is_existing_dir(): raise DirectoryUrlHashUnsupported() if not req.original_link and not req.is_pinned: # Unpinned packages are asking for trouble when a new @@ -255,26 +174,15 @@ class RequirementPreparer(object): # showing the user what the hash should be. hashes = MissingHashes() + download_dir = self.download_dir + if link.is_wheel and self.wheel_download_dir: + # when doing 'pip wheel` we download wheels to a + # dedicated dir. + download_dir = self.wheel_download_dir + try: - download_dir = self.download_dir - # We always delete unpacked sdists after pip ran. - autodelete_unpacked = True - if req.link.is_wheel and self.wheel_download_dir: - # when doing 'pip wheel` we download wheels to a - # dedicated dir. - download_dir = self.wheel_download_dir - if req.link.is_wheel: - if download_dir: - # When downloading, we only unpack wheels to get - # metadata. - autodelete_unpacked = True - else: - # When installing a wheel, we use the unpacked - # wheel. - autodelete_unpacked = False unpack_url( - req.link, req.source_dir, - download_dir, autodelete_unpacked, + link, req.source_dir, download_dir, session=session, hashes=hashes, progress_bar=self.progress_bar ) @@ -287,19 +195,42 @@ class RequirementPreparer(object): raise InstallationError( 'Could not install requirement %s because of HTTP ' 'error %s for URL %s' % - (req, exc, req.link) + (req, exc, link) ) - abstract_dist = make_abstract_dist(req) - with self.req_tracker.track(req): - abstract_dist.prep_for_dist(finder, self.build_isolation) + + if link.is_wheel: + if download_dir: + # When downloading, we only unpack wheels to get + # metadata. + autodelete_unpacked = True + else: + # When installing a wheel, we use the unpacked + # wheel. + autodelete_unpacked = False + else: + # We always delete unpacked sdists after pip runs. + autodelete_unpacked = True + if autodelete_unpacked: + write_delete_marker_file(req.source_dir) + + abstract_dist = _get_prepared_distribution( + req, self.req_tracker, finder, self.build_isolation, + ) + if self._download_should_save: # Make a .zip of the source_dir we already created. - if req.link.scheme in vcs.all_schemes: + if link.is_vcs: req.archive(self.download_dir) return abstract_dist - def prepare_editable_requirement(self, req, require_hashes, use_user_site, - finder): + def prepare_editable_requirement( + self, + req, # type: InstallRequirement + require_hashes, # type: bool + use_user_site, # type: bool + finder # type: PackageFinder + ): + # type: (...) -> AbstractDistribution """Prepare an editable requirement """ assert req.editable, "cannot prepare a non-editable req as editable" @@ -316,9 +247,9 @@ class RequirementPreparer(object): req.ensure_has_source_dir(self.src_dir) req.update_editable(not self._download_should_save) - abstract_dist = make_abstract_dist(req) - with self.req_tracker.track(req): - abstract_dist.prep_for_dist(finder, self.build_isolation) + abstract_dist = _get_prepared_distribution( + req, self.req_tracker, finder, self.build_isolation, + ) if self._download_should_save: req.archive(self.download_dir) @@ -326,7 +257,13 @@ class RequirementPreparer(object): return abstract_dist - def prepare_installed_requirement(self, req, require_hashes, skip_reason): + def prepare_installed_requirement( + self, + req, # type: InstallRequirement + require_hashes, # type: bool + skip_reason # type: str + ): + # type: (...) -> AbstractDistribution """Prepare an already-installed requirement """ assert req.satisfied_by, "req should have been satisfied but isn't" @@ -346,6 +283,6 @@ class RequirementPreparer(object): 'completely repeatable environment, install into an ' 'empty virtualenv.' ) - abstract_dist = Installed(req) + abstract_dist = InstalledDistribution(req) return abstract_dist diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py index 182c1c88..c2a1e346 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py @@ -10,12 +10,16 @@ import sysconfig import warnings from collections import OrderedDict -try: - import pipenv.patched.notpip._internal.utils.glibc -except ImportError: - import pipenv.patched.notpip.utils.glibc - +import pipenv.patched.notpip._internal.utils.glibc from pipenv.patched.notpip._internal.utils.compat import get_extension_suffixes +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Tuple, Callable, List, Optional, Union, Dict, Set + ) + + Pep425Tag = Tuple[str, str, str] logger = logging.getLogger(__name__) @@ -23,6 +27,7 @@ _osx_arch_pat = re.compile(r'(.+)_(\d+)_(\d+)_(.+)') def get_config_var(var): + # type: (str) -> Optional[str] try: return sysconfig.get_config_var(var) except IOError as e: # Issue #1074 @@ -31,6 +36,7 @@ def get_config_var(var): def get_abbr_impl(): + # type: () -> str """Return abbreviated implementation name.""" if hasattr(sys, 'pypy_version_info'): pyimpl = 'pp' @@ -43,7 +49,14 @@ def get_abbr_impl(): return pyimpl +def version_info_to_nodot(version_info): + # type: (Tuple[int, ...]) -> str + # Only use up to the first two numbers. + return ''.join(map(str, version_info[:2])) + + def get_impl_ver(): + # type: () -> str """Return implementation version.""" impl_ver = get_config_var("py_version_nodot") if not impl_ver or get_abbr_impl() == 'pp': @@ -52,17 +65,21 @@ def get_impl_ver(): def get_impl_version_info(): + # type: () -> Tuple[int, ...] """Return sys.version_info-like tuple for use in decrementing the minor version.""" if get_abbr_impl() == 'pp': # as per https://github.com/pypa/pip/issues/2882 - return (sys.version_info[0], sys.pypy_version_info.major, - sys.pypy_version_info.minor) + # attrs exist only on pypy + return (sys.version_info[0], + sys.pypy_version_info.major, # type: ignore + sys.pypy_version_info.minor) # type: ignore else: return sys.version_info[0], sys.version_info[1] def get_impl_tag(): + # type: () -> str """ Returns the Tag for this specific implementation. """ @@ -70,6 +87,7 @@ def get_impl_tag(): def get_flag(var, fallback, expected=True, warn=True): + # type: (str, Callable[..., bool], Union[bool, int], bool) -> bool """Use a fallback method for determining SOABI flags if the needed config var is unset or unavailable.""" val = get_config_var(var) @@ -82,44 +100,45 @@ def get_flag(var, fallback, expected=True, warn=True): def get_abi_tag(): + # type: () -> Optional[str] """Return the ABI tag based on SOABI (if available) or emulate SOABI (CPython 2, PyPy).""" soabi = get_config_var('SOABI') impl = get_abbr_impl() + abi = None # type: Optional[str] + if not soabi and impl in {'cp', 'pp'} and hasattr(sys, 'maxunicode'): d = '' m = '' u = '' - if get_flag('Py_DEBUG', - lambda: hasattr(sys, 'gettotalrefcount'), - warn=(impl == 'cp')): + is_cpython = (impl == 'cp') + if get_flag( + 'Py_DEBUG', lambda: hasattr(sys, 'gettotalrefcount'), + warn=is_cpython): d = 'd' - if get_flag('WITH_PYMALLOC', - lambda: impl == 'cp', - warn=(impl == 'cp')): + if sys.version_info < (3, 8) and get_flag( + 'WITH_PYMALLOC', lambda: is_cpython, warn=is_cpython): m = 'm' - if get_flag('Py_UNICODE_SIZE', - lambda: sys.maxunicode == 0x10ffff, - expected=4, - warn=(impl == 'cp' and - sys.version_info < (3, 3))) \ - and sys.version_info < (3, 3): + if sys.version_info < (3, 3) and get_flag( + 'Py_UNICODE_SIZE', lambda: sys.maxunicode == 0x10ffff, + expected=4, warn=is_cpython): u = 'u' abi = '%s%s%s%s%s' % (impl, get_impl_ver(), d, m, u) elif soabi and soabi.startswith('cpython-'): abi = 'cp' + soabi.split('-')[1] elif soabi: abi = soabi.replace('.', '_').replace('-', '_') - else: - abi = None + return abi def _is_running_32bit(): + # type: () -> bool return sys.maxsize == 2147483647 def get_platform(): + # type: () -> str """Return our platform name 'win32', 'linux_x86_64'""" if sys.platform == 'darwin': # distutils.util.get_platform() returns the release based on the value @@ -145,7 +164,35 @@ def get_platform(): return result +def is_linux_armhf(): + # type: () -> bool + if get_platform() != "linux_armv7l": + return False + # hard-float ABI can be detected from the ELF header of the running + # process + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + try: + with open(sys_executable, 'rb') as f: + elf_header_raw = f.read(40) # read 40 first bytes of ELF header + except (IOError, OSError, TypeError): + return False + if elf_header_raw is None or len(elf_header_raw) < 40: + return False + if isinstance(elf_header_raw, str): + elf_header = [ord(c) for c in elf_header_raw] + else: + elf_header = [b for b in elf_header_raw] + result = elf_header[0:4] == [0x7f, 0x45, 0x4c, 0x46] # ELF magic number + result &= elf_header[4:5] == [1] # 32-bit ELF + result &= elf_header[5:6] == [1] # little-endian + result &= elf_header[18:20] == [0x28, 0] # ARM machine + result &= elf_header[39:40] == [5] # ARM EABIv5 + result &= (elf_header[37:38][0] & 4) == 4 # EF_ARM_ABI_FLOAT_HARD + return result + + def is_manylinux1_compatible(): + # type: () -> bool # Only Linux, and only x86-64 / i686 if get_platform() not in {"linux_x86_64", "linux_i686"}: return False @@ -162,13 +209,59 @@ def is_manylinux1_compatible(): return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) +def is_manylinux2010_compatible(): + # type: () -> bool + # Only Linux, and only x86-64 / i686 + if get_platform() not in {"linux_x86_64", "linux_i686"}: + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2010_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + +def is_manylinux2014_compatible(): + # type: () -> bool + # Only Linux, and only supported architectures + platform = get_platform() + if platform not in {"linux_x86_64", "linux_i686", "linux_aarch64", + "linux_armv7l", "linux_ppc64", "linux_ppc64le", + "linux_s390x"}: + return False + + # check for hard-float ABI in case we're running linux_armv7l not to + # install hard-float ABI wheel in a soft-float ABI environment + if platform == "linux_armv7l" and not is_linux_armhf(): + return False + + # Check for presence of _manylinux module + try: + import _manylinux + return bool(_manylinux.manylinux2014_compatible) + except (ImportError, AttributeError): + # Fall through to heuristic check below + pass + + # Check glibc version. CentOS 7 uses glibc 2.17. + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 17) + + def get_darwin_arches(major, minor, machine): + # type: (int, int, str) -> List[str] """Return a list of supported arches (including group arches) for the given major, minor and machine architecture of an macOS machine. """ arches = [] def _supports_arch(major, minor, arch): + # type: (int, int, str) -> bool # Looking at the application support for macOS versions in the chart # provided by https://en.wikipedia.org/wiki/OS_X#Versions it appears # our timeline looks roughly like: @@ -209,7 +302,7 @@ def get_darwin_arches(major, minor, machine): ("intel", ("x86_64", "i386")), ("fat64", ("x86_64", "ppc64")), ("fat32", ("x86_64", "i386", "ppc")), - ]) + ]) # type: Dict[str, Tuple[str, ...]] if _supports_arch(major, minor, machine): arches.append(machine) @@ -223,8 +316,24 @@ def get_darwin_arches(major, minor, machine): return arches -def get_supported(versions=None, noarch=False, platform=None, - impl=None, abi=None): +def get_all_minor_versions_as_strings(version_info): + # type: (Tuple[int, ...]) -> List[str] + versions = [] + major = version_info[:-1] + # Support all previous minor Python versions. + for minor in range(version_info[-1], -1, -1): + versions.append(''.join(map(str, major + (minor,)))) + return versions + + +def get_supported( + versions=None, # type: Optional[List[str]] + noarch=False, # type: bool + platform=None, # type: Optional[str] + impl=None, # type: Optional[str] + abi=None # type: Optional[str] +): + # type: (...) -> List[Pep425Tag] """Return a list of supported tags for each version specified in `versions`. @@ -241,22 +350,18 @@ def get_supported(versions=None, noarch=False, platform=None, # Versions must be given with respect to the preference if versions is None: - versions = [] version_info = get_impl_version_info() - major = version_info[:-1] - # Support all previous minor Python versions. - for minor in range(version_info[-1], -1, -1): - versions.append(''.join(map(str, major + (minor,)))) + versions = get_all_minor_versions_as_strings(version_info) impl = impl or get_abbr_impl() - abis = [] + abis = [] # type: List[str] abi = abi or get_abi_tag() if abi: abis[0:0] = [abi] - abi3s = set() + abi3s = set() # type: Set[str] for suffix in get_extension_suffixes(): if suffix.startswith('.abi'): abi3s.add(suffix.split('.', 2)[1]) @@ -267,6 +372,7 @@ def get_supported(versions=None, noarch=False, platform=None, if not noarch: arch = platform or get_platform() + arch_prefix, arch_sep, arch_suffix = arch.partition('_') if arch.startswith('macosx'): # support macosx-10.6-intel on macosx-10.9-x86_64 match = _osx_arch_pat.match(arch) @@ -280,8 +386,31 @@ def get_supported(versions=None, noarch=False, platform=None, else: # arch pattern didn't match (?!) arches = [arch] - elif platform is None and is_manylinux1_compatible(): - arches = [arch.replace('linux', 'manylinux1'), arch] + elif arch_prefix == 'manylinux2014': + arches = [arch] + # manylinux1/manylinux2010 wheels run on most manylinux2014 systems + # with the exception of wheels depending on ncurses. PEP 599 states + # manylinux1/manylinux2010 wheels should be considered + # manylinux2014 wheels: + # https://www.python.org/dev/peps/pep-0599/#backwards-compatibility-with-manylinux2010-wheels + if arch_suffix in {'i686', 'x86_64'}: + arches.append('manylinux2010' + arch_sep + arch_suffix) + arches.append('manylinux1' + arch_sep + arch_suffix) + elif arch_prefix == 'manylinux2010': + # manylinux1 wheels run on most manylinux2010 systems with the + # exception of wheels depending on ncurses. PEP 571 states + # manylinux1 wheels should be considered manylinux2010 wheels: + # https://www.python.org/dev/peps/pep-0571/#backwards-compatibility-with-manylinux1-wheels + arches = [arch, 'manylinux1' + arch_sep + arch_suffix] + elif platform is None: + arches = [] + if is_manylinux2014_compatible(): + arches.append('manylinux2014' + arch_sep + arch_suffix) + if is_manylinux2010_compatible(): + arches.append('manylinux2010' + arch_sep + arch_suffix) + if is_manylinux1_compatible(): + arches.append('manylinux1' + arch_sep + arch_suffix) + arches.append(arch) else: arches = [arch] diff --git a/pipenv/patched/notpip/_internal/pyproject.py b/pipenv/patched/notpip/_internal/pyproject.py index a47e0f05..bef9c378 100644 --- a/pipenv/patched/notpip/_internal/pyproject.py +++ b/pipenv/patched/notpip/_internal/pyproject.py @@ -2,20 +2,43 @@ from __future__ import absolute_import import io import os +import sys from pipenv.patched.notpip._vendor import pytoml, six from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Any, Tuple, Optional, List def _is_list_of_str(obj): + # type: (Any) -> bool return ( isinstance(obj, list) and all(isinstance(item, six.string_types) for item in obj) ) -def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): +def make_pyproject_path(unpacked_source_directory): + # type: (str) -> str + path = os.path.join(unpacked_source_directory, 'pyproject.toml') + + # Python2 __file__ should not be unicode + if six.PY2 and isinstance(path, six.text_type): + path = path.encode(sys.getfilesystemencoding()) + + return path + + +def load_pyproject_toml( + use_pep517, # type: Optional[bool] + pyproject_toml, # type: str + setup_py, # type: str + req_name # type: str +): + # type: (...) -> Optional[Tuple[List[str], str, List[str]]] """Load the pyproject.toml file. Parameters: @@ -46,17 +69,20 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): build_system = None # The following cases must use PEP 517 - # We check for use_pep517 equalling False because that - # means the user explicitly requested --no-use-pep517 + # We check for use_pep517 being non-None and falsey because that means + # the user explicitly requested --no-use-pep517. The value 0 as + # opposed to False can occur when the value is provided via an + # environment variable or config file option (due to the quirk of + # strtobool() returning an integer in pip's configuration code). if has_pyproject and not has_setup: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project does not have a setup.py" ) use_pep517 = True elif build_system and "build-backend" in build_system: - if use_pep517 is False: + if use_pep517 is not None and not use_pep517: raise InstallationError( "Disabling PEP 517 processing is invalid: " "project specifies a build backend of {} " @@ -85,11 +111,13 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): # section, or the user has no pyproject.toml, but has opted in # explicitly via --use-pep517. # In the absence of any explicit backend specification, we - # assume the setuptools backend, and require wheel and a version - # of setuptools that supports that backend. + # assume the setuptools backend that most closely emulates the + # traditional direct setup.py execution, and require wheel and + # a version of setuptools that supports that backend. + build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } # If we're using PEP 517, we have build system information (either @@ -123,22 +151,21 @@ def load_pyproject_toml(use_pep517, pyproject_toml, setup_py, req_name): )) backend = build_system.get("build-backend") - check = [] + check = [] # type: List[str] if backend is None: # If the user didn't specify a backend, we assume they want to use # the setuptools backend. But we can't be sure they have included # a version of setuptools which supplies the backend, or wheel - # (which is neede by the backend) in their requirements. So we + # (which is needed by the backend) in their requirements. So we # make a note to check that those requirements are present once # we have set up the environment. - # TODO: Review this - it's quite a lot of work to check for a very - # specific case. The problem is, that case is potentially quite - # common - projects that adopted PEP 518 early for the ability to - # specify requirements to execute setup.py, but never considered - # needing to mention the build tools themselves. The original PEP - # 518 code had a similar check (but implemented in a different - # way). - backend = "setuptools.build_meta" - check = ["setuptools>=38.2.5", "wheel"] + # This is quite a lot of work to check for a very specific case. But + # the problem is, that case is potentially quite common - projects that + # adopted PEP 518 early for the ability to specify requirements to + # execute setup.py, but never considered needing to mention the build + # tools themselves. The original PEP 518 code had a similar check (but + # implemented in a different way). + backend = "setuptools.build_meta:__legacy__" + check = ["setuptools>=40.8.0", "wheel"] return (requires, backend, check) diff --git a/pipenv/patched/notpip/_internal/req/__init__.py b/pipenv/patched/notpip/_internal/req/__init__.py index 72aecb7c..998be6a2 100644 --- a/pipenv/patched/notpip/_internal/req/__init__.py +++ b/pipenv/patched/notpip/_internal/req/__init__.py @@ -1,12 +1,19 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + from __future__ import absolute_import import logging +from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +from .req_file import parse_requirements from .req_install import InstallRequirement from .req_set import RequirementSet -from .req_file import parse_requirements -from pipenv.patched.notpip._internal.utils.logging import indent_log +if MYPY_CHECK_RUNNING: + from typing import Any, List, Sequence __all__ = [ "RequirementSet", "InstallRequirement", @@ -16,8 +23,14 @@ __all__ = [ logger = logging.getLogger(__name__) -def install_given_reqs(to_install, install_options, global_options=(), - *args, **kwargs): +def install_given_reqs( + to_install, # type: List[InstallRequirement] + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + *args, # type: Any + **kwargs # type: Any +): + # type: (...) -> List[InstallRequirement] """ Install everything in the given list. diff --git a/pipenv/patched/notpip/_internal/req/constructors.py b/pipenv/patched/notpip/_internal/req/constructors.py index 9fe28d88..b1a2abe7 100644 --- a/pipenv/patched/notpip/_internal/req/constructors.py +++ b/pipenv/patched/notpip/_internal/req/constructors.py @@ -8,27 +8,38 @@ These are meant to be used elsewhere within pip to create instances of InstallRequirement. """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + import logging import os import re -import traceback from pipenv.patched.notpip._vendor.packaging.markers import Marker from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement from pipenv.patched.notpip._vendor.packaging.specifiers import Specifier from pipenv.patched.notpip._vendor.pkg_resources import RequirementParseError, parse_requirements -from pipenv.patched.notpip._internal.download import ( - is_archive_file, is_url, path_to_url, url_to_path, -) from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.models.index import PyPI, TestPyPI from pipenv.patched.notpip._internal.models.link import Link +from pipenv.patched.notpip._internal.pyproject import make_pyproject_path from pipenv.patched.notpip._internal.req.req_install import InstallRequirement -from pipenv.patched.notpip._internal.utils.misc import is_installable_dir -from pipenv.patched.notpip._internal.vcs import vcs +from pipenv.patched.notpip._internal.utils.filetypes import ARCHIVE_EXTENSIONS +from pipenv.patched.notpip._internal.utils.misc import is_installable_dir, splitext +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url +from pipenv.patched.notpip._internal.vcs import is_url, vcs from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Optional, Set, Tuple, Union, + ) + from pipenv.patched.notpip._internal.cache import WheelCache + + __all__ = [ "install_req_from_editable", "install_req_from_line", "parse_editable" @@ -38,7 +49,17 @@ logger = logging.getLogger(__name__) operators = Specifier._operators.keys() +def is_archive_file(name): + # type: (str) -> bool + """Return True if `name` is a considered as an archive file.""" + ext = splitext(name)[1].lower() + if ext in ARCHIVE_EXTENSIONS: + return True + return False + + def _strip_extras(path): + # type: (str) -> Tuple[str, Optional[str]] m = re.match(r'^(.+)(\[[^\]]+\])$', path) extras = None if m: @@ -50,7 +71,15 @@ def _strip_extras(path): return path_no_extras, extras +def convert_extras(extras): + # type: (Optional[str]) -> Set[str] + if not extras: + return set() + return Requirement("placeholder" + extras.lower()).extras + + def parse_editable(editable_req): + # type: (str) -> Tuple[Optional[str], str, Optional[Set[str]]] """Parses an editable requirement into: - a requirement name - an URL @@ -68,10 +97,18 @@ def parse_editable(editable_req): if os.path.isdir(url_no_extras): if not os.path.exists(os.path.join(url_no_extras, 'setup.py')): - raise InstallationError( - "Directory %r is not installable. File 'setup.py' not found." % - url_no_extras + msg = ( + 'File "setup.py" not found. Directory cannot be installed ' + 'in editable mode: {}'.format(os.path.abspath(url_no_extras)) ) + pyproject_path = make_pyproject_path(url_no_extras) + if os.path.isfile(pyproject_path): + msg += ( + '\n(A "pyproject.toml" file was found, but editable ' + 'mode currently requires a setup.py based build.)' + ) + raise InstallationError(msg) + # Treating it as code that has already been checked out url_no_extras = path_to_url(url_no_extras) @@ -93,9 +130,9 @@ def parse_editable(editable_req): if '+' not in url: raise InstallationError( - '%s should either be a path to a local project or a VCS url ' - 'beginning with svn+, git+, hg+, or bzr+' % - editable_req + '{} is not a valid editable requirement. ' + 'It should either be a path to a local project or a VCS URL ' + '(beginning with svn+, git+, hg+, or bzr+).'.format(editable_req) ) vc_type = url.split('+', 1)[0].lower() @@ -116,6 +153,7 @@ def parse_editable(editable_req): def deduce_helpful_msg(req): + # type: (str) -> str """Returns helpful msg in case requirements file does not exist, or cannot be parsed. @@ -136,24 +174,29 @@ def deduce_helpful_msg(req): " the packages specified within it." except RequirementParseError: logger.debug("Cannot parse '%s' as requirements \ - file" % (req), exc_info=1) + file" % (req), exc_info=True) else: msg += " File '%s' does not exist." % (req) return msg -# ---- The actual constructors follow ---- +class RequirementParts(object): + def __init__( + self, + requirement, # type: Optional[Requirement] + link, # type: Optional[Link] + markers, # type: Optional[Marker] + extras, # type: Set[str] + ): + self.requirement = requirement + self.link = link + self.markers = markers + self.extras = extras -def install_req_from_editable( - editable_req, comes_from=None, isolated=False, options=None, - wheel_cache=None, constraint=False -): +def parse_req_from_editable(editable_req): + # type: (str) -> RequirementParts name, url, extras_override = parse_editable(editable_req) - if url.startswith('file:'): - source_dir = url_to_path(url) - else: - source_dir = None if name is not None: try: @@ -162,68 +205,125 @@ def install_req_from_editable( raise InstallationError("Invalid requirement: '%s'" % name) else: req = None + + link = Link(url) + + return RequirementParts(req, link, None, extras_override) + + +# ---- The actual constructors follow ---- + + +def install_req_from_editable( + editable_req, # type: str + comes_from=None, # type: Optional[str] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False # type: bool +): + # type: (...) -> InstallRequirement + + parts = parse_req_from_editable(editable_req) + + source_dir = parts.link.file_path if parts.link.scheme == 'file' else None + return InstallRequirement( - req, comes_from, source_dir=source_dir, + parts.requirement, comes_from, source_dir=source_dir, editable=True, - link=Link(url), + link=parts.link, constraint=constraint, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, - extras=extras_override or (), + extras=parts.extras, ) -def install_req_from_line( - name, comes_from=None, isolated=False, options=None, wheel_cache=None, - constraint=False -): - """Creates an InstallRequirement from a name, which might be a - requirement, directory containing 'setup.py', filename, or URL. +def _looks_like_path(name): + # type: (str) -> bool + """Checks whether the string "looks like" a path on the filesystem. + + This does not check whether the target actually exists, only judge from the + appearance. + + Returns true if any of the following conditions is true: + * a path separator is found (either os.path.sep or os.path.altsep); + * a dot is found (which represents the current directory). """ + if os.path.sep in name: + return True + if os.path.altsep is not None and os.path.altsep in name: + return True + if name.startswith("."): + return True + return False + + +def _get_url_from_path(path, name): + # type: (str, str) -> str + """ + First, it checks whether a provided path is an installable directory + (e.g. it has a setup.py). If it is, returns the path. + + If false, check if the path is an archive file (such as a .whl). + The function checks if the path is a file. If false, if the path has + an @, it will treat it as a PEP 440 URL requirement and return the path. + """ + if _looks_like_path(name) and os.path.isdir(path): + if is_installable_dir(path): + return path_to_url(path) + raise InstallationError( + "Directory %r is not installable. Neither 'setup.py' " + "nor 'pyproject.toml' found." % name + ) + if not is_archive_file(path): + return None + if os.path.isfile(path): + return path_to_url(path) + urlreq_parts = name.split('@', 1) + if len(urlreq_parts) >= 2 and not _looks_like_path(urlreq_parts[0]): + # If the path contains '@' and the part before it does not look + # like a path, try to treat it as a PEP 440 URL req instead. + return None + logger.warning( + 'Requirement %r looks like a filename, but the ' + 'file does not exist', + name + ) + return path_to_url(path) + + +def parse_req_from_line(name, line_source): + # type: (str, Optional[str]) -> RequirementParts if is_url(name): marker_sep = '; ' else: marker_sep = ';' if marker_sep in name: - name, markers = name.split(marker_sep, 1) - markers = markers.strip() - if not markers: + name, markers_as_string = name.split(marker_sep, 1) + markers_as_string = markers_as_string.strip() + if not markers_as_string: markers = None else: - markers = Marker(markers) + markers = Marker(markers_as_string) else: markers = None name = name.strip() - req = None + req_as_string = None path = os.path.normpath(os.path.abspath(name)) link = None - extras = None + extras_as_string = None if is_url(name): link = Link(name) else: - p, extras = _strip_extras(path) - looks_like_dir = os.path.isdir(p) and ( - os.path.sep in name or - (os.path.altsep is not None and os.path.altsep in name) or - name.startswith('.') - ) - if looks_like_dir: - if not is_installable_dir(p): - raise InstallationError( - "Directory %r is not installable. Neither 'setup.py' " - "nor 'pyproject.toml' found." % name - ) - link = Link(path_to_url(p)) - elif is_archive_file(p): - if not os.path.isfile(p): - logger.warning( - 'Requirement %r looks like a filename, but the ' - 'file does not exist', - name - ) - link = Link(path_to_url(p)) + p, extras_as_string = _strip_extras(path) + url = _get_url_from_path(p, name) + if url is not None: + link = Link(url) # it's a local file, dir, or url if link: @@ -234,58 +334,95 @@ def install_req_from_line( # wheel file if link.is_wheel: wheel = Wheel(link.filename) # can raise InvalidWheelFilename - req = "%s==%s" % (wheel.name, wheel.version) + req_as_string = "%s==%s" % (wheel.name, wheel.version) else: # set the req to the egg fragment. when it's not there, this # will become an 'unnamed' requirement - req = link.egg_fragment + req_as_string = link.egg_fragment # a requirement specifier else: - req = name + req_as_string = name - if extras: - extras = Requirement("placeholder" + extras.lower()).extras - else: - extras = () - if req is not None: + extras = convert_extras(extras_as_string) + + def with_source(text): + if not line_source: + return text + return '{} (from {})'.format(text, line_source) + + if req_as_string is not None: try: - req = Requirement(req) + req = Requirement(req_as_string) except InvalidRequirement: - if os.path.sep in req: + if os.path.sep in req_as_string: add_msg = "It looks like a path." - add_msg += deduce_helpful_msg(req) - elif '=' in req and not any(op in req for op in operators): + add_msg += deduce_helpful_msg(req_as_string) + elif ('=' in req_as_string and + not any(op in req_as_string for op in operators)): add_msg = "= is not a valid operator. Did you mean == ?" else: - add_msg = traceback.format_exc() - raise InstallationError( - "Invalid requirement: '%s'\n%s" % (req, add_msg) + add_msg = '' + msg = with_source( + 'Invalid requirement: {!r}'.format(req_as_string) ) + if add_msg: + msg += '\nHint: {}'.format(add_msg) + raise InstallationError(msg) + else: + req = None + + return RequirementParts(req, link, markers, extras) + + +def install_req_from_line( + name, # type: str + comes_from=None, # type: Optional[Union[str, InstallRequirement]] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + line_source=None, # type: Optional[str] +): + # type: (...) -> InstallRequirement + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + + :param line_source: An optional string describing where the line is from, + for logging purposes in case of an error. + """ + parts = parse_req_from_line(name, line_source) return InstallRequirement( - req, comes_from, link=link, markers=markers, - isolated=isolated, + parts.requirement, comes_from, link=parts.link, markers=parts.markers, + use_pep517=use_pep517, isolated=isolated, options=options if options else {}, wheel_cache=wheel_cache, constraint=constraint, - extras=extras, + extras=parts.extras, ) -def install_req_from_req( - req, comes_from=None, isolated=False, wheel_cache=None +def install_req_from_req_string( + req_string, # type: str + comes_from=None, # type: Optional[InstallRequirement] + isolated=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] ): + # type: (...) -> InstallRequirement try: - req = Requirement(req) + req = Requirement(req_string) except InvalidRequirement: - raise InstallationError("Invalid requirement: '%s'" % req) + raise InstallationError("Invalid requirement: '%s'" % req_string) domains_not_allowed = [ PyPI.file_storage_domain, TestPyPI.file_storage_domain, ] - if req.url and comes_from.link.netloc in domains_not_allowed: + if (req.url and comes_from and comes_from.link and + comes_from.link.netloc in domains_not_allowed): # Explicitly disallow pypi packages that depend on external urls raise InstallationError( "Packages installed from PyPI cannot depend on packages " @@ -294,5 +431,6 @@ def install_req_from_req( ) return InstallRequirement( - req, comes_from, isolated=isolated, wheel_cache=wheel_cache + req, comes_from, isolated=isolated, wheel_cache=wheel_cache, + use_pep517=use_pep517 ) diff --git a/pipenv/patched/notpip/_internal/req/req_file.py b/pipenv/patched/notpip/_internal/req/req_file.py index 5f23cd3a..ece54986 100644 --- a/pipenv/patched/notpip/_internal/req/req_file.py +++ b/pipenv/patched/notpip/_internal/req/req_file.py @@ -2,6 +2,9 @@ Requirements file parsing """ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + from __future__ import absolute_import import optparse @@ -16,14 +19,28 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._internal.cli import cmdoptions from pipenv.patched.notpip._internal.download import get_file_content from pipenv.patched.notpip._internal.exceptions import RequirementsFileParseError +from pipenv.patched.notpip._internal.models.search_scope import SearchScope from pipenv.patched.notpip._internal.req.constructors import ( - install_req_from_editable, install_req_from_line, + install_req_from_editable, + install_req_from_line, ) +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Iterator, List, NoReturn, Optional, Text, Tuple, + ) + from pipenv.patched.notpip._internal.req import InstallRequirement + from pipenv.patched.notpip._internal.cache import WheelCache + from pipenv.patched.notpip._internal.index import PackageFinder + from pipenv.patched.notpip._internal.network.session import PipSession + + ReqFileLines = Iterator[Tuple[int, Text]] __all__ = ['parse_requirements'] SCHEME_RE = re.compile(r'^(http|https|file):', re.I) -COMMENT_RE = re.compile(r'(^|\s)+#.*$') +COMMENT_RE = re.compile(r'(^|\s+)#.*$') # Matches environment variable-style values in '${MY_VARIABLE_1}' with the # variable name consisting of only uppercase letters, digits or the '_' @@ -43,24 +60,32 @@ SUPPORTED_OPTIONS = [ cmdoptions.no_binary, cmdoptions.only_binary, cmdoptions.pre, - cmdoptions.process_dependency_links, cmdoptions.trusted_host, cmdoptions.require_hashes, -] +] # type: List[Callable[..., optparse.Option]] # options to be passed to requirements SUPPORTED_OPTIONS_REQ = [ cmdoptions.install_options, cmdoptions.global_options, cmdoptions.hash, -] +] # type: List[Callable[..., optparse.Option]] # the 'dest' string values -SUPPORTED_OPTIONS_REQ_DEST = [o().dest for o in SUPPORTED_OPTIONS_REQ] +SUPPORTED_OPTIONS_REQ_DEST = [str(o().dest) for o in SUPPORTED_OPTIONS_REQ] -def parse_requirements(filename, finder=None, comes_from=None, options=None, - session=None, constraint=False, wheel_cache=None): +def parse_requirements( + filename, # type: str + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + constraint=False, # type: bool + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None # type: Optional[bool] +): + # type: (...) -> Iterator[InstallRequirement] """Parse a requirements file and yield InstallRequirement instances. :param filename: Path or url of requirements file. @@ -71,6 +96,7 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, :param constraint: If true, parsing a constraint file rather than requirements file. :param wheel_cache: Instance of pip.wheel.WheelCache + :param use_pep517: Value of the --use-pep517 option. """ if session is None: raise TypeError( @@ -87,18 +113,19 @@ def parse_requirements(filename, finder=None, comes_from=None, options=None, for line_number, line in lines_enum: req_iter = process_line(line, filename, line_number, finder, comes_from, options, session, wheel_cache, - constraint=constraint) + use_pep517=use_pep517, constraint=constraint) for req in req_iter: yield req def preprocess(content, options): + # type: (Text, Optional[optparse.Values]) -> ReqFileLines """Split, filter, and join lines, and return a line iterator :param content: the content of the requirements file :param options: cli options """ - lines_enum = enumerate(content.splitlines(), start=1) + lines_enum = enumerate(content.splitlines(), start=1) # type: ReqFileLines lines_enum = join_lines(lines_enum) lines_enum = ignore_comments(lines_enum) lines_enum = skip_regex(lines_enum, options) @@ -106,9 +133,19 @@ def preprocess(content, options): return lines_enum -def process_line(line, filename, line_number, finder=None, comes_from=None, - options=None, session=None, wheel_cache=None, - constraint=False): +def process_line( + line, # type: Text + filename, # type: str + line_number, # type: int + finder=None, # type: Optional[PackageFinder] + comes_from=None, # type: Optional[str] + options=None, # type: Optional[optparse.Values] + session=None, # type: Optional[PipSession] + wheel_cache=None, # type: Optional[WheelCache] + use_pep517=None, # type: Optional[bool] + constraint=False, # type: bool +): + # type: (...) -> Iterator[InstallRequirement] """Process a single requirements line; This can result in creating/yielding requirements, or updating the finder. @@ -130,13 +167,15 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, defaults = parser.get_default_values() defaults.index_url = None if finder: - # `finder.format_control` will be updated during parsing defaults.format_control = finder.format_control args_str, options_str = break_args_options(line) + # Prior to 2.7.3, shlex cannot deal with unicode entries if sys.version_info < (2, 7, 3): - # Prior to 2.7.3, shlex cannot deal with unicode entries - options_str = options_str.encode('utf8') - opts, _ = parser.parse_args(shlex.split(options_str), defaults) + # https://github.com/python/mypy/issues/1174 + options_str = options_str.encode('utf8') # type: ignore + # https://github.com/python/mypy/issues/1174 + opts, _ = parser.parse_args( + shlex.split(options_str), defaults) # type: ignore # preserve for the nested code path line_comes_from = '%s %s (line %s)' % ( @@ -153,9 +192,16 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, for dest in SUPPORTED_OPTIONS_REQ_DEST: if dest in opts.__dict__ and opts.__dict__[dest]: req_options[dest] = opts.__dict__[dest] + line_source = 'line {} of {}'.format(line_number, filename) yield install_req_from_line( - args_str, line_comes_from, constraint=constraint, - isolated=isolated, options=req_options, wheel_cache=wheel_cache + args_str, + comes_from=line_comes_from, + use_pep517=use_pep517, + isolated=isolated, + options=req_options, + wheel_cache=wheel_cache, + constraint=constraint, + line_source=line_source, ) # yield an editable requirement @@ -163,6 +209,7 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, isolated = options.isolated_mode if options else False yield install_req_from_editable( opts.editables[0], comes_from=line_comes_from, + use_pep517=use_pep517, constraint=constraint, isolated=isolated, wheel_cache=wheel_cache ) @@ -183,11 +230,11 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # do a join so relative paths work req_path = os.path.join(os.path.dirname(filename), req_path) # TODO: Why not use `comes_from='-r {} (line {})'` here as well? - parser = parse_requirements( + parsed_reqs = parse_requirements( req_path, finder, comes_from, options, session, constraint=nested_constraint, wheel_cache=wheel_cache ) - for req in parser: + for req in parsed_reqs: yield req # percolate hash-checking option upward @@ -196,12 +243,14 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, # set finder options elif finder: + find_links = finder.find_links + index_urls = finder.index_urls if opts.index_url: - finder.index_urls = [opts.index_url] + index_urls = [opts.index_url] if opts.no_index is True: - finder.index_urls = [] + index_urls = [] if opts.extra_index_urls: - finder.index_urls.extend(opts.extra_index_urls) + index_urls.extend(opts.extra_index_urls) if opts.find_links: # FIXME: it would be nice to keep track of the source # of the find_links: support a find-links local path @@ -211,17 +260,23 @@ def process_line(line, filename, line_number, finder=None, comes_from=None, relative_to_reqs_file = os.path.join(req_dir, value) if os.path.exists(relative_to_reqs_file): value = relative_to_reqs_file - finder.find_links.append(value) + find_links.append(value) + + search_scope = SearchScope( + find_links=find_links, + index_urls=index_urls, + ) + finder.search_scope = search_scope + if opts.pre: - finder.allow_all_prereleases = True - if opts.process_dependency_links: - finder.process_dependency_links = True - if opts.trusted_hosts: - finder.secure_origins.extend( - ("*", host, "*") for host in opts.trusted_hosts) + finder.set_allow_all_prereleases() + for host in opts.trusted_hosts or []: + source = 'line {} of {}'.format(line_number, filename) + session.add_trusted_host(host, source=source) def break_args_options(line): + # type: (Text) -> Tuple[str, Text] """Break up the line into an args and options string. We only want to shlex (and then optparse) the options, not the args. args can contain markers which are corrupted by shlex. @@ -235,10 +290,11 @@ def break_args_options(line): else: args.append(token) options.pop(0) - return ' '.join(args), ' '.join(options) + return ' '.join(args), ' '.join(options) # type: ignore def build_parser(line): + # type: (Text) -> optparse.OptionParser """ Return a parser for parsing requirement lines """ @@ -252,20 +308,24 @@ def build_parser(line): # By default optparse sys.exits on parsing errors. We want to wrap # that in our own exception. def parser_exit(self, msg): + # type: (Any, str) -> NoReturn # add offending line msg = 'Invalid requirement: %s\n%s' % (line, msg) raise RequirementsFileParseError(msg) - parser.exit = parser_exit + # NOTE: mypy disallows assigning to a method + # https://github.com/python/mypy/issues/2427 + parser.exit = parser_exit # type: ignore return parser def join_lines(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Joins a line ending in '\' with the previous line (except when following comments). The joined line takes on the index of the first line. """ primary_line_number = None - new_line = [] + new_line = [] # type: List[Text] for line_number, line in lines_enum: if not line.endswith('\\') or COMMENT_RE.match(line): if COMMENT_RE.match(line): @@ -290,6 +350,7 @@ def join_lines(lines_enum): def ignore_comments(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """ Strips comments and filter empty lines. """ @@ -301,6 +362,7 @@ def ignore_comments(lines_enum): def skip_regex(lines_enum, options): + # type: (ReqFileLines, Optional[optparse.Values]) -> ReqFileLines """ Skip lines that match '--skip-requirements-regex' pattern @@ -314,6 +376,7 @@ def skip_regex(lines_enum, options): def expand_env_variables(lines_enum): + # type: (ReqFileLines) -> ReqFileLines """Replace all environment variables that can be retrieved via `os.getenv`. The only allowed format for environment variables defined in the @@ -322,7 +385,7 @@ def expand_env_variables(lines_enum): 1. Strings that contain a `$` aren't accidentally (partially) expanded. 2. Ensure consistency across platforms for requirement files. - These points are the result of a discusssion on the `github pull + These points are the result of a discussion on the `github pull request #3514 `_. Valid characters in variable names follow the `POSIX standard diff --git a/pipenv/patched/notpip/_internal/req/req_install.py b/pipenv/patched/notpip/_internal/req/req_install.py index 3f32892c..2da04659 100644 --- a/pipenv/patched/notpip/_internal/req/req_install.py +++ b/pipenv/patched/notpip/_internal/req/req_install.py @@ -1,5 +1,10 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import +import atexit import logging import os import shutil @@ -15,29 +20,55 @@ from pipenv.patched.notpip._vendor.packaging.version import Version from pipenv.patched.notpip._vendor.packaging.version import parse as parse_version from pipenv.patched.notpip._vendor.pep517.wrappers import Pep517HookCaller -from pipenv.patched.notpip._internal import wheel +from pipenv.patched.notpip._internal import pep425tags, wheel from pipenv.patched.notpip._internal.build_env import NoOpBuildEnvironment from pipenv.patched.notpip._internal.exceptions import InstallationError -from pipenv.patched.notpip._internal.locations import ( - PIP_DELETE_MARKER_FILENAME, running_under_virtualenv, -) from pipenv.patched.notpip._internal.models.link import Link -from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml +from pipenv.patched.notpip._internal.operations.generate_metadata import get_metadata_generator +from pipenv.patched.notpip._internal.pyproject import load_pyproject_toml, make_pyproject_path from pipenv.patched.notpip._internal.req.req_uninstall import UninstallPathSet from pipenv.patched.notpip._internal.utils.compat import native_str from pipenv.patched.notpip._internal.utils.hashes import Hashes from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.marker_files import ( + PIP_DELETE_MARKER_FILENAME, + has_delete_marker_file, +) from pipenv.patched.notpip._internal.utils.misc import ( - _make_build_dir, ask_path_exists, backup_dir, call_subprocess, - display_path, dist_in_site_packages, dist_in_usersite, ensure_dir, - get_installed_version, rmtree, + _make_build_dir, + ask_path_exists, + backup_dir, + display_path, + dist_in_site_packages, + dist_in_usersite, + ensure_dir, + get_installed_version, + hide_url, + redact_auth_from_url, + rmtree, ) from pipenv.patched.notpip._internal.utils.packaging import get_metadata -from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM +from pipenv.patched.notpip._internal.utils.setuptools_build import make_setuptools_shim_args +from pipenv.patched.notpip._internal.utils.subprocess import ( + call_subprocess, + runner_with_spinner_message, +) from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.utils.ui import open_spinner +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.virtualenv import running_under_virtualenv from pipenv.patched.notpip._internal.vcs import vcs -from pipenv.patched.notpip._internal.wheel import move_wheel_files + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Optional, Sequence, Union, + ) + from pipenv.patched.notpip._internal.build_env import BuildEnvironment + from pipenv.patched.notpip._internal.cache import WheelCache + from pipenv.patched.notpip._internal.index import PackageFinder + from pipenv.patched.notpip._vendor.pkg_resources import Distribution + from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet + from pipenv.patched.notpip._vendor.packaging.markers import Marker + logger = logging.getLogger(__name__) @@ -45,29 +76,41 @@ logger = logging.getLogger(__name__) class InstallRequirement(object): """ Represents something that may be installed later on, may have information - about where to fetch the relavant requirement and also contains logic for + about where to fetch the relevant requirement and also contains logic for installing the said requirement. """ - def __init__(self, req, comes_from, source_dir=None, editable=False, - link=None, update=True, markers=None, - isolated=False, options=None, wheel_cache=None, - constraint=False, extras=()): + def __init__( + self, + req, # type: Optional[Requirement] + comes_from, # type: Optional[Union[str, InstallRequirement]] + source_dir=None, # type: Optional[str] + editable=False, # type: bool + link=None, # type: Optional[Link] + markers=None, # type: Optional[Marker] + use_pep517=None, # type: Optional[bool] + isolated=False, # type: bool + options=None, # type: Optional[Dict[str, Any]] + wheel_cache=None, # type: Optional[WheelCache] + constraint=False, # type: bool + extras=() # type: Iterable[str] + ): + # type: (...) -> None assert req is None or isinstance(req, Requirement), req self.req = req self.comes_from = comes_from self.constraint = constraint - if source_dir is not None: - self.source_dir = os.path.normpath(os.path.abspath(source_dir)) + if source_dir is None: + self.source_dir = None # type: Optional[str] else: - self.source_dir = None + self.source_dir = os.path.normpath(os.path.abspath(source_dir)) self.editable = editable self._wheel_cache = wheel_cache - if link is not None: - self.link = self.original_link = link - else: - self.link = self.original_link = req and req.url and Link(req.url) + if link is None and req and req.url: + # PEP 508 URL requirement + link = Link(req.url) + self.link = self.original_link = link if extras: self.extras = extras @@ -77,11 +120,10 @@ class InstallRequirement(object): } else: self.extras = set() - if markers is not None: - self.markers = markers - else: - self.markers = req and req.marker - self._egg_info_path = None + if markers is None and req: + markers = req.marker + self.markers = markers + # This holds the pkg_resources.Distribution object if this requirement # is already available: self.satisfied_by = None @@ -89,55 +131,57 @@ class InstallRequirement(object): # conflicts with another installed distribution: self.conflicts_with = None # Temporary build location - self._temp_build_dir = TempDirectory(kind="req-build") + self._temp_build_dir = None # type: Optional[TempDirectory] # Used to store the global directory where the _temp_build_dir should - # have been created. Cf _correct_build_location method. - self._ideal_build_dir = None - # True if the editable should be updated: - self.update = update + # have been created. Cf move_to_correct_build_directory method. + self._ideal_build_dir = None # type: Optional[str] # Set to True after successful installation - self.install_succeeded = None - # UninstallPathSet of uninstalled distribution (for possible rollback) - self.uninstalled_pathset = None + self.install_succeeded = None # type: Optional[bool] self.options = options if options else {} # Set to True after successful preparation of this requirement self.prepared = False self.is_direct = False self.isolated = isolated - self.build_env = NoOpBuildEnvironment() + self.build_env = NoOpBuildEnvironment() # type: BuildEnvironment + + # For PEP 517, the directory where we request the project metadata + # gets stored. We need this to pass to build_wheel, so the backend + # can ensure that the wheel matches the metadata (see the PEP for + # details). + self.metadata_directory = None # type: Optional[str] # The static build requirements (from pyproject.toml) - self.pyproject_requires = None + self.pyproject_requires = None # type: Optional[List[str]] # Build requirements that we will check are available - # TODO: We don't do this for --no-build-isolation. Should we? - self.requirements_to_check = [] + self.requirements_to_check = [] # type: List[str] # The PEP 517 backend we should use to build the project - self.pep517_backend = None + self.pep517_backend = None # type: Optional[Pep517HookCaller] # Are we using PEP 517 for this requirement? # After pyproject.toml has been loaded, the only valid values are True # and False. Before loading, None is valid (meaning "use the default"). # Setting an explicit value before loading pyproject.toml is supported, # but after loading this flag should be treated as read only. - self.use_pep517 = None + self.use_pep517 = use_pep517 def __str__(self): + # type: () -> str if self.req: s = str(self.req) if self.link: - s += ' from %s' % self.link.url + s += ' from %s' % redact_auth_from_url(self.link.url) elif self.link: - s = self.link.url + s = redact_auth_from_url(self.link.url) else: s = '' if self.satisfied_by is not None: s += ' in %s' % display_path(self.satisfied_by.location) if self.comes_from: if isinstance(self.comes_from, six.string_types): - comes_from = self.comes_from + comes_from = self.comes_from # type: Optional[str] else: comes_from = self.comes_from.from_path() if comes_from: @@ -145,10 +189,27 @@ class InstallRequirement(object): return s def __repr__(self): + # type: () -> str return '<%s object: %s editable=%r>' % ( self.__class__.__name__, str(self), self.editable) + def format_debug(self): + # type: () -> str + """An un-tested helper for getting state, for debugging. + """ + attributes = vars(self) + names = sorted(attributes) + + state = ( + "{}={!r}".format(attr, attributes[attr]) for attr in sorted(names) + ) + return '<{name} object: {{{state}}}>'.format( + name=self.__class__.__name__, + state=", ".join(state), + ) + def populate_link(self, finder, upgrade, require_hashes): + # type: (PackageFinder, bool, bool) -> None """Ensure that if a link can be found for this, that it is found. Note that self.link may still be None - if Upgrade is False and the @@ -164,23 +225,31 @@ class InstallRequirement(object): self.link = finder.find_requirement(self, upgrade) if self._wheel_cache is not None and not require_hashes: old_link = self.link - self.link = self._wheel_cache.get(self.link, self.name) + supported_tags = pep425tags.get_supported() + self.link = self._wheel_cache.get( + link=self.link, + package_name=self.name, + supported_tags=supported_tags, + ) if old_link != self.link: logger.debug('Using cached wheel link: %s', self.link) # Things that are valid for all kinds of requirements? @property def name(self): + # type: () -> Optional[str] if self.req is None: return None return native_str(pkg_resources.safe_name(self.req.name)) @property def specifier(self): + # type: () -> SpecifierSet return self.req.specifier @property def is_pinned(self): + # type: () -> bool """Return whether I am pinned to an exact version. For example, some-package==1.2 is pinned; some-package>1.2 is not. @@ -191,9 +260,11 @@ class InstallRequirement(object): @property def installed_version(self): + # type: () -> Optional[str] return get_installed_version(self.name) def match_markers(self, extras_requested=None): + # type: (Optional[Iterable[str]]) -> bool if not extras_requested: # Provide an extra to safely evaluate the markers # without matching any extra @@ -207,6 +278,7 @@ class InstallRequirement(object): @property def has_hash_options(self): + # type: () -> bool """Return whether any known-good hashes are specified as options. These activate --require-hashes mode; hashes specified as part of a @@ -216,6 +288,7 @@ class InstallRequirement(object): return bool(self.options.get('hashes', {})) def hashes(self, trust_internet=True): + # type: (bool) -> Hashes """Return a hash-comparer that considers my option- and URL-based hashes to be known-good. @@ -237,6 +310,7 @@ class InstallRequirement(object): return Hashes(good_hashes) def from_path(self): + # type: () -> Optional[str] """Format a nice indicator to show where this "comes from" """ if self.req is None: @@ -251,19 +325,21 @@ class InstallRequirement(object): s += '->' + comes_from return s - def build_location(self, build_dir): + def ensure_build_location(self, build_dir): + # type: (str) -> str assert build_dir is not None - if self._temp_build_dir.path is not None: + if self._temp_build_dir is not None: + assert self._temp_build_dir.path return self._temp_build_dir.path if self.req is None: # for requirement via a path to a directory: the name of the # package is not available yet so we create a temp directory - # Once run_egg_info will have run, we'll be able - # to fix it via _correct_build_location + # Once run_egg_info will have run, we'll be able to fix it via + # move_to_correct_build_directory(). # Some systems have /tmp as a symlink which confuses custom # builds (such as numpy). Thus, we ensure that the real path # is returned. - self._temp_build_dir.create() + self._temp_build_dir = TempDirectory(kind="req-build") self._ideal_build_dir = build_dir return self._temp_build_dir.path @@ -278,51 +354,77 @@ class InstallRequirement(object): _make_build_dir(build_dir) return os.path.join(build_dir, name) - def _correct_build_location(self): - """Move self._temp_build_dir to self._ideal_build_dir/self.req.name + def move_to_correct_build_directory(self): + # type: () -> None + """Move self._temp_build_dir to "self._ideal_build_dir/self.req.name" For some requirements (e.g. a path to a directory), the name of the package is not available until we run egg_info, so the build_location will return a temporary directory and store the _ideal_build_dir. - This is only called by self.run_egg_info to fix the temporary build - directory. + This is only called to "fix" the build directory after generating + metadata. """ if self.source_dir is not None: return assert self.req is not None - assert self._temp_build_dir.path - assert self._ideal_build_dir.path - old_location = self._temp_build_dir.path - self._temp_build_dir.path = None + assert self._temp_build_dir + assert ( + self._ideal_build_dir is not None and + self._ideal_build_dir.path # type: ignore + ) + old_location = self._temp_build_dir + self._temp_build_dir = None # checked inside ensure_build_location - new_location = self.build_location(self._ideal_build_dir) + # Figure out the correct place to put the files. + new_location = self.ensure_build_location(self._ideal_build_dir) if os.path.exists(new_location): raise InstallationError( 'A package already exists in %s; please remove it to continue' - % display_path(new_location)) + % display_path(new_location) + ) + + # Move the files to the correct location. logger.debug( 'Moving package %s from %s to new location %s', - self, display_path(old_location), display_path(new_location), + self, display_path(old_location.path), display_path(new_location), ) - shutil.move(old_location, new_location) - self._temp_build_dir.path = new_location - self._ideal_build_dir = None + shutil.move(old_location.path, new_location) + + # Update directory-tracking variables, to be in line with new_location self.source_dir = os.path.normpath(os.path.abspath(new_location)) - self._egg_info_path = None + self._temp_build_dir = TempDirectory( + path=new_location, kind="req-install", + ) + + # Correct the metadata directory, if it exists + if self.metadata_directory: + old_meta = self.metadata_directory + rel = os.path.relpath(old_meta, start=old_location.path) + new_meta = os.path.join(new_location, rel) + new_meta = os.path.normpath(os.path.abspath(new_meta)) + self.metadata_directory = new_meta + + # Done with any "move built files" work, since have moved files to the + # "ideal" build location. Setting to None allows to clearly flag that + # no more moves are needed. + self._ideal_build_dir = None def remove_temporary_source(self): + # type: () -> None """Remove the source files from this requirement, if they are marked for deletion""" - if self.source_dir and os.path.exists( - os.path.join(self.source_dir, PIP_DELETE_MARKER_FILENAME)): + if self.source_dir and has_delete_marker_file(self.source_dir): logger.debug('Removing source in %s', self.source_dir) rmtree(self.source_dir) self.source_dir = None - self._temp_build_dir.cleanup() + if self._temp_build_dir: + self._temp_build_dir.cleanup() + self._temp_build_dir = None self.build_env.cleanup() def check_if_exists(self, use_user_site): + # type: (bool) -> bool """Find an installed distribution that satisfies or conflicts with this requirement, and set self.satisfied_by or self.conflicts_with appropriately. @@ -366,12 +468,23 @@ class InstallRequirement(object): # Things valid for wheels @property def is_wheel(self): - return self.link and self.link.is_wheel + # type: () -> bool + if not self.link: + return False + return self.link.is_wheel - def move_wheel_files(self, wheeldir, root=None, home=None, prefix=None, - warn_script_location=True, use_user_site=False, - pycompile=True): - move_wheel_files( + def move_wheel_files( + self, + wheeldir, # type: str + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None + wheel.move_wheel_files( self.name, self.req, wheeldir, user=use_user_site, home=home, @@ -384,16 +497,17 @@ class InstallRequirement(object): # Things valid for sdists @property - def setup_py_dir(self): + def unpacked_source_directory(self): + # type: () -> str return os.path.join( self.source_dir, self.link and self.link.subdirectory_fragment or '') @property - def setup_py(self): + def setup_py_path(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self - - setup_py = os.path.join(self.setup_py_dir, 'setup.py') + setup_py = os.path.join(self.unpacked_source_directory, 'setup.py') # Python2 __file__ should not be unicode if six.PY2 and isinstance(setup_py, six.text_type): @@ -402,18 +516,13 @@ class InstallRequirement(object): return setup_py @property - def pyproject_toml(self): + def pyproject_toml_path(self): + # type: () -> str assert self.source_dir, "No source dir for %s" % self - - pp_toml = os.path.join(self.setup_py_dir, 'pyproject.toml') - - # Python2 __file__ should not be unicode - if six.PY2 and isinstance(pp_toml, six.text_type): - pp_toml = pp_toml.encode(sys.getfilesystemencoding()) - - return pp_toml + return make_pyproject_path(self.unpacked_source_directory) def load_pyproject_toml(self): + # type: () -> None """Load the pyproject.toml file. After calling this routine, all of the attributes related to PEP 517 @@ -421,57 +530,37 @@ class InstallRequirement(object): use_pep517 attribute can be used to determine whether we should follow the PEP 517 or legacy (setup.py) code path. """ - pep517_data = load_pyproject_toml( + pyproject_toml_data = load_pyproject_toml( self.use_pep517, - self.pyproject_toml, - self.setup_py, + self.pyproject_toml_path, + self.setup_py_path, str(self) ) - if pep517_data is None: + if pyproject_toml_data is None: self.use_pep517 = False - else: - self.use_pep517 = True - requires, backend, check = pep517_data - self.requirements_to_check = check - self.pyproject_requires = requires - self.pep517_backend = Pep517HookCaller(self.setup_py_dir, backend) + return - def run_egg_info(self): + self.use_pep517 = True + requires, backend, check = pyproject_toml_data + self.requirements_to_check = check + self.pyproject_requires = requires + self.pep517_backend = Pep517HookCaller( + self.unpacked_source_directory, backend + ) + + def prepare_metadata(self): + # type: () -> None + """Ensure that project metadata is available. + + Under PEP 517, call the backend hook to prepare the metadata. + Under legacy processing, call setup.py egg-info. + """ assert self.source_dir - if self.name: - logger.debug( - 'Running setup.py (path:%s) egg_info for package %s', - self.setup_py, self.name, - ) - else: - logger.debug( - 'Running setup.py (path:%s) egg_info for package from %s', - self.setup_py, self.link, - ) + metadata_generator = get_metadata_generator(self) with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] - # We can't put the .egg-info files at the root, because then the - # source code will be mistaken for an installed egg, causing - # problems - if self.editable: - egg_base_option = [] - else: - egg_info_dir = os.path.join(self.setup_py_dir, 'pip-egg-info') - ensure_dir(egg_info_dir) - egg_base_option = ['--egg-base', 'pip-egg-info'] - with self.build_env: - call_subprocess( - egg_info_cmd + egg_base_option, - cwd=self.setup_py_dir, - show_stdout=False, - command_desc='python setup.py egg_info') + self.metadata_directory = metadata_generator(self) if not self.req: if isinstance(parse_version(self.metadata["Version"]), Version): @@ -485,90 +574,74 @@ class InstallRequirement(object): self.metadata["Version"], ]) ) - self._correct_build_location() + self.move_to_correct_build_directory() else: metadata_name = canonicalize_name(self.metadata["Name"]) if canonicalize_name(self.req.name) != metadata_name: logger.warning( - 'Running setup.py (path:%s) egg_info for package %s ' + 'Generating metadata for package %s ' 'produced metadata for project name %s. Fix your ' '#egg=%s fragments.', - self.setup_py, self.name, metadata_name, self.name + self.name, metadata_name, self.name ) self.req = Requirement(metadata_name) - @property - def egg_info_path(self): - if self._egg_info_path is None: - if self.editable: - base = self.source_dir - else: - base = os.path.join(self.setup_py_dir, 'pip-egg-info') - filenames = os.listdir(base) - if self.editable: - filenames = [] - for root, dirs, files in os.walk(base): - for dir in vcs.dirnames: - if dir in dirs: - dirs.remove(dir) - # Iterate over a copy of ``dirs``, since mutating - # a list while iterating over it can cause trouble. - # (See https://github.com/pypa/pip/pull/462.) - for dir in list(dirs): - # Don't search in anything that looks like a virtualenv - # environment - if ( - os.path.lexists( - os.path.join(root, dir, 'bin', 'python') - ) or - os.path.exists( - os.path.join( - root, dir, 'Scripts', 'Python.exe' - ) - )): - dirs.remove(dir) - # Also don't search through tests - elif dir == 'test' or dir == 'tests': - dirs.remove(dir) - filenames.extend([os.path.join(root, dir) - for dir in dirs]) - filenames = [f for f in filenames if f.endswith('.egg-info')] + def prepare_pep517_metadata(self): + # type: () -> str + assert self.pep517_backend is not None - if not filenames: - raise InstallationError( - "Files/directories not found in %s" % base + # NOTE: This needs to be refactored to stop using atexit + metadata_tmpdir = TempDirectory(kind="modern-metadata") + atexit.register(metadata_tmpdir.cleanup) + + metadata_dir = metadata_tmpdir.path + + with self.build_env: + # Note that Pep517HookCaller implements a fallback for + # prepare_metadata_for_build_wheel, so we don't have to + # consider the possibility that this hook doesn't exist. + runner = runner_with_spinner_message("Preparing wheel metadata") + backend = self.pep517_backend + with backend.subprocess_runner(runner): + distinfo_dir = backend.prepare_metadata_for_build_wheel( + metadata_dir ) - # if we have more than one match, we pick the toplevel one. This - # can easily be the case if there is a dist folder which contains - # an extracted tarball for testing purposes. - if len(filenames) > 1: - filenames.sort( - key=lambda x: x.count(os.path.sep) + - (os.path.altsep and x.count(os.path.altsep) or 0) - ) - self._egg_info_path = os.path.join(base, filenames[0]) - return self._egg_info_path + + return os.path.join(metadata_dir, distinfo_dir) @property def metadata(self): + # type: () -> Any if not hasattr(self, '_metadata'): self._metadata = get_metadata(self.get_dist()) return self._metadata def get_dist(self): - """Return a pkg_resources.Distribution built from self.egg_info_path""" - egg_info = self.egg_info_path.rstrip(os.path.sep) - base_dir = os.path.dirname(egg_info) - metadata = pkg_resources.PathMetadata(base_dir, egg_info) - dist_name = os.path.splitext(os.path.basename(egg_info))[0] - return pkg_resources.Distribution( - os.path.dirname(egg_info), + # type: () -> Distribution + """Return a pkg_resources.Distribution for this requirement""" + dist_dir = self.metadata_directory.rstrip(os.sep) + + # Determine the correct Distribution object type. + if dist_dir.endswith(".egg-info"): + dist_cls = pkg_resources.Distribution + else: + assert dist_dir.endswith(".dist-info") + dist_cls = pkg_resources.DistInfoDistribution + + # Build a PathMetadata object, from path to metadata. :wink: + base_dir, dist_dir_name = os.path.split(dist_dir) + dist_name = os.path.splitext(dist_dir_name)[0] + metadata = pkg_resources.PathMetadata(base_dir, dist_dir) + + return dist_cls( + base_dir, project_name=dist_name, metadata=metadata, ) def assert_source_matches_version(self): + # type: () -> None assert self.source_dir version = self.metadata['version'] if self.req.specifier and version not in self.req.specifier: @@ -587,6 +660,7 @@ class InstallRequirement(object): # For both source distributions and editables def ensure_has_source_dir(self, parent_dir): + # type: (str) -> None """Ensure that a source_dir is set. This will create a temporary build dir if the name of the requirement @@ -597,42 +671,39 @@ class InstallRequirement(object): :return: self.source_dir """ if self.source_dir is None: - self.source_dir = self.build_location(parent_dir) - return self.source_dir + self.source_dir = self.ensure_build_location(parent_dir) # For editable installations - def install_editable(self, install_options, - global_options=(), prefix=None): + def install_editable( + self, + install_options, # type: List[str] + global_options=(), # type: Sequence[str] + prefix=None # type: Optional[str] + ): + # type: (...) -> None logger.info('Running setup.py develop for %s', self.name) - if self.isolated: - global_options = list(global_options) + ["--no-user-cfg"] - if prefix: prefix_param = ['--prefix={}'.format(prefix)] install_options = list(install_options) + prefix_param - + base_cmd = make_setuptools_shim_args( + self.setup_py_path, + global_options=global_options, + no_user_config=self.isolated + ) with indent_log(): - # FIXME: should we do --install-headers here too? - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) with self.build_env: call_subprocess( - [ - sys_executable, - '-c', - SETUPTOOLS_SHIM % self.setup_py - ] + - list(global_options) + + base_cmd + ['develop', '--no-deps'] + list(install_options), - - cwd=self.setup_py_dir, - show_stdout=False, + cwd=self.unpacked_source_directory, ) self.install_succeeded = True def update_editable(self, obtain=True): + # type: (bool) -> None if not self.link: logger.debug( "Cannot update repository at %s; repository location is " @@ -646,16 +717,14 @@ class InstallRequirement(object): # Static paths don't get updated return assert '+' in self.link.url, "bad url: %r" % self.link.url - if not self.update: - return vc_type, url = self.link.url.split('+', 1) - backend = vcs.get_backend(vc_type) - if backend: - vcs_backend = backend(self.link.url) + vcs_backend = vcs.get_backend(vc_type) + if vcs_backend: + hidden_url = hide_url(self.link.url) if obtain: - vcs_backend.obtain(self.source_dir) + vcs_backend.obtain(self.source_dir, url=hidden_url) else: - vcs_backend.export(self.source_dir) + vcs_backend.export(self.source_dir, url=hidden_url) else: assert 0, ( 'Unexpected version control type (in %s): %s' @@ -664,6 +733,7 @@ class InstallRequirement(object): # Top-level Actions def uninstall(self, auto_confirm=False, verbose=False, use_user_site=False): + # type: (bool, bool, bool) -> Optional[UninstallPathSet] """ Uninstall the distribution currently satisfying this requirement. @@ -678,7 +748,7 @@ class InstallRequirement(object): """ if not self.check_if_exists(use_user_site): logger.warning("Skipping %s as it is not installed.", self.name) - return + return None dist = self.satisfied_by or self.conflicts_with uninstalled_pathset = UninstallPathSet.from_dist(dist) @@ -686,6 +756,7 @@ class InstallRequirement(object): return uninstalled_pathset def _clean_zip_name(self, name, prefix): # only used by archive. + # type: (str, str) -> str assert name.startswith(prefix + os.path.sep), ( "name %r doesn't start with prefix %r" % (name, prefix) ) @@ -693,13 +764,24 @@ class InstallRequirement(object): name = name.replace(os.path.sep, '/') return name - # TODO: Investigate if this should be kept in InstallRequirement - # Seems to be used only when VCS + downloads + def _get_archive_name(self, path, parentdir, rootdir): + # type: (str, str, str) -> str + path = os.path.join(parentdir, path) + name = self._clean_zip_name(path, rootdir) + return self.name + '/' + name + def archive(self, build_dir): + # type: (str) -> None + """Saves archive to provided build_dir. + + Used for saving downloaded VCS requirements as part of `pip download`. + """ assert self.source_dir + create_archive = True archive_name = '%s-%s.zip' % (self.name, self.metadata["version"]) archive_path = os.path.join(build_dir, archive_name) + if os.path.exists(archive_path): response = ask_path_exists( 'The file %s exists. (i)gnore, (w)ipe, (b)ackup, (a)bort ' % @@ -719,33 +801,50 @@ class InstallRequirement(object): shutil.move(archive_path, dest_file) elif response == 'a': sys.exit(-1) - if create_archive: - zip = zipfile.ZipFile( - archive_path, 'w', zipfile.ZIP_DEFLATED, - allowZip64=True + + if not create_archive: + return + + zip_output = zipfile.ZipFile( + archive_path, 'w', zipfile.ZIP_DEFLATED, allowZip64=True, + ) + with zip_output: + dir = os.path.normcase( + os.path.abspath(self.unpacked_source_directory) ) - dir = os.path.normcase(os.path.abspath(self.setup_py_dir)) for dirpath, dirnames, filenames in os.walk(dir): if 'pip-egg-info' in dirnames: dirnames.remove('pip-egg-info') for dirname in dirnames: - dirname = os.path.join(dirpath, dirname) - name = self._clean_zip_name(dirname, dir) - zipdir = zipfile.ZipInfo(self.name + '/' + name + '/') + dir_arcname = self._get_archive_name( + dirname, parentdir=dirpath, rootdir=dir, + ) + zipdir = zipfile.ZipInfo(dir_arcname + '/') zipdir.external_attr = 0x1ED << 16 # 0o755 - zip.writestr(zipdir, '') + zip_output.writestr(zipdir, '') for filename in filenames: if filename == PIP_DELETE_MARKER_FILENAME: continue + file_arcname = self._get_archive_name( + filename, parentdir=dirpath, rootdir=dir, + ) filename = os.path.join(dirpath, filename) - name = self._clean_zip_name(filename, dir) - zip.write(filename, self.name + '/' + name) - zip.close() - logger.info('Saved %s', display_path(archive_path)) + zip_output.write(filename, file_arcname) - def install(self, install_options, global_options=None, root=None, - home=None, prefix=None, warn_script_location=True, - use_user_site=False, pycompile=True): + logger.info('Saved %s', display_path(archive_path)) + + def install( + self, + install_options, # type: List[str] + global_options=None, # type: Optional[Sequence[str]] + root=None, # type: Optional[str] + home=None, # type: Optional[str] + prefix=None, # type: Optional[str] + warn_script_location=True, # type: bool + use_user_site=False, # type: bool + pycompile=True # type: bool + ): + # type: (...) -> None global_options = global_options if global_options is not None else [] if self.editable: self.install_editable( @@ -774,24 +873,20 @@ class InstallRequirement(object): install_options = list(install_options) + \ self.options.get('install_options', []) - if self.isolated: - global_options = global_options + ["--no-user-cfg"] - with TempDirectory(kind="record") as temp_dir: record_filename = os.path.join(temp_dir.path, 'install-record.txt') install_args = self.get_install_args( global_options, record_filename, root, prefix, pycompile, ) - msg = 'Running setup.py install for %s' % (self.name,) - with open_spinner(msg) as spinner: - with indent_log(): - with self.build_env: - call_subprocess( - install_args + install_options, - cwd=self.setup_py_dir, - show_stdout=False, - spinner=spinner, - ) + + runner = runner_with_spinner_message( + "Running setup.py install for {}".format(self.name) + ) + with indent_log(), self.build_env: + runner( + cmd=install_args + install_options, + cwd=self.unpacked_source_directory, + ) if not os.path.exists(record_filename): logger.debug('Record file %s not found', record_filename) @@ -799,6 +894,7 @@ class InstallRequirement(object): self.install_succeeded = True def prepend_root(path): + # type: (str) -> str if root is None or not os.path.isabs(path): return path else: @@ -817,7 +913,6 @@ class InstallRequirement(object): self, ) # FIXME: put the record somewhere - # FIXME: should this be an error? return new_lines = [] with open(record_filename) as f: @@ -834,14 +929,22 @@ class InstallRequirement(object): with open(inst_files_path, 'w') as f: f.write('\n'.join(new_lines) + '\n') - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): - sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - install_args = [sys_executable, "-u"] - install_args.append('-c') - install_args.append(SETUPTOOLS_SHIM % self.setup_py) - install_args += list(global_options) + \ - ['install', '--record', record_filename] + def get_install_args( + self, + global_options, # type: Sequence[str] + record_filename, # type: str + root, # type: Optional[str] + prefix, # type: Optional[str] + pycompile # type: bool + ): + # type: (...) -> List[str] + install_args = make_setuptools_shim_args( + self.setup_py_path, + global_options=global_options, + no_user_config=self.isolated, + unbuffered_output=True + ) + install_args += ['install', '--record', record_filename] install_args += ['--single-version-externally-managed'] if root is not None: diff --git a/pipenv/patched/notpip/_internal/req/req_set.py b/pipenv/patched/notpip/_internal/req/req_set.py index a65851ff..d99dc434 100644 --- a/pipenv/patched/notpip/_internal/req/req_set.py +++ b/pipenv/patched/notpip/_internal/req/req_set.py @@ -1,49 +1,86 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + from __future__ import absolute_import import logging from collections import OrderedDict +from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name + +from pipenv.patched.notpip._internal import pep425tags from pipenv.patched.notpip._internal.exceptions import InstallationError from pipenv.patched.notpip._internal.utils.logging import indent_log +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.wheel import Wheel +if MYPY_CHECK_RUNNING: + from typing import Dict, Iterable, List, Optional, Tuple + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + + logger = logging.getLogger(__name__) class RequirementSet(object): def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): + # type: (bool) -> None """Create a RequirementSet. """ - self.requirements = OrderedDict() + self.requirements = OrderedDict() # type: Dict[str, InstallRequirement] # noqa: E501 self.require_hashes = require_hashes self.check_supported_wheels = check_supported_wheels + + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] if ignore_compatibility: self.check_supported_wheels = False - self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - - # Mapping of alias: real_name - self.requirement_aliases = {} - self.unnamed_requirements = [] - self.successfully_downloaded = [] - self.reqs_to_cleanup = [] + self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) def __str__(self): - reqs = [req for req in self.requirements.values() - if not req.comes_from] - reqs.sort(key=lambda req: req.name.lower()) - return ' '.join([str(req.req) for req in reqs]) + # type: () -> str + requirements = sorted( + (req for req in self.requirements.values() if not req.comes_from), + key=lambda req: canonicalize_name(req.name), + ) + return ' '.join(str(req.req) for req in requirements) def __repr__(self): - reqs = [req for req in self.requirements.values()] - reqs.sort(key=lambda req: req.name.lower()) - reqs_str = ', '.join([str(req.req) for req in reqs]) - return ('<%s object; %d requirement(s): %s>' - % (self.__class__.__name__, len(reqs), reqs_str)) + # type: () -> str + requirements = sorted( + self.requirements.values(), + key=lambda req: canonicalize_name(req.name), + ) - def add_requirement(self, install_req, parent_req_name=None, - extras_requested=None): + format_string = '<{classname} object; {count} requirement(s): {reqs}>' + return format_string.format( + classname=self.__class__.__name__, + count=len(requirements), + reqs=', '.join(str(req.req) for req in requirements), + ) + + def add_unnamed_requirement(self, install_req): + # type: (InstallRequirement) -> None + assert not install_req.name + self.unnamed_requirements.append(install_req) + + def add_named_requirement(self, install_req): + # type: (InstallRequirement) -> None + assert install_req.name + + project_name = canonicalize_name(install_req.name) + self.requirements[project_name] = install_req + + def add_requirement( + self, + install_req, # type: InstallRequirement + parent_req_name=None, # type: Optional[str] + extras_requested=None # type: Optional[Iterable[str]] + ): + # type: (...) -> Tuple[List[InstallRequirement], Optional[InstallRequirement]] # noqa: E501 """Add install_req as a requirement to install. :param parent_req_name: The name of the requirement that needed this @@ -58,13 +95,11 @@ class RequirementSet(object): the requirement is not applicable, or [install_req] if the requirement is applicable and has just been added. """ - name = install_req.name - # If the markers do not match, ignore this requirement. if not install_req.match_markers(extras_requested): logger.info( "Ignoring %s: markers '%s' don't match your environment", - name, install_req.markers, + install_req.name, install_req.markers, ) return [], None @@ -74,7 +109,8 @@ class RequirementSet(object): # single requirements file. if install_req.link and install_req.link.is_wheel: wheel = Wheel(install_req.link.filename) - if self.check_supported_wheels and not wheel.supported(): + tags = pep425tags.get_supported() + if (self.check_supported_wheels and not wheel.supported(tags)): raise InstallationError( "%s is not a supported wheel on this platform." % wheel.filename @@ -88,13 +124,12 @@ class RequirementSet(object): # Unnamed requirements are scanned again and the requirement won't be # added as a dependency until after scanning. - if not name: - # url or path requirement w/o an egg fragment - self.unnamed_requirements.append(install_req) + if not install_req.name: + self.add_unnamed_requirement(install_req) return [install_req], None try: - existing_req = self.get_requirement(name) + existing_req = self.get_requirement(install_req.name) except KeyError: existing_req = None @@ -108,17 +143,14 @@ class RequirementSet(object): if has_conflicting_requirement: raise InstallationError( "Double requirement given: %s (already in %s, name=%r)" - % (install_req, existing_req, name) + % (install_req, existing_req, install_req.name) ) # When no existing requirement exists, add the requirement as a # dependency and it will be scanned again after. if not existing_req: - self.requirements[name] = install_req - # FIXME: what about other normalizations? E.g., _ vs. -? - if name.lower() != name: - self.requirement_aliases[name.lower()] = name - # We'd want to rescan this requirements later + self.add_named_requirement(install_req) + # We'd want to rescan this requirement later return [install_req], install_req # Assume there's no need to scan, and that we've already @@ -138,7 +170,7 @@ class RequirementSet(object): raise InstallationError( "Could not satisfy constraints for '%s': " "installation from path or url cannot be " - "constrained to a version" % name, + "constrained to a version" % install_req.name, ) # If we're now installing a constraint, mark the existing # object for real installation. @@ -154,29 +186,26 @@ class RequirementSet(object): # scanning again. return [existing_req], existing_req - def has_requirement(self, project_name): - name = project_name.lower() - if (name in self.requirements and - not self.requirements[name].constraint or - name in self.requirement_aliases and - not self.requirements[self.requirement_aliases[name]].constraint): - return True - return False + def has_requirement(self, name): + # type: (str) -> bool + project_name = canonicalize_name(name) - @property - def has_requirements(self): - return list(req for req in self.requirements.values() if not - req.constraint) or self.unnamed_requirements + return ( + project_name in self.requirements and + not self.requirements[project_name].constraint + ) + + def get_requirement(self, name): + # type: (str) -> InstallRequirement + project_name = canonicalize_name(name) + + if project_name in self.requirements: + return self.requirements[project_name] - def get_requirement(self, project_name): - for name in project_name, project_name.lower(): - if name in self.requirements: - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] pass def cleanup_files(self): + # type: () -> None """Clean up files, remove builds.""" logger.debug('Cleaning up...') with indent_log(): diff --git a/pipenv/patched/notpip/_internal/req/req_tracker.py b/pipenv/patched/notpip/_internal/req/req_tracker.py index 6e0201fe..1fa4fe7e 100644 --- a/pipenv/patched/notpip/_internal/req/req_tracker.py +++ b/pipenv/patched/notpip/_internal/req/req_tracker.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + from __future__ import absolute_import import contextlib @@ -7,6 +10,13 @@ import logging import os from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from types import TracebackType + from typing import Iterator, Optional, Set, Type + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from pipenv.patched.notpip._internal.models.link import Link logger = logging.getLogger(__name__) @@ -14,28 +24,37 @@ logger = logging.getLogger(__name__) class RequirementTracker(object): def __init__(self): + # type: () -> None self._root = os.environ.get('PIP_REQ_TRACKER') if self._root is None: self._temp_dir = TempDirectory(delete=False, kind='req-tracker') - self._temp_dir.create() self._root = os.environ['PIP_REQ_TRACKER'] = self._temp_dir.path logger.debug('Created requirements tracker %r', self._root) else: self._temp_dir = None logger.debug('Re-using requirements tracker %r', self._root) - self._entries = set() + self._entries = set() # type: Set[InstallRequirement] def __enter__(self): + # type: () -> RequirementTracker return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type, # type: Optional[Type[BaseException]] + exc_val, # type: Optional[BaseException] + exc_tb # type: Optional[TracebackType] + ): + # type: (...) -> None self.cleanup() def _entry_path(self, link): + # type: (Link) -> str hashed = hashlib.sha224(link.url_without_fragment.encode()).hexdigest() return os.path.join(self._root, hashed) def add(self, req): + # type: (InstallRequirement) -> None link = req.link info = str(req) entry_path = self._entry_path(link) @@ -54,12 +73,14 @@ class RequirementTracker(object): logger.debug('Added %s to build tracker %r', req, self._root) def remove(self, req): + # type: (InstallRequirement) -> None link = req.link self._entries.remove(req) os.unlink(self._entry_path(link)) logger.debug('Removed %s from build tracker %r', req, self._root) def cleanup(self): + # type: () -> None for req in set(self._entries): self.remove(req) remove = self._temp_dir is not None @@ -71,6 +92,7 @@ class RequirementTracker(object): @contextlib.contextmanager def track(self, req): + # type: (InstallRequirement) -> Iterator[None] self.add(req) yield self.remove(req) diff --git a/pipenv/patched/notpip/_internal/req/req_uninstall.py b/pipenv/patched/notpip/_internal/req/req_uninstall.py index 4cd15d84..add3418c 100644 --- a/pipenv/patched/notpip/_internal/req/req_uninstall.py +++ b/pipenv/patched/notpip/_internal/req/req_uninstall.py @@ -14,15 +14,30 @@ from pipenv.patched.notpip._internal.locations import bin_py, bin_user from pipenv.patched.notpip._internal.utils.compat import WINDOWS, cache_from_source, uses_pycache from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - FakeFile, ask, dist_in_usersite, dist_is_local, egg_link_path, is_local, - normalize_path, renames, + FakeFile, + ask, + dist_in_usersite, + dist_is_local, + egg_link_path, + is_local, + normalize_path, + renames, + rmtree, ) -from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.notpip._internal.utils.temp_dir import AdjacentTempDirectory, TempDirectory +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Dict, Iterable, Iterator, List, Optional, Set, Tuple, + ) + from pipenv.patched.notpip._vendor.pkg_resources import Distribution logger = logging.getLogger(__name__) def _script_names(dist, script_name, is_gui): + # type: (Distribution, str, bool) -> List[str] """Create the fully qualified name of the files created by {console,gui}_scripts for the given ``dist``. Returns the list of file names @@ -44,9 +59,11 @@ def _script_names(dist, script_name, is_gui): def _unique(fn): + # type: (Callable) -> Callable[..., Iterator[Any]] @functools.wraps(fn) def unique(*args, **kw): - seen = set() + # type: (Any, Any) -> Iterator[Any] + seen = set() # type: Set[Any] for item in fn(*args, **kw): if item not in seen: seen.add(item) @@ -56,6 +73,7 @@ def _unique(fn): @_unique def uninstallation_paths(dist): + # type: (Distribution) -> Iterator[str] """ Yield all the uninstallation paths for dist based on RECORD-without-.py[co] @@ -78,25 +96,67 @@ def uninstallation_paths(dist): def compact(paths): + # type: (Iterable[str]) -> Set[str] """Compact a path set to contain the minimal number of paths necessary to contain all paths in the set. If /a/path/ and /a/path/to/a/file.txt are both in the set, leave only the shorter path.""" sep = os.path.sep - short_paths = set() + short_paths = set() # type: Set[str] for path in sorted(paths, key=len): - should_add = any( + should_skip = any( path.startswith(shortpath.rstrip("*")) and path[len(shortpath.rstrip("*").rstrip(sep))] == sep for shortpath in short_paths ) - if not should_add: + if not should_skip: short_paths.add(path) return short_paths +def compress_for_rename(paths): + # type: (Iterable[str]) -> Set[str] + """Returns a set containing the paths that need to be renamed. + + This set may include directories when the original sequence of paths + included every file on disk. + """ + case_map = dict((os.path.normcase(p), p) for p in paths) + remaining = set(case_map) + unchecked = sorted(set(os.path.split(p)[0] + for p in case_map.values()), key=len) + wildcards = set() # type: Set[str] + + def norm_join(*a): + # type: (str) -> str + return os.path.normcase(os.path.join(*a)) + + for root in unchecked: + if any(os.path.normcase(root).startswith(w) + for w in wildcards): + # This directory has already been handled. + continue + + all_files = set() # type: Set[str] + all_subdirs = set() # type: Set[str] + for dirname, subdirs, files in os.walk(root): + all_subdirs.update(norm_join(root, dirname, d) + for d in subdirs) + all_files.update(norm_join(root, dirname, f) + for f in files) + # If all the files we found are in our remaining set of files to + # remove, then remove them from the latter set and add a wildcard + # for the directory. + if not (all_files - remaining): + remaining.difference_update(all_files) + wildcards.add(root + os.sep) + + return set(map(case_map.__getitem__, remaining)) | wildcards + + def compress_for_output_listing(paths): + # type: (Iterable[str]) -> Tuple[Set[str], Set[str]] """Returns a tuple of 2 sets of which paths to display to user The first set contains paths that would be deleted. Files of a package @@ -107,7 +167,7 @@ def compress_for_output_listing(paths): folders. """ - will_remove = list(paths) + will_remove = set(paths) will_skip = set() # Determine folders and files @@ -120,7 +180,8 @@ def compress_for_output_listing(paths): folders.add(os.path.dirname(path)) files.add(path) - _normcased_files = set(map(os.path.normcase, files)) + # probably this one https://github.com/python/mypy/issues/390 + _normcased_files = set(map(os.path.normcase, files)) # type: ignore folders = compact(folders) @@ -145,18 +206,130 @@ def compress_for_output_listing(paths): return will_remove, will_skip +class StashedUninstallPathSet(object): + """A set of file rename operations to stash files while + tentatively uninstalling them.""" + def __init__(self): + # type: () -> None + # Mapping from source file root to [Adjacent]TempDirectory + # for files under that directory. + self._save_dirs = {} # type: Dict[str, TempDirectory] + # (old path, new path) tuples for each move that may need + # to be undone. + self._moves = [] # type: List[Tuple[str, str]] + + def _get_directory_stash(self, path): + # type: (str) -> str + """Stashes a directory. + + Directories are stashed adjacent to their original location if + possible, or else moved/copied into the user's temp dir.""" + + try: + save_dir = AdjacentTempDirectory(path) # type: TempDirectory + except OSError: + save_dir = TempDirectory(kind="uninstall") + self._save_dirs[os.path.normcase(path)] = save_dir + + return save_dir.path + + def _get_file_stash(self, path): + # type: (str) -> str + """Stashes a file. + + If no root has been provided, one will be created for the directory + in the user's temp directory.""" + path = os.path.normcase(path) + head, old_head = os.path.dirname(path), None + save_dir = None + + while head != old_head: + try: + save_dir = self._save_dirs[head] + break + except KeyError: + pass + head, old_head = os.path.dirname(head), head + else: + # Did not find any suitable root + head = os.path.dirname(path) + save_dir = TempDirectory(kind='uninstall') + self._save_dirs[head] = save_dir + + relpath = os.path.relpath(path, head) + if relpath and relpath != os.path.curdir: + return os.path.join(save_dir.path, relpath) + return save_dir.path + + def stash(self, path): + # type: (str) -> str + """Stashes the directory or file and returns its new location. + Handle symlinks as files to avoid modifying the symlink targets. + """ + path_is_dir = os.path.isdir(path) and not os.path.islink(path) + if path_is_dir: + new_path = self._get_directory_stash(path) + else: + new_path = self._get_file_stash(path) + + self._moves.append((path, new_path)) + if (path_is_dir and os.path.isdir(new_path)): + # If we're moving a directory, we need to + # remove the destination first or else it will be + # moved to inside the existing directory. + # We just created new_path ourselves, so it will + # be removable. + os.rmdir(new_path) + renames(path, new_path) + return new_path + + def commit(self): + # type: () -> None + """Commits the uninstall by removing stashed files.""" + for _, save_dir in self._save_dirs.items(): + save_dir.cleanup() + self._moves = [] + self._save_dirs = {} + + def rollback(self): + # type: () -> None + """Undoes the uninstall by moving stashed files back.""" + for p in self._moves: + logging.info("Moving to %s\n from %s", *p) + + for new_path, path in self._moves: + try: + logger.debug('Replacing %s from %s', new_path, path) + if os.path.isfile(new_path) or os.path.islink(new_path): + os.unlink(new_path) + elif os.path.isdir(new_path): + rmtree(new_path) + renames(path, new_path) + except OSError as ex: + logger.error("Failed to restore %s", new_path) + logger.debug("Exception: %s", ex) + + self.commit() + + @property + def can_rollback(self): + # type: () -> bool + return bool(self._moves) + + class UninstallPathSet(object): """A set of file paths to be removed in the uninstallation of a requirement.""" def __init__(self, dist): - self.paths = set() - self._refuse = set() - self.pth = {} + # type: (Distribution) -> None + self.paths = set() # type: Set[str] + self._refuse = set() # type: Set[str] + self.pth = {} # type: Dict[str, UninstallPthEntries] self.dist = dist - self.save_dir = TempDirectory(kind="uninstall") - self._moved_paths = [] + self._moved_paths = StashedUninstallPathSet() def _permitted(self, path): + # type: (str) -> bool """ Return True if the given path is one we are permitted to remove/modify, False otherwise. @@ -165,6 +338,7 @@ class UninstallPathSet(object): return is_local(path) def add(self, path): + # type: (str) -> None head, tail = os.path.split(path) # we normalize the head to resolve parent directory symlinks, but not @@ -184,6 +358,7 @@ class UninstallPathSet(object): self.add(cache_from_source(path)) def add_pth(self, pth_file, entry): + # type: (str, str) -> None pth_file = normalize_path(pth_file) if self._permitted(pth_file): if pth_file not in self.pth: @@ -192,12 +367,8 @@ class UninstallPathSet(object): else: self._refuse.add(pth_file) - def _stash(self, path): - return os.path.join( - self.save_dir.path, os.path.splitdrive(path)[1].lstrip(os.path.sep) - ) - def remove(self, auto_confirm=False, verbose=False): + # type: (bool, bool) -> None """Remove paths in ``self.paths`` with confirmation (unless ``auto_confirm`` is True).""" @@ -215,23 +386,26 @@ class UninstallPathSet(object): with indent_log(): if auto_confirm or self._allowed_to_proceed(verbose): - self.save_dir.create() + moved = self._moved_paths - for path in sorted(compact(self.paths)): - new_path = self._stash(path) + for_rename = compress_for_rename(self.paths) + + for path in sorted(compact(for_rename)): + moved.stash(path) logger.debug('Removing file or directory %s', path) - self._moved_paths.append(path) - renames(path, new_path) + for pth in self.pth.values(): pth.remove() logger.info('Successfully uninstalled %s', dist_name_version) def _allowed_to_proceed(self, verbose): + # type: (bool) -> bool """Display which files would be deleted and prompt for confirmation """ def _display(msg, paths): + # type: (str, Iterable[str]) -> None if not paths: return @@ -245,38 +419,39 @@ class UninstallPathSet(object): else: # In verbose mode, display all the files that are going to be # deleted. - will_remove = list(self.paths) + will_remove = set(self.paths) will_skip = set() _display('Would remove:', will_remove) _display('Would not remove (might be manually added):', will_skip) _display('Would not remove (outside of prefix):', self._refuse) + if verbose: + _display('Will actually move:', compress_for_rename(self.paths)) return ask('Proceed (y/n)? ', ('y', 'n')) == 'y' def rollback(self): + # type: () -> None """Rollback the changes previously made by remove().""" - if self.save_dir.path is None: + if not self._moved_paths.can_rollback: logger.error( "Can't roll back %s; was not uninstalled", self.dist.project_name, ) - return False + return logger.info('Rolling back uninstall of %s', self.dist.project_name) - for path in self._moved_paths: - tmp_path = self._stash(path) - logger.debug('Replacing %s', path) - renames(tmp_path, path) + self._moved_paths.rollback() for pth in self.pth.values(): pth.rollback() def commit(self): + # type: () -> None """Remove temporary save dir: rollback will no longer be possible.""" - self.save_dir.cleanup() - self._moved_paths = [] + self._moved_paths.commit() @classmethod def from_dist(cls, dist): + # type: (Distribution) -> UninstallPathSet dist_path = normalize_path(dist.location) if not dist_is_local(dist): logger.info( @@ -408,25 +583,33 @@ class UninstallPathSet(object): class UninstallPthEntries(object): def __init__(self, pth_file): + # type: (str) -> None if not os.path.isfile(pth_file): raise UninstallationError( "Cannot remove entries from nonexistent file %s" % pth_file ) self.file = pth_file - self.entries = set() - self._saved_lines = None + self.entries = set() # type: Set[str] + self._saved_lines = None # type: Optional[List[bytes]] def add(self, entry): + # type: (str) -> None entry = os.path.normcase(entry) # On Windows, os.path.normcase converts the entry to use # backslashes. This is correct for entries that describe absolute # paths outside of site-packages, but all the others use forward # slashes. + # os.path.splitdrive is used instead of os.path.isabs because isabs + # treats non-absolute paths with drive letter markings like c:foo\bar + # as absolute paths. It also does not recognize UNC paths if they don't + # have more than "\\sever\share". Valid examples: "\\server\share\" or + # "\\server\share\folder". Python 2.7.8+ support UNC in splitdrive. if WINDOWS and not os.path.splitdrive(entry)[0]: entry = entry.replace('\\', '/') self.entries.add(entry) def remove(self): + # type: () -> None logger.debug('Removing pth entries from %s:', self.file) with open(self.file, 'rb') as fh: # windows uses '\r\n' with py3k, but uses '\n' with py2.x @@ -449,6 +632,7 @@ class UninstallPthEntries(object): fh.writelines(lines) def rollback(self): + # type: () -> bool if self._saved_lines is None: logger.error( 'Cannot roll back changes to %s, none were made', self.file diff --git a/pipenv/patched/notpip/_internal/self_outdated_check.py b/pipenv/patched/notpip/_internal/self_outdated_check.py new file mode 100644 index 00000000..8bf5e9f7 --- /dev/null +++ b/pipenv/patched/notpip/_internal/self_outdated_check.py @@ -0,0 +1,244 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import datetime +import hashlib +import json +import logging +import os.path +import sys + +from pipenv.patched.notpip._vendor import pkg_resources +from pipenv.patched.notpip._vendor.packaging import version as packaging_version +from pipenv.patched.notpip._vendor.six import ensure_binary + +from pipenv.patched.notpip._internal.collector import LinkCollector +from pipenv.patched.notpip._internal.index import PackageFinder +from pipenv.patched.notpip._internal.models.search_scope import SearchScope +from pipenv.patched.notpip._internal.models.selection_prefs import SelectionPreferences +from pipenv.patched.notpip._internal.utils.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.filesystem import ( + adjacent_tmp_file, + check_path_owner, + replace, +) +from pipenv.patched.notpip._internal.utils.misc import ( + ensure_dir, + get_installed_version, + redact_auth_from_url, +) +from pipenv.patched.notpip._internal.utils.packaging import get_installer +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + import optparse + from optparse import Values + from typing import Any, Dict, Text, Union + + from pipenv.patched.notpip._internal.network.session import PipSession + + +SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" + + +logger = logging.getLogger(__name__) + + +def make_link_collector( + session, # type: PipSession + options, # type: Values + suppress_no_index=False, # type: bool +): + # type: (...) -> LinkCollector + """ + :param session: The Session to use to make requests. + :param suppress_no_index: Whether to ignore the --no-index option + when constructing the SearchScope object. + """ + index_urls = [options.index_url] + options.extra_index_urls + if options.no_index and not suppress_no_index: + logger.debug( + 'Ignoring indexes: %s', + ','.join(redact_auth_from_url(url) for url in index_urls), + ) + index_urls = [] + + # Make sure find_links is a list before passing to create(). + find_links = options.find_links or [] + + search_scope = SearchScope.create( + find_links=find_links, index_urls=index_urls, + ) + + link_collector = LinkCollector(session=session, search_scope=search_scope) + + return link_collector + + +def _get_statefile_name(key): + # type: (Union[str, Text]) -> str + key_bytes = ensure_binary(key) + name = hashlib.sha224(key_bytes).hexdigest() + return name + + +class SelfCheckState(object): + def __init__(self, cache_dir): + # type: (str) -> None + self.state = {} # type: Dict[str, Any] + self.statefile_path = None + + # Try to load the existing state + if cache_dir: + self.statefile_path = os.path.join( + cache_dir, "selfcheck", _get_statefile_name(self.key) + ) + try: + with open(self.statefile_path) as statefile: + self.state = json.load(statefile) + except (IOError, ValueError, KeyError): + # Explicitly suppressing exceptions, since we don't want to + # error out if the cache file is invalid. + pass + + @property + def key(self): + return sys.prefix + + def save(self, pypi_version, current_time): + # type: (str, datetime.datetime) -> None + # If we do not have a path to cache in, don't bother saving. + if not self.statefile_path: + return + + # Check to make sure that we own the directory + if not check_path_owner(os.path.dirname(self.statefile_path)): + return + + # Now that we've ensured the directory is owned by this user, we'll go + # ahead and make sure that all our directories are created. + ensure_dir(os.path.dirname(self.statefile_path)) + + state = { + # Include the key so it's easy to tell which pip wrote the + # file. + "key": self.key, + "last_check": current_time.strftime(SELFCHECK_DATE_FMT), + "pypi_version": pypi_version, + } + + text = json.dumps(state, sort_keys=True, separators=(",", ":")) + + with adjacent_tmp_file(self.statefile_path) as f: + f.write(ensure_binary(text)) + + try: + # Since we have a prefix-specific state file, we can just + # overwrite whatever is there, no need to check. + replace(f.name, self.statefile_path) + except OSError: + # Best effort. + pass + + +def was_installed_by_pip(pkg): + # type: (str) -> bool + """Checks whether pkg was installed by pip + + This is used not to display the upgrade message when pip is in fact + installed by system package manager, such as dnf on Fedora. + """ + try: + dist = pkg_resources.get_distribution(pkg) + return "pip" == get_installer(dist) + except pkg_resources.DistributionNotFound: + return False + + +def pip_self_version_check(session, options): + # type: (PipSession, optparse.Values) -> None + """Check for an update for pip. + + Limit the frequency of checks to once per week. State is stored either in + the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix + of the pip script path. + """ + installed_version = get_installed_version("pip") + if not installed_version: + return + + pip_version = packaging_version.parse(installed_version) + pypi_version = None + + try: + state = SelfCheckState(cache_dir=options.cache_dir) + + current_time = datetime.datetime.utcnow() + # Determine if we need to refresh the state + if "last_check" in state.state and "pypi_version" in state.state: + last_check = datetime.datetime.strptime( + state.state["last_check"], + SELFCHECK_DATE_FMT + ) + if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60: + pypi_version = state.state["pypi_version"] + + # Refresh the version if we need to or just see if we need to warn + if pypi_version is None: + # Lets use PackageFinder to see what the latest pip version is + link_collector = make_link_collector( + session, + options=options, + suppress_no_index=True, + ) + + # Pass allow_yanked=False so we don't suggest upgrading to a + # yanked version. + selection_prefs = SelectionPreferences( + allow_yanked=False, + allow_all_prereleases=False, # Explicitly set to False + ) + + finder = PackageFinder.create( + link_collector=link_collector, + selection_prefs=selection_prefs, + ) + best_candidate = finder.find_best_candidate("pip").best_candidate + if best_candidate is None: + return + pypi_version = str(best_candidate.version) + + # save that we've performed a check + state.save(pypi_version, current_time) + + remote_version = packaging_version.parse(pypi_version) + + local_version_is_older = ( + pip_version < remote_version and + pip_version.base_version != remote_version.base_version and + was_installed_by_pip('pip') + ) + + # Determine if our pypi_version is older + if not local_version_is_older: + return + + # Advise "python -m pip" on Windows to avoid issues + # with overwriting pip.exe. + if WINDOWS: + pip_cmd = "python -m pip" + else: + pip_cmd = "pip" + logger.warning( + "You are using pip version %s; however, version %s is " + "available.\nYou should consider upgrading via the " + "'%s install --upgrade pip' command.", + pip_version, pypi_version, pip_cmd + ) + except Exception: + logger.debug( + "There was an error checking the latest version of pip", + exc_info=True, + ) diff --git a/pipenv/patched/notpip/_internal/utils/appdirs.py b/pipenv/patched/notpip/_internal/utils/appdirs.py index e8e14526..c1ba02d2 100644 --- a/pipenv/patched/notpip/_internal/utils/appdirs.py +++ b/pipenv/patched/notpip/_internal/utils/appdirs.py @@ -2,6 +2,10 @@ This code was taken from https://github.com/ActiveState/appdirs and modified to suit our purposes. """ + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import os @@ -10,9 +14,14 @@ import sys from pipenv.patched.notpip._vendor.six import PY2, text_type from pipenv.patched.notpip._internal.utils.compat import WINDOWS, expanduser +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List def user_cache_dir(appname): + # type: (str) -> str r""" Return full path to the user-specific cache dir for this application. @@ -61,6 +70,7 @@ def user_cache_dir(appname): def user_data_dir(appname, roaming=False): + # type: (str, bool) -> str r""" Return full path to the user-specific data dir for this application. @@ -113,6 +123,7 @@ def user_data_dir(appname, roaming=False): def user_config_dir(appname, roaming=True): + # type: (str, bool) -> str """Return full path to the user-specific config dir for this application. "appname" is the name of application. @@ -146,6 +157,7 @@ def user_config_dir(appname, roaming=True): # for the discussion regarding site_config_dirs locations # see def site_config_dirs(appname): + # type: (str) -> List[str] r"""Return a list of potential user-shared config dirs for this application. "appname" is the name of application. @@ -186,6 +198,7 @@ def site_config_dirs(appname): # -- Windows support functions -- def _get_win_folder_from_registry(csidl_name): + # type: (str) -> str """ This is a fallback technique at best. I'm not sure if using the registry for this guarantees us the correct answer for all CSIDL_* @@ -208,6 +221,9 @@ def _get_win_folder_from_registry(csidl_name): def _get_win_folder_with_ctypes(csidl_name): + # type: (str) -> str + # On Python 2, ctypes.create_unicode_buffer().value returns "unicode", + # which isn't the same as str in the annotation above. csidl_const = { "CSIDL_APPDATA": 26, "CSIDL_COMMON_APPDATA": 35, @@ -215,7 +231,8 @@ def _get_win_folder_with_ctypes(csidl_name): }[csidl_name] buf = ctypes.create_unicode_buffer(1024) - ctypes.windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) + windll = ctypes.windll # type: ignore + windll.shell32.SHGetFolderPathW(None, csidl_const, None, 0, buf) # Downgrade to short path name if have highbit chars. See # . @@ -226,10 +243,11 @@ def _get_win_folder_with_ctypes(csidl_name): break if has_high_char: buf2 = ctypes.create_unicode_buffer(1024) - if ctypes.windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): + if windll.kernel32.GetShortPathNameW(buf.value, buf2, 1024): buf = buf2 - return buf.value + # The type: ignore is explained under the type annotation for this function + return buf.value # type: ignore if WINDOWS: diff --git a/pipenv/patched/notpip/_internal/utils/compat.py b/pipenv/patched/notpip/_internal/utils/compat.py index 483bfdc8..758aa0d3 100644 --- a/pipenv/patched/notpip/_internal/utils/compat.py +++ b/pipenv/patched/notpip/_internal/utils/compat.py @@ -1,5 +1,9 @@ """Stuff that differs in different Python versions and platform distributions.""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import, division import codecs @@ -9,7 +13,21 @@ import os import shutil import sys -from pipenv.patched.notpip._vendor.six import text_type +from pipenv.patched.notpip._vendor.six import PY2, text_type +from pipenv.patched.notpip._vendor.urllib3.util import IS_PYOPENSSL + +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Tuple, Union + +try: + import _ssl # noqa +except ImportError: + ssl = None +else: + # This additional assignment was needed to prevent a mypy error. + ssl = _ssl try: import ipaddress @@ -18,8 +36,8 @@ except ImportError: from pipenv.patched.notpip._vendor import ipaddress # type: ignore except ImportError: import ipaddr as ipaddress # type: ignore - ipaddress.ip_address = ipaddress.IPAddress - ipaddress.ip_network = ipaddress.IPNetwork + ipaddress.ip_address = ipaddress.IPAddress # type: ignore + ipaddress.ip_network = ipaddress.IPNetwork # type: ignore __all__ = [ @@ -31,10 +49,9 @@ __all__ = [ logger = logging.getLogger(__name__) -if sys.version_info >= (3, 4): - uses_pycache = True - from importlib.util import cache_from_source -else: +HAS_TLS = (ssl is not None) or IS_PYOPENSSL + +if PY2: import imp try: @@ -44,40 +61,54 @@ else: cache_from_source = None uses_pycache = cache_from_source is not None - - -if sys.version_info >= (3, 5): - backslashreplace_decode = "backslashreplace" else: - # In version 3.4 and older, backslashreplace exists + uses_pycache = True + from importlib.util import cache_from_source + + +if PY2: + # In Python 2.7, backslashreplace exists # but does not support use for decoding. # We implement our own replace handler for this # situation, so that we can consistently use # backslash replacement for all versions. def backslashreplace_decode_fn(err): raw_bytes = (err.object[i] for i in range(err.start, err.end)) - if sys.version_info[0] == 2: - # Python 2 gave us characters - convert to numeric bytes - raw_bytes = (ord(b) for b in raw_bytes) + # Python 2 gave us characters - convert to numeric bytes + raw_bytes = (ord(b) for b in raw_bytes) return u"".join(u"\\x%x" % c for c in raw_bytes), err.end codecs.register_error( "backslashreplace_decode", backslashreplace_decode_fn, ) backslashreplace_decode = "backslashreplace_decode" +else: + backslashreplace_decode = "backslashreplace" -def console_to_str(data): - """Return a string, safe for output, of subprocess output. - - We assume the data is in the locale preferred encoding. - If it won't decode properly, we warn the user but decode as - best we can. - - We also ensure that the output can be safely written to - standard output without encoding errors. +def str_to_display(data, desc=None): + # type: (Union[bytes, Text], Optional[str]) -> Text """ + For display or logging purposes, convert a bytes object (or text) to + text (e.g. unicode in Python 2) safe for output. + :param desc: An optional phrase describing the input data, for use in + the log message if a warning is logged. Defaults to "Bytes object". + + This function should never error out and so can take a best effort + approach. It is okay to be lossy if needed since the return value is + just for display. + + We assume the data is in the locale preferred encoding. If it won't + decode properly, we warn the user but decode as best we can. + + We also ensure that the output can be safely written to standard output + without encoding errors. + """ + if isinstance(data, text_type): + return data + + # Otherwise, data is a bytes object (str in Python 2). # First, get the encoding we assume. This is the preferred # encoding for the locale, unless that is not found, or # it is ASCII, in which case assume UTF-8 @@ -88,13 +119,13 @@ def console_to_str(data): # Now try to decode the data - if we fail, warn the user and # decode with replacement. try: - s = data.decode(encoding) + decoded_data = data.decode(encoding) except UnicodeDecodeError: - logger.warning( - "Subprocess output does not appear to be encoded as %s", - encoding, - ) - s = data.decode(encoding, errors=backslashreplace_decode) + if desc is None: + desc = 'Bytes object' + msg_format = '{} does not appear to be encoded as %s'.format(desc) + logger.warning(msg_format, encoding) + decoded_data = data.decode(encoding, errors=backslashreplace_decode) # Make sure we can print the output, by encoding it to the output # encoding with replacement of unencodable characters, and then @@ -112,27 +143,40 @@ def console_to_str(data): "encoding", None) if output_encoding: - s = s.encode(output_encoding, errors="backslashreplace") - s = s.decode(output_encoding) + output_encoded = decoded_data.encode( + output_encoding, + errors="backslashreplace" + ) + decoded_data = output_encoded.decode(output_encoding) - return s + return decoded_data -if sys.version_info >= (3,): - def native_str(s, replace=False): - if isinstance(s, bytes): - return s.decode('utf-8', 'replace' if replace else 'strict') - return s - -else: +def console_to_str(data): + # type: (bytes) -> Text + """Return a string, safe for output, of subprocess output. + """ + return str_to_display(data, desc='Subprocess output') + + +if PY2: def native_str(s, replace=False): + # type: (str, bool) -> str # Replace is ignored -- unicode to UTF-8 can't fail if isinstance(s, text_type): return s.encode('utf-8') return s +else: + def native_str(s, replace=False): + # type: (str, bool) -> str + if isinstance(s, bytes): + return s.decode('utf-8', 'replace' if replace else 'strict') + return s + def get_path_uid(path): + # type: (str) -> int """ Return path's uid. @@ -161,19 +205,21 @@ def get_path_uid(path): return file_uid -if sys.version_info >= (3, 4): - from importlib.machinery import EXTENSION_SUFFIXES - - def get_extension_suffixes(): - return EXTENSION_SUFFIXES -else: +if PY2: from imp import get_suffixes def get_extension_suffixes(): return [suffix[0] for suffix in get_suffixes()] +else: + from importlib.machinery import EXTENSION_SUFFIXES + + def get_extension_suffixes(): + return EXTENSION_SUFFIXES + def expanduser(path): + # type: (str) -> str """ Expand ~ and ~user constructions. @@ -199,6 +245,7 @@ WINDOWS = (sys.platform.startswith("win") or def samefile(file1, file2): + # type: (str, str) -> bool """Provide an alternative for os.path.samefile on Windows/Python2""" if hasattr(os.path, 'samefile'): return os.path.samefile(file1, file2) @@ -210,13 +257,15 @@ def samefile(file1, file2): if hasattr(shutil, 'get_terminal_size'): def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. """ - return tuple(shutil.get_terminal_size()) + return tuple(shutil.get_terminal_size()) # type: ignore else: def get_terminal_size(): + # type: () -> Tuple[int, int] """ Returns a tuple (x, y) representing the width(x) and the height(y) in characters of the terminal window. diff --git a/pipenv/patched/notpip/_internal/utils/deprecation.py b/pipenv/patched/notpip/_internal/utils/deprecation.py index b140ac71..dcb407be 100644 --- a/pipenv/patched/notpip/_internal/utils/deprecation.py +++ b/pipenv/patched/notpip/_internal/utils/deprecation.py @@ -1,6 +1,10 @@ """ A module that implements tooling to enable easy warnings about deprecations. """ + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -12,7 +16,10 @@ from pipenv.patched.notpip import __version__ as current_version from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any, Optional # noqa: F401 + from typing import Any, Optional + + +DEPRECATION_MSG_PREFIX = "DEPRECATION: " class PipDeprecationWarning(Warning): @@ -41,6 +48,7 @@ def _showwarning(message, category, filename, lineno, file=None, line=None): def install_warning_logger(): + # type: () -> None # Enable our Deprecation Warnings warnings.simplefilter("default", PipDeprecationWarning, append=True) @@ -74,16 +82,23 @@ def deprecated(reason, replacement, gone_in, issue=None): """ # Construct a nice message. - # This is purposely eagerly formatted as we want it to appear as if someone - # typed this entire message out. - message = "DEPRECATION: " + reason - if replacement is not None: - message += " A possible replacement is {}.".format(replacement) - if issue is not None: - url = "https://github.com/pypa/pip/issues/" + str(issue) - message += " You can find discussion regarding this at {}.".format(url) + # This is eagerly formatted as we want it to get logged as if someone + # typed this entire message out. + sentences = [ + (reason, DEPRECATION_MSG_PREFIX + "{}"), + (gone_in, "pip {} will remove support for this functionality."), + (replacement, "A possible replacement is {}."), + (issue, ( + "You can find discussion regarding this at " + "https://github.com/pypa/pip/issues/{}." + )), + ] + message = " ".join( + template.format(val) for val, template in sentences if val is not None + ) # Raise as an error if it has to be removed. if gone_in is not None and parse(current_version) >= parse(gone_in): raise PipDeprecationWarning(message) + warnings.warn(message, category=PipDeprecationWarning, stacklevel=2) diff --git a/pipenv/patched/notpip/_internal/utils/encoding.py b/pipenv/patched/notpip/_internal/utils/encoding.py index 56f60361..5abba6f9 100644 --- a/pipenv/patched/notpip/_internal/utils/encoding.py +++ b/pipenv/patched/notpip/_internal/utils/encoding.py @@ -1,22 +1,31 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + import codecs import locale import re import sys +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Tuple, Text + BOMS = [ - (codecs.BOM_UTF8, 'utf8'), - (codecs.BOM_UTF16, 'utf16'), - (codecs.BOM_UTF16_BE, 'utf16-be'), - (codecs.BOM_UTF16_LE, 'utf16-le'), - (codecs.BOM_UTF32, 'utf32'), - (codecs.BOM_UTF32_BE, 'utf32-be'), - (codecs.BOM_UTF32_LE, 'utf32-le'), -] + (codecs.BOM_UTF8, 'utf-8'), + (codecs.BOM_UTF16, 'utf-16'), + (codecs.BOM_UTF16_BE, 'utf-16-be'), + (codecs.BOM_UTF16_LE, 'utf-16-le'), + (codecs.BOM_UTF32, 'utf-32'), + (codecs.BOM_UTF32_BE, 'utf-32-be'), + (codecs.BOM_UTF32_LE, 'utf-32-le'), +] # type: List[Tuple[bytes, Text]] ENCODING_RE = re.compile(br'coding[:=]\s*([-\w.]+)') def auto_decode(data): + # type: (bytes) -> Text """Check a bytes string for a BOM to correctly detect the encoding Fallback to locale.getpreferredencoding(False) like open() on Python3""" diff --git a/pipenv/patched/notpip/_internal/utils/filesystem.py b/pipenv/patched/notpip/_internal/utils/filesystem.py index e8d6a2bb..c1e4507d 100644 --- a/pipenv/patched/notpip/_internal/utils/filesystem.py +++ b/pipenv/patched/notpip/_internal/utils/filesystem.py @@ -1,10 +1,31 @@ import os import os.path +import shutil +import stat +from contextlib import contextmanager +from tempfile import NamedTemporaryFile + +# NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is +# why we ignore the type on this import. +from pipenv.patched.notpip._vendor.retrying import retry # type: ignore +from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._internal.utils.compat import get_path_uid +from pipenv.patched.notpip._internal.utils.misc import cast +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import BinaryIO, Iterator + + class NamedTemporaryFileResult(BinaryIO): + @property + def file(self): + # type: () -> BinaryIO + pass def check_path_owner(path): + # type: (str) -> bool # If we don't have a way to check the effective uid of this process, then # we'll just assume that we own the directory. if not hasattr(os, "geteuid"): @@ -26,3 +47,69 @@ def check_path_owner(path): return os.access(path, os.W_OK) else: previous, path = path, os.path.dirname(path) + return False # assume we don't own the path + + +def copy2_fixed(src, dest): + # type: (str, str) -> None + """Wrap shutil.copy2() but map errors copying socket files to + SpecialFileError as expected. + + See also https://bugs.python.org/issue37700. + """ + try: + shutil.copy2(src, dest) + except (OSError, IOError): + for f in [src, dest]: + try: + is_socket_file = is_socket(f) + except OSError: + # An error has already occurred. Another error here is not + # a problem and we can ignore it. + pass + else: + if is_socket_file: + raise shutil.SpecialFileError("`%s` is a socket" % f) + + raise + + +def is_socket(path): + # type: (str) -> bool + return stat.S_ISSOCK(os.lstat(path).st_mode) + + +@contextmanager +def adjacent_tmp_file(path): + # type: (str) -> Iterator[NamedTemporaryFileResult] + """Given a path to a file, open a temp file next to it securely and ensure + it is written to disk after the context reaches its end. + """ + with NamedTemporaryFile( + delete=False, + dir=os.path.dirname(path), + prefix=os.path.basename(path), + suffix='.tmp', + ) as f: + result = cast('NamedTemporaryFileResult', f) + try: + yield result + finally: + result.file.flush() + os.fsync(result.file.fileno()) + + +_replace_retry = retry(stop_max_delay=1000, wait_fixed=250) + +if PY2: + @_replace_retry + def replace(src, dest): + # type: (str, str) -> None + try: + os.rename(src, dest) + except OSError: + os.remove(dest) + os.rename(src, dest) + +else: + replace = _replace_retry(os.replace) diff --git a/pipenv/patched/notpip/_internal/utils/filetypes.py b/pipenv/patched/notpip/_internal/utils/filetypes.py new file mode 100644 index 00000000..482925d1 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/filetypes.py @@ -0,0 +1,16 @@ +"""Filetype information. +""" +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Tuple + +WHEEL_EXTENSION = '.whl' +BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') # type: Tuple[str, ...] +XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', + '.tar.lz', '.tar.lzma') # type: Tuple[str, ...] +ZIP_EXTENSIONS = ('.zip', WHEEL_EXTENSION) # type: Tuple[str, ...] +TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') # type: Tuple[str, ...] +ARCHIVE_EXTENSIONS = ( + ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS +) diff --git a/pipenv/patched/notpip/_internal/utils/glibc.py b/pipenv/patched/notpip/_internal/utils/glibc.py index ebcfc5be..5e3c6b1a 100644 --- a/pipenv/patched/notpip/_internal/utils/glibc.py +++ b/pipenv/patched/notpip/_internal/utils/glibc.py @@ -1,12 +1,48 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + from __future__ import absolute_import -import ctypes +import os import re import warnings +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + def glibc_version_string(): + # type: () -> Optional[str] "Returns glibc version string, or None if not using glibc." + return glibc_version_string_confstr() or glibc_version_string_ctypes() + + +def glibc_version_string_confstr(): + # type: () -> Optional[str] + "Primary implementation of glibc_version_string using os.confstr." + # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely + # to be broken or missing. This strategy is used in the standard library + # platform module: + # https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183 + try: + # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17": + _, version = os.confstr("CS_GNU_LIBC_VERSION").split() + except (AttributeError, OSError, ValueError): + # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... + return None + return version + + +def glibc_version_string_ctypes(): + # type: () -> Optional[str] + "Fallback implementation of glibc_version_string using ctypes." + + try: + import ctypes + except ImportError: + return None # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen # manpage says, "If filename is NULL, then the returned handle is for the @@ -32,6 +68,7 @@ def glibc_version_string(): # Separated out from have_compatible_glibc for easier unit testing def check_glibc_version(version_str, required_major, minimum_minor): + # type: (str, int, int) -> bool # Parse string and check against requested version. # # We use a regexp instead of str.split because we want to discard any @@ -48,6 +85,7 @@ def check_glibc_version(version_str, required_major, minimum_minor): def have_compatible_glibc(required_major, minimum_minor): + # type: (int, int) -> bool version_str = glibc_version_string() if version_str is None: return False @@ -72,6 +110,7 @@ def have_compatible_glibc(required_major, minimum_minor): # misleading. Solution: instead of using platform, use our code that actually # works. def libc_ver(): + # type: () -> Tuple[str, str] """Try to determine the glibc version Returns a tuple of strings (lib, version) which default to empty strings diff --git a/pipenv/patched/notpip/_internal/utils/hashes.py b/pipenv/patched/notpip/_internal/utils/hashes.py index 818f5505..ef81612b 100644 --- a/pipenv/patched/notpip/_internal/utils/hashes.py +++ b/pipenv/patched/notpip/_internal/utils/hashes.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import hashlib @@ -5,9 +8,23 @@ import hashlib from pipenv.patched.notpip._vendor.six import iteritems, iterkeys, itervalues from pipenv.patched.notpip._internal.exceptions import ( - HashMismatch, HashMissing, InstallationError, + HashMismatch, + HashMissing, + InstallationError, ) from pipenv.patched.notpip._internal.utils.misc import read_chunks +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import ( + Dict, List, BinaryIO, NoReturn, Iterator + ) + from pipenv.patched.notpip._vendor.six import PY3 + if PY3: + from hashlib import _Hash + else: + from hashlib import _hash as _Hash + # The recommended hash algo of the moment. Change this whenever the state of # the art changes; it won't hurt backward compatibility. @@ -25,13 +42,28 @@ class Hashes(object): """ def __init__(self, hashes=None): + # type: (Dict[str, List[str]]) -> None """ :param hashes: A dict of algorithm names pointing to lists of allowed hex digests """ self._allowed = {} if hashes is None else hashes + @property + def digest_count(self): + # type: () -> int + return sum(len(digests) for digests in self._allowed.values()) + + def is_hash_allowed( + self, + hash_name, # type: str + hex_digest, # type: str + ): + """Return whether the given hex digest is allowed.""" + return hex_digest in self._allowed.get(hash_name, []) + def check_against_chunks(self, chunks): + # type: (Iterator[bytes]) -> None """Check good hashes against ones built from iterable of chunks of data. @@ -55,9 +87,11 @@ class Hashes(object): self._raise(gots) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMismatch(self._allowed, gots) def check_against_file(self, file): + # type: (BinaryIO) -> None """Check good hashes against a file-like object Raise HashMismatch if none match. @@ -66,14 +100,17 @@ class Hashes(object): return self.check_against_chunks(read_chunks(file)) def check_against_path(self, path): + # type: (str) -> None with open(path, 'rb') as file: return self.check_against_file(file) def __nonzero__(self): + # type: () -> bool """Return whether I know any known-good hashes.""" return bool(self._allowed) def __bool__(self): + # type: () -> bool return self.__nonzero__() @@ -85,10 +122,12 @@ class MissingHashes(Hashes): """ def __init__(self): + # type: () -> None """Don't offer the ``hashes`` kwarg.""" # Pass our favorite hash in to generate a "gotten hash". With the # empty list, it will never match, so an error will always raise. super(MissingHashes, self).__init__(hashes={FAVORITE_HASH: []}) def _raise(self, gots): + # type: (Dict[str, _Hash]) -> NoReturn raise HashMissing(gots[FAVORITE_HASH].hexdigest()) diff --git a/pipenv/patched/notpip/_internal/utils/inject_securetransport.py b/pipenv/patched/notpip/_internal/utils/inject_securetransport.py new file mode 100644 index 00000000..f367b64e --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/inject_securetransport.py @@ -0,0 +1,36 @@ +"""A helper module that injects SecureTransport, on import. + +The import should be done as early as possible, to ensure all requests and +sessions (or whatever) are created after injecting SecureTransport. + +Note that we only do the injection on macOS, when the linked OpenSSL is too +old to handle TLSv1.2. +""" + +import sys + + +def inject_securetransport(): + # type: () -> None + # Only relevant on macOS + if sys.platform != "darwin": + return + + try: + import ssl + except ImportError: + return + + # Checks for OpenSSL 1.0.1 + if ssl.OPENSSL_VERSION_NUMBER >= 0x1000100f: + return + + try: + from pipenv.patched.notpip._vendor.urllib3.contrib import securetransport + except (ImportError, OSError): + return + + securetransport.inject_into_urllib3() + + +inject_securetransport() diff --git a/pipenv/patched/notpip/_internal/utils/logging.py b/pipenv/patched/notpip/_internal/utils/logging.py index 576c4fa0..d956b6a7 100644 --- a/pipenv/patched/notpip/_internal/utils/logging.py +++ b/pipenv/patched/notpip/_internal/utils/logging.py @@ -1,11 +1,20 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import contextlib +import errno import logging import logging.handlers import os +import sys +from logging import Filter, getLogger + +from pipenv.patched.notpip._vendor.six import PY2 from pipenv.patched.notpip._internal.utils.compat import WINDOWS +from pipenv.patched.notpip._internal.utils.deprecation import DEPRECATION_MSG_PREFIX from pipenv.patched.notpip._internal.utils.misc import ensure_dir try: @@ -15,15 +24,78 @@ except ImportError: try: - from pipenv.patched.notpip._vendor import colorama + # Use "import as" and set colorama in the else clause to avoid mypy + # errors and get the following correct revealed type for colorama: + # `Union[_importlib_modulespec.ModuleType, None]` + # Otherwise, we get an error like the following in the except block: + # > Incompatible types in assignment (expression has type "None", + # variable has type Module) + # TODO: eliminate the need to use "import as" once mypy addresses some + # of its issues with conditional imports. Here is an umbrella issue: + # https://github.com/python/mypy/issues/1297 + from pipenv.patched.notpip._vendor import colorama as _colorama # Lots of different errors can come from this, including SystemError and # ImportError. except Exception: colorama = None +else: + # Import Fore explicitly rather than accessing below as colorama.Fore + # to avoid the following error running mypy: + # > Module has no attribute "Fore" + # TODO: eliminate the need to import Fore once mypy addresses some of its + # issues with conditional imports. This particular case could be an + # instance of the following issue (but also see the umbrella issue above): + # https://github.com/python/mypy/issues/3500 + from pipenv.patched.notpip._vendor.colorama import Fore + + colorama = _colorama _log_state = threading.local() _log_state.indentation = 0 +subprocess_logger = getLogger('pip.subprocessor') + + +class BrokenStdoutLoggingError(Exception): + """ + Raised if BrokenPipeError occurs for the stdout stream while logging. + """ + pass + + +# BrokenPipeError does not exist in Python 2 and, in addition, manifests +# differently in Windows and non-Windows. +if WINDOWS: + # In Windows, a broken pipe can show up as EINVAL rather than EPIPE: + # https://bugs.python.org/issue19612 + # https://bugs.python.org/issue30418 + if PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and + exc.errno in (errno.EINVAL, errno.EPIPE)) + else: + # In Windows, a broken pipe IOError became OSError in Python 3. + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return ((exc_class is BrokenPipeError) or # noqa: F821 + (exc_class is OSError and + exc.errno in (errno.EINVAL, errno.EPIPE))) +elif PY2: + def _is_broken_pipe_error(exc_class, exc): + """See the docstring for non-Windows Python 3 below.""" + return (exc_class is IOError and exc.errno == errno.EPIPE) +else: + # Then we are in the non-Windows Python 3 case. + def _is_broken_pipe_error(exc_class, exc): + """ + Return whether an exception is a broken pipe error. + + Args: + exc_class: an exception class. + exc: an exception instance. + """ + return (exc_class is BrokenPipeError) # noqa: F821 @contextlib.contextmanager @@ -45,14 +117,49 @@ def get_indentation(): class IndentingFormatter(logging.Formatter): + def __init__(self, *args, **kwargs): + """ + A logging.Formatter that obeys the indent_log() context manager. + + :param add_timestamp: A bool indicating output lines should be prefixed + with their record's timestamp. + """ + self.add_timestamp = kwargs.pop("add_timestamp", False) + super(IndentingFormatter, self).__init__(*args, **kwargs) + + def get_message_start(self, formatted, levelno): + """ + Return the start of the formatted log message (not counting the + prefix to add to each line). + """ + if levelno < logging.WARNING: + return '' + if formatted.startswith(DEPRECATION_MSG_PREFIX): + # Then the message already has a prefix. We don't want it to + # look like "WARNING: DEPRECATION: ...." + return '' + if levelno < logging.ERROR: + return 'WARNING: ' + + return 'ERROR: ' + def format(self, record): """ - Calls the standard formatter, but will indent all of the log messages - by our current indentation level. + Calls the standard formatter, but will indent all of the log message + lines by our current indentation level. """ - formatted = logging.Formatter.format(self, record) + formatted = super(IndentingFormatter, self).format(record) + message_start = self.get_message_start(formatted, record.levelno) + formatted = message_start + formatted + + prefix = '' + if self.add_timestamp: + # TODO: Use Formatter.default_time_format after dropping PY2. + t = self.formatTime(record, "%Y-%m-%dT%H:%M:%S") + prefix = '%s,%03d ' % (t, record.msecs) + prefix += " " * get_indentation() formatted = "".join([ - (" " * get_indentation()) + line + prefix + line for line in formatted.splitlines(True) ]) return formatted @@ -70,8 +177,8 @@ class ColorizedStreamHandler(logging.StreamHandler): if colorama: COLORS = [ # This needs to be in order from highest logging level to lowest. - (logging.ERROR, _color_wrap(colorama.Fore.RED)), - (logging.WARNING, _color_wrap(colorama.Fore.YELLOW)), + (logging.ERROR, _color_wrap(Fore.RED)), + (logging.WARNING, _color_wrap(Fore.YELLOW)), ] else: COLORS = [] @@ -83,6 +190,16 @@ class ColorizedStreamHandler(logging.StreamHandler): if WINDOWS and colorama: self.stream = colorama.AnsiToWin32(self.stream) + def _using_stdout(self): + """ + Return whether the handler is using sys.stdout. + """ + if WINDOWS and colorama: + # Then self.stream is an AnsiToWin32 object. + return self.stream.wrapped is sys.stdout + + return self.stream is sys.stdout + def should_color(self): # Don't colorize things if we do not have colorama or if told not to if not colorama or self._no_color: @@ -115,6 +232,19 @@ class ColorizedStreamHandler(logging.StreamHandler): return msg + # The logging module says handleError() can be customized. + def handleError(self, record): + exc_class, exc = sys.exc_info()[:2] + # If a broken pipe occurred while calling write() or flush() on the + # stdout stream in logging's Handler.emit(), then raise our special + # exception so we can handle it in main() instead of logging the + # broken pipe error and continuing. + if (exc_class and self._using_stdout() and + _is_broken_pipe_error(exc_class, exc)): + raise BrokenStdoutLoggingError() + + return super(ColorizedStreamHandler, self).handleError(record) + class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): @@ -123,7 +253,7 @@ class BetterRotatingFileHandler(logging.handlers.RotatingFileHandler): return logging.handlers.RotatingFileHandler._open(self) -class MaxLevelFilter(logging.Filter): +class MaxLevelFilter(Filter): def __init__(self, level): self.level = level @@ -132,8 +262,22 @@ class MaxLevelFilter(logging.Filter): return record.levelno < self.level +class ExcludeLoggerFilter(Filter): + + """ + A logging Filter that excludes records from a logger (or its children). + """ + + def filter(self, record): + # The base Filter class allows only records from a logger (or its + # children). + return not super(ExcludeLoggerFilter, self).filter(record) + + def setup_logging(verbosity, no_color, user_log_file): """Configures and sets up all of the logging + + Returns the requested logging level, as its integer value. """ # Determine the level to be logging at. @@ -148,6 +292,8 @@ def setup_logging(verbosity, no_color, user_log_file): else: level = "INFO" + level_number = getattr(logging, level) + # The "root" logger should match the "console" level *unless* we also need # to log to a user log file. include_user_log = user_log_file is not None @@ -168,24 +314,40 @@ def setup_logging(verbosity, no_color, user_log_file): "stderr": "ext://sys.stderr", } handler_classes = { - "stream": "pip._internal.utils.logging.ColorizedStreamHandler", - "file": "pip._internal.utils.logging.BetterRotatingFileHandler", + "stream": "pipenv.patched.notpip._internal.utils.logging.ColorizedStreamHandler", + "file": "pipenv.patched.notpip._internal.utils.logging.BetterRotatingFileHandler", } + handlers = ["console", "console_errors", "console_subprocess"] + ( + ["user_log"] if include_user_log else [] + ) logging.config.dictConfig({ "version": 1, "disable_existing_loggers": False, "filters": { "exclude_warnings": { - "()": "pip._internal.utils.logging.MaxLevelFilter", + "()": "pipenv.patched.notpip._internal.utils.logging.MaxLevelFilter", "level": logging.WARNING, }, + "restrict_to_subprocess": { + "()": "logging.Filter", + "name": subprocess_logger.name, + }, + "exclude_subprocess": { + "()": "pipenv.patched.notpip._internal.utils.logging.ExcludeLoggerFilter", + "name": subprocess_logger.name, + }, }, "formatters": { "indent": { "()": IndentingFormatter, "format": "%(message)s", }, + "indent_with_timestamp": { + "()": IndentingFormatter, + "format": "%(message)s", + "add_timestamp": True, + }, }, "handlers": { "console": { @@ -193,7 +355,7 @@ def setup_logging(verbosity, no_color, user_log_file): "class": handler_classes["stream"], "no_color": no_color, "stream": log_streams["stdout"], - "filters": ["exclude_warnings"], + "filters": ["exclude_subprocess", "exclude_warnings"], "formatter": "indent", }, "console_errors": { @@ -201,6 +363,17 @@ def setup_logging(verbosity, no_color, user_log_file): "class": handler_classes["stream"], "no_color": no_color, "stream": log_streams["stderr"], + "filters": ["exclude_subprocess"], + "formatter": "indent", + }, + # A handler responsible for logging to the console messages + # from the "subprocessor" logger. + "console_subprocess": { + "level": level, + "class": handler_classes["stream"], + "no_color": no_color, + "stream": log_streams["stderr"], + "filters": ["restrict_to_subprocess"], "formatter": "indent", }, "user_log": { @@ -208,14 +381,12 @@ def setup_logging(verbosity, no_color, user_log_file): "class": handler_classes["file"], "filename": additional_log_file, "delay": True, - "formatter": "indent", + "formatter": "indent_with_timestamp", }, }, "root": { "level": root_level, - "handlers": ["console", "console_errors"] + ( - ["user_log"] if include_user_log else [] - ), + "handlers": handlers, }, "loggers": { "pip._vendor": { @@ -223,3 +394,5 @@ def setup_logging(verbosity, no_color, user_log_file): } }, }) + + return level_number diff --git a/pipenv/patched/notpip/_internal/utils/marker_files.py b/pipenv/patched/notpip/_internal/utils/marker_files.py new file mode 100644 index 00000000..734cba4c --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/marker_files.py @@ -0,0 +1,27 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +import os.path + +DELETE_MARKER_MESSAGE = '''\ +This file is placed here by pip to indicate the source was put +here by pip. + +Once this package is successfully installed this source code will be +deleted (unless you remove this file). +''' +PIP_DELETE_MARKER_FILENAME = 'pip-delete-this-directory.txt' + + +def has_delete_marker_file(directory): + return os.path.exists(os.path.join(directory, PIP_DELETE_MARKER_FILENAME)) + + +def write_delete_marker_file(directory): + # type: (str) -> None + """ + Write the pip delete marker file into this directory. + """ + filepath = os.path.join(directory, PIP_DELETE_MARKER_FILENAME) + with open(filepath, 'w') as marker_fp: + marker_fp.write(DELETE_MARKER_MESSAGE) diff --git a/pipenv/patched/notpip/_internal/utils/misc.py b/pipenv/patched/notpip/_internal/utils/misc.py index 45e5204c..87af02a4 100644 --- a/pipenv/patched/notpip/_internal/utils/misc.py +++ b/pipenv/patched/notpip/_internal/utils/misc.py @@ -1,38 +1,48 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import contextlib import errno +import getpass import io -import locale -# we have a submodule named 'logging' which would shadow this if we used the -# regular name: -import logging as std_logging +import logging import os import posixpath -import re import shutil import stat -import subprocess import sys -import tarfile -import zipfile from collections import deque from pipenv.patched.notpip._vendor import pkg_resources # NOTE: retrying is not annotated in typeshed as on 2017-07-17, which is # why we ignore the type on this import. from pipenv.patched.notpip._vendor.retrying import retry # type: ignore -from pipenv.patched.notpip._vendor.six import PY2 +from pipenv.patched.notpip._vendor.six import PY2, text_type from pipenv.patched.notpip._vendor.six.moves import input from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.six.moves.urllib.parse import unquote as urllib_unquote -from pipenv.patched.notpip._internal.exceptions import CommandError, InstallationError +from pipenv.patched.notpip import __version__ +from pipenv.patched.notpip._internal.exceptions import CommandError from pipenv.patched.notpip._internal.locations import ( - running_under_virtualenv, site_packages, user_site, virtualenv_no_global, - write_delete_marker_file, + get_major_minor_version, + site_packages, + user_site, ) from pipenv.patched.notpip._internal.utils.compat import ( - WINDOWS, console_to_str, expanduser, stdlib_pkgs, + WINDOWS, + expanduser, + stdlib_pkgs, + str_to_display, +) +from pipenv.patched.notpip._internal.utils.marker_files import write_delete_marker_file +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.virtualenv import ( + running_under_virtualenv, + virtualenv_no_global, ) if PY2: @@ -40,50 +50,67 @@ if PY2: else: from io import StringIO +if MYPY_CHECK_RUNNING: + from typing import ( + Any, AnyStr, Container, Iterable, List, Optional, Text, + Tuple, Union, cast, + ) + from pipenv.patched.notpip._vendor.pkg_resources import Distribution + + VersionInfo = Tuple[int, int, int] +else: + # typing's cast() is needed at runtime, but we don't want to import typing. + # Thus, we use a dummy no-op version, which we tell mypy to ignore. + def cast(type_, value): # type: ignore + return value + + __all__ = ['rmtree', 'display_path', 'backup_dir', 'ask', 'splitext', 'format_size', 'is_installable_dir', - 'is_svn_page', 'file_contents', - 'split_leading_dir', 'has_leading_dir', 'normalize_path', 'renames', 'get_prog', - 'unzip_file', 'untar_file', 'unpack_file', 'call_subprocess', 'captured_stdout', 'ensure_dir', - 'ARCHIVE_EXTENSIONS', 'SUPPORTED_EXTENSIONS', 'get_installed_version', 'remove_auth_from_url'] -logger = std_logging.getLogger(__name__) - -BZ2_EXTENSIONS = ('.tar.bz2', '.tbz') -XZ_EXTENSIONS = ('.tar.xz', '.txz', '.tlz', '.tar.lz', '.tar.lzma') -ZIP_EXTENSIONS = ('.zip', '.whl') -TAR_EXTENSIONS = ('.tar.gz', '.tgz', '.tar') -ARCHIVE_EXTENSIONS = ( - ZIP_EXTENSIONS + BZ2_EXTENSIONS + TAR_EXTENSIONS + XZ_EXTENSIONS) -SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS -try: - import bz2 # noqa - SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS -except ImportError: - logger.debug('bz2 module is not available') - -try: - # Only for Python 3.3+ - import lzma # noqa - SUPPORTED_EXTENSIONS += XZ_EXTENSIONS -except ImportError: - logger.debug('lzma module is not available') +logger = logging.getLogger(__name__) -def import_or_raise(pkg_or_module_string, ExceptionType, *args, **kwargs): - try: - return __import__(pkg_or_module_string) - except ImportError: - raise ExceptionType(*args, **kwargs) +def get_pip_version(): + # type: () -> str + pip_pkg_dir = os.path.join(os.path.dirname(__file__), "..", "..") + pip_pkg_dir = os.path.abspath(pip_pkg_dir) + + return ( + 'pip {} from {} (python {})'.format( + __version__, pip_pkg_dir, get_major_minor_version(), + ) + ) + + +def normalize_version_info(py_version_info): + # type: (Tuple[int, ...]) -> Tuple[int, int, int] + """ + Convert a tuple of ints representing a Python version to one of length + three. + + :param py_version_info: a tuple of ints representing a Python version, + or None to specify no version. The tuple can have any length. + + :return: a tuple of length three if `py_version_info` is non-None. + Otherwise, return `py_version_info` unchanged (i.e. None). + """ + if len(py_version_info) < 3: + py_version_info += (3 - len(py_version_info)) * (0,) + elif len(py_version_info) > 3: + py_version_info = py_version_info[:3] + + return cast('VersionInfo', py_version_info) def ensure_dir(path): + # type: (AnyStr) -> None """os.path.makedirs without EEXIST.""" try: os.makedirs(path) @@ -93,6 +120,7 @@ def ensure_dir(path): def get_prog(): + # type: () -> str try: prog = os.path.basename(sys.argv[0]) if prog in ('__main__.py', '-c'): @@ -107,16 +135,22 @@ def get_prog(): # Retry every half second for up to 3 seconds @retry(stop_max_delay=3000, wait_fixed=500) def rmtree(dir, ignore_errors=False): - shutil.rmtree(dir, ignore_errors=ignore_errors, - onerror=rmtree_errorhandler) + # type: (str, bool) -> None + from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly + vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) def rmtree_errorhandler(func, path, exc_info): """On Windows, the files in .svn are read-only, so when rmtree() tries to remove them, an exception is thrown. We catch that here, remove the read-only attribute, and hopefully continue without problems.""" - # if file type currently read only - if os.stat(path).st_mode & stat.S_IREAD: + try: + has_attr_readonly = not (os.stat(path).st_mode & stat.S_IWRITE) + except (IOError, OSError): + # it's equivalent to os.path.exists + return + + if has_attr_readonly: # convert to read/write os.chmod(path, stat.S_IWRITE) # use the original function to repeat the operation @@ -126,7 +160,42 @@ def rmtree_errorhandler(func, path, exc_info): raise +def path_to_display(path): + # type: (Optional[Union[str, Text]]) -> Optional[Text] + """ + Convert a bytes (or text) path to text (unicode in Python 2) for display + and logging purposes. + + This function should never error out. Also, this function is mainly needed + for Python 2 since in Python 3 str paths are already text. + """ + if path is None: + return None + if isinstance(path, text_type): + return path + # Otherwise, path is a bytes object (str in Python 2). + try: + display_path = path.decode(sys.getfilesystemencoding(), 'strict') + except UnicodeDecodeError: + # Include the full bytes to make troubleshooting easier, even though + # it may not be very human readable. + if PY2: + # Convert the bytes to a readable str representation using + # repr(), and then convert the str to unicode. + # Also, we add the prefix "b" to the repr() return value both + # to make the Python 2 output look like the Python 3 output, and + # to signal to the user that this is a bytes representation. + display_path = str_to_display('b{!r}'.format(path)) + else: + # Silence the "F821 undefined name 'ascii'" flake8 error since + # in Python 3 ascii() is a built-in. + display_path = ascii(path) # noqa: F821 + + return display_path + + def display_path(path): + # type: (Union[str, Text]) -> str """Gives the display value for a given path, making it relative to cwd if possible.""" path = os.path.normcase(os.path.abspath(path)) @@ -139,6 +208,7 @@ def display_path(path): def backup_dir(dir, ext='.bak'): + # type: (str, str) -> str """Figure out the name of a directory to back up the given dir to (adding .bak, .bak2, etc)""" n = 1 @@ -150,20 +220,28 @@ def backup_dir(dir, ext='.bak'): def ask_path_exists(message, options): + # type: (str, Iterable[str]) -> str for action in os.environ.get('PIP_EXISTS_ACTION', '').split(): if action in options: return action return ask(message, options) +def _check_no_input(message): + # type: (str) -> None + """Raise an error if no input is allowed.""" + if os.environ.get('PIP_NO_INPUT'): + raise Exception( + 'No input was expected ($PIP_NO_INPUT set); question: %s' % + message + ) + + def ask(message, options): + # type: (str, Iterable[str]) -> str """Ask the message interactively, with the given possible responses""" while 1: - if os.environ.get('PIP_NO_INPUT'): - raise Exception( - 'No input was expected ($PIP_NO_INPUT set); question: %s' % - message - ) + _check_no_input(message) response = input(message) response = response.strip().lower() if response not in options: @@ -175,7 +253,22 @@ def ask(message, options): return response +def ask_input(message): + # type: (str) -> str + """Ask for input interactively.""" + _check_no_input(message) + return input(message) + + +def ask_password(message): + # type: (str) -> str + """Ask for a password interactively.""" + _check_no_input(message) + return getpass.getpass(message) + + def format_size(bytes): + # type: (float) -> str if bytes > 1000 * 1000: return '%.1fMB' % (bytes / 1000.0 / 1000) elif bytes > 10 * 1000: @@ -187,6 +280,7 @@ def format_size(bytes): def is_installable_dir(path): + # type: (str) -> bool """Is path is a directory containing setup.py or pyproject.toml? """ if not os.path.isdir(path): @@ -200,19 +294,6 @@ def is_installable_dir(path): return False -def is_svn_page(html): - """ - Returns true if the page appears to be the index page of an svn repository - """ - return (re.search(r'[^<]*Revision \d+:', html) and - re.search(r'Powered by (?:<a[^>]*?>)?Subversion', html, re.I)) - - -def file_contents(filename): - with open(filename, 'rb') as fp: - return fp.read().decode('utf-8') - - def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): """Yield pieces of data from a file-like object until EOF.""" while True: @@ -222,33 +303,8 @@ def read_chunks(file, size=io.DEFAULT_BUFFER_SIZE): yield chunk -def split_leading_dir(path): - path = path.lstrip('/').lstrip('\\') - if '/' in path and (('\\' in path and path.find('/') < path.find('\\')) or - '\\' not in path): - return path.split('/', 1) - elif '\\' in path: - return path.split('\\', 1) - else: - return path, '' - - -def has_leading_dir(paths): - """Returns true if all the paths have the same leading path name - (i.e., everything is in one subdirectory in an archive)""" - common_prefix = None - for path in paths: - prefix, rest = split_leading_dir(path) - if not prefix: - return False - elif common_prefix is None: - common_prefix = prefix - elif prefix != common_prefix: - return False - return True - - def normalize_path(path, resolve_symlinks=True): + # type: (str, bool) -> str """ Convert a path to its canonical, case-normalized, absolute version. @@ -262,6 +318,7 @@ def normalize_path(path, resolve_symlinks=True): def splitext(path): + # type: (str) -> Tuple[str, str] """Like os.path.splitext, but take off .tar too""" base, ext = posixpath.splitext(path) if base.lower().endswith('.tar'): @@ -271,6 +328,7 @@ def splitext(path): def renames(old, new): + # type: (str, str) -> None """Like os.renames(), but handles renaming across devices.""" # Implementation borrowed from os.renames(). head, tail = os.path.split(new) @@ -288,18 +346,22 @@ def renames(old, new): def is_local(path): + # type: (str) -> bool """ Return True if path is within sys.prefix, if we're running in a virtualenv. If we're not in a virtualenv, all paths are considered "local." + Caution: this function assumes the head of path has been normalized + with normalize_path. """ if not running_under_virtualenv(): return True - return normalize_path(path).startswith(normalize_path(sys.prefix)) + return path.startswith(normalize_path(sys.prefix)) def dist_is_local(dist): + # type: (Distribution) -> bool """ Return True if given Distribution object is installed locally (i.e. within current virtualenv). @@ -311,25 +373,27 @@ def dist_is_local(dist): def dist_in_usersite(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in user site. """ - norm_path = normalize_path(dist_location(dist)) - return norm_path.startswith(normalize_path(user_site)) + return dist_location(dist).startswith(normalize_path(user_site)) def dist_in_site_packages(dist): + # type: (Distribution) -> bool """ Return True if given Distribution is installed in sysconfig.get_python_lib(). """ - return normalize_path( - dist_location(dist) - ).startswith(normalize_path(site_packages)) + return dist_location(dist).startswith(normalize_path(site_packages)) def dist_is_editable(dist): - """Is distribution an editable install?""" + # type: (Distribution) -> bool + """ + Return True if given Distribution is an editable install. + """ for path_item in sys.path: egg_link = os.path.join(path_item, dist.project_name + '.egg-link') if os.path.isfile(egg_link): @@ -337,11 +401,15 @@ def dist_is_editable(dist): return False -def get_installed_distributions(local_only=True, - skip=stdlib_pkgs, - include_editables=True, - editables_only=False, - user_only=False): +def get_installed_distributions( + local_only=True, # type: bool + skip=stdlib_pkgs, # type: Container[str] + include_editables=True, # type: bool + editables_only=False, # type: bool + user_only=False, # type: bool + paths=None # type: Optional[List[str]] +): + # type: (...) -> List[Distribution] """ Return a list of installed Distribution objects. @@ -358,7 +426,14 @@ def get_installed_distributions(local_only=True, If ``user_only`` is True , only report installations in the user site directory. + If ``paths`` is set, only report the distributions present at the + specified list of locations. """ + if paths: + working_set = pkg_resources.WorkingSet(paths) + else: + working_set = pkg_resources.working_set + if local_only: local_test = dist_is_local else: @@ -385,7 +460,8 @@ def get_installed_distributions(local_only=True, def user_test(d): return True - return [d for d in pkg_resources.working_set + # because of pkg_resources vendoring, mypy cannot find stub in typeshed + return [d for d in working_set # type: ignore if local_test(d) and d.key not in skip and editable_test(d) and @@ -395,6 +471,7 @@ def get_installed_distributions(local_only=True, def egg_link_path(dist): + # type: (Distribution) -> Optional[str] """ Return the path for the .egg-link file if it exists, otherwise, None. @@ -414,12 +491,9 @@ def egg_link_path(dist): """ sites = [] if running_under_virtualenv(): - if virtualenv_no_global(): - sites.append(site_packages) - else: - sites.append(site_packages) - if user_site: - sites.append(user_site) + sites.append(site_packages) + if not virtualenv_no_global() and user_site: + sites.append(user_site) else: if user_site: sites.append(user_site) @@ -429,316 +503,28 @@ def egg_link_path(dist): egglink = os.path.join(site, dist.project_name) + '.egg-link' if os.path.isfile(egglink): return egglink + return None def dist_location(dist): + # type: (Distribution) -> str """ Get the site-packages location of this distribution. Generally this is dist.location, except in the case of develop-installed packages, where dist.location is the source code location, and we want to know where the egg-link file is. + The returned location is normalized (in particular, with symlinks removed). """ egg_link = egg_link_path(dist) if egg_link: - return egg_link - return dist.location + return normalize_path(egg_link) + return normalize_path(dist.location) -def current_umask(): - """Get the current umask which involves having to set it temporarily.""" - mask = os.umask(0) - os.umask(mask) - return mask - - -def unzip_file(filename, location, flatten=True): - """ - Unzip the file (with path `filename`) to the destination `location`. All - files are written based on system defaults and umask (i.e. permissions are - not preserved), except that regular file members with any execute - permissions (user, group, or world) have "chmod +x" applied after being - written. Note that for windows, any execute changes using os.chmod are - no-ops per the python docs. - """ - ensure_dir(location) - zipfp = open(filename, 'rb') - try: - zip = zipfile.ZipFile(zipfp, allowZip64=True) - leading = has_leading_dir(zip.namelist()) and flatten - for info in zip.infolist(): - name = info.filename - data = zip.read(name) - fn = name - if leading: - fn = split_leading_dir(name)[1] - fn = os.path.join(location, fn) - dir = os.path.dirname(fn) - if fn.endswith('/') or fn.endswith('\\'): - # A directory - ensure_dir(fn) - else: - ensure_dir(dir) - fp = open(fn, 'wb') - try: - fp.write(data) - finally: - fp.close() - mode = info.external_attr >> 16 - # if mode and regular file and any execute permissions for - # user/group/world? - if mode and stat.S_ISREG(mode) and mode & 0o111: - # make dest file have execute for user/group/world - # (chmod +x) no-op on windows per python docs - os.chmod(fn, (0o777 - current_umask() | 0o111)) - finally: - zipfp.close() - - -def untar_file(filename, location): - """ - Untar the file (with path `filename`) to the destination `location`. - All files are written based on system defaults and umask (i.e. permissions - are not preserved), except that regular file members with any execute - permissions (user, group, or world) have "chmod +x" applied after being - written. Note that for windows, any execute changes using os.chmod are - no-ops per the python docs. - """ - ensure_dir(location) - if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): - mode = 'r:gz' - elif filename.lower().endswith(BZ2_EXTENSIONS): - mode = 'r:bz2' - elif filename.lower().endswith(XZ_EXTENSIONS): - mode = 'r:xz' - elif filename.lower().endswith('.tar'): - mode = 'r' - else: - logger.warning( - 'Cannot determine compression type for file %s', filename, - ) - mode = 'r:*' - tar = tarfile.open(filename, mode) - try: - # note: python<=2.5 doesn't seem to know about pax headers, filter them - leading = has_leading_dir([ - member.name for member in tar.getmembers() - if member.name != 'pax_global_header' - ]) - for member in tar.getmembers(): - fn = member.name - if fn == 'pax_global_header': - continue - if leading: - fn = split_leading_dir(fn)[1] - path = os.path.join(location, fn) - if member.isdir(): - ensure_dir(path) - elif member.issym(): - try: - tar._extract_member(member, path) - except Exception as exc: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warning( - 'In the tar file %s the member %s is invalid: %s', - filename, member.name, exc, - ) - continue - else: - try: - fp = tar.extractfile(member) - except (KeyError, AttributeError) as exc: - # Some corrupt tar files seem to produce this - # (specifically bad symlinks) - logger.warning( - 'In the tar file %s the member %s is invalid: %s', - filename, member.name, exc, - ) - continue - ensure_dir(os.path.dirname(path)) - with open(path, 'wb') as destfp: - shutil.copyfileobj(fp, destfp) - fp.close() - # Update the timestamp (useful for cython compiled files) - tar.utime(member, path) - # member have any execute permissions for user/group/world? - if member.mode & 0o111: - # make dest file have execute for user/group/world - # no-op on windows per python docs - os.chmod(path, (0o777 - current_umask() | 0o111)) - finally: - tar.close() - - -def unpack_file(filename, location, content_type, link): - filename = os.path.realpath(filename) - if (content_type == 'application/zip' or - filename.lower().endswith(ZIP_EXTENSIONS) or - zipfile.is_zipfile(filename)): - unzip_file( - filename, - location, - flatten=not filename.endswith('.whl') - ) - elif (content_type == 'application/x-gzip' or - tarfile.is_tarfile(filename) or - filename.lower().endswith( - TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS)): - untar_file(filename, location) - elif (content_type and content_type.startswith('text/html') and - is_svn_page(file_contents(filename))): - # We don't really care about this - from pipenv.patched.notpip._internal.vcs.subversion import Subversion - Subversion('svn+' + link.url).unpack(location) - else: - # FIXME: handle? - # FIXME: magic signatures? - logger.critical( - 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' - 'cannot detect archive format', - filename, location, content_type, - ) - raise InstallationError( - 'Cannot determine archive format of %s' % location - ) - - -def call_subprocess(cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, unset_environ=None, spinner=None): - """ - Args: - unset_environ: an iterable of environment variable names to unset - prior to calling subprocess.Popen(). - """ - if unset_environ is None: - unset_environ = [] - # This function's handling of subprocess output is confusing and I - # previously broke it terribly, so as penance I will write a long comment - # explaining things. - # - # The obvious thing that affects output is the show_stdout= - # kwarg. show_stdout=True means, let the subprocess write directly to our - # stdout. Even though it is nominally the default, it is almost never used - # inside pip (and should not be used in new code without a very good - # reason); as of 2016-02-22 it is only used in a few places inside the VCS - # wrapper code. Ideally we should get rid of it entirely, because it - # creates a lot of complexity here for a rarely used feature. - # - # Most places in pip set show_stdout=False. What this means is: - # - We connect the child stdout to a pipe, which we read. - # - By default, we hide the output but show a spinner -- unless the - # subprocess exits with an error, in which case we show the output. - # - If the --verbose option was passed (= loglevel is DEBUG), then we show - # the output unconditionally. (But in this case we don't want to show - # the output a second time if it turns out that there was an error.) - # - # stderr is always merged with stdout (even if show_stdout=True). - if show_stdout: - stdout = None - else: - stdout = subprocess.PIPE - if command_desc is None: - cmd_parts = [] - for part in cmd: - if ' ' in part or '\n' in part or '"' in part or "'" in part: - part = '"%s"' % part.replace('"', '\\"') - cmd_parts.append(part) - command_desc = ' '.join(cmd_parts) - logger.debug("Running command %s", command_desc) - env = os.environ.copy() - if extra_environ: - env.update(extra_environ) - for name in unset_environ: - env.pop(name, None) - try: - proc = subprocess.Popen( - cmd, stderr=subprocess.STDOUT, stdin=subprocess.PIPE, - stdout=stdout, cwd=cwd, env=env, - ) - proc.stdin.close() - except Exception as exc: - logger.critical( - "Error %s while executing command %s", exc, command_desc, - ) - raise - all_output = [] - if stdout is not None: - while True: - line = console_to_str(proc.stdout.readline()) - if not line: - break - line = line.rstrip() - all_output.append(line + '\n') - if logger.getEffectiveLevel() <= std_logging.DEBUG: - # Show the line immediately - logger.debug(line) - else: - # Update the spinner - if spinner is not None: - spinner.spin() - try: - proc.wait() - finally: - if proc.stdout: - proc.stdout.close() - if spinner is not None: - if proc.returncode: - spinner.finish("error") - else: - spinner.finish("done") - if proc.returncode: - if on_returncode == 'raise': - if (logger.getEffectiveLevel() > std_logging.DEBUG and - not show_stdout): - logger.info( - 'Complete output from command %s:', command_desc, - ) - logger.info( - ''.join(all_output) + - '\n----------------------------------------' - ) - raise InstallationError( - 'Command "%s" failed with error code %s in %s' - % (command_desc, proc.returncode, cwd)) - elif on_returncode == 'warn': - logger.warning( - 'Command "%s" had error code %s in %s', - command_desc, proc.returncode, cwd, - ) - elif on_returncode == 'ignore': - pass - else: - raise ValueError('Invalid value: on_returncode=%s' % - repr(on_returncode)) - if not show_stdout: - return ''.join(all_output) - - -def read_text_file(filename): - """Return the contents of *filename*. - - Try to decode the file contents with utf-8, the preferred system encoding - (e.g., cp1252 on some Windows machines), and latin1, in that order. - Decoding a byte string with latin1 will never raise an error. In the worst - case, the returned string will contain some garbage characters. - - """ - with open(filename, 'rb') as fp: - data = fp.read() - - encodings = ['utf-8', locale.getpreferredencoding(False), 'latin1'] - for enc in encodings: - try: - data = data.decode(enc) - except UnicodeDecodeError: - continue - break - - assert type(data) != bytes # Latin1 should have worked. - return data +def write_output(msg, *args): + # type: (str, str) -> None + logger.info(msg, *args) def _make_build_dir(build_dir): @@ -805,6 +591,13 @@ def captured_stdout(): return captured_output('stdout') +def captured_stderr(): + """ + See captured_stdout(). + """ + return captured_output('stderr') + + class cached_property(object): """A property that is only computed once per instance and then replaces itself with an ordinary attribute. Deleting the attribute resets the @@ -856,18 +649,38 @@ def enum(*sequential, **named): return type('Enum', (), enums) -def make_vcs_requirement_url(repo_url, rev, egg_project_name, subdir=None): +def build_netloc(host, port): + # type: (str, Optional[int]) -> str """ - Return the URL for a VCS requirement. - - Args: - repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + Build a netloc from a host-port pair """ - req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) - if subdir: - req += '&subdirectory={}'.format(subdir) + if port is None: + return host + if ':' in host: + # Only wrap host with square brackets when it is IPv6 + host = '[{}]'.format(host) + return '{}:{}'.format(host, port) - return req + +def build_url_from_netloc(netloc, scheme='https'): + # type: (str, str) -> str + """ + Build a full URL from a netloc. + """ + if netloc.count(':') >= 2 and '@' not in netloc and '[' not in netloc: + # It must be a bare IPv6 address, so wrap it with brackets. + netloc = '[{}]'.format(netloc) + return '{}://{}'.format(scheme, netloc) + + +def parse_netloc(netloc): + # type: (str) -> Tuple[str, Optional[int]] + """ + Return the host-port pair from a netloc. + """ + url = build_url_from_netloc(netloc) + parsed = urllib_parse.urlparse(url) + return parsed.hostname, parsed.port def split_auth_from_netloc(netloc): @@ -887,41 +700,151 @@ def split_auth_from_netloc(netloc): # Split from the left because that's how urllib.parse.urlsplit() # behaves if more than one : is present (which again can be checked # using the password attribute of the return value) - user_pass = tuple(auth.split(':', 1)) + user_pass = auth.split(':', 1) else: user_pass = auth, None + user_pass = tuple( + None if x is None else urllib_unquote(x) for x in user_pass + ) + return netloc, user_pass -def remove_auth_from_url(url): - # Return a copy of url with 'username:password@' removed. - # username/pass params are passed to subversion through flags - # and are not recognized in the url. +def redact_netloc(netloc): + # type: (str) -> str + """ + Replace the sensitive data in a netloc with "****", if it exists. - # parsed url + For example: + - "user:pass@example.com" returns "user:****@example.com" + - "accesstoken@example.com" returns "****@example.com" + """ + netloc, (user, password) = split_auth_from_netloc(netloc) + if user is None: + return netloc + if password is None: + user = '****' + password = '' + else: + user = urllib_parse.quote(user) + password = ':****' + return '{user}{password}@{netloc}'.format(user=user, + password=password, + netloc=netloc) + + +def _transform_url(url, transform_netloc): + """Transform and replace netloc in a url. + + transform_netloc is a function taking the netloc and returning a + tuple. The first element of this tuple is the new netloc. The + entire tuple is returned. + + Returns a tuple containing the transformed url as item 0 and the + original tuple returned by transform_netloc as item 1. + """ purl = urllib_parse.urlsplit(url) - netloc, user_pass = split_auth_from_netloc(purl.netloc) - + netloc_tuple = transform_netloc(purl.netloc) # stripped url url_pieces = ( - purl.scheme, netloc, purl.path, purl.query, purl.fragment + purl.scheme, netloc_tuple[0], purl.path, purl.query, purl.fragment ) surl = urllib_parse.urlunsplit(url_pieces) - return surl + return surl, netloc_tuple + + +def _get_netloc(netloc): + return split_auth_from_netloc(netloc) + + +def _redact_netloc(netloc): + return (redact_netloc(netloc),) + + +def split_auth_netloc_from_url(url): + # type: (str) -> Tuple[str, str, Tuple[str, str]] + """ + Parse a url into separate netloc, auth, and url with no auth. + + Returns: (url_without_auth, netloc, (username, password)) + """ + url_without_auth, (netloc, auth) = _transform_url(url, _get_netloc) + return url_without_auth, netloc, auth + + +def remove_auth_from_url(url): + # type: (str) -> str + """Return a copy of url with 'username:password@' removed.""" + # username/pass params are passed to subversion through flags + # and are not recognized in the url. + return _transform_url(url, _get_netloc)[0] + + +def redact_auth_from_url(url): + # type: (str) -> str + """Replace the password in a given url with ****.""" + return _transform_url(url, _redact_netloc)[0] + + +class HiddenText(object): + def __init__( + self, + secret, # type: str + redacted, # type: str + ): + # type: (...) -> None + self.secret = secret + self.redacted = redacted + + def __repr__(self): + # type: (...) -> str + return '<HiddenText {!r}>'.format(str(self)) + + def __str__(self): + # type: (...) -> str + return self.redacted + + # This is useful for testing. + def __eq__(self, other): + # type: (Any) -> bool + if type(self) != type(other): + return False + + # The string being used for redaction doesn't also have to match, + # just the raw, original string. + return (self.secret == other.secret) + + # We need to provide an explicit __ne__ implementation for Python 2. + # TODO: remove this when we drop PY2 support. + def __ne__(self, other): + # type: (Any) -> bool + return not self == other + + +def hide_value(value): + # type: (str) -> HiddenText + return HiddenText(value, redacted='****') + + +def hide_url(url): + # type: (str) -> HiddenText + redacted = redact_auth_from_url(url) + return HiddenText(url, redacted=redacted) def protect_pip_from_modification_on_windows(modifying_pip): + # type: (bool) -> None """Protection of pip.exe from modification on Windows On Windows, any operation modifying pip should be run as: python -m pip ... """ - pip_names = [ - "pip.exe", - "pip{}.exe".format(sys.version_info[0]), - "pip{}.{}.exe".format(*sys.version_info[:2]) - ] + pip_names = set() + for ext in ('', '.exe'): + pip_names.add('pip{ext}'.format(ext=ext)) + pip_names.add('pip{}{ext}'.format(sys.version_info[0], ext=ext)) + pip_names.add('pip{}.{}{ext}'.format(*sys.version_info[:2], ext=ext)) # See https://github.com/pypa/pip/issues/1299 for more discussion should_show_use_python_msg = ( @@ -938,3 +861,10 @@ def protect_pip_from_modification_on_windows(modifying_pip): 'To modify pip, please run the following command:\n{}' .format(" ".join(new_command)) ) + + +def is_console_interactive(): + # type: () -> bool + """Is this console interactive? + """ + return sys.stdin is not None and sys.stdin.isatty() diff --git a/pipenv/patched/notpip/_internal/utils/models.py b/pipenv/patched/notpip/_internal/utils/models.py index d5cb80a7..29e14411 100644 --- a/pipenv/patched/notpip/_internal/utils/models.py +++ b/pipenv/patched/notpip/_internal/utils/models.py @@ -1,11 +1,13 @@ """Utilities for defining models """ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False import operator class KeyBasedCompareMixin(object): - """Provides comparision capabilities that is based on a key + """Provides comparison capabilities that is based on a key """ def __init__(self, key, defining_class): diff --git a/pipenv/patched/notpip/_internal/utils/outdated.py b/pipenv/patched/notpip/_internal/utils/outdated.py deleted file mode 100644 index f8b1fe04..00000000 --- a/pipenv/patched/notpip/_internal/utils/outdated.py +++ /dev/null @@ -1,154 +0,0 @@ -from __future__ import absolute_import - -import datetime -import json -import logging -import os.path -import sys - -from pipenv.patched.notpip._vendor import lockfile, pkg_resources -from pipenv.patched.notpip._vendor.packaging import version as packaging_version - -from pipenv.patched.notpip._internal.index import PackageFinder -from pipenv.patched.notpip._internal.utils.compat import WINDOWS -from pipenv.patched.notpip._internal.utils.filesystem import check_path_owner -from pipenv.patched.notpip._internal.utils.misc import ensure_dir, get_installed_version - -SELFCHECK_DATE_FMT = "%Y-%m-%dT%H:%M:%SZ" - - -logger = logging.getLogger(__name__) - - -class SelfCheckState(object): - def __init__(self, cache_dir): - self.state = {} - self.statefile_path = None - - # Try to load the existing state - if cache_dir: - self.statefile_path = os.path.join(cache_dir, "selfcheck.json") - try: - with open(self.statefile_path) as statefile: - self.state = json.load(statefile)[sys.prefix] - except (IOError, ValueError, KeyError): - # Explicitly suppressing exceptions, since we don't want to - # error out if the cache file is invalid. - pass - - def save(self, pypi_version, current_time): - # If we do not have a path to cache in, don't bother saving. - if not self.statefile_path: - return - - # Check to make sure that we own the directory - if not check_path_owner(os.path.dirname(self.statefile_path)): - return - - # Now that we've ensured the directory is owned by this user, we'll go - # ahead and make sure that all our directories are created. - ensure_dir(os.path.dirname(self.statefile_path)) - - # Attempt to write out our version check file - with lockfile.LockFile(self.statefile_path): - if os.path.exists(self.statefile_path): - with open(self.statefile_path) as statefile: - state = json.load(statefile) - else: - state = {} - - state[sys.prefix] = { - "last_check": current_time.strftime(SELFCHECK_DATE_FMT), - "pypi_version": pypi_version, - } - - with open(self.statefile_path, "w") as statefile: - json.dump(state, statefile, sort_keys=True, - separators=(",", ":")) - - -def was_installed_by_pip(pkg): - """Checks whether pkg was installed by pip - - This is used not to display the upgrade message when pip is in fact - installed by system package manager, such as dnf on Fedora. - """ - try: - dist = pkg_resources.get_distribution(pkg) - return (dist.has_metadata('INSTALLER') and - 'pip' in dist.get_metadata_lines('INSTALLER')) - except pkg_resources.DistributionNotFound: - return False - - -def pip_version_check(session, options): - """Check for an update for pip. - - Limit the frequency of checks to once per week. State is stored either in - the active virtualenv or in the user's USER_CACHE_DIR keyed off the prefix - of the pip script path. - """ - installed_version = get_installed_version("pip") - if not installed_version: - return - - pip_version = packaging_version.parse(installed_version) - pypi_version = None - - try: - state = SelfCheckState(cache_dir=options.cache_dir) - - current_time = datetime.datetime.utcnow() - # Determine if we need to refresh the state - if "last_check" in state.state and "pypi_version" in state.state: - last_check = datetime.datetime.strptime( - state.state["last_check"], - SELFCHECK_DATE_FMT - ) - if (current_time - last_check).total_seconds() < 7 * 24 * 60 * 60: - pypi_version = state.state["pypi_version"] - - # Refresh the version if we need to or just see if we need to warn - if pypi_version is None: - # Lets use PackageFinder to see what the latest pip version is - finder = PackageFinder( - find_links=options.find_links, - index_urls=[options.index_url] + options.extra_index_urls, - allow_all_prereleases=False, # Explicitly set to False - trusted_hosts=options.trusted_hosts, - process_dependency_links=options.process_dependency_links, - session=session, - ) - all_candidates = finder.find_all_candidates("pip") - if not all_candidates: - return - pypi_version = str( - max(all_candidates, key=lambda c: c.version).version - ) - - # save that we've performed a check - state.save(pypi_version, current_time) - - remote_version = packaging_version.parse(pypi_version) - - # Determine if our pypi_version is older - if (pip_version < remote_version and - pip_version.base_version != remote_version.base_version and - was_installed_by_pip('pip')): - # Advise "python -m pip" on Windows to avoid issues - # with overwriting pip.exe. - if WINDOWS: - pip_cmd = "python -m pip" - else: - pip_cmd = "pip" - logger.warning( - "You are using pip version %s, however version %s is " - "available.\nYou should consider upgrading via the " - "'%s install --upgrade pip' command.", - pip_version, pypi_version, pip_cmd - ) - except Exception: - logger.debug( - "There was an error checking the latest version of pip", - exc_info=True, - ) diff --git a/pipenv/patched/notpip/_internal/utils/packaging.py b/pipenv/patched/notpip/_internal/utils/packaging.py index d1e8ecaa..dd32f70d 100644 --- a/pipenv/patched/notpip/_internal/utils/packaging.py +++ b/pipenv/patched/notpip/_internal/utils/packaging.py @@ -2,74 +2,92 @@ from __future__ import absolute_import import logging import sys -from email.parser import FeedParser # type: ignore +from email.parser import FeedParser from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.packaging import specifiers, version -from pipenv.patched.notpip._internal import exceptions +from pipenv.patched.notpip._internal.exceptions import NoneMetadataError from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from email.message import Message + from pipenv.patched.notpip._vendor.pkg_resources import Distribution + logger = logging.getLogger(__name__) -def check_requires_python(requires_python): +def check_requires_python(requires_python, version_info): + # type: (Optional[str], Tuple[int, ...]) -> bool """ - Check if the python version in use match the `requires_python` specifier. + Check if the given Python version matches a "Requires-Python" specifier. - Returns `True` if the version of python in use matches the requirement. - Returns `False` if the version of python in use does not matches the - requirement. + :param version_info: A 3-tuple of ints representing a Python + major-minor-micro version to check (e.g. `sys.version_info[:3]`). - Raises an InvalidSpecifier if `requires_python` have an invalid format. + :return: `True` if the given Python version satisfies the requirement. + Otherwise, return `False`. + + :raises InvalidSpecifier: If `requires_python` has an invalid format. """ if requires_python is None: # The package provides no information return True requires_python_specifier = specifiers.SpecifierSet(requires_python) - # We only use major.minor.micro python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) return python_version in requires_python_specifier def get_metadata(dist): + # type: (Distribution) -> Message + """ + :raises NoneMetadataError: if the distribution reports `has_metadata()` + True but `get_metadata()` returns None. + """ + metadata_name = 'METADATA' if (isinstance(dist, pkg_resources.DistInfoDistribution) and - dist.has_metadata('METADATA')): - metadata = dist.get_metadata('METADATA') + dist.has_metadata(metadata_name)): + metadata = dist.get_metadata(metadata_name) elif dist.has_metadata('PKG-INFO'): - metadata = dist.get_metadata('PKG-INFO') + metadata_name = 'PKG-INFO' + metadata = dist.get_metadata(metadata_name) else: logger.warning("No metadata found in %s", display_path(dist.location)) metadata = '' + if metadata is None: + raise NoneMetadataError(dist, metadata_name) + feed_parser = FeedParser() + # The following line errors out if with a "NoneType" TypeError if + # passed metadata=None. feed_parser.feed(metadata) return feed_parser.close() -def check_dist_requires_python(dist, absorb=True): +def get_requires_python(dist): + # type: (pkg_resources.Distribution) -> Optional[str] + """ + Return the "Requires-Python" metadata for a distribution, or None + if not present. + """ pkg_info_dict = get_metadata(dist) requires_python = pkg_info_dict.get('Requires-Python') - if absorb: - return requires_python - try: - if not check_requires_python(requires_python): - raise exceptions.UnsupportedPythonVersion( - "%s requires Python '%s' but the running Python is %s" % ( - dist.project_name, - requires_python, - '.'.join(map(str, sys.version_info[:3])),) - ) - except specifiers.InvalidSpecifier as e: - logger.warning( - "Package %s has an invalid Requires-Python entry %s - %s", - dist.project_name, requires_python, e, - ) - return + + if requires_python is not None: + # Convert to a str to satisfy the type checker, since requires_python + # can be a Header object. + requires_python = str(requires_python) + + return requires_python def get_installer(dist): + # type: (Distribution) -> str if dist.has_metadata('INSTALLER'): for line in dist.get_metadata_lines('INSTALLER'): if line.strip(): diff --git a/pipenv/patched/notpip/_internal/utils/setuptools_build.py b/pipenv/patched/notpip/_internal/utils/setuptools_build.py index 03973e97..0151ee2f 100644 --- a/pipenv/patched/notpip/_internal/utils/setuptools_build.py +++ b/pipenv/patched/notpip/_internal/utils/setuptools_build.py @@ -1,8 +1,49 @@ +import os +import sys + +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import List, Sequence + # Shim to wrap setup.py invocation with setuptools -SETUPTOOLS_SHIM = ( - "import setuptools, tokenize;__file__=%r;" +# +# We set sys.argv[0] to the path to the underlying setup.py file so +# setuptools / distutils don't take the path to the setup.py to be "-c" when +# invoking via the shim. This avoids e.g. the following manifest_maker +# warning: "warning: manifest_maker: standard file '-c' not found". +_SETUPTOOLS_SHIM = ( + "import sys, setuptools, tokenize; sys.argv[0] = {0!r}; __file__={0!r};" "f=getattr(tokenize, 'open', open)(__file__);" "code=f.read().replace('\\r\\n', '\\n');" "f.close();" "exec(compile(code, __file__, 'exec'))" ) + + +def make_setuptools_shim_args( + setup_py_path, # type: str + global_options=None, # type: Sequence[str] + no_user_config=False, # type: bool + unbuffered_output=False # type: bool +): + # type: (...) -> List[str] + """ + Get setuptools command arguments with shim wrapped setup file invocation. + + :param setup_py_path: The path to setup.py to be wrapped. + :param global_options: Additional global options. + :param no_user_config: If True, disables personal user configuration. + :param unbuffered_output: If True, adds the unbuffered switch to the + argument list. + """ + sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + args = [sys_executable] + if unbuffered_output: + args.append('-u') + args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path)]) + if global_options: + args.extend(global_options) + if no_user_config: + args.append('--no-user-cfg') + return args diff --git a/pipenv/patched/notpip/_internal/utils/subprocess.py b/pipenv/patched/notpip/_internal/utils/subprocess.py new file mode 100644 index 00000000..1cf2faa9 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/subprocess.py @@ -0,0 +1,278 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False + +from __future__ import absolute_import + +import logging +import os +import subprocess + +from pipenv.patched.notpip._vendor.six.moves import shlex_quote + +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.compat import console_to_str, str_to_display +from pipenv.patched.notpip._internal.utils.logging import subprocess_logger +from pipenv.patched.notpip._internal.utils.misc import HiddenText, path_to_display +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.ui import open_spinner + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Callable, Iterable, List, Mapping, Optional, Text, Union, + ) + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface + + CommandArgs = List[Union[str, HiddenText]] + + +LOG_DIVIDER = '----------------------------------------' + + +def make_command(*args): + # type: (Union[str, HiddenText, CommandArgs]) -> CommandArgs + """ + Create a CommandArgs object. + """ + command_args = [] # type: CommandArgs + for arg in args: + # Check for list instead of CommandArgs since CommandArgs is + # only known during type-checking. + if isinstance(arg, list): + command_args.extend(arg) + else: + # Otherwise, arg is str or HiddenText. + command_args.append(arg) + + return command_args + + +def format_command_args(args): + # type: (Union[List[str], CommandArgs]) -> str + """ + Format command arguments for display. + """ + # For HiddenText arguments, display the redacted form by calling str(). + # Also, we don't apply str() to arguments that aren't HiddenText since + # this can trigger a UnicodeDecodeError in Python 2 if the argument + # has type unicode and includes a non-ascii character. (The type + # checker doesn't ensure the annotations are correct in all cases.) + return ' '.join( + shlex_quote(str(arg)) if isinstance(arg, HiddenText) + else shlex_quote(arg) for arg in args + ) + + +def reveal_command_args(args): + # type: (Union[List[str], CommandArgs]) -> List[str] + """ + Return the arguments in their raw, unredacted form. + """ + return [ + arg.secret if isinstance(arg, HiddenText) else arg for arg in args + ] + + +def make_subprocess_output_error( + cmd_args, # type: Union[List[str], CommandArgs] + cwd, # type: Optional[str] + lines, # type: List[Text] + exit_status, # type: int +): + # type: (...) -> Text + """ + Create and return the error message to use to log a subprocess error + with command output. + + :param lines: A list of lines, each ending with a newline. + """ + command = format_command_args(cmd_args) + # Convert `command` and `cwd` to text (unicode in Python 2) so we can use + # them as arguments in the unicode format string below. This avoids + # "UnicodeDecodeError: 'ascii' codec can't decode byte ..." in Python 2 + # if either contains a non-ascii character. + command_display = str_to_display(command, desc='command bytes') + cwd_display = path_to_display(cwd) + + # We know the joined output value ends in a newline. + output = ''.join(lines) + msg = ( + # Use a unicode string to avoid "UnicodeEncodeError: 'ascii' + # codec can't encode character ..." in Python 2 when a format + # argument (e.g. `output`) has a non-ascii character. + u'Command errored out with exit status {exit_status}:\n' + ' command: {command_display}\n' + ' cwd: {cwd_display}\n' + 'Complete output ({line_count} lines):\n{output}{divider}' + ).format( + exit_status=exit_status, + command_display=command_display, + cwd_display=cwd_display, + line_count=len(lines), + output=output, + divider=LOG_DIVIDER, + ) + return msg + + +def call_subprocess( + cmd, # type: Union[List[str], CommandArgs] + show_stdout=False, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + unset_environ=None, # type: Optional[Iterable[str]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True # type: Optional[bool] +): + # type: (...) -> Text + """ + Args: + show_stdout: if true, use INFO to log the subprocess's stderr and + stdout streams. Otherwise, use DEBUG. Defaults to False. + extra_ok_returncodes: an iterable of integer return codes that are + acceptable, in addition to 0. Defaults to None, which means []. + unset_environ: an iterable of environment variable names to unset + prior to calling subprocess.Popen(). + log_failed_cmd: if false, failed commands are not logged, only raised. + """ + if extra_ok_returncodes is None: + extra_ok_returncodes = [] + if unset_environ is None: + unset_environ = [] + # Most places in pip use show_stdout=False. What this means is-- + # + # - We connect the child's output (combined stderr and stdout) to a + # single pipe, which we read. + # - We log this output to stderr at DEBUG level as it is received. + # - If DEBUG logging isn't enabled (e.g. if --verbose logging wasn't + # requested), then we show a spinner so the user can still see the + # subprocess is in progress. + # - If the subprocess exits with an error, we log the output to stderr + # at ERROR level if it hasn't already been displayed to the console + # (e.g. if --verbose logging wasn't enabled). This way we don't log + # the output to the console twice. + # + # If show_stdout=True, then the above is still done, but with DEBUG + # replaced by INFO. + if show_stdout: + # Then log the subprocess output at INFO level. + log_subprocess = subprocess_logger.info + used_level = logging.INFO + else: + # Then log the subprocess output using DEBUG. This also ensures + # it will be logged to the log file (aka user_log), if enabled. + log_subprocess = subprocess_logger.debug + used_level = logging.DEBUG + + # Whether the subprocess will be visible in the console. + showing_subprocess = subprocess_logger.getEffectiveLevel() <= used_level + + # Only use the spinner if we're not showing the subprocess output + # and we have a spinner. + use_spinner = not showing_subprocess and spinner is not None + + if command_desc is None: + command_desc = format_command_args(cmd) + + log_subprocess("Running command %s", command_desc) + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + for name in unset_environ: + env.pop(name, None) + try: + proc = subprocess.Popen( + # Convert HiddenText objects to the underlying str. + reveal_command_args(cmd), + stderr=subprocess.STDOUT, stdin=subprocess.PIPE, + stdout=subprocess.PIPE, cwd=cwd, env=env, + ) + proc.stdin.close() + except Exception as exc: + if log_failed_cmd: + subprocess_logger.critical( + "Error %s while executing command %s", exc, command_desc, + ) + raise + all_output = [] + while True: + # The "line" value is a unicode string in Python 2. + line = console_to_str(proc.stdout.readline()) + if not line: + break + line = line.rstrip() + all_output.append(line + '\n') + + # Show the line immediately. + log_subprocess(line) + # Update the spinner. + if use_spinner: + spinner.spin() + try: + proc.wait() + finally: + if proc.stdout: + proc.stdout.close() + proc_had_error = ( + proc.returncode and proc.returncode not in extra_ok_returncodes + ) + if use_spinner: + if proc_had_error: + spinner.finish("error") + else: + spinner.finish("done") + if proc_had_error: + if on_returncode == 'raise': + if not showing_subprocess and log_failed_cmd: + # Then the subprocess streams haven't been logged to the + # console yet. + msg = make_subprocess_output_error( + cmd_args=cmd, + cwd=cwd, + lines=all_output, + exit_status=proc.returncode, + ) + subprocess_logger.error(msg) + exc_msg = ( + 'Command errored out with exit status {}: {} ' + 'Check the logs for full command output.' + ).format(proc.returncode, command_desc) + raise InstallationError(exc_msg) + elif on_returncode == 'warn': + subprocess_logger.warning( + 'Command "%s" had error code %s in %s', + command_desc, proc.returncode, cwd, + ) + elif on_returncode == 'ignore': + pass + else: + raise ValueError('Invalid value: on_returncode=%s' % + repr(on_returncode)) + return ''.join(all_output) + + +def runner_with_spinner_message(message): + # type: (str) -> Callable + """Provide a subprocess_runner that shows a spinner message. + + Intended for use with for pep517's Pep517HookCaller. Thus, the runner has + an API that matches what's expected by Pep517HookCaller.subprocess_runner. + """ + + def runner( + cmd, # type: List[str] + cwd=None, # type: Optional[str] + extra_environ=None # type: Optional[Mapping[str, Any]] + ): + # type: (...) -> None + with open_spinner(message) as spinner: + call_subprocess( + cmd, + cwd=cwd, + extra_environ=extra_environ, + spinner=spinner, + ) + + return runner diff --git a/pipenv/patched/notpip/_internal/utils/temp_dir.py b/pipenv/patched/notpip/_internal/utils/temp_dir.py index 893dc975..84bba3ac 100644 --- a/pipenv/patched/notpip/_internal/utils/temp_dir.py +++ b/pipenv/patched/notpip/_internal/utils/temp_dir.py @@ -1,13 +1,23 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import +import errno +import itertools import logging import os.path import tempfile import warnings from pipenv.patched.notpip._internal.utils.misc import rmtree +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.vendor.vistir.compat import finalize, ResourceWarning +if MYPY_CHECK_RUNNING: + from typing import Optional + + logger = logging.getLogger(__name__) @@ -19,24 +29,25 @@ class TempDirectory(object): Attributes: path - Location to the created temporary directory or None + Location to the created temporary directory delete Whether the directory should be deleted when exiting (when used as a contextmanager) Methods: - create() - Creates a temporary directory and stores its path in the path - attribute. cleanup() - Deletes the temporary directory and sets path attribute to None + Deletes the temporary directory - When used as a context manager, a temporary directory is created on - entering the context and, if the delete attribute is True, on exiting the - context the created directory is deleted. + When used as a context manager, if the delete attribute is True, on + exiting the context the temporary directory is deleted. """ - def __init__(self, path=None, delete=None, kind="temp"): + def __init__( + self, + path=None, # type: Optional[str] + delete=None, # type: Optional[bool] + kind="temp" + ): super(TempDirectory, self).__init__() if path is None and delete is None: @@ -44,55 +55,63 @@ class TempDirectory(object): # an explicit delete option, then we'll default to deleting. delete = True - self.path = path + if path is None: + path = self._create(kind) + + self._path = path + self._deleted = False self.delete = delete self.kind = kind self._finalizer = None - if path: + if self._path: self._register_finalizer() def _register_finalizer(self): - if self.delete and self.path: + if self.delete and self._path: self._finalizer = finalize( self, self._cleanup, - self.path, - warn_message=None + self._path, + warn_message = None ) else: self._finalizer = None + @property + def path(self): + # type: () -> str + assert not self._deleted, ( + "Attempted to access deleted path: {}".format(self._path) + ) + return self._path + def __repr__(self): return "<{} {!r}>".format(self.__class__.__name__, self.path) def __enter__(self): - self.create() return self def __exit__(self, exc, value, tb): if self.delete: self.cleanup() - def create(self): - """Create a temporary directory and store it's path in self.path + def _create(self, kind): + """Create a temporary directory and store its path in self.path """ - if self.path is not None: - logger.debug( - "Skipped creation of temporary directory: {}".format(self.path) - ) - return # We realpath here because some systems have their default tmpdir # symlinked to another directory. This tends to confuse build # scripts, so we canonicalize the path by traversing potential # symlinks here. - self.path = os.path.realpath( - tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) + path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(kind)) ) - self._register_finalizer() - logger.debug("Created temporary directory: {}".format(self.path)) + logger.debug("Created temporary directory: {}".format(path)) + return path @classmethod def _cleanup(cls, name, warn_message=None): + if not os.path.exists(name): + return try: rmtree(name) except OSError: @@ -105,10 +124,81 @@ class TempDirectory(object): """Remove the temporary directory created and reset state """ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): - if os.path.exists(self.path): + if os.path.exists(self._path): + self._deleted = True try: - rmtree(self.path) + rmtree(self._path) except OSError: pass - else: - self.path = None + + +class AdjacentTempDirectory(TempDirectory): + """Helper class that creates a temporary directory adjacent to a real one. + + Attributes: + original + The original directory to create a temp directory for. + path + After calling create() or entering, contains the full + path to the temporary directory. + delete + Whether the directory should be deleted when exiting + (when used as a contextmanager) + + """ + # The characters that may be used to name the temp directory + # We always prepend a ~ and then rotate through these until + # a usable name is found. + # pkg_resources raises a different error for .dist-info folder + # with leading '-' and invalid metadata + LEADING_CHARS = "-~.=%0123456789" + + def __init__(self, original, delete=None): + self.original = original.rstrip('/\\') + super(AdjacentTempDirectory, self).__init__(delete=delete) + + @classmethod + def _generate_names(cls, name): + """Generates a series of temporary names. + + The algorithm replaces the leading characters in the name + with ones that are valid filesystem characters, but are not + valid package names (for both Python and pip definitions of + package). + """ + for i in range(1, len(name)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i - 1): + new_name = '~' + ''.join(candidate) + name[i:] + if new_name != name: + yield new_name + + # If we make it this far, we will have to make a longer name + for i in range(len(cls.LEADING_CHARS)): + for candidate in itertools.combinations_with_replacement( + cls.LEADING_CHARS, i): + new_name = '~' + ''.join(candidate) + name + if new_name != name: + yield new_name + + def _create(self, kind): + root, name = os.path.split(self.original) + for candidate in self._generate_names(name): + path = os.path.join(root, candidate) + try: + os.mkdir(path) + except OSError as ex: + # Continue if the name exists already + if ex.errno != errno.EEXIST: + raise + else: + path = os.path.realpath(path) + break + else: + # Final fallback on the default behavior. + path = os.path.realpath( + tempfile.mkdtemp(prefix="pip-{}-".format(kind)) + ) + + logger.debug("Created temporary directory: {}".format(path)) + return path \ No newline at end of file diff --git a/pipenv/patched/notpip/_internal/utils/typing.py b/pipenv/patched/notpip/_internal/utils/typing.py index 56f2fa87..ec11da8a 100644 --- a/pipenv/patched/notpip/_internal/utils/typing.py +++ b/pipenv/patched/notpip/_internal/utils/typing.py @@ -21,7 +21,7 @@ In pip, all static-typing related imports should be guarded as follows: from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import ... # noqa: F401 + from typing import ... Ref: https://github.com/python/mypy/issues/3216 """ diff --git a/pipenv/patched/notpip/_internal/utils/ui.py b/pipenv/patched/notpip/_internal/utils/ui.py index 6eebd17d..78a960cf 100644 --- a/pipenv/patched/notpip/_internal/utils/ui.py +++ b/pipenv/patched/notpip/_internal/utils/ui.py @@ -1,3 +1,7 @@ +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import, division import contextlib @@ -8,11 +12,8 @@ import time from signal import SIGINT, default_int_handler, signal from pipenv.patched.notpip._vendor import six -from pipenv.patched.notpip._vendor.progress.bar import ( - Bar, ChargingBar, FillingCirclesBar, FillingSquaresBar, IncrementalBar, - ShadyBar, -) -from pipenv.patched.notpip._vendor.progress.helpers import HIDE_CURSOR, SHOW_CURSOR, WritelnMixin +from pipenv.patched.notpip._vendor.progress import HIDE_CURSOR, SHOW_CURSOR +from pipenv.patched.notpip._vendor.progress.bar import Bar, FillingCirclesBar, IncrementalBar from pipenv.patched.notpip._vendor.progress.spinner import Spinner from pipenv.patched.notpip._internal.utils.compat import WINDOWS @@ -21,7 +22,7 @@ from pipenv.patched.notpip._internal.utils.misc import format_size from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING if MYPY_CHECK_RUNNING: - from typing import Any # noqa: F401 + from typing import Any, Iterator, IO try: from pipenv.patched.notpip._vendor import colorama @@ -168,7 +169,7 @@ class WindowsMixin(object): # The Windows terminal does not support the hide/show cursor ANSI codes # even with colorama. So we'll ensure that hide_cursor is False on # Windows. - # This call neds to go before the super() call, so that hide_cursor + # This call needs to go before the super() call, so that hide_cursor # is set in time. The base progress bar class writes the "hide cursor" # code to the terminal in its init, so if we don't set this soon # enough, we get a "hide" with no corresponding "show"... @@ -203,7 +204,7 @@ class BaseDownloadProgressBar(WindowsMixin, InterruptibleMixin, class DefaultDownloadProgressBar(BaseDownloadProgressBar, - _BaseBar): # type: ignore + _BaseBar): pass @@ -211,22 +212,8 @@ class DownloadSilentBar(BaseDownloadProgressBar, SilentBar): # type: ignore pass -class DownloadIncrementalBar(BaseDownloadProgressBar, # type: ignore - IncrementalBar): - pass - - -class DownloadChargingBar(BaseDownloadProgressBar, # type: ignore - ChargingBar): - pass - - -class DownloadShadyBar(BaseDownloadProgressBar, ShadyBar): # type: ignore - pass - - -class DownloadFillingSquaresBar(BaseDownloadProgressBar, # type: ignore - FillingSquaresBar): +class DownloadBar(BaseDownloadProgressBar, # type: ignore + Bar): pass @@ -241,7 +228,7 @@ class DownloadBlueEmojiProgressBar(BaseDownloadProgressBar, # type: ignore class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin, - DownloadProgressMixin, WritelnMixin, Spinner): + DownloadProgressMixin, Spinner): file = sys.stdout suffix = "%(downloaded)s %(download_speed)s" @@ -269,7 +256,7 @@ class DownloadProgressSpinner(WindowsMixin, InterruptibleMixin, BAR_TYPES = { "off": (DownloadSilentBar, DownloadSilentBar), "on": (DefaultDownloadProgressBar, DownloadProgressSpinner), - "ascii": (DownloadIncrementalBar, DownloadProgressSpinner), + "ascii": (DownloadBar, DownloadProgressSpinner), "pretty": (DownloadFillingCirclesBar, DownloadProgressSpinner), "emoji": (DownloadBlueEmojiProgressBar, DownloadProgressSpinner) } @@ -292,6 +279,7 @@ def DownloadProgressProvider(progress_bar, max=None): @contextlib.contextmanager def hidden_cursor(file): + # type: (IO) -> Iterator[None] # The Windows terminal does not support the hide/show cursor ANSI codes, # even via colorama. So don't even try. if WINDOWS: @@ -311,19 +299,32 @@ def hidden_cursor(file): class RateLimiter(object): def __init__(self, min_update_interval_seconds): + # type: (float) -> None self._min_update_interval_seconds = min_update_interval_seconds - self._last_update = 0 + self._last_update = 0 # type: float def ready(self): + # type: () -> bool now = time.time() delta = now - self._last_update return delta >= self._min_update_interval_seconds def reset(self): + # type: () -> None self._last_update = time.time() -class InteractiveSpinner(object): +class SpinnerInterface(object): + def spin(self): + # type: () -> None + raise NotImplementedError() + + def finish(self, final_status): + # type: (str) -> None + raise NotImplementedError() + + +class InteractiveSpinner(SpinnerInterface): def __init__(self, message, file=None, spin_chars="-\\|/", # Empirically, 8 updates/second looks nice min_update_interval_seconds=0.125): @@ -352,6 +353,7 @@ class InteractiveSpinner(object): self._rate_limiter.reset() def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -359,6 +361,7 @@ class InteractiveSpinner(object): self._write(next(self._spin_cycle)) def finish(self, final_status): + # type: (str) -> None if self._finished: return self._write(final_status) @@ -371,8 +374,9 @@ class InteractiveSpinner(object): # We still print updates occasionally (once every 60 seconds by default) to # act as a keep-alive for systems like Travis-CI that take lack-of-output as # an indication that a task has frozen. -class NonInteractiveSpinner(object): +class NonInteractiveSpinner(SpinnerInterface): def __init__(self, message, min_update_interval_seconds=60): + # type: (str, float) -> None self._message = message self._finished = False self._rate_limiter = RateLimiter(min_update_interval_seconds) @@ -384,6 +388,7 @@ class NonInteractiveSpinner(object): logger.info("%s: %s", self._message, status) def spin(self): + # type: () -> None if self._finished: return if not self._rate_limiter.ready(): @@ -391,6 +396,7 @@ class NonInteractiveSpinner(object): self._update("still running...") def finish(self, final_status): + # type: (str) -> None if self._finished: return self._update("finished with status '%s'" % (final_status,)) @@ -399,13 +405,14 @@ class NonInteractiveSpinner(object): @contextlib.contextmanager def open_spinner(message): + # type: (str) -> Iterator[SpinnerInterface] # Interactive spinner goes directly to sys.stdout rather than being routed # through the logging system, but it acts like it has level INFO, # i.e. it's only displayed if we're at level INFO or better. # Non-interactive spinner goes through the logging system, so it is always # in sync with logging configuration. if sys.stdout.isatty() and logger.getEffectiveLevel() <= logging.INFO: - spinner = InteractiveSpinner(message) + spinner = InteractiveSpinner(message) # type: SpinnerInterface else: spinner = NonInteractiveSpinner(message) try: diff --git a/pipenv/patched/notpip/_internal/utils/unpacking.py b/pipenv/patched/notpip/_internal/utils/unpacking.py new file mode 100644 index 00000000..af0b2674 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/unpacking.py @@ -0,0 +1,272 @@ +"""Utilities related archives. +""" + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import logging +import os +import shutil +import stat +import tarfile +import zipfile + +from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.utils.filetypes import ( + BZ2_EXTENSIONS, + TAR_EXTENSIONS, + XZ_EXTENSIONS, + ZIP_EXTENSIONS, +) +from pipenv.patched.notpip._internal.utils.misc import ensure_dir +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Iterable, List, Optional, Text, Union + + +logger = logging.getLogger(__name__) + + +SUPPORTED_EXTENSIONS = ZIP_EXTENSIONS + TAR_EXTENSIONS + +try: + import bz2 # noqa + SUPPORTED_EXTENSIONS += BZ2_EXTENSIONS +except ImportError: + logger.debug('bz2 module is not available') + +try: + # Only for Python 3.3+ + import lzma # noqa + SUPPORTED_EXTENSIONS += XZ_EXTENSIONS +except ImportError: + logger.debug('lzma module is not available') + + +def current_umask(): + """Get the current umask which involves having to set it temporarily.""" + mask = os.umask(0) + os.umask(mask) + return mask + + +def split_leading_dir(path): + # type: (Union[str, Text]) -> List[Union[str, Text]] + path = path.lstrip('/').lstrip('\\') + if ( + '/' in path and ( + ('\\' in path and path.find('/') < path.find('\\')) or + '\\' not in path + ) + ): + return path.split('/', 1) + elif '\\' in path: + return path.split('\\', 1) + else: + return [path, ''] + + +def has_leading_dir(paths): + # type: (Iterable[Union[str, Text]]) -> bool + """Returns true if all the paths have the same leading path name + (i.e., everything is in one subdirectory in an archive)""" + common_prefix = None + for path in paths: + prefix, rest = split_leading_dir(path) + if not prefix: + return False + elif common_prefix is None: + common_prefix = prefix + elif prefix != common_prefix: + return False + return True + + +def is_within_directory(directory, target): + # type: ((Union[str, Text]), (Union[str, Text])) -> bool + """ + Return true if the absolute path of target is within the directory + """ + abs_directory = os.path.abspath(directory) + abs_target = os.path.abspath(target) + + prefix = os.path.commonprefix([abs_directory, abs_target]) + return prefix == abs_directory + + +def unzip_file(filename, location, flatten=True): + # type: (str, str, bool) -> None + """ + Unzip the file (with path `filename`) to the destination `location`. All + files are written based on system defaults and umask (i.e. permissions are + not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + zipfp = open(filename, 'rb') + try: + zip = zipfile.ZipFile(zipfp, allowZip64=True) + leading = has_leading_dir(zip.namelist()) and flatten + for info in zip.infolist(): + name = info.filename + fn = name + if leading: + fn = split_leading_dir(name)[1] + fn = os.path.join(location, fn) + dir = os.path.dirname(fn) + if not is_within_directory(location, fn): + message = ( + 'The zip file ({}) has a file ({}) trying to install ' + 'outside target directory ({})' + ) + raise InstallationError(message.format(filename, fn, location)) + if fn.endswith('/') or fn.endswith('\\'): + # A directory + ensure_dir(fn) + else: + ensure_dir(dir) + # Don't use read() to avoid allocating an arbitrarily large + # chunk of memory for the file's content + fp = zip.open(name) + try: + with open(fn, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + finally: + fp.close() + mode = info.external_attr >> 16 + # if mode and regular file and any execute permissions for + # user/group/world? + if mode and stat.S_ISREG(mode) and mode & 0o111: + # make dest file have execute for user/group/world + # (chmod +x) no-op on windows per python docs + os.chmod(fn, (0o777 - current_umask() | 0o111)) + finally: + zipfp.close() + + +def untar_file(filename, location): + # type: (str, str) -> None + """ + Untar the file (with path `filename`) to the destination `location`. + All files are written based on system defaults and umask (i.e. permissions + are not preserved), except that regular file members with any execute + permissions (user, group, or world) have "chmod +x" applied after being + written. Note that for windows, any execute changes using os.chmod are + no-ops per the python docs. + """ + ensure_dir(location) + if filename.lower().endswith('.gz') or filename.lower().endswith('.tgz'): + mode = 'r:gz' + elif filename.lower().endswith(BZ2_EXTENSIONS): + mode = 'r:bz2' + elif filename.lower().endswith(XZ_EXTENSIONS): + mode = 'r:xz' + elif filename.lower().endswith('.tar'): + mode = 'r' + else: + logger.warning( + 'Cannot determine compression type for file %s', filename, + ) + mode = 'r:*' + tar = tarfile.open(filename, mode) + try: + leading = has_leading_dir([ + member.name for member in tar.getmembers() + ]) + for member in tar.getmembers(): + fn = member.name + if leading: + # https://github.com/python/mypy/issues/1174 + fn = split_leading_dir(fn)[1] # type: ignore + path = os.path.join(location, fn) + if not is_within_directory(location, path): + message = ( + 'The tar file ({}) has a file ({}) trying to install ' + 'outside target directory ({})' + ) + raise InstallationError( + message.format(filename, path, location) + ) + if member.isdir(): + ensure_dir(path) + elif member.issym(): + try: + # https://github.com/python/typeshed/issues/2673 + tar._extract_member(member, path) # type: ignore + except Exception as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + else: + try: + fp = tar.extractfile(member) + except (KeyError, AttributeError) as exc: + # Some corrupt tar files seem to produce this + # (specifically bad symlinks) + logger.warning( + 'In the tar file %s the member %s is invalid: %s', + filename, member.name, exc, + ) + continue + ensure_dir(os.path.dirname(path)) + with open(path, 'wb') as destfp: + shutil.copyfileobj(fp, destfp) + fp.close() + # Update the timestamp (useful for cython compiled files) + # https://github.com/python/typeshed/issues/2673 + tar.utime(member, path) # type: ignore + # member have any execute permissions for user/group/world? + if member.mode & 0o111: + # make dest file have execute for user/group/world + # no-op on windows per python docs + os.chmod(path, (0o777 - current_umask() | 0o111)) + finally: + tar.close() + + +def unpack_file( + filename, # type: str + location, # type: str + content_type=None, # type: Optional[str] +): + # type: (...) -> None + filename = os.path.realpath(filename) + if ( + content_type == 'application/zip' or + filename.lower().endswith(ZIP_EXTENSIONS) or + zipfile.is_zipfile(filename) + ): + unzip_file( + filename, + location, + flatten=not filename.endswith('.whl') + ) + elif ( + content_type == 'application/x-gzip' or + tarfile.is_tarfile(filename) or + filename.lower().endswith( + TAR_EXTENSIONS + BZ2_EXTENSIONS + XZ_EXTENSIONS + ) + ): + untar_file(filename, location) + else: + # FIXME: handle? + # FIXME: magic signatures? + logger.critical( + 'Cannot unpack file %s (downloaded from %s, content-type: %s); ' + 'cannot detect archive format', + filename, location, content_type, + ) + raise InstallationError( + 'Cannot determine archive format of {}'.format(location) + ) diff --git a/pipenv/patched/notpip/_internal/utils/urls.py b/pipenv/patched/notpip/_internal/utils/urls.py new file mode 100644 index 00000000..5a78dadc --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/urls.py @@ -0,0 +1,54 @@ +import os +import sys + +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse +from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request + +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: + from typing import Optional, Text, Union + + +def get_url_scheme(url): + # type: (Union[str, Text]) -> Optional[Text] + if ':' not in url: + return None + return url.split(':', 1)[0].lower() + + +def path_to_url(path): + # type: (Union[str, Text]) -> str + """ + Convert a path to a file: URL. The path will be made absolute and have + quoted path parts. + """ + path = os.path.normpath(os.path.abspath(path)) + url = urllib_parse.urljoin('file:', urllib_request.pathname2url(path)) + return url + + +def url_to_path(url): + # type: (str) -> str + """ + Convert a file: URL to a path. + """ + assert url.startswith('file:'), ( + "You can only turn file: urls into filenames (not %r)" % url) + + _, netloc, path, _, _ = urllib_parse.urlsplit(url) + + if not netloc or netloc == 'localhost': + # According to RFC 8089, same as empty authority. + netloc = '' + elif sys.platform == 'win32': + # If we have a UNC path, prepend UNC share notation. + netloc = '\\\\' + netloc + else: + raise ValueError( + 'non-local file URIs are not supported on this platform: %r' + % url + ) + + path = urllib_request.url2pathname(netloc + path) + return path diff --git a/pipenv/patched/notpip/_internal/utils/virtualenv.py b/pipenv/patched/notpip/_internal/utils/virtualenv.py new file mode 100644 index 00000000..380db1c3 --- /dev/null +++ b/pipenv/patched/notpip/_internal/utils/virtualenv.py @@ -0,0 +1,34 @@ +import os.path +import site +import sys + + +def running_under_virtualenv(): + # type: () -> bool + """ + Return True if we're running inside a virtualenv, False otherwise. + + """ + if hasattr(sys, 'real_prefix'): + # pypa/virtualenv case + return True + elif sys.prefix != getattr(sys, "base_prefix", sys.prefix): + # PEP 405 venv + return True + + return False + + +def virtualenv_no_global(): + # type: () -> bool + """ + Return True if in a venv and no system site packages. + """ + # this mirrors the logic in virtualenv.py for locating the + # no-global-site-packages.txt file + site_mod_dir = os.path.dirname(os.path.abspath(site.__file__)) + no_global_file = os.path.join(site_mod_dir, 'no-global-site-packages.txt') + if running_under_virtualenv() and os.path.isfile(no_global_file): + return True + else: + return False diff --git a/pipenv/patched/notpip/_internal/vcs/__init__.py b/pipenv/patched/notpip/_internal/vcs/__init__.py index 5aeac633..91d7b350 100644 --- a/pipenv/patched/notpip/_internal/vcs/__init__.py +++ b/pipenv/patched/notpip/_internal/vcs/__init__.py @@ -1,509 +1,15 @@ -"""Handles all VCS (version control) support""" -from __future__ import absolute_import - -import errno -import logging -import os -import shutil -import sys - -from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse - -from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.misc import ( - display_path, backup_dir, call_subprocess, rmtree, ask_path_exists, +# Expose a limited set of classes and functions so callers outside of +# the vcs package don't need to import deeper than `pip._internal.vcs`. +# (The test directory and imports protected by MYPY_CHECK_RUNNING may +# still need to import from a vcs sub-package.) +# Import all vcs modules to register each VCS in the VcsSupport object. +import pipenv.patched.notpip._internal.vcs.bazaar +import pipenv.patched.notpip._internal.vcs.git +import pipenv.patched.notpip._internal.vcs.mercurial +import pipenv.patched.notpip._internal.vcs.subversion # noqa: F401 +from pipenv.patched.notpip._internal.vcs.versioncontrol import ( # noqa: F401 + RemoteNotFoundError, + is_url, + make_vcs_requirement_url, + vcs, ) -from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING - -if MYPY_CHECK_RUNNING: - from typing import Dict, Optional, Tuple # noqa: F401 - from pipenv.patched.notpip._internal.cli.base_command import Command # noqa: F401 - -__all__ = ['vcs', 'get_src_requirement'] - - -logger = logging.getLogger(__name__) - - -class RevOptions(object): - - """ - Encapsulates a VCS-specific revision to install, along with any VCS - install options. - - Instances of this class should be treated as if immutable. - """ - - def __init__(self, vcs, rev=None, extra_args=None): - """ - Args: - vcs: a VersionControl object. - rev: the name of the revision to install. - extra_args: a list of extra options. - """ - if extra_args is None: - extra_args = [] - - self.extra_args = extra_args - self.rev = rev - self.vcs = vcs - - def __repr__(self): - return '<RevOptions {}: rev={!r}>'.format(self.vcs.name, self.rev) - - @property - def arg_rev(self): - if self.rev is None: - return self.vcs.default_arg_rev - - return self.rev - - def to_args(self): - """ - Return the VCS-specific command arguments. - """ - args = [] - rev = self.arg_rev - if rev is not None: - args += self.vcs.get_base_rev_args(rev) - args += self.extra_args - - return args - - def to_display(self): - if not self.rev: - return '' - - return ' (to revision {})'.format(self.rev) - - def make_new(self, rev): - """ - Make a copy of the current instance, but with a new rev. - - Args: - rev: the name of the revision for the new object. - """ - return self.vcs.make_rev_options(rev, extra_args=self.extra_args) - - -class VcsSupport(object): - _registry = {} # type: Dict[str, Command] - schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] - - def __init__(self): - # Register more schemes with urlparse for various version control - # systems - urllib_parse.uses_netloc.extend(self.schemes) - # Python >= 2.7.4, 3.3 doesn't have uses_fragment - if getattr(urllib_parse, 'uses_fragment', None): - urllib_parse.uses_fragment.extend(self.schemes) - super(VcsSupport, self).__init__() - - def __iter__(self): - return self._registry.__iter__() - - @property - def backends(self): - return list(self._registry.values()) - - @property - def dirnames(self): - return [backend.dirname for backend in self.backends] - - @property - def all_schemes(self): - schemes = [] - for backend in self.backends: - schemes.extend(backend.schemes) - return schemes - - def register(self, cls): - if not hasattr(cls, 'name'): - logger.warning('Cannot register VCS %s', cls.__name__) - return - if cls.name not in self._registry: - self._registry[cls.name] = cls - logger.debug('Registered VCS backend: %s', cls.name) - - def unregister(self, cls=None, name=None): - if name in self._registry: - del self._registry[name] - elif cls in self._registry.values(): - del self._registry[cls.name] - else: - logger.warning('Cannot unregister because no class or name given') - - def get_backend_name(self, location): - """ - Return the name of the version control backend if found at given - location, e.g. vcs.get_backend_name('/path/to/vcs/checkout') - """ - for vc_type in self._registry.values(): - if vc_type.controls_location(location): - logger.debug('Determine that %s uses VCS: %s', - location, vc_type.name) - return vc_type.name - return None - - def get_backend(self, name): - name = name.lower() - if name in self._registry: - return self._registry[name] - - def get_backend_from_location(self, location): - vc_type = self.get_backend_name(location) - if vc_type: - return self.get_backend(vc_type) - return None - - -vcs = VcsSupport() - - -class VersionControl(object): - name = '' - dirname = '' - # List of supported schemes for this Version Control - schemes = () # type: Tuple[str, ...] - # Iterable of environment variable names to pass to call_subprocess(). - unset_environ = () # type: Tuple[str, ...] - default_arg_rev = None # type: Optional[str] - - def __init__(self, url=None, *args, **kwargs): - self.url = url - super(VersionControl, self).__init__(*args, **kwargs) - - def get_base_rev_args(self, rev): - """ - Return the base revision arguments for a vcs command. - - Args: - rev: the name of a revision to install. Cannot be None. - """ - raise NotImplementedError - - def make_rev_options(self, rev=None, extra_args=None): - """ - Return a RevOptions object. - - Args: - rev: the name of a revision to install. - extra_args: a list of extra options. - """ - return RevOptions(self, rev, extra_args=extra_args) - - def _is_local_repository(self, repo): - """ - posix absolute paths start with os.path.sep, - win32 ones start with drive (like c:\\folder) - """ - drive, tail = os.path.splitdrive(repo) - return repo.startswith(os.path.sep) or drive - - def export(self, location): - """ - Export the repository at the url to the destination location - i.e. only download the files, without vcs informations - """ - raise NotImplementedError - - def get_netloc_and_auth(self, netloc, scheme): - """ - Parse the repository URL's netloc, and return the new netloc to use - along with auth information. - - Args: - netloc: the original repository URL netloc. - scheme: the repository URL's scheme without the vcs prefix. - - This is mainly for the Subversion class to override, so that auth - information can be provided via the --username and --password options - instead of through the URL. For other subclasses like Git without - such an option, auth information must stay in the URL. - - Returns: (netloc, (username, password)). - """ - return netloc, (None, None) - - def get_url_rev_and_auth(self, url): - """ - Parse the repository URL to use, and return the URL, revision, - and auth info to use. - - Returns: (url, rev, (username, password)). - """ - scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) - if '+' not in scheme: - raise ValueError( - "Sorry, {!r} is a malformed VCS url. " - "The format is <vcs>+<protocol>://<url>, " - "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) - ) - # Remove the vcs prefix. - scheme = scheme.split('+', 1)[1] - netloc, user_pass = self.get_netloc_and_auth(netloc, scheme) - rev = None - if '@' in path: - path, rev = path.rsplit('@', 1) - url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) - return url, rev, user_pass - - def make_rev_args(self, username, password): - """ - Return the RevOptions "extra arguments" to use in obtain(). - """ - return [] - - def get_url_rev_options(self, url): - """ - Return the URL and RevOptions object to use in obtain() and in - some cases export(), as a tuple (url, rev_options). - """ - url, rev, user_pass = self.get_url_rev_and_auth(url) - username, password = user_pass - extra_args = self.make_rev_args(username, password) - rev_options = self.make_rev_options(rev, extra_args=extra_args) - - return url, rev_options - - def normalize_url(self, url): - """ - Normalize a URL for comparison by unquoting it and removing any - trailing slash. - """ - return urllib_parse.unquote(url).rstrip('/') - - def compare_urls(self, url1, url2): - """ - Compare two repo URLs for identity, ignoring incidental differences. - """ - return (self.normalize_url(url1) == self.normalize_url(url2)) - - def fetch_new(self, dest, url, rev_options): - """ - Fetch a revision from a repository, in the case that this is the - first fetch from the repository. - - Args: - dest: the directory to fetch the repository to. - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def switch(self, dest, url, rev_options): - """ - Switch the repo at ``dest`` to point to ``URL``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def update(self, dest, url, rev_options): - """ - Update an already-existing repo to the given ``rev_options``. - - Args: - rev_options: a RevOptions object. - """ - raise NotImplementedError - - def is_commit_id_equal(self, dest, name): - """ - Return whether the id of the current commit equals the given name. - - Args: - dest: the repository directory. - name: a string name. - """ - raise NotImplementedError - - def obtain(self, dest): - """ - Install or update in editable mode the package represented by this - VersionControl object. - - Args: - dest: the repository directory in which to install or update. - """ - url, rev_options = self.get_url_rev_options(self.url) - - if not os.path.exists(dest): - self.fetch_new(dest, url, rev_options) - return - - rev_display = rev_options.to_display() - if self.is_repository_directory(dest): - existing_url = self.get_url(dest) - if self.compare_urls(existing_url, url): - logger.debug( - '%s in %s exists, and has correct URL (%s)', - self.repo_name.title(), - display_path(dest), - url, - ) - if not self.is_commit_id_equal(dest, rev_options.rev): - logger.info( - 'Updating %s %s%s', - display_path(dest), - self.repo_name, - rev_display, - ) - self.update(dest, url, rev_options) - else: - logger.info('Skipping because already up-to-date.') - return - - logger.warning( - '%s %s in %s exists with URL %s', - self.name, - self.repo_name, - display_path(dest), - existing_url, - ) - prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', - ('s', 'i', 'w', 'b')) - else: - logger.warning( - 'Directory %s already exists, and is not a %s %s.', - dest, - self.name, - self.repo_name, - ) - prompt = ('(i)gnore, (w)ipe, (b)ackup ', ('i', 'w', 'b')) - - logger.warning( - 'The plan is to install the %s repository %s', - self.name, - url, - ) - response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) - - if response == 'a': - sys.exit(-1) - - if response == 'w': - logger.warning('Deleting %s', display_path(dest)) - rmtree(dest) - self.fetch_new(dest, url, rev_options) - return - - if response == 'b': - dest_dir = backup_dir(dest) - logger.warning( - 'Backing up %s to %s', display_path(dest), dest_dir, - ) - shutil.move(dest, dest_dir) - self.fetch_new(dest, url, rev_options) - return - - # Do nothing if the response is "i". - if response == 's': - logger.info( - 'Switching %s %s to %s%s', - self.repo_name, - display_path(dest), - url, - rev_display, - ) - self.switch(dest, url, rev_options) - - def unpack(self, location): - """ - Clean up current location and download the url repository - (and vcs infos) into location - """ - if os.path.exists(location): - rmtree(location) - self.obtain(location) - - def get_src_requirement(self, dist, location): - """ - Return a string representing the requirement needed to - redownload the files currently present in location, something - like: - {repository_url}@{revision}#egg={project_name}-{version_identifier} - """ - raise NotImplementedError - - def get_url(self, location): - """ - Return the url used at location - """ - raise NotImplementedError - - def get_revision(self, location): - """ - Return the current commit id of the files at the given location. - """ - raise NotImplementedError - - def run_command(self, cmd, show_stdout=True, cwd=None, - on_returncode='raise', - command_desc=None, - extra_environ=None, spinner=None): - """ - Run a VCS subcommand - This is simply a wrapper around call_subprocess that adds the VCS - command name, and checks that the VCS is available - """ - cmd = [self.name] + cmd - try: - return call_subprocess(cmd, show_stdout, cwd, - on_returncode, - command_desc, extra_environ, - unset_environ=self.unset_environ, - spinner=spinner) - except OSError as e: - # errno.ENOENT = no such file or directory - # In other words, the VCS executable isn't available - if e.errno == errno.ENOENT: - raise BadCommand( - 'Cannot find command %r - do you have ' - '%r installed and in your ' - 'PATH?' % (self.name, self.name)) - else: - raise # re-raise exception if a different error occurred - - @classmethod - def is_repository_directory(cls, path): - """ - Return whether a directory path is a repository directory. - """ - logger.debug('Checking in %s for %s (%s)...', - path, cls.dirname, cls.name) - return os.path.exists(os.path.join(path, cls.dirname)) - - @classmethod - def controls_location(cls, location): - """ - Check if a location is controlled by the vcs. - It is meant to be overridden to implement smarter detection - mechanisms for specific vcs. - - This can do more than is_repository_directory() alone. For example, - the Git override checks that Git is actually available. - """ - return cls.is_repository_directory(location) - - -def get_src_requirement(dist, location): - version_control = vcs.get_backend_from_location(location) - if version_control: - try: - return version_control().get_src_requirement(dist, - location) - except BadCommand: - logger.warning( - 'cannot determine version of editable source in %s ' - '(%s command not found in path)', - location, - version_control.name, - ) - return dist.as_requirement() - logger.warning( - 'cannot determine version of editable source in %s (is not SVN ' - 'checkout, Git clone, Mercurial clone or Bazaar branch)', - location, - ) - return dist.as_requirement() diff --git a/pipenv/patched/notpip/_internal/vcs/bazaar.py b/pipenv/patched/notpip/_internal/vcs/bazaar.py index 890448ed..1342ceeb 100644 --- a/pipenv/patched/notpip/_internal/vcs/bazaar.py +++ b/pipenv/patched/notpip/_internal/vcs/bazaar.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -5,12 +8,17 @@ import os from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse -from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import ( - display_path, make_vcs_requirement_url, rmtree, -) -from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.utils.misc import display_path, rmtree +from pipenv.patched.notpip._internal.utils.subprocess import make_command +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url +from pipenv.patched.notpip._internal.vcs.versioncontrol import VersionControl, vcs + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pipenv.patched.notpip._internal.utils.misc import HiddenText + from pipenv.patched.notpip._internal.vcs.versioncontrol import AuthInfo, RevOptions + logger = logging.getLogger(__name__) @@ -24,17 +32,19 @@ class Bazaar(VersionControl): 'bzr+lp', ) - def __init__(self, url=None, *args, **kwargs): - super(Bazaar, self).__init__(url, *args, **kwargs) + def __init__(self, *args, **kwargs): + super(Bazaar, self).__init__(*args, **kwargs) # This is only needed for python <2.7.5 # Register lp but do not expose as a scheme to support bzr+lp. if getattr(urllib_parse, 'uses_fragment', None): urllib_parse.uses_fragment.extend(['lp']) - def get_base_rev_args(self, rev): + @staticmethod + def get_base_rev_args(rev): return ['-r', rev] - def export(self, location): + def export(self, location, url): + # type: (str, HiddenText) -> None """ Export the Bazaar repository at the url to the destination location """ @@ -42,15 +52,14 @@ class Bazaar(VersionControl): if os.path.exists(location): rmtree(location) - with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path) - - self.run_command( - ['export', location], - cwd=temp_dir.path, show_stdout=False, - ) + url, rev_options = self.get_url_rev_options(url) + self.run_command( + make_command('export', location, url, rev_options.to_args()), + show_stdout=False, + ) def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None rev_display = rev_options.to_display() logger.info( 'Checking out %s%s to %s', @@ -58,53 +67,52 @@ class Bazaar(VersionControl): rev_display, display_path(dest), ) - cmd_args = ['branch', '-q'] + rev_options.to_args() + [url, dest] + cmd_args = ( + make_command('branch', '-q', rev_options.to_args(), url, dest) + ) self.run_command(cmd_args) def switch(self, dest, url, rev_options): - self.run_command(['switch', url], cwd=dest) + # type: (str, HiddenText, RevOptions) -> None + self.run_command(make_command('switch', url), cwd=dest) def update(self, dest, url, rev_options): - cmd_args = ['pull', '-q'] + rev_options.to_args() + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command('pull', '-q', rev_options.to_args()) self.run_command(cmd_args, cwd=dest) - def get_url_rev_and_auth(self, url): + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] # hotfix the URL scheme after removing bzr+ from bzr+ssh:// readd it - url, rev, user_pass = super(Bazaar, self).get_url_rev_and_auth(url) + url, rev, user_pass = super(Bazaar, cls).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'bzr+' + url return url, rev, user_pass - def get_url(self, location): - urls = self.run_command(['info'], show_stdout=False, cwd=location) + @classmethod + def get_remote_url(cls, location): + urls = cls.run_command(['info'], show_stdout=False, cwd=location) for line in urls.splitlines(): line = line.strip() for x in ('checkout of branch: ', 'parent branch: '): if line.startswith(x): repo = line.split(x)[1] - if self._is_local_repository(repo): + if cls._is_local_repository(repo): return path_to_url(repo) return repo return None - def get_revision(self, location): - revision = self.run_command( + @classmethod + def get_revision(cls, location): + revision = cls.run_command( ['revno'], show_stdout=False, cwd=location, ) return revision.splitlines()[-1] - def get_src_requirement(self, dist, location): - repo = self.get_url(location) - if not repo: - return None - if not repo.lower().startswith('bzr:'): - repo = 'bzr+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev, egg_project_name) - - def is_commit_id_equal(self, dest, name): + @classmethod + def is_commit_id_equal(cls, dest, name): """Always assume the versions don't match""" return False diff --git a/pipenv/patched/notpip/_internal/vcs/git.py b/pipenv/patched/notpip/_internal/vcs/git.py index 3db56144..6855afb2 100644 --- a/pipenv/patched/notpip/_internal/vcs/git.py +++ b/pipenv/patched/notpip/_internal/vcs/git.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -9,10 +12,22 @@ from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse from pipenv.patched.notpip._vendor.six.moves.urllib import request as urllib_request from pipenv.patched.notpip._internal.exceptions import BadCommand -from pipenv.patched.notpip._internal.utils.compat import samefile -from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url +from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.subprocess import make_command from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.vcs.versioncontrol import ( + RemoteNotFoundError, + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pipenv.patched.notpip._internal.utils.misc import HiddenText + from pipenv.patched.notpip._internal.vcs.versioncontrol import AuthInfo, RevOptions + urlsplit = urllib_parse.urlsplit urlunsplit = urllib_parse.urlunsplit @@ -21,7 +36,7 @@ urlunsplit = urllib_parse.urlunsplit logger = logging.getLogger(__name__) -HASH_REGEX = re.compile('[a-fA-F0-9]{40}') +HASH_REGEX = re.compile('^[a-fA-F0-9]{40}$') def looks_like_hash(sha): @@ -40,28 +55,8 @@ class Git(VersionControl): unset_environ = ('GIT_DIR', 'GIT_WORK_TREE') default_arg_rev = 'HEAD' - def __init__(self, url=None, *args, **kwargs): - - # Works around an apparent Git bug - # (see https://article.gmane.org/gmane.comp.version-control.git/146500) - if url: - scheme, netloc, path, query, fragment = urlsplit(url) - if scheme.endswith('file'): - initial_slashes = path[:-len(path.lstrip('/'))] - newpath = ( - initial_slashes + - urllib_request.url2pathname(path) - .replace('\\', '/').lstrip('/') - ) - url = urlunsplit((scheme, netloc, newpath, query, fragment)) - after_plus = scheme.find('+') + 1 - url = scheme[:after_plus] + urlunsplit( - (scheme[after_plus:], netloc, newpath, query, fragment), - ) - - super(Git, self).__init__(url, *args, **kwargs) - - def get_base_rev_args(self, rev): + @staticmethod + def get_base_rev_args(rev): return [rev] def get_git_version(self): @@ -71,39 +66,48 @@ class Git(VersionControl): version = version[len(VERSION_PFX):].split()[0] else: version = '' - # get first 3 positions of the git version becasue + # get first 3 positions of the git version because # on windows it is x.y.z.windows.t, and this parses as # LegacyVersion which always smaller than a Version. version = '.'.join(version.split('.')[:3]) return parse_version(version) - def get_branch(self, location): + @classmethod + def get_current_branch(cls, location): """ Return the current branch, or None if HEAD isn't at a branch (e.g. detached HEAD). """ - args = ['rev-parse', '--abbrev-ref', 'HEAD'] - output = self.run_command(args, show_stdout=False, cwd=location) - branch = output.strip() + # git-symbolic-ref exits with empty stdout if "HEAD" is a detached + # HEAD rather than a symbolic ref. In addition, the -q causes the + # command to exit with status code 1 instead of 128 in this case + # and to suppress the message to stderr. + args = ['symbolic-ref', '-q', 'HEAD'] + output = cls.run_command( + args, extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, + ) + ref = output.strip() - if branch == 'HEAD': - return None + if ref.startswith('refs/heads/'): + return ref[len('refs/heads/'):] - return branch + return None - def export(self, location): + def export(self, location, url): + # type: (str, HiddenText) -> None """Export the Git repository at the url to the destination location""" if not location.endswith('/'): location = location + '/' with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path) + self.unpack(temp_dir.path, url=url) self.run_command( ['checkout-index', '-a', '-f', '--prefix', location], show_stdout=False, cwd=temp_dir.path ) - def get_revision_sha(self, dest, rev): + @classmethod + def get_revision_sha(cls, dest, rev): """ Return (sha_or_none, is_branch), where sha_or_none is a commit hash if the revision names a remote branch or tag, otherwise None. @@ -113,8 +117,8 @@ class Git(VersionControl): rev: the revision name. """ # Pass rev to pre-filter the list. - output = self.run_command(['show-ref', rev], cwd=dest, - show_stdout=False, on_returncode='ignore') + output = cls.run_command(['show-ref', rev], cwd=dest, + show_stdout=False, on_returncode='ignore') refs = {} for line in output.strip().splitlines(): try: @@ -137,7 +141,9 @@ class Git(VersionControl): return (sha, False) - def resolve_revision(self, dest, url, rev_options): + @classmethod + def resolve_revision(cls, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> RevOptions """ Resolve a revision to a new RevOptions object with the SHA1 of the branch, tag, or ref if found. @@ -146,7 +152,11 @@ class Git(VersionControl): rev_options: a RevOptions object. """ rev = rev_options.arg_rev - sha, is_branch = self.get_revision_sha(dest, rev) + # The arg_rev property's implementation for Git ensures that the + # rev return value is always non-None. + assert rev is not None + + sha, is_branch = cls.get_revision_sha(dest, rev) if sha is not None: rev_options = rev_options.make_new(sha) @@ -166,17 +176,18 @@ class Git(VersionControl): return rev_options # If it looks like a ref, we have to fetch it explicitly. - self.run_command( - ['fetch', '-q', url] + rev_options.to_args(), + cls.run_command( + make_command('fetch', '-q', url, rev_options.to_args()), cwd=dest, ) # Change the revision to the SHA of the ref we fetched - sha = self.get_revision(dest, rev='FETCH_HEAD') + sha = cls.get_revision(dest, rev='FETCH_HEAD') rev_options = rev_options.make_new(sha) return rev_options - def is_commit_id_equal(self, dest, name): + @classmethod + def is_commit_id_equal(cls, dest, name): """ Return whether the current commit hash equals the given name. @@ -188,14 +199,13 @@ class Git(VersionControl): # Then avoid an unnecessary subprocess call. return False - return self.get_revision(dest) == name + return cls.get_revision(dest) == name def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None rev_display = rev_options.to_display() - logger.info( - 'Cloning %s%s to %s', url, rev_display, display_path(dest), - ) - self.run_command(['clone', '-q', url, dest]) + logger.info('Cloning %s%s to %s', url, rev_display, display_path(dest)) + self.run_command(make_command('clone', '-q', url, dest)) if rev_options.rev: # Then a specific revision was requested. @@ -205,9 +215,11 @@ class Git(VersionControl): # Only do a checkout if the current commit id doesn't match # the requested revision. if not self.is_commit_id_equal(dest, rev_options.rev): - cmd_args = ['checkout', '-q'] + rev_options.to_args() + cmd_args = make_command( + 'checkout', '-q', rev_options.to_args(), + ) self.run_command(cmd_args, cwd=dest) - elif self.get_branch(dest) != branch_name: + elif self.get_current_branch(dest) != branch_name: # Then a specific branch was requested, and that branch # is not yet checked out. track_branch = 'origin/{}'.format(branch_name) @@ -220,13 +232,18 @@ class Git(VersionControl): self.update_submodules(dest) def switch(self, dest, url, rev_options): - self.run_command(['config', 'remote.origin.url', url], cwd=dest) - cmd_args = ['checkout', '-q'] + rev_options.to_args() + # type: (str, HiddenText, RevOptions) -> None + self.run_command( + make_command('config', 'remote.origin.url', url), + cwd=dest, + ) + cmd_args = make_command('checkout', '-q', rev_options.to_args()) self.run_command(cmd_args, cwd=dest) self.update_submodules(dest) def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None # First fetch changes from the default remote if self.get_git_version() >= parse_version('1.9.0'): # fetch tags in addition to everything else @@ -235,19 +252,31 @@ class Git(VersionControl): self.run_command(['fetch', '-q'], cwd=dest) # Then reset to wanted revision (maybe even origin/master) rev_options = self.resolve_revision(dest, url, rev_options) - cmd_args = ['reset', '--hard', '-q'] + rev_options.to_args() + cmd_args = make_command('reset', '--hard', '-q', rev_options.to_args()) self.run_command(cmd_args, cwd=dest) #: update submodules self.update_submodules(dest) - def get_url(self, location): - """Return URL of the first remote encountered.""" - remotes = self.run_command( + @classmethod + def get_remote_url(cls, location): + """ + Return URL of the first remote encountered. + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + # We need to pass 1 for extra_ok_returncodes since the command + # exits with return code 1 if there are no matching lines. + stdout = cls.run_command( ['config', '--get-regexp', r'remote\..*\.url'], - show_stdout=False, cwd=location, + extra_ok_returncodes=(1, ), show_stdout=False, cwd=location, ) - remotes = remotes.splitlines() - found_remote = remotes[0] + remotes = stdout.splitlines() + try: + found_remote = remotes[0] + except IndexError: + raise RemoteNotFoundError + for remote in remotes: if remote.startswith('remote.origin.url '): found_remote = remote @@ -255,74 +284,70 @@ class Git(VersionControl): url = found_remote.split(' ')[1] return url.strip() - def get_revision(self, location, rev=None): + @classmethod + def get_revision(cls, location, rev=None): if rev is None: rev = 'HEAD' - current_rev = self.run_command( + current_rev = cls.run_command( ['rev-parse', rev], show_stdout=False, cwd=location, ) return current_rev.strip() - def _get_subdirectory(self, location): - """Return the relative path of setup.py to the git repo root.""" + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ # find the repo root - git_dir = self.run_command(['rev-parse', '--git-dir'], - show_stdout=False, cwd=location).strip() + git_dir = cls.run_command( + ['rev-parse', '--git-dir'], + show_stdout=False, cwd=location).strip() if not os.path.isabs(git_dir): git_dir = os.path.join(location, git_dir) - root_dir = os.path.join(git_dir, '..') - # find setup.py - orig_location = location - while not os.path.exists(os.path.join(location, 'setup.py')): - last_location = location - location = os.path.dirname(location) - if location == last_location: - # We've traversed up to the root of the filesystem without - # finding setup.py - logger.warning( - "Could not find setup.py for directory %s (tried all " - "parent directories)", - orig_location, - ) - return None - # relative path of setup.py to repo root - if samefile(root_dir, location): - return None - return os.path.relpath(location, root_dir) + repo_root = os.path.abspath(os.path.join(git_dir, '..')) + return find_path_to_setup_from_repo_root(location, repo_root) - def get_src_requirement(self, dist, location): - repo = self.get_url(location) - if not repo.lower().startswith('git:'): - repo = 'git+' + repo - current_rev = self.get_revision(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - subdir = self._get_subdirectory(location) - req = make_vcs_requirement_url(repo, current_rev, egg_project_name, - subdir=subdir) - - return req - - def get_url_rev_and_auth(self, url): + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] """ Prefixes stub URLs like 'user@hostname:user/repo.git' with 'ssh://'. That's required because although they use SSH they sometimes don't work with a ssh:// scheme (e.g. GitHub). But we need a scheme for parsing. Hence we remove it again afterwards and return it as a stub. """ + # Works around an apparent Git bug + # (see https://article.gmane.org/gmane.comp.version-control.git/146500) + scheme, netloc, path, query, fragment = urlsplit(url) + if scheme.endswith('file'): + initial_slashes = path[:-len(path.lstrip('/'))] + newpath = ( + initial_slashes + + urllib_request.url2pathname(path) + .replace('\\', '/').lstrip('/') + ) + url = urlunsplit((scheme, netloc, newpath, query, fragment)) + after_plus = scheme.find('+') + 1 + url = scheme[:after_plus] + urlunsplit( + (scheme[after_plus:], netloc, newpath, query, fragment), + ) + if '://' not in url: assert 'file:' not in url url = url.replace('git+', 'git+ssh://') - url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) url = url.replace('ssh://', '') else: - url, rev, user_pass = super(Git, self).get_url_rev_and_auth(url) + url, rev, user_pass = super(Git, cls).get_url_rev_and_auth(url) return url, rev, user_pass - def update_submodules(self, location): + @classmethod + def update_submodules(cls, location): if not os.path.exists(os.path.join(location, '.gitmodules')): return - self.run_command( + cls.run_command( ['submodule', 'update', '--init', '--recursive', '-q'], cwd=location, ) @@ -332,10 +357,11 @@ class Git(VersionControl): if super(Git, cls).controls_location(location): return True try: - r = cls().run_command(['rev-parse'], - cwd=location, - show_stdout=False, - on_returncode='ignore') + r = cls.run_command(['rev-parse'], + cwd=location, + show_stdout=False, + on_returncode='ignore', + log_failed_cmd=False) return not r except BadCommand: logger.debug("could not determine if %s is under git control " diff --git a/pipenv/patched/notpip/_internal/vcs/mercurial.py b/pipenv/patched/notpip/_internal/vcs/mercurial.py index d76d47f3..e36f7130 100644 --- a/pipenv/patched/notpip/_internal/vcs/mercurial.py +++ b/pipenv/patched/notpip/_internal/vcs/mercurial.py @@ -1,3 +1,6 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging @@ -5,10 +8,22 @@ import os from pipenv.patched.notpip._vendor.six.moves import configparser -from pipenv.patched.notpip._internal.download import path_to_url -from pipenv.patched.notpip._internal.utils.misc import display_path, make_vcs_requirement_url +from pipenv.patched.notpip._internal.exceptions import BadCommand, InstallationError +from pipenv.patched.notpip._internal.utils.misc import display_path +from pipenv.patched.notpip._internal.utils.subprocess import make_command from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import path_to_url +from pipenv.patched.notpip._internal.vcs.versioncontrol import ( + VersionControl, + find_path_to_setup_from_repo_root, + vcs, +) + +if MYPY_CHECK_RUNNING: + from pipenv.patched.notpip._internal.utils.misc import HiddenText + from pipenv.patched.notpip._internal.vcs.versioncontrol import RevOptions + logger = logging.getLogger(__name__) @@ -17,21 +32,26 @@ class Mercurial(VersionControl): name = 'hg' dirname = '.hg' repo_name = 'clone' - schemes = ('hg', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http') + schemes = ( + 'hg', 'hg+file', 'hg+http', 'hg+https', 'hg+ssh', 'hg+static-http', + ) - def get_base_rev_args(self, rev): + @staticmethod + def get_base_rev_args(rev): return [rev] - def export(self, location): + def export(self, location, url): + # type: (str, HiddenText) -> None """Export the Hg repository at the url to the destination location""" with TempDirectory(kind="export") as temp_dir: - self.unpack(temp_dir.path) + self.unpack(temp_dir.path, url=url) self.run_command( ['archive', location], show_stdout=False, cwd=temp_dir.path ) def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None rev_display = rev_options.to_display() logger.info( 'Cloning hg %s%s to %s', @@ -39,16 +59,19 @@ class Mercurial(VersionControl): rev_display, display_path(dest), ) - self.run_command(['clone', '--noupdate', '-q', url, dest]) - cmd_args = ['update', '-q'] + rev_options.to_args() - self.run_command(cmd_args, cwd=dest) + self.run_command(make_command('clone', '--noupdate', '-q', url, dest)) + self.run_command( + make_command('update', '-q', rev_options.to_args()), + cwd=dest, + ) def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None repo_config = os.path.join(dest, self.dirname, 'hgrc') - config = configparser.SafeConfigParser() + config = configparser.RawConfigParser() try: config.read(repo_config) - config.set('paths', 'default', url) + config.set('paths', 'default', url.secret) with open(repo_config, 'w') as config_file: config.write(config_file) except (OSError, configparser.NoSectionError) as exc: @@ -56,46 +79,77 @@ class Mercurial(VersionControl): 'Could not switch Mercurial repository to %s: %s', url, exc, ) else: - cmd_args = ['update', '-q'] + rev_options.to_args() + cmd_args = make_command('update', '-q', rev_options.to_args()) self.run_command(cmd_args, cwd=dest) def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None self.run_command(['pull', '-q'], cwd=dest) - cmd_args = ['update', '-q'] + rev_options.to_args() + cmd_args = make_command('update', '-q', rev_options.to_args()) self.run_command(cmd_args, cwd=dest) - def get_url(self, location): - url = self.run_command( + @classmethod + def get_remote_url(cls, location): + url = cls.run_command( ['showconfig', 'paths.default'], show_stdout=False, cwd=location).strip() - if self._is_local_repository(url): + if cls._is_local_repository(url): url = path_to_url(url) return url.strip() - def get_revision(self, location): - current_revision = self.run_command( + @classmethod + def get_revision(cls, location): + """ + Return the repository-local changeset revision number, as an integer. + """ + current_revision = cls.run_command( ['parents', '--template={rev}'], show_stdout=False, cwd=location).strip() return current_revision - def get_revision_hash(self, location): - current_rev_hash = self.run_command( + @classmethod + def get_requirement_revision(cls, location): + """ + Return the changeset identification hash, as a 40-character + hexadecimal string + """ + current_rev_hash = cls.run_command( ['parents', '--template={node}'], show_stdout=False, cwd=location).strip() return current_rev_hash - def get_src_requirement(self, dist, location): - repo = self.get_url(location) - if not repo.lower().startswith('hg:'): - repo = 'hg+' + repo - current_rev_hash = self.get_revision_hash(location) - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, current_rev_hash, - egg_project_name) - - def is_commit_id_equal(self, dest, name): + @classmethod + def is_commit_id_equal(cls, dest, name): """Always assume the versions don't match""" return False + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + # find the repo root + repo_root = cls.run_command( + ['root'], show_stdout=False, cwd=location).strip() + if not os.path.isabs(repo_root): + repo_root = os.path.abspath(os.path.join(location, repo_root)) + return find_path_to_setup_from_repo_root(location, repo_root) + + @classmethod + def controls_location(cls, location): + if super(Mercurial, cls).controls_location(location): + return True + try: + cls.run_command( + ['identify'], + cwd=location, + show_stdout=False, + on_returncode='raise', + log_failed_cmd=False) + return True + except (BadCommand, InstallationError): + return False + vcs.register(Mercurial) diff --git a/pipenv/patched/notpip/_internal/vcs/subversion.py b/pipenv/patched/notpip/_internal/vcs/subversion.py index f3c3db4d..90e53c11 100644 --- a/pipenv/patched/notpip/_internal/vcs/subversion.py +++ b/pipenv/patched/notpip/_internal/vcs/subversion.py @@ -1,15 +1,22 @@ +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import logging import os import re -from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log from pipenv.patched.notpip._internal.utils.misc import ( - display_path, make_vcs_requirement_url, rmtree, split_auth_from_netloc, + display_path, + is_console_interactive, + rmtree, + split_auth_from_netloc, ) -from pipenv.patched.notpip._internal.vcs import VersionControl, vcs +from pipenv.patched.notpip._internal.utils.subprocess import make_command +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.vcs.versioncontrol import VersionControl, vcs _svn_xml_url_re = re.compile('url="([^"]+)"') _svn_rev_re = re.compile(r'committed-rev="(\d+)"') @@ -17,6 +24,13 @@ _svn_info_xml_rev_re = re.compile(r'\s*revision="(\d+)"') _svn_info_xml_url_re = re.compile(r'<url>(.*)</url>') +if MYPY_CHECK_RUNNING: + from typing import Optional, Tuple + from pipenv.patched.notpip._internal.utils.subprocess import CommandArgs + from pipenv.patched.notpip._internal.utils.misc import HiddenText + from pipenv.patched.notpip._internal.vcs.versioncontrol import AuthInfo, RevOptions + + logger = logging.getLogger(__name__) @@ -26,56 +40,16 @@ class Subversion(VersionControl): repo_name = 'checkout' schemes = ('svn', 'svn+ssh', 'svn+http', 'svn+https', 'svn+svn') - def get_base_rev_args(self, rev): + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + return True + + @staticmethod + def get_base_rev_args(rev): return ['-r', rev] - def export(self, location): - """Export the svn repository at the url to the destination location""" - url, rev_options = self.get_url_rev_options(self.url) - - logger.info('Exporting svn repository %s to %s', url, location) - with indent_log(): - if os.path.exists(location): - # Subversion doesn't like to check out over an existing - # directory --force fixes this, but was only added in svn 1.5 - rmtree(location) - cmd_args = ['export'] + rev_options.to_args() + [url, location] - self.run_command(cmd_args, show_stdout=False) - - def fetch_new(self, dest, url, rev_options): - rev_display = rev_options.to_display() - logger.info( - 'Checking out %s%s to %s', - url, - rev_display, - display_path(dest), - ) - cmd_args = ['checkout', '-q'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) - - def switch(self, dest, url, rev_options): - cmd_args = ['switch'] + rev_options.to_args() + [url, dest] - self.run_command(cmd_args) - - def update(self, dest, url, rev_options): - cmd_args = ['update'] + rev_options.to_args() + [dest] - self.run_command(cmd_args) - - def get_location(self, dist, dependency_links): - for url in dependency_links: - egg_fragment = Link(url).egg_fragment - if not egg_fragment: - continue - if '-' in egg_fragment: - # FIXME: will this work when a package has - in the name? - key = '-'.join(egg_fragment.split('-')[:-1]).lower() - else: - key = egg_fragment - if key == dist.key: - return url.split('#', 1)[0] - return None - - def get_revision(self, location): + @classmethod + def get_revision(cls, location): """ Return the maximum revision for all files under a given location """ @@ -83,16 +57,16 @@ class Subversion(VersionControl): revision = 0 for base, dirs, files in os.walk(location): - if self.dirname not in dirs: + if cls.dirname not in dirs: dirs[:] = [] continue # no sense walking uncontrolled subdirs - dirs.remove(self.dirname) - entries_fn = os.path.join(base, self.dirname, 'entries') + dirs.remove(cls.dirname) + entries_fn = os.path.join(base, cls.dirname, 'entries') if not os.path.exists(entries_fn): # FIXME: should we warn? continue - dirurl, localrev = self._get_svn_url_rev(base) + dirurl, localrev = cls._get_svn_url_rev(base) if base == location: base = dirurl + '/' # save the root url @@ -102,7 +76,8 @@ class Subversion(VersionControl): revision = max(revision, localrev) return revision - def get_netloc_and_auth(self, netloc, scheme): + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): """ This override allows the auth information to be passed to svn via the --username and --password options instead of via the URL. @@ -110,20 +85,23 @@ class Subversion(VersionControl): if scheme == 'ssh': # The --username and --password options can't be used for # svn+ssh URLs, so keep the auth information in the URL. - return super(Subversion, self).get_netloc_and_auth( - netloc, scheme) + return super(Subversion, cls).get_netloc_and_auth(netloc, scheme) return split_auth_from_netloc(netloc) - def get_url_rev_and_auth(self, url): + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] # hotfix the URL scheme after removing svn+ from svn+ssh:// readd it - url, rev, user_pass = super(Subversion, self).get_url_rev_and_auth(url) + url, rev, user_pass = super(Subversion, cls).get_url_rev_and_auth(url) if url.startswith('ssh://'): url = 'svn+' + url return url, rev, user_pass - def make_rev_args(self, username, password): - extra_args = [] + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + extra_args = [] # type: CommandArgs if username: extra_args += ['--username', username] if password: @@ -131,7 +109,8 @@ class Subversion(VersionControl): return extra_args - def get_url(self, location): + @classmethod + def get_remote_url(cls, location): # In cases where the source is in a subdirectory, not alongside # setup.py we have to look up in the location until we find a real # setup.py @@ -149,12 +128,13 @@ class Subversion(VersionControl): ) return None - return self._get_svn_url_rev(location)[0] + return cls._get_svn_url_rev(location)[0] - def _get_svn_url_rev(self, location): + @classmethod + def _get_svn_url_rev(cls, location): from pipenv.patched.notpip._internal.exceptions import InstallationError - entries_path = os.path.join(location, self.dirname, 'entries') + entries_path = os.path.join(location, cls.dirname, 'entries') if os.path.exists(entries_path): with open(entries_path) as f: data = f.read() @@ -177,7 +157,12 @@ class Subversion(VersionControl): else: try: # subversion >= 1.7 - xml = self.run_command( + # Note that using get_remote_call_options is not necessary here + # because `svn info` is being run against a local directory. + # We don't need to worry about making sure interactive mode + # is being used to prompt for passwords, because passwords + # are only potentially needed for remote server requests. + xml = cls.run_command( ['info', '--xml', location], show_stdout=False, ) @@ -195,19 +180,154 @@ class Subversion(VersionControl): return url, rev - def get_src_requirement(self, dist, location): - repo = self.get_url(location) - if repo is None: - return None - repo = 'svn+' + repo - rev = self.get_revision(location) - # FIXME: why not project name? - egg_project_name = dist.egg_name().split('-', 1)[0] - return make_vcs_requirement_url(repo, rev, egg_project_name) - - def is_commit_id_equal(self, dest, name): + @classmethod + def is_commit_id_equal(cls, dest, name): """Always assume the versions don't match""" return False + def __init__(self, use_interactive=None): + # type: (bool) -> None + if use_interactive is None: + use_interactive = is_console_interactive() + self.use_interactive = use_interactive + + # This member is used to cache the fetched version of the current + # ``svn`` client. + # Special value definitions: + # None: Not evaluated yet. + # Empty tuple: Could not parse version. + self._vcs_version = None # type: Optional[Tuple[int, ...]] + + super(Subversion, self).__init__() + + def call_vcs_version(self): + # type: () -> Tuple[int, ...] + """Query the version of the currently installed Subversion client. + + :return: A tuple containing the parts of the version information or + ``()`` if the version returned from ``svn`` could not be parsed. + :raises: BadCommand: If ``svn`` is not installed. + """ + # Example versions: + # svn, version 1.10.3 (r1842928) + # compiled Feb 25 2019, 14:20:39 on x86_64-apple-darwin17.0.0 + # svn, version 1.7.14 (r1542130) + # compiled Mar 28 2018, 08:49:13 on x86_64-pc-linux-gnu + version_prefix = 'svn, version ' + version = self.run_command(['--version'], show_stdout=False) + if not version.startswith(version_prefix): + return () + + version = version[len(version_prefix):].split()[0] + version_list = version.split('.') + try: + parsed_version = tuple(map(int, version_list)) + except ValueError: + return () + + return parsed_version + + def get_vcs_version(self): + # type: () -> Tuple[int, ...] + """Return the version of the currently installed Subversion client. + + If the version of the Subversion client has already been queried, + a cached value will be used. + + :return: A tuple containing the parts of the version information or + ``()`` if the version returned from ``svn`` could not be parsed. + :raises: BadCommand: If ``svn`` is not installed. + """ + if self._vcs_version is not None: + # Use cached version, if available. + # If parsing the version failed previously (empty tuple), + # do not attempt to parse it again. + return self._vcs_version + + vcs_version = self.call_vcs_version() + self._vcs_version = vcs_version + return vcs_version + + def get_remote_call_options(self): + # type: () -> CommandArgs + """Return options to be used on calls to Subversion that contact the server. + + These options are applicable for the following ``svn`` subcommands used + in this class. + + - checkout + - export + - switch + - update + + :return: A list of command line arguments to pass to ``svn``. + """ + if not self.use_interactive: + # --non-interactive switch is available since Subversion 0.14.4. + # Subversion < 1.8 runs in interactive mode by default. + return ['--non-interactive'] + + svn_version = self.get_vcs_version() + # By default, Subversion >= 1.8 runs in non-interactive mode if + # stdin is not a TTY. Since that is how pip invokes SVN, in + # call_subprocess(), pip must pass --force-interactive to ensure + # the user can be prompted for a password, if required. + # SVN added the --force-interactive option in SVN 1.8. Since + # e.g. RHEL/CentOS 7, which is supported until 2024, ships with + # SVN 1.7, pip should continue to support SVN 1.7. Therefore, pip + # can't safely add the option if the SVN version is < 1.8 (or unknown). + if svn_version >= (1, 8): + return ['--force-interactive'] + + return [] + + def export(self, location, url): + # type: (str, HiddenText) -> None + """Export the svn repository at the url to the destination location""" + url, rev_options = self.get_url_rev_options(url) + + logger.info('Exporting svn repository %s to %s', url, location) + with indent_log(): + if os.path.exists(location): + # Subversion doesn't like to check out over an existing + # directory --force fixes this, but was only added in svn 1.5 + rmtree(location) + cmd_args = make_command( + 'export', self.get_remote_call_options(), + rev_options.to_args(), url, location, + ) + self.run_command(cmd_args, show_stdout=False) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + rev_display = rev_options.to_display() + logger.info( + 'Checking out %s%s to %s', + url, + rev_display, + display_path(dest), + ) + cmd_args = make_command( + 'checkout', '-q', self.get_remote_call_options(), + rev_options.to_args(), url, dest, + ) + self.run_command(cmd_args) + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command( + 'switch', self.get_remote_call_options(), rev_options.to_args(), + url, dest, + ) + self.run_command(cmd_args) + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + cmd_args = make_command( + 'update', self.get_remote_call_options(), rev_options.to_args(), + dest, + ) + self.run_command(cmd_args) + vcs.register(Subversion) diff --git a/pipenv/patched/notpip/_internal/vcs/versioncontrol.py b/pipenv/patched/notpip/_internal/vcs/versioncontrol.py new file mode 100644 index 00000000..efe27c12 --- /dev/null +++ b/pipenv/patched/notpip/_internal/vcs/versioncontrol.py @@ -0,0 +1,665 @@ +"""Handles all VCS (version control) support""" + +# The following comment should be removed at some point in the future. +# mypy: disallow-untyped-defs=False + +from __future__ import absolute_import + +import errno +import logging +import os +import shutil +import sys + +from pipenv.patched.notpip._vendor import pkg_resources +from pipenv.patched.notpip._vendor.six.moves.urllib import parse as urllib_parse + +from pipenv.patched.notpip._internal.exceptions import BadCommand +from pipenv.patched.notpip._internal.utils.compat import samefile +from pipenv.patched.notpip._internal.utils.misc import ( + ask_path_exists, + backup_dir, + display_path, + hide_url, + hide_value, + rmtree, +) +from pipenv.patched.notpip._internal.utils.subprocess import call_subprocess, make_command +from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING +from pipenv.patched.notpip._internal.utils.urls import get_url_scheme + +if MYPY_CHECK_RUNNING: + from typing import ( + Any, Dict, Iterable, List, Mapping, Optional, Text, Tuple, Type, Union + ) + from pipenv.patched.notpip._internal.utils.ui import SpinnerInterface + from pipenv.patched.notpip._internal.utils.misc import HiddenText + from pipenv.patched.notpip._internal.utils.subprocess import CommandArgs + + AuthInfo = Tuple[Optional[str], Optional[str]] + + +__all__ = ['vcs'] + + +logger = logging.getLogger(__name__) + + +def is_url(name): + # type: (Union[str, Text]) -> bool + """ + Return true if the name looks like a URL. + """ + scheme = get_url_scheme(name) + if scheme is None: + return False + return scheme in ['http', 'https', 'file', 'ftp'] + vcs.all_schemes + + +def make_vcs_requirement_url(repo_url, rev, project_name, subdir=None): + """ + Return the URL for a VCS requirement. + + Args: + repo_url: the remote VCS url, with any needed VCS prefix (e.g. "git+"). + project_name: the (unescaped) project name. + """ + egg_project_name = pkg_resources.to_filename(project_name) + req = '{}@{}#egg={}'.format(repo_url, rev, egg_project_name) + if subdir: + req += '&subdirectory={}'.format(subdir) + + return req + + +def find_path_to_setup_from_repo_root(location, repo_root): + """ + Find the path to `setup.py` by searching up the filesystem from `location`. + Return the path to `setup.py` relative to `repo_root`. + Return None if `setup.py` is in `repo_root` or cannot be found. + """ + # find setup.py + orig_location = location + while not os.path.exists(os.path.join(location, 'setup.py')): + last_location = location + location = os.path.dirname(location) + if location == last_location: + # We've traversed up to the root of the filesystem without + # finding setup.py + logger.warning( + "Could not find setup.py for directory %s (tried all " + "parent directories)", + orig_location, + ) + return None + + if samefile(repo_root, location): + return None + + return os.path.relpath(location, repo_root) + + +class RemoteNotFoundError(Exception): + pass + + +class RevOptions(object): + + """ + Encapsulates a VCS-specific revision to install, along with any VCS + install options. + + Instances of this class should be treated as if immutable. + """ + + def __init__( + self, + vc_class, # type: Type[VersionControl] + rev=None, # type: Optional[str] + extra_args=None, # type: Optional[CommandArgs] + ): + # type: (...) -> None + """ + Args: + vc_class: a VersionControl subclass. + rev: the name of the revision to install. + extra_args: a list of extra options. + """ + if extra_args is None: + extra_args = [] + + self.extra_args = extra_args + self.rev = rev + self.vc_class = vc_class + self.branch_name = None # type: Optional[str] + + def __repr__(self): + return '<RevOptions {}: rev={!r}>'.format(self.vc_class.name, self.rev) + + @property + def arg_rev(self): + # type: () -> Optional[str] + if self.rev is None: + return self.vc_class.default_arg_rev + + return self.rev + + def to_args(self): + # type: () -> CommandArgs + """ + Return the VCS-specific command arguments. + """ + args = [] # type: CommandArgs + rev = self.arg_rev + if rev is not None: + args += self.vc_class.get_base_rev_args(rev) + args += self.extra_args + + return args + + def to_display(self): + # type: () -> str + if not self.rev: + return '' + + return ' (to revision {})'.format(self.rev) + + def make_new(self, rev): + # type: (str) -> RevOptions + """ + Make a copy of the current instance, but with a new rev. + + Args: + rev: the name of the revision for the new object. + """ + return self.vc_class.make_rev_options(rev, extra_args=self.extra_args) + + +class VcsSupport(object): + _registry = {} # type: Dict[str, VersionControl] + schemes = ['ssh', 'git', 'hg', 'bzr', 'sftp', 'svn'] + + def __init__(self): + # type: () -> None + # Register more schemes with urlparse for various version control + # systems + urllib_parse.uses_netloc.extend(self.schemes) + # Python >= 2.7.4, 3.3 doesn't have uses_fragment + if getattr(urllib_parse, 'uses_fragment', None): + urllib_parse.uses_fragment.extend(self.schemes) + super(VcsSupport, self).__init__() + + def __iter__(self): + return self._registry.__iter__() + + @property + def backends(self): + # type: () -> List[VersionControl] + return list(self._registry.values()) + + @property + def dirnames(self): + # type: () -> List[str] + return [backend.dirname for backend in self.backends] + + @property + def all_schemes(self): + # type: () -> List[str] + schemes = [] # type: List[str] + for backend in self.backends: + schemes.extend(backend.schemes) + return schemes + + def register(self, cls): + # type: (Type[VersionControl]) -> None + if not hasattr(cls, 'name'): + logger.warning('Cannot register VCS %s', cls.__name__) + return + if cls.name not in self._registry: + self._registry[cls.name] = cls() + logger.debug('Registered VCS backend: %s', cls.name) + + def unregister(self, name): + # type: (str) -> None + if name in self._registry: + del self._registry[name] + + def get_backend_for_dir(self, location): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object if a repository of that type is found + at the given directory. + """ + for vcs_backend in self._registry.values(): + if vcs_backend.controls_location(location): + logger.debug('Determine that %s uses VCS: %s', + location, vcs_backend.name) + return vcs_backend + return None + + def get_backend(self, name): + # type: (str) -> Optional[VersionControl] + """ + Return a VersionControl object or None. + """ + name = name.lower() + return self._registry.get(name) + + +vcs = VcsSupport() + + +class VersionControl(object): + name = '' + dirname = '' + repo_name = '' + # List of supported schemes for this Version Control + schemes = () # type: Tuple[str, ...] + # Iterable of environment variable names to pass to call_subprocess(). + unset_environ = () # type: Tuple[str, ...] + default_arg_rev = None # type: Optional[str] + + @classmethod + def should_add_vcs_url_prefix(cls, remote_url): + """ + Return whether the vcs prefix (e.g. "git+") should be added to a + repository's remote url when used in a requirement. + """ + return not remote_url.lower().startswith('{}:'.format(cls.name)) + + @classmethod + def get_subdirectory(cls, location): + """ + Return the path to setup.py, relative to the repo root. + Return None if setup.py is in the repo root. + """ + return None + + @classmethod + def get_requirement_revision(cls, repo_dir): + """ + Return the revision string that should be used in a requirement. + """ + return cls.get_revision(repo_dir) + + @classmethod + def get_src_requirement(cls, repo_dir, project_name): + """ + Return the requirement string to use to redownload the files + currently at the given repository directory. + + Args: + project_name: the (unescaped) project name. + + The return value has a form similar to the following: + + {repository_url}@{revision}#egg={project_name} + """ + repo_url = cls.get_remote_url(repo_dir) + if repo_url is None: + return None + + if cls.should_add_vcs_url_prefix(repo_url): + repo_url = '{}+{}'.format(cls.name, repo_url) + + revision = cls.get_requirement_revision(repo_dir) + subdir = cls.get_subdirectory(repo_dir) + req = make_vcs_requirement_url(repo_url, revision, project_name, + subdir=subdir) + + return req + + @staticmethod + def get_base_rev_args(rev): + """ + Return the base revision arguments for a vcs command. + + Args: + rev: the name of a revision to install. Cannot be None. + """ + raise NotImplementedError + + @classmethod + def make_rev_options(cls, rev=None, extra_args=None): + # type: (Optional[str], Optional[CommandArgs]) -> RevOptions + """ + Return a RevOptions object. + + Args: + rev: the name of a revision to install. + extra_args: a list of extra options. + """ + return RevOptions(cls, rev, extra_args=extra_args) + + @classmethod + def _is_local_repository(cls, repo): + # type: (str) -> bool + """ + posix absolute paths start with os.path.sep, + win32 ones start with drive (like c:\\folder) + """ + drive, tail = os.path.splitdrive(repo) + return repo.startswith(os.path.sep) or bool(drive) + + def export(self, location, url): + # type: (str, HiddenText) -> None + """ + Export the repository at the url to the destination location + i.e. only download the files, without vcs informations + + :param url: the repository URL starting with a vcs prefix. + """ + raise NotImplementedError + + @classmethod + def get_netloc_and_auth(cls, netloc, scheme): + """ + Parse the repository URL's netloc, and return the new netloc to use + along with auth information. + + Args: + netloc: the original repository URL netloc. + scheme: the repository URL's scheme without the vcs prefix. + + This is mainly for the Subversion class to override, so that auth + information can be provided via the --username and --password options + instead of through the URL. For other subclasses like Git without + such an option, auth information must stay in the URL. + + Returns: (netloc, (username, password)). + """ + return netloc, (None, None) + + @classmethod + def get_url_rev_and_auth(cls, url): + # type: (str) -> Tuple[str, Optional[str], AuthInfo] + """ + Parse the repository URL to use, and return the URL, revision, + and auth info to use. + + Returns: (url, rev, (username, password)). + """ + scheme, netloc, path, query, frag = urllib_parse.urlsplit(url) + if '+' not in scheme: + raise ValueError( + "Sorry, {!r} is a malformed VCS url. " + "The format is <vcs>+<protocol>://<url>, " + "e.g. svn+http://myrepo/svn/MyApp#egg=MyApp".format(url) + ) + # Remove the vcs prefix. + scheme = scheme.split('+', 1)[1] + netloc, user_pass = cls.get_netloc_and_auth(netloc, scheme) + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + url = urllib_parse.urlunsplit((scheme, netloc, path, query, '')) + return url, rev, user_pass + + @staticmethod + def make_rev_args(username, password): + # type: (Optional[str], Optional[HiddenText]) -> CommandArgs + """ + Return the RevOptions "extra arguments" to use in obtain(). + """ + return [] + + def get_url_rev_options(self, url): + # type: (HiddenText) -> Tuple[HiddenText, RevOptions] + """ + Return the URL and RevOptions object to use in obtain() and in + some cases export(), as a tuple (url, rev_options). + """ + secret_url, rev, user_pass = self.get_url_rev_and_auth(url.secret) + username, secret_password = user_pass + password = None # type: Optional[HiddenText] + if secret_password is not None: + password = hide_value(secret_password) + extra_args = self.make_rev_args(username, password) + rev_options = self.make_rev_options(rev, extra_args=extra_args) + + return hide_url(secret_url), rev_options + + @staticmethod + def normalize_url(url): + # type: (str) -> str + """ + Normalize a URL for comparison by unquoting it and removing any + trailing slash. + """ + return urllib_parse.unquote(url).rstrip('/') + + @classmethod + def compare_urls(cls, url1, url2): + # type: (str, str) -> bool + """ + Compare two repo URLs for identity, ignoring incidental differences. + """ + return (cls.normalize_url(url1) == cls.normalize_url(url2)) + + def fetch_new(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Fetch a revision from a repository, in the case that this is the + first fetch from the repository. + + Args: + dest: the directory to fetch the repository to. + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def switch(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Switch the repo at ``dest`` to point to ``URL``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + def update(self, dest, url, rev_options): + # type: (str, HiddenText, RevOptions) -> None + """ + Update an already-existing repo to the given ``rev_options``. + + Args: + rev_options: a RevOptions object. + """ + raise NotImplementedError + + @classmethod + def is_commit_id_equal(cls, dest, name): + """ + Return whether the id of the current commit equals the given name. + + Args: + dest: the repository directory. + name: a string name. + """ + raise NotImplementedError + + def obtain(self, dest, url): + # type: (str, HiddenText) -> None + """ + Install or update in editable mode the package represented by this + VersionControl object. + + :param dest: the repository directory in which to install or update. + :param url: the repository URL starting with a vcs prefix. + """ + url, rev_options = self.get_url_rev_options(url) + + if not os.path.exists(dest): + self.fetch_new(dest, url, rev_options) + return + + rev_display = rev_options.to_display() + if self.is_repository_directory(dest): + existing_url = self.get_remote_url(dest) + if self.compare_urls(existing_url, url.secret): + logger.debug( + '%s in %s exists, and has correct URL (%s)', + self.repo_name.title(), + display_path(dest), + url, + ) + if not self.is_commit_id_equal(dest, rev_options.rev): + logger.info( + 'Updating %s %s%s', + display_path(dest), + self.repo_name, + rev_display, + ) + self.update(dest, url, rev_options) + else: + logger.info('Skipping because already up-to-date.') + return + + logger.warning( + '%s %s in %s exists with URL %s', + self.name, + self.repo_name, + display_path(dest), + existing_url, + ) + prompt = ('(s)witch, (i)gnore, (w)ipe, (b)ackup ', + ('s', 'i', 'w', 'b')) + else: + logger.warning( + 'Directory %s already exists, and is not a %s %s.', + dest, + self.name, + self.repo_name, + ) + # https://github.com/python/mypy/issues/1174 + prompt = ('(i)gnore, (w)ipe, (b)ackup ', # type: ignore + ('i', 'w', 'b')) + + logger.warning( + 'The plan is to install the %s repository %s', + self.name, + url, + ) + response = ask_path_exists('What to do? %s' % prompt[0], prompt[1]) + + if response == 'a': + sys.exit(-1) + + if response == 'w': + logger.warning('Deleting %s', display_path(dest)) + rmtree(dest) + self.fetch_new(dest, url, rev_options) + return + + if response == 'b': + dest_dir = backup_dir(dest) + logger.warning( + 'Backing up %s to %s', display_path(dest), dest_dir, + ) + shutil.move(dest, dest_dir) + self.fetch_new(dest, url, rev_options) + return + + # Do nothing if the response is "i". + if response == 's': + logger.info( + 'Switching %s %s to %s%s', + self.repo_name, + display_path(dest), + url, + rev_display, + ) + self.switch(dest, url, rev_options) + + def unpack(self, location, url): + # type: (str, HiddenText) -> None + """ + Clean up current location and download the url repository + (and vcs infos) into location + + :param url: the repository URL starting with a vcs prefix. + """ + if os.path.exists(location): + rmtree(location) + self.obtain(location, url=url) + + @classmethod + def get_remote_url(cls, location): + """ + Return the url used at location + + Raises RemoteNotFoundError if the repository does not have a remote + url configured. + """ + raise NotImplementedError + + @classmethod + def get_revision(cls, location): + """ + Return the current commit id of the files at the given location. + """ + raise NotImplementedError + + @classmethod + def run_command( + cls, + cmd, # type: Union[List[str], CommandArgs] + show_stdout=True, # type: bool + cwd=None, # type: Optional[str] + on_returncode='raise', # type: str + extra_ok_returncodes=None, # type: Optional[Iterable[int]] + command_desc=None, # type: Optional[str] + extra_environ=None, # type: Optional[Mapping[str, Any]] + spinner=None, # type: Optional[SpinnerInterface] + log_failed_cmd=True + ): + # type: (...) -> Text + """ + Run a VCS subcommand + This is simply a wrapper around call_subprocess that adds the VCS + command name, and checks that the VCS is available + """ + cmd = make_command(cls.name, *cmd) + try: + return call_subprocess(cmd, show_stdout, cwd, + on_returncode=on_returncode, + extra_ok_returncodes=extra_ok_returncodes, + command_desc=command_desc, + extra_environ=extra_environ, + unset_environ=cls.unset_environ, + spinner=spinner, + log_failed_cmd=log_failed_cmd) + except OSError as e: + # errno.ENOENT = no such file or directory + # In other words, the VCS executable isn't available + if e.errno == errno.ENOENT: + raise BadCommand( + 'Cannot find command %r - do you have ' + '%r installed and in your ' + 'PATH?' % (cls.name, cls.name)) + else: + raise # re-raise exception if a different error occurred + + @classmethod + def is_repository_directory(cls, path): + # type: (str) -> bool + """ + Return whether a directory path is a repository directory. + """ + logger.debug('Checking in %s for %s (%s)...', + path, cls.dirname, cls.name) + return os.path.exists(os.path.join(path, cls.dirname)) + + @classmethod + def controls_location(cls, location): + # type: (str) -> bool + """ + Check if a location is controlled by the vcs. + It is meant to be overridden to implement smarter detection + mechanisms for specific vcs. + + This can do more than is_repository_directory() alone. For example, + the Git override checks that Git is actually available. + """ + return cls.is_repository_directory(location) diff --git a/pipenv/patched/notpip/_internal/wheel.py b/pipenv/patched/notpip/_internal/wheel.py index 6df5a3a3..d4c155b4 100644 --- a/pipenv/patched/notpip/_internal/wheel.py +++ b/pipenv/patched/notpip/_internal/wheel.py @@ -1,6 +1,11 @@ """ Support for installing and building the "wheel" binary package format. """ + +# The following comment should be removed at some point in the future. +# mypy: strict-optional=False +# mypy: disallow-untyped-defs=False + from __future__ import absolute_import import collections @@ -19,30 +24,51 @@ from email.parser import Parser from pipenv.patched.notpip._vendor import pkg_resources from pipenv.patched.notpip._vendor.distlib.scripts import ScriptMaker +from pipenv.patched.notpip._vendor.distlib.util import get_export_entry from pipenv.patched.notpip._vendor.packaging.utils import canonicalize_name from pipenv.patched.notpip._vendor.six import StringIO from pipenv.patched.notpip._internal import pep425tags -from pipenv.patched.notpip._internal.download import path_to_url, unpack_url from pipenv.patched.notpip._internal.exceptions import ( - InstallationError, InvalidWheelFilename, UnsupportedWheel, -) -from pipenv.patched.notpip._internal.locations import ( - PIP_DELETE_MARKER_FILENAME, distutils_scheme, + InstallationError, + InvalidWheelFilename, + UnsupportedWheel, ) +from pipenv.patched.notpip._internal.locations import distutils_scheme, get_major_minor_version +from pipenv.patched.notpip._internal.models.link import Link from pipenv.patched.notpip._internal.utils.logging import indent_log -from pipenv.patched.notpip._internal.utils.misc import ( - call_subprocess, captured_stdout, ensure_dir, read_chunks, +from pipenv.patched.notpip._internal.utils.marker_files import has_delete_marker_file +from pipenv.patched.notpip._internal.utils.misc import captured_stdout, ensure_dir, read_chunks +from pipenv.patched.notpip._internal.utils.setuptools_build import make_setuptools_shim_args +from pipenv.patched.notpip._internal.utils.subprocess import ( + LOG_DIVIDER, + call_subprocess, + format_command_args, + runner_with_spinner_message, ) -from pipenv.patched.notpip._internal.utils.setuptools_build import SETUPTOOLS_SHIM from pipenv.patched.notpip._internal.utils.temp_dir import TempDirectory from pipenv.patched.notpip._internal.utils.typing import MYPY_CHECK_RUNNING from pipenv.patched.notpip._internal.utils.ui import open_spinner +from pipenv.patched.notpip._internal.utils.unpacking import unpack_file +from pipenv.patched.notpip._internal.utils.urls import path_to_url if MYPY_CHECK_RUNNING: - from typing import Dict, List, Optional # noqa: F401 + from typing import ( + Dict, List, Optional, Sequence, Mapping, Tuple, IO, Text, Any, + Iterable, Callable, Set, + ) + from pipenv.patched.notpip._vendor.packaging.requirements import Requirement + from pipenv.patched.notpip._internal.req.req_install import InstallRequirement + from pipenv.patched.notpip._internal.operations.prepare import ( + RequirementPreparer + ) + from pipenv.patched.notpip._internal.cache import WheelCache + from pipenv.patched.notpip._internal.pep425tags import Pep425Tag + + InstalledCSVRow = Tuple[str, ...] + + BinaryAllowedPredicate = Callable[[InstallRequirement], bool] -wheel_ext = '.whl' VERSION_COMPATIBLE = (1, 0) @@ -50,7 +76,12 @@ VERSION_COMPATIBLE = (1, 0) logger = logging.getLogger(__name__) -def rehash(path, blocksize=1 << 20): +def normpath(src, p): + return os.path.relpath(src, p).replace(os.path.sep, '/') + + +def hash_file(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[Any, int] """Return (hash, length) for path using hashlib.sha256()""" h = hashlib.sha256() length = 0 @@ -58,23 +89,42 @@ def rehash(path, blocksize=1 << 20): for block in read_chunks(f, size=blocksize): length += len(block) h.update(block) + return (h, length) # type: ignore + + +def rehash(path, blocksize=1 << 20): + # type: (str, int) -> Tuple[str, str] + """Return (encoded_digest, length) for path using hashlib.sha256()""" + h, length = hash_file(path, blocksize) digest = 'sha256=' + urlsafe_b64encode( h.digest() ).decode('latin1').rstrip('=') - return (digest, length) + # unicode/str python2 issues + return (digest, str(length)) # type: ignore def open_for_csv(name, mode): + # type: (str, Text) -> IO if sys.version_info[0] < 3: - nl = {} + nl = {} # type: Dict[str, Any] bin = 'b' else: - nl = {'newline': ''} + nl = {'newline': ''} # type: Dict[str, Any] bin = '' return open(name, mode + bin, **nl) +def replace_python_tag(wheelname, new_tag): + # type: (str, str) -> str + """Replace the Python tag in a wheel file name with a new value. + """ + parts = wheelname.split('-') + parts[-3] = new_tag + return '-'.join(parts) + + def fix_script(path): + # type: (str) -> Optional[bool] """Replace #!python with #!/path/to/python Return True if file was changed.""" # XXX RECORD hashes will need to be updated @@ -83,13 +133,14 @@ def fix_script(path): firstline = script.readline() if not firstline.startswith(b'#!python'): return False - exename = os.environ.get('PIP_PYTHON_PATH', sys.executable).encode(sys.getfilesystemencoding()) + exename = sys.executable.encode(sys.getfilesystemencoding()) firstline = b'#!' + exename + os.linesep.encode("ascii") rest = script.read() with open(path, 'wb') as script: script.write(firstline) script.write(rest) return True + return None dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>.+?))?) @@ -97,6 +148,7 @@ dist_info_re = re.compile(r"""^(?P<namever>(?P<name>.+?)(-(?P<ver>.+?))?) def root_is_purelib(name, wheeldir): + # type: (str, str) -> bool """ Return True if the extracted wheel in wheeldir should go into purelib. """ @@ -113,6 +165,7 @@ def root_is_purelib(name, wheeldir): def get_entrypoints(filename): + # type: (str) -> Tuple[Dict[str, str], Dict[str, str]] if not os.path.exists(filename): return {}, {} @@ -144,7 +197,7 @@ def get_entrypoints(filename): def message_about_scripts_not_on_PATH(scripts): - # type: (List[str]) -> Optional[str] + # type: (Sequence[str]) -> Optional[str] """Determine if any scripts are not on PATH and format a warning. Returns a warning message if one or more scripts are not on PATH, @@ -154,7 +207,7 @@ def message_about_scripts_not_on_PATH(scripts): return None # Group scripts by the path they were installed in - grouped_by_dir = collections.defaultdict(set) # type: Dict[str, set] + grouped_by_dir = collections.defaultdict(set) # type: Dict[str, Set[str]] for destfile in scripts: parent_dir = os.path.dirname(destfile) script_name = os.path.basename(destfile) @@ -167,24 +220,23 @@ def message_about_scripts_not_on_PATH(scripts): ] # If an executable sits with sys.executable, we don't warn for it. # This covers the case of venv invocations without activating the venv. - executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) - not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) + not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) warn_for = { parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() if os.path.normcase(parent_dir) not in not_warn_dirs - } + } # type: Dict[str, Set[str]] if not warn_for: return None # Format a message msg_lines = [] - for parent_dir, scripts in warn_for.items(): - scripts = sorted(scripts) - if len(scripts) == 1: - start_text = "script {} is".format(scripts[0]) + for parent_dir, dir_scripts in warn_for.items(): + sorted_scripts = sorted(dir_scripts) # type: List[str] + if len(sorted_scripts) == 1: + start_text = "script {} is".format(sorted_scripts[0]) else: start_text = "scripts {} are".format( - ", ".join(scripts[:-1]) + " and " + scripts[-1] + ", ".join(sorted_scripts[:-1]) + " and " + sorted_scripts[-1] ) msg_lines.append( @@ -205,10 +257,97 @@ def message_about_scripts_not_on_PATH(scripts): return "\n".join(msg_lines) -def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, - pycompile=True, scheme=None, isolated=False, prefix=None, - warn_script_location=True): +def sorted_outrows(outrows): + # type: (Iterable[InstalledCSVRow]) -> List[InstalledCSVRow] + """ + Return the given rows of a RECORD file in sorted order. + + Each row is a 3-tuple (path, hash, size) and corresponds to a record of + a RECORD file (see PEP 376 and PEP 427 for details). For the rows + passed to this function, the size can be an integer as an int or string, + or the empty string. + """ + # Normally, there should only be one row per path, in which case the + # second and third elements don't come into play when sorting. + # However, in cases in the wild where a path might happen to occur twice, + # we don't want the sort operation to trigger an error (but still want + # determinism). Since the third element can be an int or string, we + # coerce each element to a string to avoid a TypeError in this case. + # For additional background, see-- + # https://github.com/pypa/pip/issues/5868 + return sorted(outrows, key=lambda row: tuple(str(x) for x in row)) + + +def get_csv_rows_for_installed( + old_csv_rows, # type: Iterable[List[str]] + installed, # type: Dict[str, str] + changed, # type: set + generated, # type: List[str] + lib_dir, # type: str +): + # type: (...) -> List[InstalledCSVRow] + """ + :param installed: A map from archive RECORD path to installation RECORD + path. + """ + installed_rows = [] # type: List[InstalledCSVRow] + for row in old_csv_rows: + if len(row) > 3: + logger.warning( + 'RECORD line has more than three elements: {}'.format(row) + ) + # Make a copy because we are mutating the row. + row = list(row) + old_path = row[0] + new_path = installed.pop(old_path, old_path) + row[0] = new_path + if new_path in changed: + digest, length = rehash(new_path) + row[1] = digest + row[2] = length + installed_rows.append(tuple(row)) + for f in generated: + digest, length = rehash(f) + installed_rows.append((normpath(f, lib_dir), digest, str(length))) + for f in installed: + installed_rows.append((installed[f], '', '')) + return installed_rows + + +class MissingCallableSuffix(Exception): + pass + + +def _raise_for_invalid_entrypoint(specification): + entry = get_export_entry(specification) + if entry is not None and entry.suffix is None: + raise MissingCallableSuffix(str(entry)) + + +class PipScriptMaker(ScriptMaker): + def make(self, specification, options=None): + _raise_for_invalid_entrypoint(specification) + return super(PipScriptMaker, self).make(specification, options) + + +def move_wheel_files( + name, # type: str + req, # type: Requirement + wheeldir, # type: str + user=False, # type: bool + home=None, # type: Optional[str] + root=None, # type: Optional[str] + pycompile=True, # type: bool + scheme=None, # type: Optional[Mapping[str, str]] + isolated=False, # type: bool + prefix=None, # type: Optional[str] + warn_script_location=True # type: bool +): + # type: (...) -> None """Install a wheel""" + # TODO: Investigate and break this up. + # TODO: Look into moving this into a dedicated class for representing an + # installation. if not scheme: scheme = distutils_scheme( @@ -221,7 +360,7 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, else: lib_dir = scheme['platlib'] - info_dir = [] + info_dir = [] # type: List[str] data_dirs = [] source = wheeldir.rstrip(os.path.sep) + os.path.sep @@ -229,9 +368,9 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, # installed = files copied from the wheel to the destination # changed = files changed while installing (scripts #! line typically) # generated = files newly generated during the install (script wrappers) - installed = {} + installed = {} # type: Dict[str, str] changed = set() - generated = [] + generated = [] # type: List[str] # Compile all of the pyc files that we're going to be installing if pycompile: @@ -241,9 +380,6 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, compileall.compile_dir(source, force=True, quiet=True) logger.debug(stdout.getvalue()) - def normpath(src, p): - return os.path.relpath(src, p).replace(os.path.sep, '/') - def record_installed(srcfile, destfile, modified=False): """Map archive RECORD paths to installation RECORD paths.""" oldpath = normpath(srcfile, wheeldir) @@ -356,7 +492,7 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, dest = scheme[subdir] clobber(source, dest, False, fixer=fixer, filter=filter) - maker = ScriptMaker(None, scheme['scripts']) + maker = PipScriptMaker(None, scheme['scripts']) # Ensure old scripts are overwritten. # See https://github.com/pypa/pip/issues/1800 @@ -372,35 +508,7 @@ def move_wheel_files(name, req, wheeldir, user=False, home=None, root=None, # See https://bitbucket.org/pypa/distlib/issue/32/ maker.set_mode = True - # Simplify the script and fix the fact that the default script swallows - # every single stack trace. - # See https://bitbucket.org/pypa/distlib/issue/34/ - # See https://bitbucket.org/pypa/distlib/issue/33/ - def _get_script_text(entry): - if entry.suffix is None: - raise InstallationError( - "Invalid script entry point: %s for req: %s - A callable " - "suffix is required. Cf https://packaging.python.org/en/" - "latest/distributing.html#console-scripts for more " - "information." % (entry, req) - ) - return maker.script_template % { - "module": entry.prefix, - "import_name": entry.suffix.split(".")[0], - "func": entry.suffix, - } - - maker._get_script_text = _get_script_text - maker.script_template = r"""# -*- coding: utf-8 -*- -import re -import sys - -from %(module)s import %(import_name)s - -if __name__ == '__main__': - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - sys.exit(%(func)s()) -""" + scripts_to_generate = [] # Special case pip and setuptools to generate versioned wrappers # @@ -438,15 +546,16 @@ if __name__ == '__main__': pip_script = console.pop('pip', None) if pip_script: if "ENSUREPIP_OPTIONS" not in os.environ: - spec = 'pip = ' + pip_script - generated.extend(maker.make(spec)) + scripts_to_generate.append('pip = ' + pip_script) if os.environ.get("ENSUREPIP_OPTIONS", "") != "altinstall": - spec = 'pip%s = %s' % (sys.version[:1], pip_script) - generated.extend(maker.make(spec)) + scripts_to_generate.append( + 'pip%s = %s' % (sys.version_info[0], pip_script) + ) - spec = 'pip%s = %s' % (sys.version[:3], pip_script) - generated.extend(maker.make(spec)) + scripts_to_generate.append( + 'pip%s = %s' % (get_major_minor_version(), pip_script) + ) # Delete any other versioned pip entry points pip_ep = [k for k in console if re.match(r'pip(\d(\.\d)?)?$', k)] for k in pip_ep: @@ -454,11 +563,15 @@ if __name__ == '__main__': easy_install_script = console.pop('easy_install', None) if easy_install_script: if "ENSUREPIP_OPTIONS" not in os.environ: - spec = 'easy_install = ' + easy_install_script - generated.extend(maker.make(spec)) + scripts_to_generate.append( + 'easy_install = ' + easy_install_script + ) - spec = 'easy_install-%s = %s' % (sys.version[:3], easy_install_script) - generated.extend(maker.make(spec)) + scripts_to_generate.append( + 'easy_install-%s = %s' % ( + get_major_minor_version(), easy_install_script + ) + ) # Delete any other versioned easy_install entry points easy_install_ep = [ k for k in console if re.match(r'easy_install(-\d\.\d)?$', k) @@ -467,24 +580,36 @@ if __name__ == '__main__': del console[k] # Generate the console and GUI entry points specified in the wheel - if len(console) > 0: - generated_console_scripts = maker.make_multiple( - ['%s = %s' % kv for kv in console.items()] - ) + scripts_to_generate.extend( + '%s = %s' % kv for kv in console.items() + ) + + gui_scripts_to_generate = [ + '%s = %s' % kv for kv in gui.items() + ] + + generated_console_scripts = [] # type: List[str] + + try: + generated_console_scripts = maker.make_multiple(scripts_to_generate) generated.extend(generated_console_scripts) - if warn_script_location: - msg = message_about_scripts_not_on_PATH(generated_console_scripts) - if msg is not None: - logger.warning(msg) - - if len(gui) > 0: generated.extend( - maker.make_multiple( - ['%s = %s' % kv for kv in gui.items()], - {'gui': True} - ) + maker.make_multiple(gui_scripts_to_generate, {'gui': True}) ) + except MissingCallableSuffix as e: + entry = e.args[0] + raise InstallationError( + "Invalid script entry point: {} for req: {} - A callable " + "suffix is required. Cf https://packaging.python.org/en/" + "latest/distributing.html#console-scripts for more " + "information.".format(entry, req) + ) + + if warn_script_location: + msg = message_about_scripts_not_on_PATH(generated_console_scripts) + if msg is not None: + logger.warning(msg) # Record pip as the installer installer = os.path.join(info_dir[0], 'INSTALLER') @@ -500,28 +625,23 @@ if __name__ == '__main__': with open_for_csv(record, 'r') as record_in: with open_for_csv(temp_record, 'w+') as record_out: reader = csv.reader(record_in) + outrows = get_csv_rows_for_installed( + reader, installed=installed, changed=changed, + generated=generated, lib_dir=lib_dir, + ) writer = csv.writer(record_out) - outrows = [] - for row in reader: - row[0] = installed.pop(row[0], row[0]) - if row[0] in changed: - row[1], row[2] = rehash(row[0]) - outrows.append(tuple(row)) - for f in generated: - digest, length = rehash(f) - outrows.append((normpath(f, lib_dir), digest, length)) - for f in installed: - outrows.append((installed[f], '', '')) - for row in sorted(outrows): + # Sort to simplify testing. + for row in sorted_outrows(outrows): writer.writerow(row) shutil.move(temp_record, record) def wheel_version(source_dir): + # type: (Optional[str]) -> Optional[Tuple[int, ...]] """ Return the Wheel-Version of an extracted wheel, if possible. - Otherwise, return False if we couldn't parse / extract it. + Otherwise, return None if we couldn't parse / extract it. """ try: dist = [d for d in pkg_resources.find_on_path(None, source_dir)][0] @@ -533,10 +653,11 @@ def wheel_version(source_dir): version = tuple(map(int, version.split('.'))) return version except Exception: - return False + return None def check_compatibility(version, name): + # type: (Optional[Tuple[int, ...]], str) -> None """ Raises errors or warns if called with an incompatible Wheel-Version. @@ -565,10 +686,21 @@ def check_compatibility(version, name): ) +def format_tag(file_tag): + # type: (Tuple[str, ...]) -> str + """ + Format three tags in the form "<python_tag>-<abi_tag>-<platform_tag>". + + :param file_tag: A 3-tuple of tags (python_tag, abi_tag, platform_tag). + """ + return '-'.join(file_tag) + + class Wheel(object): """A wheel file""" - # TODO: maybe move the install code into this class + # TODO: Maybe move the class into the models sub-package + # TODO: Maybe move the install code into this class wheel_file_re = re.compile( r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?)) @@ -578,6 +710,7 @@ class Wheel(object): ) def __init__(self, filename): + # type: (str) -> None """ :raises InvalidWheelFilename: when the filename is invalid for a wheel """ @@ -602,31 +735,183 @@ class Wheel(object): for y in self.abis for z in self.plats } - def support_index_min(self, tags=None): + def get_formatted_file_tags(self): + # type: () -> List[str] + """ + Return the wheel's tags as a sorted list of strings. + """ + return sorted(format_tag(tag) for tag in self.file_tags) + + def support_index_min(self, tags): + # type: (List[Pep425Tag]) -> int """ Return the lowest index that one of the wheel's file_tag combinations - achieves in the supported_tags list e.g. if there are 8 supported tags, - and one of the file tags is first in the list, then return 0. Returns - None is the wheel is not supported. - """ - if tags is None: # for mock - tags = pep425tags.get_supported() - indexes = [tags.index(c) for c in self.file_tags if c in tags] - return min(indexes) if indexes else None + achieves in the given list of supported tags. - def supported(self, tags=None): - """Is this wheel supported on this system?""" - if tags is None: # for mock - tags = pep425tags.get_supported() - return bool(set(tags).intersection(self.file_tags)) + For example, if there are 8 supported tags and one of the file tags + is first in the list, then return 0. + + :param tags: the PEP 425 tags to check the wheel against, in order + with most preferred first. + + :raises ValueError: If none of the wheel's file tags match one of + the supported tags. + """ + return min(tags.index(tag) for tag in self.file_tags if tag in tags) + + def supported(self, tags): + # type: (List[Pep425Tag]) -> bool + """ + Return whether the wheel is compatible with one of the given tags. + + :param tags: the PEP 425 tags to check the wheel against. + """ + return not self.file_tags.isdisjoint(tags) + + +def _contains_egg_info( + s, _egg_info_re=re.compile(r'([a-z0-9_.]+)-([a-z0-9_.!+-]+)', re.I)): + """Determine whether the string looks like an egg_info. + + :param s: The string to parse. E.g. foo-2.1 + """ + return bool(_egg_info_re.search(s)) + + +def should_use_ephemeral_cache( + req, # type: InstallRequirement + should_unpack, # type: bool + cache_available, # type: bool + check_binary_allowed, # type: BinaryAllowedPredicate +): + # type: (...) -> Optional[bool] + """ + Return whether to build an InstallRequirement object using the + ephemeral cache. + + :param cache_available: whether a cache directory is available for the + should_unpack=True case. + + :return: True or False to build the requirement with ephem_cache=True + or False, respectively; or None not to build the requirement. + """ + if req.constraint: + # never build requirements that are merely constraints + return None + if req.is_wheel: + if not should_unpack: + logger.info( + 'Skipping %s, due to already being wheel.', req.name, + ) + return None + if not should_unpack: + # i.e. pip wheel, not pip install; + # return False, knowing that the caller will never cache + # in this case anyway, so this return merely means "build it". + # TODO improve this behavior + return False + + if req.editable or not req.source_dir: + return None + + if not check_binary_allowed(req): + logger.info( + "Skipping wheel build for %s, due to binaries " + "being disabled for it.", req.name, + ) + return None + + if req.link and req.link.is_vcs: + # VCS checkout. Build wheel just for this run. + return True + + link = req.link + base, ext = link.splitext() + if cache_available and _contains_egg_info(base): + return False + + # Otherwise, build the wheel just for this run using the ephemeral + # cache since we are either in the case of e.g. a local directory, or + # no cache directory is available to use. + return True + + +def format_command_result( + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> str + """ + Format command information for logging. + """ + command_desc = format_command_args(command_args) + text = 'Command arguments: {}\n'.format(command_desc) + + if not command_output: + text += 'Command output: None' + elif logger.getEffectiveLevel() > logging.DEBUG: + text += 'Command output: [use --verbose to show]' + else: + if not command_output.endswith('\n'): + command_output += '\n' + text += 'Command output:\n{}{}'.format(command_output, LOG_DIVIDER) + + return text + + +def get_legacy_build_wheel_path( + names, # type: List[str] + temp_dir, # type: str + req, # type: InstallRequirement + command_args, # type: List[str] + command_output, # type: str +): + # type: (...) -> Optional[str] + """ + Return the path to the wheel in the temporary build directory. + """ + # Sort for determinism. + names = sorted(names) + if not names: + msg = ( + 'Legacy build of wheel for {!r} created no files.\n' + ).format(req.name) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + return None + + if len(names) > 1: + msg = ( + 'Legacy build of wheel for {!r} created more than one file.\n' + 'Filenames (choosing first): {}\n' + ).format(req.name, names) + msg += format_command_result(command_args, command_output) + logger.warning(msg) + + return os.path.join(temp_dir, names[0]) + + +def _always_true(_): + return True class WheelBuilder(object): """Build wheels from a RequirementSet.""" - def __init__(self, finder, preparer, wheel_cache, - build_options=None, global_options=None, no_clean=False): - self.finder = finder + def __init__( + self, + preparer, # type: RequirementPreparer + wheel_cache, # type: WheelCache + build_options=None, # type: Optional[List[str]] + global_options=None, # type: Optional[List[str]] + check_binary_allowed=None, # type: Optional[BinaryAllowedPredicate] + no_clean=False # type: bool + ): + # type: (...) -> None + if check_binary_allowed is None: + # Binaries allowed by default. + check_binary_allowed = _always_true + self.preparer = preparer self.wheel_cache = wheel_cache @@ -634,6 +919,7 @@ class WheelBuilder(object): self.build_options = build_options or [] self.global_options = global_options or [] + self.check_binary_allowed = check_binary_allowed self.no_clean = no_clean def _build_one(self, req, output_dir, python_tag=None): @@ -648,15 +934,23 @@ class WheelBuilder(object): def _build_one_inside_env(self, req, output_dir, python_tag=None): with TempDirectory(kind="wheel") as temp_dir: - if self.__build_one(req, temp_dir.path, python_tag=python_tag): + if req.use_pep517: + builder = self._build_one_pep517 + else: + builder = self._build_one_legacy + wheel_path = builder(req, temp_dir.path, python_tag=python_tag) + if wheel_path is not None: + wheel_name = os.path.basename(wheel_path) + dest_path = os.path.join(output_dir, wheel_name) try: - wheel_name = os.listdir(temp_dir.path)[0] - wheel_path = os.path.join(output_dir, wheel_name) - shutil.move( - os.path.join(temp_dir.path, wheel_name), wheel_path - ) + wheel_hash, length = hash_file(wheel_path) + shutil.move(wheel_path, dest_path) + logger.info('Created wheel for %s: ' + 'filename=%s size=%d sha256=%s', + req.name, wheel_name, length, + wheel_hash.hexdigest()) logger.info('Stored in directory: %s', output_dir) - return wheel_path + return dest_path except Exception: pass # Ignore return, we can't do anything else useful. @@ -668,16 +962,59 @@ class WheelBuilder(object): # isolating. Currently, it breaks Python in virtualenvs, because it # relies on site.py to find parts of the standard library outside the # virtualenv. - executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) - return [ - executable_loc, '-u', '-c', - SETUPTOOLS_SHIM % req.setup_py - ] + list(self.global_options) + return make_setuptools_shim_args( + req.setup_py_path, + global_options=self.global_options, + unbuffered_output=True + ) - def __build_one(self, req, tempd, python_tag=None): + def _build_one_pep517(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the PEP 517 build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ + assert req.metadata_directory is not None + if self.build_options: + # PEP 517 does not support --build-options + logger.error('Cannot build wheel for %s using PEP 517 when ' + '--build-options is present' % (req.name,)) + return None + try: + logger.debug('Destination directory: %s', tempd) + + runner = runner_with_spinner_message( + 'Building wheel for {} (PEP 517)'.format(req.name) + ) + backend = req.pep517_backend + with backend.subprocess_runner(runner): + wheel_name = backend.build_wheel( + tempd, + metadata_directory=req.metadata_directory, + ) + if python_tag: + # General PEP 517 backends don't necessarily support + # a "--python-tag" option, so we rename the wheel + # file directly. + new_name = replace_python_tag(wheel_name, python_tag) + os.rename( + os.path.join(tempd, wheel_name), + os.path.join(tempd, new_name) + ) + # Reassign to simplify the return at the end of function + wheel_name = new_name + except Exception: + logger.error('Failed building wheel for %s', req.name) + return None + return os.path.join(tempd, wheel_name) + + def _build_one_legacy(self, req, tempd, python_tag=None): + """Build one InstallRequirement using the "legacy" build process. + + Returns path to wheel if successfully built. Otherwise, returns None. + """ base_args = self._base_setup_args(req) - spin_message = 'Running setup.py bdist_wheel for %s' % (req.name,) + spin_message = 'Building wheel for %s (setup.py)' % (req.name,) with open_spinner(spin_message) as spinner: logger.debug('Destination directory: %s', tempd) wheel_args = base_args + ['bdist_wheel', '-d', tempd] \ @@ -687,13 +1024,25 @@ class WheelBuilder(object): wheel_args += ["--python-tag", python_tag] try: - call_subprocess(wheel_args, cwd=req.setup_py_dir, - show_stdout=False, spinner=spinner) - return True + output = call_subprocess( + wheel_args, + cwd=req.unpacked_source_directory, + spinner=spinner, + ) except Exception: spinner.finish("error") logger.error('Failed building wheel for %s', req.name) - return False + return None + + names = os.listdir(tempd) + wheel_path = get_legacy_build_wheel_path( + names=names, + temp_dir=tempd, + req=req, + command_args=wheel_args, + command_output=output, + ) + return wheel_path def _clean_one(self, req): base_args = self._base_setup_args(req) @@ -701,120 +1050,119 @@ class WheelBuilder(object): logger.info('Running setup.py clean for %s', req.name) clean_args = base_args + ['clean', '--all'] try: - call_subprocess(clean_args, cwd=req.source_dir, show_stdout=False) + call_subprocess(clean_args, cwd=req.source_dir) return True except Exception: logger.error('Failed cleaning build dir for %s', req.name) return False - def build(self, requirements, session, autobuilding=False): + def build( + self, + requirements, # type: Iterable[InstallRequirement] + should_unpack=False # type: bool + ): + # type: (...) -> List[InstallRequirement] """Build wheels. - :param unpack: If True, replace the sdist we built from with the - newly built wheel, in preparation for installation. + :param should_unpack: If True, after building the wheel, unpack it + and replace the sdist with the unpacked version in preparation + for installation. :return: True if all the wheels built correctly. """ - from pipenv.patched.notpip._internal import index - from pipenv.patched.notpip._internal.models.link import Link - - building_is_possible = self._wheel_dir or ( - autobuilding and self.wheel_cache.cache_dir + # pip install uses should_unpack=True. + # pip install never provides a _wheel_dir. + # pip wheel uses should_unpack=False. + # pip wheel always provides a _wheel_dir (via the preparer). + assert ( + (should_unpack and not self._wheel_dir) or + (not should_unpack and self._wheel_dir) ) - assert building_is_possible buildset = [] - format_control = self.finder.format_control + cache_available = bool(self.wheel_cache.cache_dir) + for req in requirements: - if req.constraint: + ephem_cache = should_use_ephemeral_cache( + req, + should_unpack=should_unpack, + cache_available=cache_available, + check_binary_allowed=self.check_binary_allowed, + ) + if ephem_cache is None: continue - if req.is_wheel: - if not autobuilding: - logger.info( - 'Skipping %s, due to already being wheel.', req.name, + + # Determine where the wheel should go. + if should_unpack: + if ephem_cache: + output_dir = self.wheel_cache.get_ephem_path_for_link( + req.link ) - elif autobuilding and req.editable: - pass - elif autobuilding and not req.source_dir: - pass - elif autobuilding and req.link and not req.link.is_artifact: - # VCS checkout. Build wheel just for this run. - buildset.append((req, True)) + else: + output_dir = self.wheel_cache.get_path_for_link(req.link) else: - ephem_cache = False - if autobuilding: - link = req.link - base, ext = link.splitext() - if index.egg_info_matches(base, None, link) is None: - # E.g. local directory. Build wheel just for this run. - ephem_cache = True - if "binary" not in format_control.get_allowed_formats( - canonicalize_name(req.name)): - logger.info( - "Skipping bdist_wheel for %s, due to binaries " - "being disabled for it.", req.name, - ) - continue - buildset.append((req, ephem_cache)) + output_dir = self._wheel_dir + + buildset.append((req, output_dir)) if not buildset: - return True + return [] + + # TODO by @pradyunsg + # Should break up this method into 2 separate methods. # Build the wheels. logger.info( 'Building wheels for collected packages: %s', ', '.join([req.name for (req, _) in buildset]), ) - _cache = self.wheel_cache # shorter name + + python_tag = None + if should_unpack: + python_tag = pep425tags.implementation_tag + with indent_log(): build_success, build_failure = [], [] - for req, ephem in buildset: - python_tag = None - if autobuilding: - python_tag = pep425tags.implementation_tag - if ephem: - output_dir = _cache.get_ephem_path_for_link(req.link) - else: - output_dir = _cache.get_path_for_link(req.link) - try: - ensure_dir(output_dir) - except OSError as e: - logger.warning("Building wheel for %s failed: %s", - req.name, e) - build_failure.append(req) - continue - else: - output_dir = self._wheel_dir + for req, output_dir in buildset: + try: + ensure_dir(output_dir) + except OSError as e: + logger.warning( + "Building wheel for %s failed: %s", + req.name, e, + ) + build_failure.append(req) + continue + wheel_file = self._build_one( req, output_dir, python_tag=python_tag, ) if wheel_file: build_success.append(req) - if autobuilding: + if should_unpack: # XXX: This is mildly duplicative with prepare_files, # but not close enough to pull out to a single common # method. # The code below assumes temporary source dirs - # prevent it doing bad things. - if req.source_dir and not os.path.exists(os.path.join( - req.source_dir, PIP_DELETE_MARKER_FILENAME)): + if ( + req.source_dir and + not has_delete_marker_file(req.source_dir) + ): raise AssertionError( "bad source dir - missing marker") # Delete the source we built the wheel from req.remove_temporary_source() # set the build directory again - name is known from # the work prepare_files did. - req.source_dir = req.build_location( + req.source_dir = req.ensure_build_location( self.preparer.build_dir ) # Update the link for this. req.link = Link(path_to_url(wheel_file)) assert req.link.is_wheel # extract the wheel into the dir - unpack_url( - req.link, req.source_dir, None, False, - session=session, - ) + unpack_file(req.link.file_path, req.source_dir) else: build_failure.append(req) @@ -829,5 +1177,5 @@ class WheelBuilder(object): 'Failed to build %s', ' '.join([req.name for req in build_failure]), ) - # Return True if all builds were successful - return len(build_failure) == 0 + # Return a list of requirements that failed to build + return build_failure diff --git a/pipenv/patched/notpip/_vendor/__init__.py b/pipenv/patched/notpip/_vendor/__init__.py index b6294b21..3b61408b 100644 --- a/pipenv/patched/notpip/_vendor/__init__.py +++ b/pipenv/patched/notpip/_vendor/__init__.py @@ -30,24 +30,21 @@ def vendored(modulename): vendored_name = "{0}.{1}".format(__name__, modulename) try: - __import__(vendored_name, globals(), locals(), level=0) + __import__(modulename, globals(), locals(), level=0) except ImportError: - try: - __import__(modulename, globals(), locals(), level=0) - except ImportError: - # We can just silently allow import failures to pass here. If we - # got to this point it means that ``import pipenv.patched.notpip._vendor.whatever`` - # failed and so did ``import whatever``. Since we're importing this - # upfront in an attempt to alias imports, not erroring here will - # just mean we get a regular import error whenever pip *actually* - # tries to import one of these modules to use it, which actually - # gives us a better error message than we would have otherwise - # gotten. - pass - else: - sys.modules[vendored_name] = sys.modules[modulename] - base, head = vendored_name.rsplit(".", 1) - setattr(sys.modules[base], head, sys.modules[modulename]) + # We can just silently allow import failures to pass here. If we + # got to this point it means that ``import pipenv.patched.notpip._vendor.whatever`` + # failed and so did ``import whatever``. Since we're importing this + # upfront in an attempt to alias imports, not erroring here will + # just mean we get a regular import error whenever pip *actually* + # tries to import one of these modules to use it, which actually + # gives us a better error message than we would have otherwise + # gotten. + pass + else: + sys.modules[vendored_name] = sys.modules[modulename] + base, head = vendored_name.rsplit(".", 1) + setattr(sys.modules[base], head, sys.modules[modulename]) # If we're operating in a debundled setup, then we want to go ahead and trigger @@ -63,10 +60,10 @@ if DEBUNDLED: # Actually alias all of our vendored dependencies. vendored("cachecontrol") vendored("colorama") + vendored("contextlib2") vendored("distlib") vendored("distro") vendored("html5lib") - vendored("lockfile") vendored("six") vendored("six.moves") vendored("six.moves.urllib") @@ -74,11 +71,13 @@ if DEBUNDLED: vendored("packaging") vendored("packaging.version") vendored("packaging.specifiers") + vendored("pep517") vendored("pkg_resources") vendored("progress") vendored("pytoml") vendored("retrying") vendored("requests") + vendored("requests.exceptions") vendored("requests.packages") vendored("requests.packages.urllib3") vendored("requests.packages.urllib3._collections") diff --git a/pipenv/patched/notpip/_vendor/cachecontrol/caches/file_cache.py b/pipenv/patched/notpip/_vendor/cachecontrol/caches/file_cache.py index 06f7d09e..607b9452 100644 --- a/pipenv/patched/notpip/_vendor/cachecontrol/caches/file_cache.py +++ b/pipenv/patched/notpip/_vendor/cachecontrol/caches/file_cache.py @@ -69,8 +69,8 @@ class FileCache(BaseCache): raise ValueError("Cannot use use_dir_lock and lock_class together") try: - from pipenv.patched.notpip._vendor.lockfile import LockFile - from pipenv.patched.notpip._vendor.lockfile.mkdirlockfile import MkdirLockFile + from lockfile import LockFile + from lockfile.mkdirlockfile import MkdirLockFile except ImportError: notice = dedent( """ diff --git a/pipenv/patched/notpip/_vendor/certifi/__init__.py b/pipenv/patched/notpip/_vendor/certifi/__init__.py index aa329fbb..8e358e4c 100644 --- a/pipenv/patched/notpip/_vendor/certifi/__init__.py +++ b/pipenv/patched/notpip/_vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.08.24" +__version__ = "2019.09.11" diff --git a/pipenv/patched/notpip/_vendor/certifi/cacert.pem b/pipenv/patched/notpip/_vendor/certifi/cacert.pem index 85de024e..70fa91f6 100644 --- a/pipenv/patched/notpip/_vendor/certifi/cacert.pem +++ b/pipenv/patched/notpip/_vendor/certifi/cacert.pem @@ -326,36 +326,6 @@ OCiNUW7dFGdTbHFcJoRNdVq2fmBWqU2t+5sel/MN2dKXVHfaPRK34B7vCAas+YWH QMAJKOSLakhT2+zNVVXxxvjpoixMptEmX36vWkzaH6byHCx+rgIW0lbQL1dTR+iS -----END CERTIFICATE----- -# Issuer: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Subject: CN=Visa eCommerce Root O=VISA OU=Visa International Service Association -# Label: "Visa eCommerce Root" -# Serial: 25952180776285836048024890241505565794 -# MD5 Fingerprint: fc:11:b8:d8:08:93:30:00:6d:23:f9:7e:eb:52:1e:02 -# SHA1 Fingerprint: 70:17:9b:86:8c:00:a4:fa:60:91:52:22:3f:9f:3e:32:bd:e0:05:62 -# SHA256 Fingerprint: 69:fa:c9:bd:55:fb:0a:c7:8d:53:bb:ee:5c:f1:d5:97:98:9f:d0:aa:ab:20:a2:51:51:bd:f1:73:3e:e7:d1:22 ------BEGIN CERTIFICATE----- -MIIDojCCAoqgAwIBAgIQE4Y1TR0/BvLB+WUF1ZAcYjANBgkqhkiG9w0BAQUFADBr -MQswCQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRl -cm5hdGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNv -bW1lcmNlIFJvb3QwHhcNMDIwNjI2MDIxODM2WhcNMjIwNjI0MDAxNjEyWjBrMQsw -CQYDVQQGEwJVUzENMAsGA1UEChMEVklTQTEvMC0GA1UECxMmVmlzYSBJbnRlcm5h -dGlvbmFsIFNlcnZpY2UgQXNzb2NpYXRpb24xHDAaBgNVBAMTE1Zpc2EgZUNvbW1l -cmNlIFJvb3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCvV95WHm6h -2mCxlCfLF9sHP4CFT8icttD0b0/Pmdjh28JIXDqsOTPHH2qLJj0rNfVIsZHBAk4E -lpF7sDPwsRROEW+1QK8bRaVK7362rPKgH1g/EkZgPI2h4H3PVz4zHvtH8aoVlwdV -ZqW1LS7YgFmypw23RuwhY/81q6UCzyr0TP579ZRdhE2o8mCP2w4lPJ9zcc+U30rq -299yOIzzlr3xF7zSujtFWsan9sYXiwGd/BmoKoMWuDpI/k4+oKsGGelT84ATB+0t -vz8KPFUgOSwsAGl0lUq8ILKpeeUYiZGo3BxN77t+Nwtd/jmliFKMAGzsGHxBvfaL -dXe6YJ2E5/4tAgMBAAGjQjBAMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQD -AgEGMB0GA1UdDgQWBBQVOIMPPyw/cDMezUb+B4wg4NfDtzANBgkqhkiG9w0BAQUF -AAOCAQEAX/FBfXxcCLkr4NWSR/pnXKUTwwMhmytMiUbPWU3J/qVAtmPN3XEolWcR -zCSs00Rsca4BIGsDoo8Ytyk6feUWYFN4PMCvFYP3j1IzJL1kk5fui/fbGKhtcbP3 -LBfQdCVp9/5rPJS+TUtBjE7ic9DjkCJzQ83z7+pzzkWKsKZJ/0x9nXGIxHYdkFsd -7v3M9+79YKWxehZx0RbQfBI8bGmX265fOZpwLwU8GUYEmSA20GBuYQa7FkKMcPcw -++DbZqMAAb3mLNqRX6BGi01qnD093QVG/na/oAo85ADmJ7f/hC3euiInlhBx6yLt -398znM/jra6O1I7mT1GvFpLgXPYHDw== ------END CERTIFICATE----- - # Issuer: CN=AAA Certificate Services O=Comodo CA Limited # Subject: CN=AAA Certificate Services O=Comodo CA Limited # Label: "Comodo AAA Services root" @@ -801,36 +771,6 @@ vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep +OkuE6N36B9K -----END CERTIFICATE----- -# Issuer: CN=Class 2 Primary CA O=Certplus -# Subject: CN=Class 2 Primary CA O=Certplus -# Label: "Certplus Class 2 Primary CA" -# Serial: 177770208045934040241468760488327595043 -# MD5 Fingerprint: 88:2c:8c:52:b8:a2:3c:f3:f7:bb:03:ea:ae:ac:42:0b -# SHA1 Fingerprint: 74:20:74:41:72:9c:dd:92:ec:79:31:d8:23:10:8d:c2:81:92:e2:bb -# SHA256 Fingerprint: 0f:99:3c:8a:ef:97:ba:af:56:87:14:0e:d5:9a:d1:82:1b:b4:af:ac:f0:aa:9a:58:b5:d5:7a:33:8a:3a:fb:cb ------BEGIN CERTIFICATE----- -MIIDkjCCAnqgAwIBAgIRAIW9S/PY2uNp9pTXX8OlRCMwDQYJKoZIhvcNAQEFBQAw -PTELMAkGA1UEBhMCRlIxETAPBgNVBAoTCENlcnRwbHVzMRswGQYDVQQDExJDbGFz -cyAyIFByaW1hcnkgQ0EwHhcNOTkwNzA3MTcwNTAwWhcNMTkwNzA2MjM1OTU5WjA9 -MQswCQYDVQQGEwJGUjERMA8GA1UEChMIQ2VydHBsdXMxGzAZBgNVBAMTEkNsYXNz -IDIgUHJpbWFyeSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANxQ -ltAS+DXSCHh6tlJw/W/uz7kRy1134ezpfgSN1sxvc0NXYKwzCkTsA18cgCSR5aiR -VhKC9+Ar9NuuYS6JEI1rbLqzAr3VNsVINyPi8Fo3UjMXEuLRYE2+L0ER4/YXJQyL -kcAbmXuZVg2v7tK8R1fjeUl7NIknJITesezpWE7+Tt9avkGtrAjFGA7v0lPubNCd -EgETjdyAYveVqUSISnFOYFWe2yMZeVYHDD9jC1yw4r5+FfyUM1hBOHTE4Y+L3yas -H7WLO7dDWWuwJKZtkIvEcupdM5i3y95ee++U8Rs+yskhwcWYAqqi9lt3m/V+llU0 -HGdpwPFC40es/CgcZlUCAwEAAaOBjDCBiTAPBgNVHRMECDAGAQH/AgEKMAsGA1Ud -DwQEAwIBBjAdBgNVHQ4EFgQU43Mt38sOKAze3bOkynm4jrvoMIkwEQYJYIZIAYb4 -QgEBBAQDAgEGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHA6Ly93d3cuY2VydHBsdXMu -Y29tL0NSTC9jbGFzczIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCnVM+IRBnL39R/ -AN9WM2K191EBkOvDP9GIROkkXe/nFL0gt5o8AP5tn9uQ3Nf0YtaLcF3n5QRIqWh8 -yfFC82x/xXp8HVGIutIKPidd3i1RTtMTZGnkLuPT55sJmabglZvOGtd/vjzOUrMR -FcEPF80Du5wlFbqidon8BvEY0JNLDnyCt6X09l/+7UCmnYR0ObncHoUW2ikbhiMA -ybuJfm6AiB4vFLQDJKgybwOaRywwvlbGp0ICcBvqQNi6BQNwB6SW//1IMwrh3KWB -kJtN3X3n57LNXMhqlfil9o3EXXgIvnsG1knPGTZQIy4I5p4FTUcY1Rbpsda2ENW7 -l7+ijrRU ------END CERTIFICATE----- - # Issuer: CN=DST Root CA X3 O=Digital Signature Trust Co. # Subject: CN=DST Root CA X3 O=Digital Signature Trust Co. # Label: "DST Root CA X3" @@ -1249,36 +1189,6 @@ t0QmwCbAr1UwnjvVNioZBPRcHv/PLLf/0P2HQBHVESO7SMAhqaQoLf0V+LBOK/Qw WyH8EZE0vkHve52Xdf+XlcCWWC/qu0bXu+TZLg== -----END CERTIFICATE----- -# Issuer: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center -# Subject: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center -# Label: "Deutsche Telekom Root CA 2" -# Serial: 38 -# MD5 Fingerprint: 74:01:4a:91:b1:08:c4:58:ce:47:cd:f0:dd:11:53:08 -# SHA1 Fingerprint: 85:a4:08:c0:9c:19:3e:5d:51:58:7d:cd:d6:13:30:fd:8c:de:37:bf -# SHA256 Fingerprint: b6:19:1a:50:d0:c3:97:7f:7d:a9:9b:cd:aa:c8:6a:22:7d:ae:b9:67:9e:c7:0b:a3:b0:c9:d9:22:71:c1:70:d3 ------BEGIN CERTIFICATE----- -MIIDnzCCAoegAwIBAgIBJjANBgkqhkiG9w0BAQUFADBxMQswCQYDVQQGEwJERTEc -MBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxlU2Vj -IFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290IENB -IDIwHhcNOTkwNzA5MTIxMTAwWhcNMTkwNzA5MjM1OTAwWjBxMQswCQYDVQQGEwJE -RTEcMBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxl -U2VjIFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290 -IENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrC6M14IspFLEU -ha88EOQ5bzVdSq7d6mGNlUn0b2SjGmBmpKlAIoTZ1KXleJMOaAGtuU1cOs7TuKhC -QN/Po7qCWWqSG6wcmtoIKyUn+WkjR/Hg6yx6m/UTAtB+NHzCnjwAWav12gz1Mjwr -rFDa1sPeg5TKqAyZMg4ISFZbavva4VhYAUlfckE8FQYBjl2tqriTtM2e66foai1S -NNs671x1Udrb8zH57nGYMsRUFUQM+ZtV7a3fGAigo4aKSe5TBY8ZTNXeWHmb0moc -QqvF1afPaA+W5OFhmHZhyJF81j4A4pFQh+GdCuatl9Idxjp9y7zaAzTVjlsB9WoH -txa2bkp/AgMBAAGjQjBAMB0GA1UdDgQWBBQxw3kbuvVT1xfgiXotF2wKsyudMzAP -BgNVHRMECDAGAQH/AgEFMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOC -AQEAlGRZrTlk5ynrE/5aw4sTV8gEJPB0d8Bg42f76Ymmg7+Wgnxu1MM9756Abrsp -tJh6sTtU6zkXR34ajgv8HzFZMQSyzhfzLMdiNlXiItiJVbSYSKpk+tYcNthEeFpa -IzpXl/V6ME+un2pMSyuOoAPjPuCp1NJ70rOo4nI8rZ7/gFnkm0W09juwzTkZmDLl -6iFhkOQxIY40sfcvNUqFENrnijchvllj4PKFiDFT1FQUhXB59C4Gdyd1Lx+4ivn+ -xbrYNuSD7Odlt79jWvNGr4GUN9RBjNYj1h7P9WgbRGOiWrqnNVmh5XAFmw4jV5mU -Cm26OWMohpLzGITY+9HPBVZkVw== ------END CERTIFICATE----- - # Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc # Subject: CN=Cybertrust Global Root O=Cybertrust, Inc # Label: "Cybertrust Global Root" @@ -3483,46 +3393,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su -----END CERTIFICATE----- -# Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 -# Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 -# Label: "Certinomis - Root CA" -# Serial: 1 -# MD5 Fingerprint: 14:0a:fd:8d:a8:28:b5:38:69:db:56:7e:61:22:03:3f -# SHA1 Fingerprint: 9d:70:bb:01:a5:a4:a0:18:11:2e:f7:1c:01:b9:32:c5:34:e7:88:a8 -# SHA256 Fingerprint: 2a:99:f5:bc:11:74:b7:3c:bb:1d:62:08:84:e0:1c:34:e5:1c:cb:39:78:da:12:5f:0e:33:26:88:83:bf:41:58 ------BEGIN CERTIFICATE----- -MIIFkjCCA3qgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJGUjET -MBEGA1UEChMKQ2VydGlub21pczEXMBUGA1UECxMOMDAwMiA0MzM5OTg5MDMxHTAb -BgNVBAMTFENlcnRpbm9taXMgLSBSb290IENBMB4XDTEzMTAyMTA5MTcxOFoXDTMz -MTAyMTA5MTcxOFowWjELMAkGA1UEBhMCRlIxEzARBgNVBAoTCkNlcnRpbm9taXMx -FzAVBgNVBAsTDjAwMDIgNDMzOTk4OTAzMR0wGwYDVQQDExRDZXJ0aW5vbWlzIC0g -Um9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANTMCQosP5L2 -fxSeC5yaah1AMGT9qt8OHgZbn1CF6s2Nq0Nn3rD6foCWnoR4kkjW4znuzuRZWJfl -LieY6pOod5tK8O90gC3rMB+12ceAnGInkYjwSond3IjmFPnVAy//ldu9n+ws+hQV -WZUKxkd8aRi5pwP5ynapz8dvtF4F/u7BUrJ1Mofs7SlmO/NKFoL21prbcpjp3vDF -TKWrteoB4owuZH9kb/2jJZOLyKIOSY008B/sWEUuNKqEUL3nskoTuLAPrjhdsKkb -5nPJWqHZZkCqqU2mNAKthH6yI8H7KsZn9DS2sJVqM09xRLWtwHkziOC/7aOgFLSc -CbAK42C++PhmiM1b8XcF4LVzbsF9Ri6OSyemzTUK/eVNfaoqoynHWmgE6OXWk6Ri -wsXm9E/G+Z8ajYJJGYrKWUM66A0ywfRMEwNvbqY/kXPLynNvEiCL7sCCeN5LLsJJ -wx3tFvYk9CcbXFcx3FXuqB5vbKziRcxXV4p1VxngtViZSTYxPDMBbRZKzbgqg4SG -m/lg0h9tkQPTYKbVPZrdd5A9NaSfD171UkRpucC63M9933zZxKyGIjK8e2uR73r4 -F2iw4lNVYC2vPsKD2NkJK/DAZNuHi5HMkesE/Xa0lZrmFAYb1TQdvtj/dBxThZng -WVJKYe2InmtJiUZ+IFrZ50rlau7SZRFDAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTvkUz1pcMw6C8I6tNxIqSSaHh0 -2TAfBgNVHSMEGDAWgBTvkUz1pcMw6C8I6tNxIqSSaHh02TANBgkqhkiG9w0BAQsF -AAOCAgEAfj1U2iJdGlg+O1QnurrMyOMaauo++RLrVl89UM7g6kgmJs95Vn6RHJk/ -0KGRHCwPT5iVWVO90CLYiF2cN/z7ZMF4jIuaYAnq1fohX9B0ZedQxb8uuQsLrbWw -F6YSjNRieOpWauwK0kDDPAUwPk2Ut59KA9N9J0u2/kTO+hkzGm2kQtHdzMjI1xZS -g081lLMSVX3l4kLr5JyTCcBMWwerx20RoFAXlCOotQqSD7J6wWAsOMwaplv/8gzj -qh8c3LigkyfeY+N/IZ865Z764BNqdeuWXGKRlI5nU7aJ+BIJy29SWwNyhlCVCNSN -h4YVH5Uk2KRvms6knZtt0rJ2BobGVgjF6wnaNsIbW0G+YSrjcOa4pvi2WsS9Iff/ -ql+hbHY5ZtbqTFXhADObE5hjyW/QASAJN1LnDE8+zbz1X5YnpyACleAu6AdBBR8V -btaw5BngDwKTACdyxYvRVB9dSsNAl35VpnzBMwQUAR1JIGkLGZOdblgi90AMRgwj -Y/M50n92Uaf0yKHxDHYiI0ZSKS3io0EHVmmY0gUJvGnHWmHNj4FgFU2A3ZDifcRQ -8ow7bkrHxuaAKzyBvBGAFhAn1/DNP3nMcyrDflOR1m749fPH0FFNjkulW+YZFzvW -gQncItzujrnEj1PhZ7szuIgVRs/taTX/dQ1G885x4cVrhkIGuUE= ------END CERTIFICATE----- - # Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed # Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed # Label: "OISTE WISeKey Global Root GB CA" @@ -4298,3 +4168,391 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- diff --git a/pipenv/patched/notpip/_vendor/certifi/core.py b/pipenv/patched/notpip/_vendor/certifi/core.py index eab9d1d1..7271acf4 100644 --- a/pipenv/patched/notpip/_vendor/certifi/core.py +++ b/pipenv/patched/notpip/_vendor/certifi/core.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ @@ -8,30 +7,9 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): f = os.path.dirname(__file__) return os.path.join(f, 'cacert.pem') - - -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - -if __name__ == '__main__': - print(where()) diff --git a/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt b/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt index 5f567799..3105888e 100644 --- a/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt +++ b/pipenv/patched/notpip/_vendor/colorama/LICENSE.txt @@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pipenv/patched/notpip/_vendor/colorama/__init__.py b/pipenv/patched/notpip/_vendor/colorama/__init__.py index f4d9ce21..2a3bf471 100644 --- a/pipenv/patched/notpip/_vendor/colorama/__init__.py +++ b/pipenv/patched/notpip/_vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.1' diff --git a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py +++ b/pipenv/patched/notpip/_vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/patched/notpip/_vendor/colorama/initialise.py b/pipenv/patched/notpip/_vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/patched/notpip/_vendor/colorama/initialise.py +++ b/pipenv/patched/notpip/_vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/patched/notpip/_vendor/colorama/win32.py b/pipenv/patched/notpip/_vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/patched/notpip/_vendor/colorama/win32.py +++ b/pipenv/patched/notpip/_vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/patched/notpip/_vendor/colorama/winterm.py b/pipenv/patched/notpip/_vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/patched/notpip/_vendor/colorama/winterm.py +++ b/pipenv/patched/notpip/_vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/patched/notpip/_vendor/contextlib2.LICENSE.txt b/pipenv/patched/notpip/_vendor/contextlib2.LICENSE.txt new file mode 100644 index 00000000..5de20277 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/contextlib2.LICENSE.txt @@ -0,0 +1,122 @@ + + +A. HISTORY OF THE SOFTWARE +========================== + +contextlib2 is a derivative of the contextlib module distributed by the PSF +as part of the Python standard library. According, it is itself redistributed +under the PSF license (reproduced in full below). As the contextlib module +was added only in Python 2.5, the licenses for earlier Python versions are +not applicable and have not been included. + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases that included the contextlib module. + + Release Derived Year Owner GPL- + from compatible? (1) + + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 3.0 2.6 2008 PSF yes + 3.0.1 3.0 2009 PSF yes + 3.1 3.0.1 2009 PSF yes + 3.1.1 3.1 2009 PSF yes + 3.1.2 3.1.1 2010 PSF yes + 3.1.3 3.1.2 2010 PSF yes + 3.1.4 3.1.3 2011 PSF yes + 3.2 3.1 2011 PSF yes + 3.2.1 3.2 2011 PSF yes + 3.2.2 3.2.1 2011 PSF yes + 3.3 3.2 2012 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011 Python Software Foundation; All Rights Reserved" are retained in Python +alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/patched/notpip/_vendor/contextlib2.py b/pipenv/patched/notpip/_vendor/contextlib2.py new file mode 100644 index 00000000..3aae8f41 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/contextlib2.py @@ -0,0 +1,518 @@ +"""contextlib2 - backports and enhancements to the contextlib module""" + +import abc +import sys +import warnings +from collections import deque +from functools import wraps + +__all__ = ["contextmanager", "closing", "nullcontext", + "AbstractContextManager", + "ContextDecorator", "ExitStack", + "redirect_stdout", "redirect_stderr", "suppress"] + +# Backwards compatibility +__all__ += ["ContextStack"] + + +# Backport abc.ABC +if sys.version_info[:2] >= (3, 4): + _abc_ABC = abc.ABC +else: + _abc_ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +# Backport classic class MRO +def _classic_mro(C, result): + if C in result: + return + result.append(C) + for B in C.__bases__: + _classic_mro(B, result) + return result + + +# Backport _collections_abc._check_methods +def _check_methods(C, *methods): + try: + mro = C.__mro__ + except AttributeError: + mro = tuple(_classic_mro(C, [])) + + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + + +class AbstractContextManager(_abc_ABC): + """An abstract base class for context managers.""" + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + """Check whether subclass is considered a subclass of this ABC.""" + if cls is AbstractContextManager: + return _check_methods(C, "__enter__", "__exit__") + return NotImplemented + + +class ContextDecorator(object): + """A base class or mixin that enables context managers to work as decorators.""" + + def refresh_cm(self): + """Returns the context manager used to actually wrap the call to the + decorated function. + + The default implementation just returns *self*. + + Overriding this method allows otherwise one-shot context managers + like _GeneratorContextManager to support use as decorators via + implicit recreation. + + DEPRECATED: refresh_cm was never added to the standard library's + ContextDecorator API + """ + warnings.warn("refresh_cm was never added to the standard library", + DeprecationWarning) + return self._recreate_cm() + + def _recreate_cm(self): + """Return a recreated instance of self. + + Allows an otherwise one-shot context manager like + _GeneratorContextManager to support use as + a decorator via implicit recreation. + + This is a private interface just for _GeneratorContextManager. + See issue #11647 for details. + """ + return self + + def __call__(self, func): + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): + return func(*args, **kwds) + return inner + + +class _GeneratorContextManager(ContextDecorator): + """Helper for @contextmanager decorator.""" + + def __init__(self, func, args, kwds): + self.gen = func(*args, **kwds) + self.func, self.args, self.kwds = func, args, kwds + # Issue 19330: ensure context manager instances have good docstrings + doc = getattr(func, "__doc__", None) + if doc is None: + doc = type(self).__doc__ + self.__doc__ = doc + # Unfortunately, this still doesn't provide good help output when + # inspecting the created context manager instances, since pydoc + # currently bypasses the instance docstring and shows the docstring + # for the class instead. + # See http://bugs.python.org/issue19404 for more details. + + def _recreate_cm(self): + # _GCM instances are one-shot context managers, so the + # CM must be recreated each time a decorated function is + # called + return self.__class__(self.func, self.args, self.kwds) + + def __enter__(self): + try: + return next(self.gen) + except StopIteration: + raise RuntimeError("generator didn't yield") + + def __exit__(self, type, value, traceback): + if type is None: + try: + next(self.gen) + except StopIteration: + return + else: + raise RuntimeError("generator didn't stop") + else: + if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back + value = type() + try: + self.gen.throw(type, value, traceback) + raise RuntimeError("generator didn't stop after throw()") + except StopIteration as exc: + # Suppress StopIteration *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed. + return exc is not value + except RuntimeError as exc: + # Don't re-raise the passed in exception + if exc is value: + return False + # Likewise, avoid suppressing if a StopIteration exception + # was passed to throw() and later wrapped into a RuntimeError + # (see PEP 479). + if _HAVE_EXCEPTION_CHAINING and exc.__cause__ is value: + return False + raise + except: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. + # + if sys.exc_info()[1] is not value: + raise + + +def contextmanager(func): + """@contextmanager decorator. + + Typical usage: + + @contextmanager + def some_generator(<arguments>): + <setup> + try: + yield <value> + finally: + <cleanup> + + This makes this: + + with some_generator(<arguments>) as <variable>: + <body> + + equivalent to this: + + <setup> + try: + <variable> = <value> + <body> + finally: + <cleanup> + + """ + @wraps(func) + def helper(*args, **kwds): + return _GeneratorContextManager(func, args, kwds) + return helper + + +class closing(object): + """Context to automatically close something at the end of a block. + + Code like this: + + with closing(<module>.open(<arguments>)) as f: + <block> + + is equivalent to this: + + f = <module>.open(<arguments>) + try: + <block> + finally: + f.close() + + """ + def __init__(self, thing): + self.thing = thing + + def __enter__(self): + return self.thing + + def __exit__(self, *exc_info): + self.thing.close() + + +class _RedirectStream(object): + + _stream = None + + def __init__(self, new_target): + self._new_target = new_target + # We use a list of old targets to make this CM re-entrant + self._old_targets = [] + + def __enter__(self): + self._old_targets.append(getattr(sys, self._stream)) + setattr(sys, self._stream, self._new_target) + return self._new_target + + def __exit__(self, exctype, excinst, exctb): + setattr(sys, self._stream, self._old_targets.pop()) + + +class redirect_stdout(_RedirectStream): + """Context manager for temporarily redirecting stdout to another file. + + # How to send help() to stderr + with redirect_stdout(sys.stderr): + help(dir) + + # How to write help() to a file + with open('help.txt', 'w') as f: + with redirect_stdout(f): + help(pow) + """ + + _stream = "stdout" + + +class redirect_stderr(_RedirectStream): + """Context manager for temporarily redirecting stderr to another file.""" + + _stream = "stderr" + + +class suppress(object): + """Context manager to suppress specified exceptions + + After the exception is suppressed, execution proceeds with the next + statement following the with statement. + + with suppress(FileNotFoundError): + os.remove(somefile) + # Execution still resumes here if the file was already removed + """ + + def __init__(self, *exceptions): + self._exceptions = exceptions + + def __enter__(self): + pass + + def __exit__(self, exctype, excinst, exctb): + # Unlike isinstance and issubclass, CPython exception handling + # currently only looks at the concrete type hierarchy (ignoring + # the instance and subclass checking hooks). While Guido considers + # that a bug rather than a feature, it's a fairly hard one to fix + # due to various internal implementation details. suppress provides + # the simpler issubclass based semantics, rather than trying to + # exactly reproduce the limitations of the CPython interpreter. + # + # See http://bugs.python.org/issue12029 for more details + return exctype is not None and issubclass(exctype, self._exceptions) + + +# Context manipulation is Python 3 only +_HAVE_EXCEPTION_CHAINING = sys.version_info[0] >= 3 +if _HAVE_EXCEPTION_CHAINING: + def _make_context_fixer(frame_exc): + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is None or exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc + return _fix_exception_context + + def _reraise_with_existing_context(exc_details): + try: + # bare "raise exc_details[1]" replaces our carefully + # set-up context + fixed_ctx = exc_details[1].__context__ + raise exc_details[1] + except BaseException: + exc_details[1].__context__ = fixed_ctx + raise +else: + # No exception context in Python 2 + def _make_context_fixer(frame_exc): + return lambda new_exc, old_exc: None + + # Use 3 argument raise in Python 2, + # but use exec to avoid SyntaxError in Python 3 + def _reraise_with_existing_context(exc_details): + exc_type, exc_value, exc_tb = exc_details + exec("raise exc_type, exc_value, exc_tb") + +# Handle old-style classes if they exist +try: + from types import InstanceType +except ImportError: + # Python 3 doesn't have old-style classes + _get_type = type +else: + # Need to handle old-style context managers on Python 2 + def _get_type(obj): + obj_type = type(obj) + if obj_type is InstanceType: + return obj.__class__ # Old-style class + return obj_type # New-style class + + +# Inspired by discussions on http://bugs.python.org/issue13585 +class ExitStack(object): + """Context manager for dynamic management of a stack of exit callbacks + + For example: + + with ExitStack() as stack: + files = [stack.enter_context(open(fname)) for fname in filenames] + # All opened files will automatically be closed at the end of + # the with statement, even if attempts to open files later + # in the list raise an exception + + """ + def __init__(self): + self._exit_callbacks = deque() + + def pop_all(self): + """Preserve the context stack by transferring it to a new instance""" + new_stack = type(self)() + new_stack._exit_callbacks = self._exit_callbacks + self._exit_callbacks = deque() + return new_stack + + def _push_cm_exit(self, cm, cm_exit): + """Helper to correctly register callbacks to __exit__ methods""" + def _exit_wrapper(*exc_details): + return cm_exit(cm, *exc_details) + _exit_wrapper.__self__ = cm + self.push(_exit_wrapper) + + def push(self, exit): + """Registers a callback with the standard __exit__ method signature + + Can suppress exceptions the same way __exit__ methods can. + + Also accepts any object with an __exit__ method (registering a call + to the method instead of the object itself) + """ + # We use an unbound method rather than a bound method to follow + # the standard lookup behaviour for special methods + _cb_type = _get_type(exit) + try: + exit_method = _cb_type.__exit__ + except AttributeError: + # Not a context manager, so assume its a callable + self._exit_callbacks.append(exit) + else: + self._push_cm_exit(exit, exit_method) + return exit # Allow use as a decorator + + def callback(self, callback, *args, **kwds): + """Registers an arbitrary callback and arguments. + + Cannot suppress exceptions. + """ + def _exit_wrapper(exc_type, exc, tb): + callback(*args, **kwds) + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection + _exit_wrapper.__wrapped__ = callback + self.push(_exit_wrapper) + return callback # Allow use as a decorator + + def enter_context(self, cm): + """Enters the supplied context manager + + If successful, also pushes its __exit__ method as a callback and + returns the result of the __enter__ method. + """ + # We look up the special methods on the type to match the with statement + _cm_type = _get_type(cm) + _exit = _cm_type.__exit__ + result = _cm_type.__enter__(cm) + self._push_cm_exit(cm, _exit) + return result + + def close(self): + """Immediately unwind the context stack""" + self.__exit__(None, None, None) + + def __enter__(self): + return self + + def __exit__(self, *exc_details): + received_exc = exc_details[0] is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exc_info()[1] + _fix_exception_context = _make_context_fixer(frame_exc) + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + cb = self._exit_callbacks.pop() + try: + if cb(*exc_details): + suppressed_exc = True + pending_raise = False + exc_details = (None, None, None) + except: + new_exc_details = sys.exc_info() + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc_details[1], exc_details[1]) + pending_raise = True + exc_details = new_exc_details + if pending_raise: + _reraise_with_existing_context(exc_details) + return received_exc and suppressed_exc + + +# Preserve backwards compatibility +class ContextStack(ExitStack): + """Backwards compatibility alias for ExitStack""" + + def __init__(self): + warnings.warn("ContextStack has been renamed to ExitStack", + DeprecationWarning) + super(ContextStack, self).__init__() + + def register_exit(self, callback): + return self.push(callback) + + def register(self, callback, *args, **kwds): + return self.callback(callback, *args, **kwds) + + def preserve(self): + return self.pop_all() + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/pipenv/patched/notpip/_vendor/distlib/__init__.py b/pipenv/patched/notpip/_vendor/distlib/__init__.py index d4aab453..a2d70d47 100644 --- a/pipenv/patched/notpip/_vendor/distlib/__init__.py +++ b/pipenv/patched/notpip/_vendor/distlib/__init__.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2017 Vinay Sajip. +# Copyright (C) 2012-2019 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # import logging -__version__ = '0.2.7' +__version__ = '0.2.9.post0' class DistlibException(Exception): pass diff --git a/pipenv/patched/notpip/_vendor/distlib/database.py b/pipenv/patched/notpip/_vendor/distlib/database.py index a19905e2..b13cdac9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/database.py +++ b/pipenv/patched/notpip/_vendor/distlib/database.py @@ -20,7 +20,8 @@ import zipimport from . import DistlibException, resources from .compat import StringIO from .version import get_scheme, UnsupportedVersionError -from .metadata import Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME +from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME) from .util import (parse_requirement, cached_property, parse_name_and_version, read_exports, write_exports, CSVReader, CSVWriter) @@ -132,7 +133,9 @@ class DistributionPath(object): if not r or r.path in seen: continue if self._include_dist and entry.endswith(DISTINFO_EXT): - possible_filenames = [METADATA_FILENAME, WHEEL_METADATA_FILENAME] + possible_filenames = [METADATA_FILENAME, + WHEEL_METADATA_FILENAME, + LEGACY_METADATA_FILENAME] for metadata_filename in possible_filenames: metadata_path = posixpath.join(entry, metadata_filename) pydist = finder.find(metadata_path) diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py index cb05b184..a7ed9469 100644 --- a/pipenv/patched/notpip/_vendor/distlib/locators.py +++ b/pipenv/patched/notpip/_vendor/distlib/locators.py @@ -255,7 +255,9 @@ class Locator(object): if path.endswith('.whl'): try: wheel = Wheel(path) - if is_compatible(wheel, self.wheel_tags): + if not is_compatible(wheel, self.wheel_tags): + logger.debug('Wheel not compatible: %s', path) + else: if project_name is None: include = True else: @@ -613,6 +615,7 @@ class SimpleScrapingLocator(Locator): # as it is for coordinating our internal threads - the ones created # in _prepare_threads. self._gplock = threading.RLock() + self.platform_check = False # See issue #112 def _prepare_threads(self): """ @@ -658,8 +661,8 @@ class SimpleScrapingLocator(Locator): del self.result return result - platform_dependent = re.compile(r'\b(linux-(i\d86|x86_64|arm\w+)|' - r'win(32|-amd64)|macosx-?\d+)\b', re.I) + platform_dependent = re.compile(r'\b(linux_(i\d86|x86_64|arm\w+)|' + r'win(32|_amd64)|macosx_?\d+)\b', re.I) def _is_platform_dependent(self, url): """ @@ -677,7 +680,7 @@ class SimpleScrapingLocator(Locator): Note that the return value isn't actually used other than as a boolean value. """ - if self._is_platform_dependent(url): + if self.platform_check and self._is_platform_dependent(url): info = None else: info = self.convert_url_to_download_info(url, self.project_name) diff --git a/pipenv/patched/notpip/_vendor/distlib/metadata.py b/pipenv/patched/notpip/_vendor/distlib/metadata.py index 6d6470ff..2d61378e 100644 --- a/pipenv/patched/notpip/_vendor/distlib/metadata.py +++ b/pipenv/patched/notpip/_vendor/distlib/metadata.py @@ -91,7 +91,11 @@ _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') -_566_FIELDS = _426_FIELDS + ('Description-Content-Type',) +# See issue #106: Sometimes 'Requires' and 'Provides' occur wrongly in +# the metadata. Include them in the tuple literal below to allow them +# (for now). +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', + 'Requires', 'Provides') _566_MARKERS = ('Description-Content-Type',) @@ -377,8 +381,8 @@ class LegacyMetadata(object): value = msg[field] if value is not None and value != 'UNKNOWN': self.set(field, value) - logger.debug('Attempting to set metadata for %s', self) - self.set_metadata_version() + # logger.debug('Attempting to set metadata for %s', self) + # self.set_metadata_version() def write(self, filepath, skip_unknown=False): """Write the metadata fields to filepath.""" @@ -648,6 +652,7 @@ class LegacyMetadata(object): METADATA_FILENAME = 'pydist.json' WHEEL_METADATA_FILENAME = 'metadata.json' +LEGACY_METADATA_FILENAME = 'METADATA' class Metadata(object): diff --git a/pipenv/patched/notpip/_vendor/distlib/scripts.py b/pipenv/patched/notpip/_vendor/distlib/scripts.py index 0b7c3d0b..5965e241 100644 --- a/pipenv/patched/notpip/_vendor/distlib/scripts.py +++ b/pipenv/patched/notpip/_vendor/distlib/scripts.py @@ -39,27 +39,12 @@ _DEFAULT_MANIFEST = ''' # check if Python is called on the first line with this expression FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$') SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*- +import re +import sys +from %(module)s import %(import_name)s if __name__ == '__main__': - import sys, re - - def _resolve(module, func): - __import__(module) - mod = sys.modules[module] - parts = func.split('.') - result = getattr(mod, parts.pop(0)) - for p in parts: - result = getattr(result, p) - return result - - try: - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - - func = _resolve('%(module)s', '%(func)s') - rc = func() # None interpreted as 0 - except Exception as e: # only supporting Python >= 2.6 - sys.stderr.write('%%s\n' %% e) - rc = 1 - sys.exit(rc) + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(%(func)s()) ''' @@ -225,6 +210,7 @@ class ScriptMaker(object): def _get_script_text(self, entry): return self.script_template % dict(module=entry.prefix, + import_name=entry.suffix.split('.')[0], func=entry.suffix) manifest = _DEFAULT_MANIFEST @@ -236,8 +222,10 @@ class ScriptMaker(object): def _write_script(self, names, shebang, script_bytes, filenames, ext): use_launcher = self.add_launchers and self._is_nt linesep = os.linesep.encode('utf-8') + if not shebang.endswith(linesep): + shebang += linesep if not use_launcher: - script_bytes = shebang + linesep + script_bytes + script_bytes = shebang + script_bytes else: # pragma: no cover if ext == 'py': launcher = self._get_launcher('t') @@ -247,7 +235,7 @@ class ScriptMaker(object): with ZipFile(stream, 'w') as zf: zf.writestr('__main__.py', script_bytes) zip_data = stream.getvalue() - script_bytes = launcher + shebang + linesep + zip_data + script_bytes = launcher + shebang + zip_data for name in names: outname = os.path.join(self.target_dir, name) if use_launcher: # pragma: no cover diff --git a/pipenv/patched/notpip/_vendor/distlib/t32.exe b/pipenv/patched/notpip/_vendor/distlib/t32.exe index a09d9268..5d5bce1f 100644 Binary files a/pipenv/patched/notpip/_vendor/distlib/t32.exe and b/pipenv/patched/notpip/_vendor/distlib/t32.exe differ diff --git a/pipenv/patched/notpip/_vendor/distlib/t64.exe b/pipenv/patched/notpip/_vendor/distlib/t64.exe index 9da9b40d..039ce441 100644 Binary files a/pipenv/patched/notpip/_vendor/distlib/t64.exe and b/pipenv/patched/notpip/_vendor/distlib/t64.exe differ diff --git a/pipenv/patched/notpip/_vendor/distlib/util.py b/pipenv/patched/notpip/_vendor/distlib/util.py index 0b14a93b..e851146c 100644 --- a/pipenv/patched/notpip/_vendor/distlib/util.py +++ b/pipenv/patched/notpip/_vendor/distlib/util.py @@ -545,16 +545,14 @@ class FileOperator(object): def write_binary_file(self, path, data): self.ensure_dir(os.path.dirname(path)) if not self.dry_run: + if os.path.exists(path): + os.remove(path) with open(path, 'wb') as f: f.write(data) self.record_as_written(path) def write_text_file(self, path, data, encoding): - self.ensure_dir(os.path.dirname(path)) - if not self.dry_run: - with open(path, 'wb') as f: - f.write(data.encode(encoding)) - self.record_as_written(path) + self.write_binary_file(path, data.encode(encoding)) def set_mode(self, bits, mask, files): if os.name == 'posix' or (os.name == 'java' and os._name == 'posix'): @@ -582,7 +580,7 @@ class FileOperator(object): if self.record: self.dirs_created.add(path) - def byte_compile(self, path, optimize=False, force=False, prefix=None): + def byte_compile(self, path, optimize=False, force=False, prefix=None, hashed_invalidation=False): dpath = cache_from_source(path, not optimize) logger.info('Byte-compiling %s to %s', path, dpath) if not self.dry_run: @@ -592,7 +590,10 @@ class FileOperator(object): else: assert path.startswith(prefix) diagpath = path[len(prefix):] - py_compile.compile(path, dpath, diagpath, True) # raise error + compile_kwargs = {} + if hashed_invalidation and hasattr(py_compile, 'PycInvalidationMode'): + compile_kwargs['invalidation_mode'] = py_compile.PycInvalidationMode.CHECKED_HASH + py_compile.compile(path, dpath, diagpath, True, **compile_kwargs) # raise error self.record_as_written(dpath) return dpath @@ -803,11 +804,15 @@ def ensure_slash(s): def parse_credentials(netloc): username = password = None if '@' in netloc: - prefix, netloc = netloc.split('@', 1) + prefix, netloc = netloc.rsplit('@', 1) if ':' not in prefix: username = prefix else: username, password = prefix.split(':', 1) + if username: + username = unquote(username) + if password: + password = unquote(password) return username, password, netloc diff --git a/pipenv/patched/notpip/_vendor/distlib/w32.exe b/pipenv/patched/notpip/_vendor/distlib/w32.exe index 732215a9..4df77001 100644 Binary files a/pipenv/patched/notpip/_vendor/distlib/w32.exe and b/pipenv/patched/notpip/_vendor/distlib/w32.exe differ diff --git a/pipenv/patched/notpip/_vendor/distlib/w64.exe b/pipenv/patched/notpip/_vendor/distlib/w64.exe index c41bd0a0..63ce483d 100644 Binary files a/pipenv/patched/notpip/_vendor/distlib/w64.exe and b/pipenv/patched/notpip/_vendor/distlib/w64.exe differ diff --git a/pipenv/patched/notpip/_vendor/distlib/wheel.py b/pipenv/patched/notpip/_vendor/distlib/wheel.py index 77372235..0c8efad9 100644 --- a/pipenv/patched/notpip/_vendor/distlib/wheel.py +++ b/pipenv/patched/notpip/_vendor/distlib/wheel.py @@ -433,6 +433,22 @@ class Wheel(object): self.build_zip(pathname, archive_paths) return pathname + def skip_entry(self, arcname): + """ + Determine whether an archive entry should be skipped when verifying + or installing. + """ + # The signature file won't be in RECORD, + # and we don't currently don't do anything with it + # We also skip directories, as they won't be in RECORD + # either. See: + # + # https://github.com/pypa/wheel/issues/294 + # https://github.com/pypa/wheel/issues/287 + # https://github.com/pypa/wheel/pull/289 + # + return arcname.endswith(('/', '/RECORD.jws')) + def install(self, paths, maker, **kwargs): """ Install a wheel to the specified paths. If kwarg ``warner`` is @@ -442,7 +458,9 @@ class Wheel(object): This can be used to issue any warnings to raise any exceptions. If kwarg ``lib_only`` is True, only the purelib/platlib files are installed, and the headers, scripts, data and dist-info metadata are - not written. + not written. If kwarg ``bytecode_hashed_invalidation`` is True, written + bytecode will try to use file-hash based invalidation (PEP-552) on + supported interpreter versions (CPython 2.7+). The return value is a :class:`InstalledDistribution` instance unless ``options.lib_only`` is True, in which case the return value is ``None``. @@ -451,6 +469,7 @@ class Wheel(object): dry_run = maker.dry_run warner = kwargs.get('warner') lib_only = kwargs.get('lib_only', False) + bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False) pathname = os.path.join(self.dirname, self.filename) name_ver = '%s-%s' % (self.name, self.version) @@ -511,9 +530,7 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: @@ -557,7 +574,8 @@ class Wheel(object): '%s' % outfile) if bc and outfile.endswith('.py'): try: - pyc = fileop.byte_compile(outfile) + pyc = fileop.byte_compile(outfile, + hashed_invalidation=bc_hashed_invalidation) outfiles.append(pyc) except Exception: # Don't give up if byte-compilation fails, @@ -782,13 +800,15 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - if '..' in u_arcname: + # See issue #115: some wheels have .. in their entries, but + # in the filename ... e.g. __main__..py ! So the check is + # updated to look for .. in the directory portions + p = u_arcname.split('/') + if '..' in p: raise DistlibException('invalid entry in ' 'wheel: %r' % u_arcname) - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: diff --git a/pipenv/patched/notpip/_vendor/distro.py b/pipenv/patched/notpip/_vendor/distro.py index aa4defc3..33061633 100644 --- a/pipenv/patched/notpip/_vendor/distro.py +++ b/pipenv/patched/notpip/_vendor/distro.py @@ -17,12 +17,12 @@ The ``distro`` package (``distro`` stands for Linux Distribution) provides information about the Linux distribution it runs on, such as a reliable machine-readable distro ID, or version information. -It is a renewed alternative implementation for Python's original +It is the recommended replacement for Python's original :py:func:`platform.linux_distribution` function, but it provides much more functionality. An alternative implementation became necessary because Python -3.5 deprecated this function, and Python 3.7 is expected to remove it -altogether. Its predecessor function :py:func:`platform.dist` was already -deprecated since Python 2.6 and is also expected to be removed in Python 3.7. +3.5 deprecated this function, and Python 3.8 will remove it altogether. +Its predecessor function :py:func:`platform.dist` was already +deprecated since Python 2.6 and will also be removed in Python 3.8. Still, there are many cases in which access to OS distribution information is needed. See `Python issue 1322 <https://bugs.python.org/issue1322>`_ for more information. @@ -48,7 +48,9 @@ _OS_RELEASE_BASENAME = 'os-release' #: with blanks translated to underscores. #: #: * Value: Normalized value. -NORMALIZED_OS_ID = {} +NORMALIZED_OS_ID = { + 'ol': 'oracle', # Oracle Enterprise Linux +} #: Translation table for normalizing the "Distributor ID" attribute returned by #: the lsb_release command, for use by the :func:`distro.id` method. @@ -812,10 +814,14 @@ class LinuxDistribution(object): For details, see :func:`distro.codename`. """ - return self.os_release_attr('codename') \ - or self.lsb_release_attr('codename') \ - or self.distro_release_attr('codename') \ - or '' + try: + # Handle os_release specially since distros might purposefully set + # this to empty string to have no codename + return self._os_release_info['codename'] + except KeyError: + return self.lsb_release_attr('codename') \ + or self.distro_release_attr('codename') \ + or '' def info(self, pretty=False, best=False): """ @@ -872,6 +878,7 @@ class LinuxDistribution(object): For details, see :func:`distro.uname_info`. """ + return self._uname_info def os_release_attr(self, attribute): """ @@ -963,23 +970,30 @@ class LinuxDistribution(object): if isinstance(v, bytes): v = v.decode('utf-8') props[k.lower()] = v - if k == 'VERSION': - # this handles cases in which the codename is in - # the `(CODENAME)` (rhel, centos, fedora) format - # or in the `, CODENAME` format (Ubuntu). - codename = re.search(r'(\(\D+\))|,(\s+)?\D+', v) - if codename: - codename = codename.group() - codename = codename.strip('()') - codename = codename.strip(',') - codename = codename.strip() - # codename appears within paranthese. - props['codename'] = codename - else: - props['codename'] = '' else: # Ignore any tokens that are not variable assignments pass + + if 'version_codename' in props: + # os-release added a version_codename field. Use that in + # preference to anything else Note that some distros purposefully + # do not have code names. They should be setting + # version_codename="" + props['codename'] = props['version_codename'] + elif 'ubuntu_codename' in props: + # Same as above but a non-standard field name used on older Ubuntus + props['codename'] = props['ubuntu_codename'] + elif 'version' in props: + # If there is no version_codename, parse it from the version + codename = re.search(r'(\(\D+\))|,(\s+)?\D+', props['version']) + if codename: + codename = codename.group() + codename = codename.strip('()') + codename = codename.strip(',') + codename = codename.strip() + # codename appears within paranthese. + props['codename'] = codename + return props @cached_property @@ -1072,7 +1086,10 @@ class LinuxDistribution(object): # file), because we want to use what was specified as best as # possible. match = _DISTRO_RELEASE_BASENAME_PATTERN.match(basename) - if match: + if 'name' in distro_info \ + and 'cloudlinux' in distro_info['name'].lower(): + distro_info['id'] = 'cloudlinux' + elif match: distro_info['id'] = match.group(1) return distro_info else: @@ -1113,6 +1130,8 @@ class LinuxDistribution(object): # The name is always present if the pattern matches self.distro_release_file = filepath distro_info['id'] = match.group(1) + if 'cloudlinux' in distro_info['name'].lower(): + distro_info['id'] = 'cloudlinux' return distro_info return {} diff --git a/pipenv/patched/notpip/_vendor/html5lib/_trie/_base.py b/pipenv/patched/notpip/_vendor/html5lib/_trie/_base.py index a1158bbb..6b71975f 100644 --- a/pipenv/patched/notpip/_vendor/html5lib/_trie/_base.py +++ b/pipenv/patched/notpip/_vendor/html5lib/_trie/_base.py @@ -1,6 +1,9 @@ from __future__ import absolute_import, division, unicode_literals -from collections import Mapping +try: + from collections.abc import Mapping +except ImportError: # Python 2.7 + from collections import Mapping class Trie(Mapping): diff --git a/pipenv/patched/notpip/_vendor/html5lib/treebuilders/dom.py b/pipenv/patched/notpip/_vendor/html5lib/treebuilders/dom.py index dcfac220..d8b53004 100644 --- a/pipenv/patched/notpip/_vendor/html5lib/treebuilders/dom.py +++ b/pipenv/patched/notpip/_vendor/html5lib/treebuilders/dom.py @@ -1,7 +1,10 @@ from __future__ import absolute_import, division, unicode_literals -from collections import MutableMapping +try: + from collections.abc import MutableMapping +except ImportError: # Python 2.7 + from collections import MutableMapping from xml.dom import minidom, Node import weakref diff --git a/pipenv/patched/notpip/_vendor/idna/core.py b/pipenv/patched/notpip/_vendor/idna/core.py index 090c2c18..104624ad 100644 --- a/pipenv/patched/notpip/_vendor/idna/core.py +++ b/pipenv/patched/notpip/_vendor/idna/core.py @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label diff --git a/pipenv/patched/notpip/_vendor/idna/idnadata.py b/pipenv/patched/notpip/_vendor/idna/idnadata.py index 17974e23..a80c959d 100644 --- a/pipenv/patched/notpip/_vendor/idna/idnadata.py +++ b/pipenv/patched/notpip/_vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "11.0.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -248,6 +248,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +523,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +692,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -1034,14 +1100,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1121,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1145,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1205,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1276,7 +1344,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1544,11 +1612,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1723,10 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,11 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1837,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1861,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1871,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e0001145f, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1807,15 +1883,17 @@ codepoint_classes = { 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, 0x11a0000011a3f, 0x11a4700011a48, 0x11a5000011a84, 0x11a8600011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1909,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,11 +1930,12 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, + 0x16e6000016e80, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x17000000187f2, 0x1880000018af3, 0x1b0000001b11f, 0x1b1700001b2fc, diff --git a/pipenv/patched/notpip/_vendor/idna/package_data.py b/pipenv/patched/notpip/_vendor/idna/package_data.py index 39c192ba..257e8989 100644 --- a/pipenv/patched/notpip/_vendor/idna/package_data.py +++ b/pipenv/patched/notpip/_vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.8' diff --git a/pipenv/patched/notpip/_vendor/idna/uts46data.py b/pipenv/patched/notpip/_vendor/idna/uts46data.py index 79731cb9..a68ed4c0 100644 --- a/pipenv/patched/notpip/_vendor/idna/uts46data.py +++ b/pipenv/patched/notpip/_vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "11.0.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1278,8 +1273,6 @@ def _seg_12(): (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,10 +1348,6 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), @@ -1366,6 +1355,10 @@ def _seg_13(): (0xE8D, 'V'), (0xE8E, 'X'), (0xE94, 'V'), + ] + +def _seg_13(): + return [ (0xE98, 'X'), (0xE99, 'V'), (0xEA0, 'X'), @@ -1459,10 +1452,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1470,6 +1459,10 @@ def _seg_14(): (0x1260, 'V'), (0x1289, 'X'), (0x128A, 'V'), + ] + +def _seg_14(): + return [ (0x128E, 'X'), (0x1290, 'V'), (0x12B1, 'X'), @@ -1538,7 +1531,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1556,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1574,6 +1563,10 @@ def _seg_15(): (0x1A7F, 'V'), (0x1A8A, 'X'), (0x1A90, 'V'), + ] + +def _seg_15(): + return [ (0x1A9A, 'X'), (0x1AA0, 'V'), (0x1AAE, 'X'), @@ -1667,10 +1660,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1678,6 +1667,10 @@ def _seg_16(): (0x1D9C, 'M', u'c'), (0x1D9D, 'M', u'ɕ'), (0x1D9E, 'M', u'ð'), + ] + +def _seg_16(): + return [ (0x1D9F, 'M', u'ɜ'), (0x1DA0, 'M', u'f'), (0x1DA1, 'M', u'ɟ'), @@ -1771,10 +1764,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1782,6 +1771,10 @@ def _seg_17(): (0x1E3D, 'V'), (0x1E3E, 'M', u'ḿ'), (0x1E3F, 'V'), + ] + +def _seg_17(): + return [ (0x1E40, 'M', u'ṁ'), (0x1E41, 'V'), (0x1E42, 'M', u'ṃ'), @@ -1875,10 +1868,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1886,6 +1875,10 @@ def _seg_18(): (0x1EA6, 'M', u'ầ'), (0x1EA7, 'V'), (0x1EA8, 'M', u'ẩ'), + ] + +def _seg_18(): + return [ (0x1EA9, 'V'), (0x1EAA, 'M', u'ẫ'), (0x1EAB, 'V'), @@ -1979,10 +1972,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1990,6 +1979,10 @@ def _seg_19(): (0x1F18, 'M', u'ἐ'), (0x1F19, 'M', u'ἑ'), (0x1F1A, 'M', u'ἒ'), + ] + +def _seg_19(): + return [ (0x1F1B, 'M', u'ἓ'), (0x1F1C, 'M', u'ἔ'), (0x1F1D, 'M', u'ἕ'), @@ -2083,10 +2076,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2094,6 +2083,10 @@ def _seg_20(): (0x1FA1, 'M', u'ὡι'), (0x1FA2, 'M', u'ὢι'), (0x1FA3, 'M', u'ὣι'), + ] + +def _seg_20(): + return [ (0x1FA4, 'M', u'ὤι'), (0x1FA5, 'M', u'ὥι'), (0x1FA6, 'M', u'ὦι'), @@ -2187,10 +2180,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2198,6 +2187,10 @@ def _seg_21(): (0x2035, 'V'), (0x2036, 'M', u'‵‵'), (0x2037, 'M', u'‵‵‵'), + ] + +def _seg_21(): + return [ (0x2038, 'V'), (0x203C, '3', u'!!'), (0x203D, 'V'), @@ -2291,10 +2284,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2302,6 +2291,10 @@ def _seg_22(): (0x2127, 'V'), (0x2128, 'M', u'z'), (0x2129, 'V'), + ] + +def _seg_22(): + return [ (0x212A, 'M', u'k'), (0x212B, 'M', u'å'), (0x212C, 'M', u'b'), @@ -2395,10 +2388,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2406,6 +2395,10 @@ def _seg_23(): (0x244B, 'X'), (0x2460, 'M', u'1'), (0x2461, 'M', u'2'), + ] + +def _seg_23(): + return [ (0x2462, 'M', u'3'), (0x2463, 'M', u'4'), (0x2464, 'M', u'5'), @@ -2499,10 +2492,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2510,6 +2499,10 @@ def _seg_24(): (0x24D6, 'M', u'g'), (0x24D7, 'M', u'h'), (0x24D8, 'M', u'i'), + ] + +def _seg_24(): + return [ (0x24D9, 'M', u'j'), (0x24DA, 'M', u'k'), (0x24DB, 'M', u'l'), @@ -2541,13 +2534,9 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), (0x2BC9, 'X'), (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), + (0x2BFF, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2603,10 +2592,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2618,6 +2603,10 @@ def _seg_25(): (0x2C6E, 'M', u'ɱ'), (0x2C6F, 'M', u'ɐ'), (0x2C70, 'M', u'ɒ'), + ] + +def _seg_25(): + return [ (0x2C71, 'V'), (0x2C72, 'M', u'ⱳ'), (0x2C73, 'V'), @@ -2707,10 +2696,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2722,6 +2707,10 @@ def _seg_26(): (0x2CD8, 'M', u'ⳙ'), (0x2CD9, 'V'), (0x2CDA, 'M', u'ⳛ'), + ] + +def _seg_26(): + return [ (0x2CDB, 'V'), (0x2CDC, 'M', u'ⳝ'), (0x2CDD, 'V'), @@ -2768,7 +2757,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E4F, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2811,10 +2800,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2826,6 +2811,10 @@ def _seg_27(): (0x2F2B, 'M', u'尸'), (0x2F2C, 'M', u'屮'), (0x2F2D, 'M', u'山'), + ] + +def _seg_27(): + return [ (0x2F2E, 'M', u'巛'), (0x2F2F, 'M', u'工'), (0x2F30, 'M', u'己'), @@ -2915,10 +2904,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2930,6 +2915,10 @@ def _seg_28(): (0x2F8F, 'M', u'行'), (0x2F90, 'M', u'衣'), (0x2F91, 'M', u'襾'), + ] + +def _seg_28(): + return [ (0x2F92, 'M', u'見'), (0x2F93, 'M', u'角'), (0x2F94, 'M', u'言'), @@ -3019,13 +3008,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3034,6 +3019,10 @@ def _seg_29(): (0x3136, 'M', u'ᆭ'), (0x3137, 'M', u'ᄃ'), (0x3138, 'M', u'ᄄ'), + ] + +def _seg_29(): + return [ (0x3139, 'M', u'ᄅ'), (0x313A, 'M', u'ᆰ'), (0x313B, 'M', u'ᆱ'), @@ -3123,10 +3112,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3138,6 +3123,10 @@ def _seg_30(): (0x319B, 'M', u'丙'), (0x319C, 'M', u'丁'), (0x319D, 'M', u'天'), + ] + +def _seg_30(): + return [ (0x319E, 'M', u'地'), (0x319F, 'M', u'人'), (0x31A0, 'V'), @@ -3227,10 +3216,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3242,6 +3227,10 @@ def _seg_31(): (0x3261, 'M', u'ᄂ'), (0x3262, 'M', u'ᄃ'), (0x3263, 'M', u'ᄅ'), + ] + +def _seg_31(): + return [ (0x3264, 'M', u'ᄆ'), (0x3265, 'M', u'ᄇ'), (0x3266, 'M', u'ᄉ'), @@ -3331,10 +3320,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3346,6 +3331,10 @@ def _seg_32(): (0x32C5, 'M', u'6月'), (0x32C6, 'M', u'7月'), (0x32C7, 'M', u'8月'), + ] + +def _seg_32(): + return [ (0x32C8, 'M', u'9月'), (0x32C9, 'M', u'10月'), (0x32CA, 'M', u'11月'), @@ -3435,10 +3424,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3450,6 +3435,10 @@ def _seg_33(): (0x3329, 'M', u'ノット'), (0x332A, 'M', u'ハイツ'), (0x332B, 'M', u'パーセント'), + ] + +def _seg_33(): + return [ (0x332C, 'M', u'パーツ'), (0x332D, 'M', u'バーレル'), (0x332E, 'M', u'ピアストル'), @@ -3539,10 +3528,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3554,6 +3539,10 @@ def _seg_34(): (0x338D, 'M', u'μg'), (0x338E, 'M', u'mg'), (0x338F, 'M', u'kg'), + ] + +def _seg_34(): + return [ (0x3390, 'M', u'hz'), (0x3391, 'M', u'khz'), (0x3392, 'M', u'mhz'), @@ -3643,10 +3632,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3658,6 +3643,10 @@ def _seg_35(): (0x33F1, 'M', u'18日'), (0x33F2, 'M', u'19日'), (0x33F3, 'M', u'20日'), + ] + +def _seg_35(): + return [ (0x33F4, 'M', u'21日'), (0x33F5, 'M', u'22日'), (0x33F6, 'M', u'23日'), @@ -3673,7 +3662,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3747,10 +3736,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3762,6 +3747,10 @@ def _seg_36(): (0xA69D, 'M', u'ь'), (0xA69E, 'V'), (0xA6F8, 'X'), + ] + +def _seg_36(): + return [ (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), (0xA723, 'V'), @@ -3851,10 +3840,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3866,6 +3851,10 @@ def _seg_37(): (0xA78E, 'V'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), + ] + +def _seg_37(): + return [ (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), (0xA796, 'M', u'ꞗ'), @@ -3893,7 +3882,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3903,6 +3892,8 @@ def _seg_37(): (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), (0xA7B8, 'X'), + (0xA7B9, 'V'), + (0xA7BA, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,8 +3908,6 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), @@ -3955,10 +3944,6 @@ def _seg_37(): (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), (0xAB66, 'X'), - ] - -def _seg_38(): - return [ (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -3970,6 +3955,10 @@ def _seg_38(): (0xAB78, 'M', u'Ꭸ'), (0xAB79, 'M', u'Ꭹ'), (0xAB7A, 'M', u'Ꭺ'), + ] + +def _seg_38(): + return [ (0xAB7B, 'M', u'Ꭻ'), (0xAB7C, 'M', u'Ꭼ'), (0xAB7D, 'M', u'Ꭽ'), @@ -4059,10 +4048,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4074,6 +4059,10 @@ def _seg_39(): (0xF913, 'M', u'邏'), (0xF914, 'M', u'樂'), (0xF915, 'M', u'洛'), + ] + +def _seg_39(): + return [ (0xF916, 'M', u'烙'), (0xF917, 'M', u'珞'), (0xF918, 'M', u'落'), @@ -4163,10 +4152,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4178,6 +4163,10 @@ def _seg_40(): (0xF977, 'M', u'亮'), (0xF978, 'M', u'兩'), (0xF979, 'M', u'凉'), + ] + +def _seg_40(): + return [ (0xF97A, 'M', u'梁'), (0xF97B, 'M', u'糧'), (0xF97C, 'M', u'良'), @@ -4267,10 +4256,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4282,6 +4267,10 @@ def _seg_41(): (0xF9DB, 'M', u'率'), (0xF9DC, 'M', u'隆'), (0xF9DD, 'M', u'利'), + ] + +def _seg_41(): + return [ (0xF9DE, 'M', u'吏'), (0xF9DF, 'M', u'履'), (0xF9E0, 'M', u'易'), @@ -4371,10 +4360,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4386,6 +4371,10 @@ def _seg_42(): (0xFA44, 'M', u'梅'), (0xFA45, 'M', u'海'), (0xFA46, 'M', u'渚'), + ] + +def _seg_42(): + return [ (0xFA47, 'M', u'漢'), (0xFA48, 'M', u'煮'), (0xFA49, 'M', u'爫'), @@ -4475,10 +4464,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4490,6 +4475,10 @@ def _seg_43(): (0xFAAA, 'M', u'着'), (0xFAAB, 'M', u'磌'), (0xFAAC, 'M', u'窱'), + ] + +def _seg_43(): + return [ (0xFAAD, 'M', u'節'), (0xFAAE, 'M', u'类'), (0xFAAF, 'M', u'絛'), @@ -4579,10 +4568,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4594,6 +4579,10 @@ def _seg_44(): (0xFB43, 'M', u'ףּ'), (0xFB44, 'M', u'פּ'), (0xFB45, 'X'), + ] + +def _seg_44(): + return [ (0xFB46, 'M', u'צּ'), (0xFB47, 'M', u'קּ'), (0xFB48, 'M', u'רּ'), @@ -4683,10 +4672,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4698,6 +4683,10 @@ def _seg_45(): (0xFC24, 'M', u'ضخ'), (0xFC25, 'M', u'ضم'), (0xFC26, 'M', u'طح'), + ] + +def _seg_45(): + return [ (0xFC27, 'M', u'طم'), (0xFC28, 'M', u'ظم'), (0xFC29, 'M', u'عج'), @@ -4787,10 +4776,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4802,6 +4787,10 @@ def _seg_46(): (0xFC88, 'M', u'ما'), (0xFC89, 'M', u'مم'), (0xFC8A, 'M', u'نر'), + ] + +def _seg_46(): + return [ (0xFC8B, 'M', u'نز'), (0xFC8C, 'M', u'نم'), (0xFC8D, 'M', u'نن'), @@ -4891,10 +4880,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4906,6 +4891,10 @@ def _seg_47(): (0xFCEC, 'M', u'كم'), (0xFCED, 'M', u'لم'), (0xFCEE, 'M', u'نم'), + ] + +def _seg_47(): + return [ (0xFCEF, 'M', u'نه'), (0xFCF0, 'M', u'يم'), (0xFCF1, 'M', u'يه'), @@ -4995,10 +4984,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5010,6 +4995,10 @@ def _seg_48(): (0xFD66, 'M', u'صمم'), (0xFD67, 'M', u'شحم'), (0xFD69, 'M', u'شجي'), + ] + +def _seg_48(): + return [ (0xFD6A, 'M', u'شمخ'), (0xFD6C, 'M', u'شمم'), (0xFD6E, 'M', u'ضحى'), @@ -5099,10 +5088,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5114,6 +5099,10 @@ def _seg_49(): (0xFDFE, 'X'), (0xFE00, 'I'), (0xFE10, '3', u','), + ] + +def _seg_49(): + return [ (0xFE11, 'M', u'、'), (0xFE12, 'X'), (0xFE13, '3', u':'), @@ -5203,10 +5192,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5218,6 +5203,10 @@ def _seg_50(): (0xFEB1, 'M', u'س'), (0xFEB5, 'M', u'ش'), (0xFEB9, 'M', u'ص'), + ] + +def _seg_50(): + return [ (0xFEBD, 'M', u'ض'), (0xFEC1, 'M', u'ط'), (0xFEC5, 'M', u'ظ'), @@ -5307,10 +5296,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5322,6 +5307,10 @@ def _seg_51(): (0xFF4C, 'M', u'l'), (0xFF4D, 'M', u'm'), (0xFF4E, 'M', u'n'), + ] + +def _seg_51(): + return [ (0xFF4F, 'M', u'o'), (0xFF50, 'M', u'p'), (0xFF51, 'M', u'q'), @@ -5411,10 +5400,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5426,6 +5411,10 @@ def _seg_52(): (0xFFB0, 'M', u'ᄚ'), (0xFFB1, 'M', u'ᄆ'), (0xFFB2, 'M', u'ᄇ'), + ] + +def _seg_52(): + return [ (0xFFB3, 'M', u'ᄈ'), (0xFFB4, 'M', u'ᄡ'), (0xFFB5, 'M', u'ᄉ'), @@ -5515,10 +5504,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5530,6 +5515,10 @@ def _seg_53(): (0x103D6, 'X'), (0x10400, 'M', u'𐐨'), (0x10401, 'M', u'𐐩'), + ] + +def _seg_53(): + return [ (0x10402, 'M', u'𐐪'), (0x10403, 'M', u'𐐫'), (0x10404, 'M', u'𐐬'), @@ -5619,10 +5608,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5634,6 +5619,10 @@ def _seg_54(): (0x1080A, 'V'), (0x10836, 'X'), (0x10837, 'V'), + ] + +def _seg_54(): + return [ (0x10839, 'X'), (0x1083C, 'V'), (0x1083D, 'X'), @@ -5666,11 +5655,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5723,10 +5712,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5738,6 +5723,10 @@ def _seg_55(): (0x10CA6, 'M', u'𐳦'), (0x10CA7, 'M', u'𐳧'), (0x10CA8, 'M', u'𐳨'), + ] + +def _seg_55(): + return [ (0x10CA9, 'M', u'𐳩'), (0x10CAA, 'M', u'𐳪'), (0x10CAB, 'M', u'𐳫'), @@ -5752,9 +5741,15 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5765,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5811,7 +5806,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5822,16 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + ] + +def _seg_56(): + return [ + (0x1145F, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5856,11 +5851,13 @@ def _seg_56(): (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5902,8 +5899,6 @@ def _seg_56(): (0x11A50, 'V'), (0x11A84, 'X'), (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,14 +5926,28 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), + (0x11D60, 'V'), + ] + +def _seg_57(): + return [ + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), (0x12000, 'V'), (0x1239A, 'X'), (0x12400, 'V'), @@ -5973,6 +5982,8 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), @@ -5982,7 +5993,7 @@ def _seg_57(): (0x16FE0, 'V'), (0x16FE2, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F2, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), @@ -6024,21 +6035,23 @@ def _seg_57(): (0x1D1C1, 'V'), (0x1D1E9, 'X'), (0x1D200, 'V'), + ] + +def _seg_58(): + return [ (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), - ] - -def _seg_58(): - return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -6126,6 +6139,10 @@ def _seg_58(): (0x1D45A, 'M', u'm'), (0x1D45B, 'M', u'n'), (0x1D45C, 'M', u'o'), + ] + +def _seg_59(): + return [ (0x1D45D, 'M', u'p'), (0x1D45E, 'M', u'q'), (0x1D45F, 'M', u'r'), @@ -6139,10 +6156,6 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), - ] - -def _seg_59(): - return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -6230,6 +6243,10 @@ def _seg_59(): (0x1D4C1, 'M', u'l'), (0x1D4C2, 'M', u'm'), (0x1D4C3, 'M', u'n'), + ] + +def _seg_60(): + return [ (0x1D4C4, 'X'), (0x1D4C5, 'M', u'p'), (0x1D4C6, 'M', u'q'), @@ -6243,10 +6260,6 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), - ] - -def _seg_60(): - return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -6334,6 +6347,10 @@ def _seg_60(): (0x1D526, 'M', u'i'), (0x1D527, 'M', u'j'), (0x1D528, 'M', u'k'), + ] + +def _seg_61(): + return [ (0x1D529, 'M', u'l'), (0x1D52A, 'M', u'm'), (0x1D52B, 'M', u'n'), @@ -6347,10 +6364,6 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), - ] - -def _seg_61(): - return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -6438,6 +6451,10 @@ def _seg_61(): (0x1D58C, 'M', u'g'), (0x1D58D, 'M', u'h'), (0x1D58E, 'M', u'i'), + ] + +def _seg_62(): + return [ (0x1D58F, 'M', u'j'), (0x1D590, 'M', u'k'), (0x1D591, 'M', u'l'), @@ -6451,10 +6468,6 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), - ] - -def _seg_62(): - return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6542,6 +6555,10 @@ def _seg_62(): (0x1D5F0, 'M', u'c'), (0x1D5F1, 'M', u'd'), (0x1D5F2, 'M', u'e'), + ] + +def _seg_63(): + return [ (0x1D5F3, 'M', u'f'), (0x1D5F4, 'M', u'g'), (0x1D5F5, 'M', u'h'), @@ -6555,10 +6572,6 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), - ] - -def _seg_63(): - return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6646,6 +6659,10 @@ def _seg_63(): (0x1D654, 'M', u'y'), (0x1D655, 'M', u'z'), (0x1D656, 'M', u'a'), + ] + +def _seg_64(): + return [ (0x1D657, 'M', u'b'), (0x1D658, 'M', u'c'), (0x1D659, 'M', u'd'), @@ -6659,10 +6676,6 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), - ] - -def _seg_64(): - return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6750,6 +6763,10 @@ def _seg_64(): (0x1D6B9, 'M', u'θ'), (0x1D6BA, 'M', u'σ'), (0x1D6BB, 'M', u'τ'), + ] + +def _seg_65(): + return [ (0x1D6BC, 'M', u'υ'), (0x1D6BD, 'M', u'φ'), (0x1D6BE, 'M', u'χ'), @@ -6763,10 +6780,6 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), - ] - -def _seg_65(): - return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6854,6 +6867,10 @@ def _seg_65(): (0x1D71F, 'M', u'δ'), (0x1D720, 'M', u'ε'), (0x1D721, 'M', u'ζ'), + ] + +def _seg_66(): + return [ (0x1D722, 'M', u'η'), (0x1D723, 'M', u'θ'), (0x1D724, 'M', u'ι'), @@ -6867,10 +6884,6 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), - ] - -def _seg_66(): - return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6958,6 +6971,10 @@ def _seg_66(): (0x1D785, 'M', u'φ'), (0x1D786, 'M', u'χ'), (0x1D787, 'M', u'ψ'), + ] + +def _seg_67(): + return [ (0x1D788, 'M', u'ω'), (0x1D789, 'M', u'∂'), (0x1D78A, 'M', u'ε'), @@ -6971,10 +6988,6 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), - ] - -def _seg_67(): - return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -7062,6 +7075,10 @@ def _seg_67(): (0x1D7EC, 'M', u'0'), (0x1D7ED, 'M', u'1'), (0x1D7EE, 'M', u'2'), + ] + +def _seg_68(): + return [ (0x1D7EF, 'M', u'3'), (0x1D7F0, 'M', u'4'), (0x1D7F1, 'M', u'5'), @@ -7075,10 +7092,6 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), - ] - -def _seg_68(): - return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), @@ -7143,6 +7156,8 @@ def _seg_68(): (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7179,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_69(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7198,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7283,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_70(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7302,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7387,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_71(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7401,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7476,6 +7491,10 @@ def _seg_71(): (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), (0x1F23B, 'M', u'配'), + ] + +def _seg_72(): + return [ (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), @@ -7491,21 +7510,17 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), (0x1F6D5, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FA, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7521,15 +7536,21 @@ def _seg_72(): (0x1F910, 'V'), (0x1F93F, 'X'), (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), + (0x1F971, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F97B, 'X'), + (0x1F97C, 'V'), + (0x1F9A3, 'X'), + (0x1F9B0, 'V'), + (0x1F9BA, 'X'), (0x1F9C0, 'V'), - (0x1F9C1, 'X'), + (0x1F9C3, 'X'), (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1FA00, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7574,6 +7595,10 @@ def _seg_72(): (0x2F81F, 'M', u'㓟'), (0x2F820, 'M', u'刻'), (0x2F821, 'M', u'剆'), + ] + +def _seg_73(): + return [ (0x2F822, 'M', u'割'), (0x2F823, 'M', u'剷'), (0x2F824, 'M', u'㔕'), @@ -7595,10 +7620,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7678,6 +7699,10 @@ def _seg_73(): (0x2F887, 'M', u'幩'), (0x2F888, 'M', u'㡢'), (0x2F889, 'M', u'𢆃'), + ] + +def _seg_74(): + return [ (0x2F88A, 'M', u'㡼'), (0x2F88B, 'M', u'庰'), (0x2F88C, 'M', u'庳'), @@ -7699,10 +7724,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7782,6 +7803,10 @@ def _seg_74(): (0x2F8ED, 'M', u'櫛'), (0x2F8EE, 'M', u'㰘'), (0x2F8EF, 'M', u'次'), + ] + +def _seg_75(): + return [ (0x2F8F0, 'M', u'𣢧'), (0x2F8F1, 'M', u'歔'), (0x2F8F2, 'M', u'㱎'), @@ -7803,10 +7828,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7886,6 +7907,10 @@ def _seg_75(): (0x2F953, 'M', u'祖'), (0x2F954, 'M', u'𥚚'), (0x2F955, 'M', u'𥛅'), + ] + +def _seg_76(): + return [ (0x2F956, 'M', u'福'), (0x2F957, 'M', u'秫'), (0x2F958, 'M', u'䄯'), @@ -7907,10 +7932,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7990,6 +8011,10 @@ def _seg_76(): (0x2F9B8, 'M', u'蚈'), (0x2F9B9, 'M', u'蜎'), (0x2F9BA, 'M', u'蛢'), + ] + +def _seg_77(): + return [ (0x2F9BB, 'M', u'蝹'), (0x2F9BC, 'M', u'蜨'), (0x2F9BD, 'M', u'蝫'), @@ -8011,10 +8036,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8094,6 +8115,10 @@ def _seg_77(): (0x2FA1D, 'M', u'𪘀'), (0x2FA1E, 'X'), (0xE0100, 'I'), + ] + +def _seg_78(): + return [ (0xE01F0, 'X'), ] @@ -8176,4 +8201,5 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() ) diff --git a/pipenv/patched/notpip/_vendor/lockfile/__init__.py b/pipenv/patched/notpip/_vendor/lockfile/__init__.py deleted file mode 100644 index a6f44a55..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/__init__.py +++ /dev/null @@ -1,347 +0,0 @@ -# -*- coding: utf-8 -*- - -""" -lockfile.py - Platform-independent advisory file locks. - -Requires Python 2.5 unless you apply 2.4.diff -Locking is done on a per-thread basis instead of a per-process basis. - -Usage: - ->>> lock = LockFile('somefile') ->>> try: -... lock.acquire() -... except AlreadyLocked: -... print 'somefile', 'is locked already.' -... except LockFailed: -... print 'somefile', 'can\\'t be locked.' -... else: -... print 'got lock' -got lock ->>> print lock.is_locked() -True ->>> lock.release() - ->>> lock = LockFile('somefile') ->>> print lock.is_locked() -False ->>> with lock: -... print lock.is_locked() -True ->>> print lock.is_locked() -False - ->>> lock = LockFile('somefile') ->>> # It is okay to lock twice from the same thread... ->>> with lock: -... lock.acquire() -... ->>> # Though no counter is kept, so you can't unlock multiple times... ->>> print lock.is_locked() -False - -Exceptions: - - Error - base class for other exceptions - LockError - base class for all locking exceptions - AlreadyLocked - Another thread or process already holds the lock - LockFailed - Lock failed for some other reason - UnlockError - base class for all unlocking exceptions - AlreadyUnlocked - File was not locked. - NotMyLock - File was locked but not by the current thread/process -""" - -from __future__ import absolute_import - -import functools -import os -import socket -import threading -import warnings - -# Work with PEP8 and non-PEP8 versions of threading module. -if not hasattr(threading, "current_thread"): - threading.current_thread = threading.currentThread -if not hasattr(threading.Thread, "get_name"): - threading.Thread.get_name = threading.Thread.getName - -__all__ = ['Error', 'LockError', 'LockTimeout', 'AlreadyLocked', - 'LockFailed', 'UnlockError', 'NotLocked', 'NotMyLock', - 'LinkFileLock', 'MkdirFileLock', 'SQLiteFileLock', - 'LockBase', 'locked'] - - -class Error(Exception): - """ - Base class for other exceptions. - - >>> try: - ... raise Error - ... except Exception: - ... pass - """ - pass - - -class LockError(Error): - """ - Base class for error arising from attempts to acquire the lock. - - >>> try: - ... raise LockError - ... except Error: - ... pass - """ - pass - - -class LockTimeout(LockError): - """Raised when lock creation fails within a user-defined period of time. - - >>> try: - ... raise LockTimeout - ... except LockError: - ... pass - """ - pass - - -class AlreadyLocked(LockError): - """Some other thread/process is locking the file. - - >>> try: - ... raise AlreadyLocked - ... except LockError: - ... pass - """ - pass - - -class LockFailed(LockError): - """Lock file creation failed for some other reason. - - >>> try: - ... raise LockFailed - ... except LockError: - ... pass - """ - pass - - -class UnlockError(Error): - """ - Base class for errors arising from attempts to release the lock. - - >>> try: - ... raise UnlockError - ... except Error: - ... pass - """ - pass - - -class NotLocked(UnlockError): - """Raised when an attempt is made to unlock an unlocked file. - - >>> try: - ... raise NotLocked - ... except UnlockError: - ... pass - """ - pass - - -class NotMyLock(UnlockError): - """Raised when an attempt is made to unlock a file someone else locked. - - >>> try: - ... raise NotMyLock - ... except UnlockError: - ... pass - """ - pass - - -class _SharedBase(object): - def __init__(self, path): - self.path = path - - def acquire(self, timeout=None): - """ - Acquire the lock. - - * If timeout is omitted (or None), wait forever trying to lock the - file. - - * If timeout > 0, try to acquire the lock for that many seconds. If - the lock period expires and the file is still locked, raise - LockTimeout. - - * If timeout <= 0, raise AlreadyLocked immediately if the file is - already locked. - """ - raise NotImplemented("implement in subclass") - - def release(self): - """ - Release the lock. - - If the file is not locked, raise NotLocked. - """ - raise NotImplemented("implement in subclass") - - def __enter__(self): - """ - Context manager support. - """ - self.acquire() - return self - - def __exit__(self, *_exc): - """ - Context manager support. - """ - self.release() - - def __repr__(self): - return "<%s: %r>" % (self.__class__.__name__, self.path) - - -class LockBase(_SharedBase): - """Base class for platform-specific lock classes.""" - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = LockBase('somefile') - >>> lock = LockBase('somefile', threaded=False) - """ - super(LockBase, self).__init__(path) - self.lock_file = os.path.abspath(path) + ".lock" - self.hostname = socket.gethostname() - self.pid = os.getpid() - if threaded: - t = threading.current_thread() - # Thread objects in Python 2.4 and earlier do not have ident - # attrs. Worm around that. - ident = getattr(t, "ident", hash(t)) - self.tname = "-%x" % (ident & 0xffffffff) - else: - self.tname = "" - dirname = os.path.dirname(self.lock_file) - - # unique name is mostly about the current process, but must - # also contain the path -- otherwise, two adjacent locked - # files conflict (one file gets locked, creating lock-file and - # unique file, the other one gets locked, creating lock-file - # and overwriting the already existing lock-file, then one - # gets unlocked, deleting both lock-file and unique file, - # finally the last lock errors out upon releasing. - self.unique_name = os.path.join(dirname, - "%s%s.%s%s" % (self.hostname, - self.tname, - self.pid, - hash(self.path))) - self.timeout = timeout - - def is_locked(self): - """ - Tell whether or not the file is locked. - """ - raise NotImplemented("implement in subclass") - - def i_am_locking(self): - """ - Return True if this object is locking the file. - """ - raise NotImplemented("implement in subclass") - - def break_lock(self): - """ - Remove a lock. Useful if a locking thread failed to unlock. - """ - raise NotImplemented("implement in subclass") - - def __repr__(self): - return "<%s: %r -- %r>" % (self.__class__.__name__, self.unique_name, - self.path) - - -def _fl_helper(cls, mod, *args, **kwds): - warnings.warn("Import from %s module instead of lockfile package" % mod, - DeprecationWarning, stacklevel=2) - # This is a bit funky, but it's only for awhile. The way the unit tests - # are constructed this function winds up as an unbound method, so it - # actually takes three args, not two. We want to toss out self. - if not isinstance(args[0], str): - # We are testing, avoid the first arg - args = args[1:] - if len(args) == 1 and not kwds: - kwds["threaded"] = True - return cls(*args, **kwds) - - -def LinkFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import LinkLockFile from the - lockfile.linklockfile module. - """ - from . import linklockfile - return _fl_helper(linklockfile.LinkLockFile, "lockfile.linklockfile", - *args, **kwds) - - -def MkdirFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import MkdirLockFile from the - lockfile.mkdirlockfile module. - """ - from . import mkdirlockfile - return _fl_helper(mkdirlockfile.MkdirLockFile, "lockfile.mkdirlockfile", - *args, **kwds) - - -def SQLiteFileLock(*args, **kwds): - """Factory function provided for backwards compatibility. - - Do not use in new code. Instead, import SQLiteLockFile from the - lockfile.mkdirlockfile module. - """ - from . import sqlitelockfile - return _fl_helper(sqlitelockfile.SQLiteLockFile, "lockfile.sqlitelockfile", - *args, **kwds) - - -def locked(path, timeout=None): - """Decorator which enables locks for decorated function. - - Arguments: - - path: path for lockfile. - - timeout (optional): Timeout for acquiring lock. - - Usage: - @locked('/var/run/myname', timeout=0) - def myname(...): - ... - """ - def decor(func): - @functools.wraps(func) - def wrapper(*args, **kwargs): - lock = FileLock(path, timeout=timeout) - lock.acquire() - try: - return func(*args, **kwargs) - finally: - lock.release() - return wrapper - return decor - - -if hasattr(os, "link"): - from . import linklockfile as _llf - LockFile = _llf.LinkLockFile -else: - from . import mkdirlockfile as _mlf - LockFile = _mlf.MkdirLockFile - -FileLock = LockFile diff --git a/pipenv/patched/notpip/_vendor/lockfile/linklockfile.py b/pipenv/patched/notpip/_vendor/lockfile/linklockfile.py deleted file mode 100644 index 2ca9be04..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/linklockfile.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import absolute_import - -import time -import os - -from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class LinkLockFile(LockBase): - """Lock access to a file using atomic property of link(2). - - >>> lock = LinkLockFile('somefile') - >>> lock = LinkLockFile('somefile', threaded=False) - """ - - def acquire(self, timeout=None): - try: - open(self.unique_name, "wb").close() - except IOError: - raise LockFailed("failed to create %s" % self.unique_name) - - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - # Try and create a hard link to it. - try: - os.link(self.unique_name, self.lock_file) - except OSError: - # Link creation failed. Maybe we've double-locked? - nlinks = os.stat(self.unique_name).st_nlink - if nlinks == 2: - # The original link plus the one I created == 2. We're - # good to go. - return - else: - # Otherwise the lock creation failed. - if timeout is not None and time.time() > end_time: - os.unlink(self.unique_name) - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout is not None and timeout / 10 or 0.1) - else: - # Link creation succeeded. We're good to go. - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not os.path.exists(self.unique_name): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.unique_name) - os.unlink(self.lock_file) - - def is_locked(self): - return os.path.exists(self.lock_file) - - def i_am_locking(self): - return (self.is_locked() and - os.path.exists(self.unique_name) and - os.stat(self.unique_name).st_nlink == 2) - - def break_lock(self): - if os.path.exists(self.lock_file): - os.unlink(self.lock_file) diff --git a/pipenv/patched/notpip/_vendor/lockfile/mkdirlockfile.py b/pipenv/patched/notpip/_vendor/lockfile/mkdirlockfile.py deleted file mode 100644 index 05a8c96c..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/mkdirlockfile.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import absolute_import, division - -import time -import os -import sys -import errno - -from . import (LockBase, LockFailed, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class MkdirLockFile(LockBase): - """Lock file by creating a directory.""" - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = MkdirLockFile('somefile') - >>> lock = MkdirLockFile('somefile', threaded=False) - """ - LockBase.__init__(self, path, threaded, timeout) - # Lock file itself is a directory. Place the unique file name into - # it. - self.unique_name = os.path.join(self.lock_file, - "%s.%s%s" % (self.hostname, - self.tname, - self.pid)) - - def acquire(self, timeout=None): - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - if timeout is None: - wait = 0.1 - else: - wait = max(0, timeout / 10) - - while True: - try: - os.mkdir(self.lock_file) - except OSError: - err = sys.exc_info()[1] - if err.errno == errno.EEXIST: - # Already locked. - if os.path.exists(self.unique_name): - # Already locked by me. - return - if timeout is not None and time.time() > end_time: - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - # Someone else has the lock. - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(wait) - else: - # Couldn't create the lock for some other reason - raise LockFailed("failed to create %s" % self.lock_file) - else: - open(self.unique_name, "wb").close() - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not os.path.exists(self.unique_name): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.unique_name) - os.rmdir(self.lock_file) - - def is_locked(self): - return os.path.exists(self.lock_file) - - def i_am_locking(self): - return (self.is_locked() and - os.path.exists(self.unique_name)) - - def break_lock(self): - if os.path.exists(self.lock_file): - for name in os.listdir(self.lock_file): - os.unlink(os.path.join(self.lock_file, name)) - os.rmdir(self.lock_file) diff --git a/pipenv/patched/notpip/_vendor/lockfile/pidlockfile.py b/pipenv/patched/notpip/_vendor/lockfile/pidlockfile.py deleted file mode 100644 index 069e85b1..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/pidlockfile.py +++ /dev/null @@ -1,190 +0,0 @@ -# -*- coding: utf-8 -*- - -# pidlockfile.py -# -# Copyright © 2008–2009 Ben Finney <ben+python@benfinney.id.au> -# -# This is free software: you may copy, modify, and/or distribute this work -# under the terms of the Python Software Foundation License, version 2 or -# later as published by the Python Software Foundation. -# No warranty expressed or implied. See the file LICENSE.PSF-2 for details. - -""" Lockfile behaviour implemented via Unix PID files. - """ - -from __future__ import absolute_import - -import errno -import os -import time - -from . import (LockBase, AlreadyLocked, LockFailed, NotLocked, NotMyLock, - LockTimeout) - - -class PIDLockFile(LockBase): - """ Lockfile implemented as a Unix PID file. - - The lock file is a normal file named by the attribute `path`. - A lock's PID file contains a single line of text, containing - the process ID (PID) of the process that acquired the lock. - - >>> lock = PIDLockFile('somefile') - >>> lock = PIDLockFile('somefile') - """ - - def __init__(self, path, threaded=False, timeout=None): - # pid lockfiles don't support threaded operation, so always force - # False as the threaded arg. - LockBase.__init__(self, path, False, timeout) - self.unique_name = self.path - - def read_pid(self): - """ Get the PID from the lock file. - """ - return read_pid_from_pidfile(self.path) - - def is_locked(self): - """ Test if the lock is currently held. - - The lock is held if the PID file for this lock exists. - - """ - return os.path.exists(self.path) - - def i_am_locking(self): - """ Test if the lock is held by the current process. - - Returns ``True`` if the current process ID matches the - number stored in the PID file. - """ - return self.is_locked() and os.getpid() == self.read_pid() - - def acquire(self, timeout=None): - """ Acquire the lock. - - Creates the PID file for this lock, or raises an error if - the lock could not be acquired. - """ - - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - try: - write_pid_to_pidfile(self.path) - except OSError as exc: - if exc.errno == errno.EEXIST: - # The lock creation failed. Maybe sleep a bit. - if time.time() > end_time: - if timeout is not None and timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout is not None and timeout / 10 or 0.1) - else: - raise LockFailed("failed to create %s" % self.path) - else: - return - - def release(self): - """ Release the lock. - - Removes the PID file to release the lock, or raises an - error if the current process does not hold the lock. - - """ - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - if not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me" % self.path) - remove_existing_pidfile(self.path) - - def break_lock(self): - """ Break an existing lock. - - Removes the PID file if it already exists, otherwise does - nothing. - - """ - remove_existing_pidfile(self.path) - - -def read_pid_from_pidfile(pidfile_path): - """ Read the PID recorded in the named PID file. - - Read and return the numeric PID recorded as text in the named - PID file. If the PID file cannot be read, or if the content is - not a valid PID, return ``None``. - - """ - pid = None - try: - pidfile = open(pidfile_path, 'r') - except IOError: - pass - else: - # According to the FHS 2.3 section on PID files in /var/run: - # - # The file must consist of the process identifier in - # ASCII-encoded decimal, followed by a newline character. - # - # Programs that read PID files should be somewhat flexible - # in what they accept; i.e., they should ignore extra - # whitespace, leading zeroes, absence of the trailing - # newline, or additional lines in the PID file. - - line = pidfile.readline().strip() - try: - pid = int(line) - except ValueError: - pass - pidfile.close() - - return pid - - -def write_pid_to_pidfile(pidfile_path): - """ Write the PID in the named PID file. - - Get the numeric process ID (“PID”) of the current process - and write it to the named file as a line of text. - - """ - open_flags = (os.O_CREAT | os.O_EXCL | os.O_WRONLY) - open_mode = 0o644 - pidfile_fd = os.open(pidfile_path, open_flags, open_mode) - pidfile = os.fdopen(pidfile_fd, 'w') - - # According to the FHS 2.3 section on PID files in /var/run: - # - # The file must consist of the process identifier in - # ASCII-encoded decimal, followed by a newline character. For - # example, if crond was process number 25, /var/run/crond.pid - # would contain three characters: two, five, and newline. - - pid = os.getpid() - pidfile.write("%s\n" % pid) - pidfile.close() - - -def remove_existing_pidfile(pidfile_path): - """ Remove the named PID file if it exists. - - Removing a PID file that doesn't already exist puts us in the - desired state, so we ignore the condition if the file does not - exist. - - """ - try: - os.remove(pidfile_path) - except OSError as exc: - if exc.errno == errno.ENOENT: - pass - else: - raise diff --git a/pipenv/patched/notpip/_vendor/lockfile/sqlitelockfile.py b/pipenv/patched/notpip/_vendor/lockfile/sqlitelockfile.py deleted file mode 100644 index f997e244..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/sqlitelockfile.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import absolute_import, division - -import time -import os - -try: - unicode -except NameError: - unicode = str - -from . import LockBase, NotLocked, NotMyLock, LockTimeout, AlreadyLocked - - -class SQLiteLockFile(LockBase): - "Demonstrate SQL-based locking." - - testdb = None - - def __init__(self, path, threaded=True, timeout=None): - """ - >>> lock = SQLiteLockFile('somefile') - >>> lock = SQLiteLockFile('somefile', threaded=False) - """ - LockBase.__init__(self, path, threaded, timeout) - self.lock_file = unicode(self.lock_file) - self.unique_name = unicode(self.unique_name) - - if SQLiteLockFile.testdb is None: - import tempfile - _fd, testdb = tempfile.mkstemp() - os.close(_fd) - os.unlink(testdb) - del _fd, tempfile - SQLiteLockFile.testdb = testdb - - import sqlite3 - self.connection = sqlite3.connect(SQLiteLockFile.testdb) - - c = self.connection.cursor() - try: - c.execute("create table locks" - "(" - " lock_file varchar(32)," - " unique_name varchar(32)" - ")") - except sqlite3.OperationalError: - pass - else: - self.connection.commit() - import atexit - atexit.register(os.unlink, SQLiteLockFile.testdb) - - def acquire(self, timeout=None): - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - if timeout is None: - wait = 0.1 - elif timeout <= 0: - wait = 0 - else: - wait = timeout / 10 - - cursor = self.connection.cursor() - - while True: - if not self.is_locked(): - # Not locked. Try to lock it. - cursor.execute("insert into locks" - " (lock_file, unique_name)" - " values" - " (?, ?)", - (self.lock_file, self.unique_name)) - self.connection.commit() - - # Check to see if we are the only lock holder. - cursor.execute("select * from locks" - " where unique_name = ?", - (self.unique_name,)) - rows = cursor.fetchall() - if len(rows) > 1: - # Nope. Someone else got there. Remove our lock. - cursor.execute("delete from locks" - " where unique_name = ?", - (self.unique_name,)) - self.connection.commit() - else: - # Yup. We're done, so go home. - return - else: - # Check to see if we are the only lock holder. - cursor.execute("select * from locks" - " where unique_name = ?", - (self.unique_name,)) - rows = cursor.fetchall() - if len(rows) == 1: - # We're the locker, so go home. - return - - # Maybe we should wait a bit longer. - if timeout is not None and time.time() > end_time: - if timeout > 0: - # No more waiting. - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - # Someone else has the lock and we are impatient.. - raise AlreadyLocked("%s is already locked" % self.path) - - # Well, okay. We'll give it a bit longer. - time.sleep(wait) - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - if not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me (by %s)" % - (self.unique_name, self._who_is_locking())) - cursor = self.connection.cursor() - cursor.execute("delete from locks" - " where unique_name = ?", - (self.unique_name,)) - self.connection.commit() - - def _who_is_locking(self): - cursor = self.connection.cursor() - cursor.execute("select unique_name from locks" - " where lock_file = ?", - (self.lock_file,)) - return cursor.fetchone()[0] - - def is_locked(self): - cursor = self.connection.cursor() - cursor.execute("select * from locks" - " where lock_file = ?", - (self.lock_file,)) - rows = cursor.fetchall() - return not not rows - - def i_am_locking(self): - cursor = self.connection.cursor() - cursor.execute("select * from locks" - " where lock_file = ?" - " and unique_name = ?", - (self.lock_file, self.unique_name)) - return not not cursor.fetchall() - - def break_lock(self): - cursor = self.connection.cursor() - cursor.execute("delete from locks" - " where lock_file = ?", - (self.lock_file,)) - self.connection.commit() diff --git a/pipenv/patched/notpip/_vendor/lockfile/symlinklockfile.py b/pipenv/patched/notpip/_vendor/lockfile/symlinklockfile.py deleted file mode 100644 index 23b41f58..00000000 --- a/pipenv/patched/notpip/_vendor/lockfile/symlinklockfile.py +++ /dev/null @@ -1,70 +0,0 @@ -from __future__ import absolute_import - -import os -import time - -from . import (LockBase, NotLocked, NotMyLock, LockTimeout, - AlreadyLocked) - - -class SymlinkLockFile(LockBase): - """Lock access to a file using symlink(2).""" - - def __init__(self, path, threaded=True, timeout=None): - # super(SymlinkLockFile).__init(...) - LockBase.__init__(self, path, threaded, timeout) - # split it back! - self.unique_name = os.path.split(self.unique_name)[1] - - def acquire(self, timeout=None): - # Hopefully unnecessary for symlink. - # try: - # open(self.unique_name, "wb").close() - # except IOError: - # raise LockFailed("failed to create %s" % self.unique_name) - timeout = timeout if timeout is not None else self.timeout - end_time = time.time() - if timeout is not None and timeout > 0: - end_time += timeout - - while True: - # Try and create a symbolic link to it. - try: - os.symlink(self.unique_name, self.lock_file) - except OSError: - # Link creation failed. Maybe we've double-locked? - if self.i_am_locking(): - # Linked to out unique name. Proceed. - return - else: - # Otherwise the lock creation failed. - if timeout is not None and time.time() > end_time: - if timeout > 0: - raise LockTimeout("Timeout waiting to acquire" - " lock for %s" % - self.path) - else: - raise AlreadyLocked("%s is already locked" % - self.path) - time.sleep(timeout / 10 if timeout is not None else 0.1) - else: - # Link creation succeeded. We're good to go. - return - - def release(self): - if not self.is_locked(): - raise NotLocked("%s is not locked" % self.path) - elif not self.i_am_locking(): - raise NotMyLock("%s is locked, but not by me" % self.path) - os.unlink(self.lock_file) - - def is_locked(self): - return os.path.islink(self.lock_file) - - def i_am_locking(self): - return (os.path.islink(self.lock_file) - and os.readlink(self.lock_file) == self.unique_name) - - def break_lock(self): - if os.path.islink(self.lock_file): # exists && link - os.unlink(self.lock_file) diff --git a/pipenv/patched/notpip/_vendor/msgpack/__init__.py b/pipenv/patched/notpip/_vendor/msgpack/__init__.py index a15e5769..4ad9c1a5 100644 --- a/pipenv/patched/notpip/_vendor/msgpack/__init__.py +++ b/pipenv/patched/notpip/_vendor/msgpack/__init__.py @@ -1,6 +1,6 @@ # coding: utf-8 -from pipenv.patched.notpip._vendor.msgpack._version import version -from pipenv.patched.notpip._vendor.msgpack.exceptions import * +from ._version import version +from .exceptions import * from collections import namedtuple @@ -19,13 +19,12 @@ class ExtType(namedtuple('ExtType', 'code data')): import os if os.environ.get('MSGPACK_PUREPYTHON'): - from pipenv.patched.notpip._vendor.msgpack.fallback import Packer, unpackb, Unpacker + from .fallback import Packer, unpackb, Unpacker else: try: - from pipenv.patched.notpip._vendor.msgpack._packer import Packer - from pipenv.patched.notpip._vendor.msgpack._unpacker import unpackb, Unpacker + from ._cmsgpack import Packer, unpackb, Unpacker except ImportError: - from pipenv.patched.notpip._vendor.msgpack.fallback import Packer, unpackb, Unpacker + from .fallback import Packer, unpackb, Unpacker def pack(o, stream, **kwargs): diff --git a/pipenv/patched/notpip/_vendor/msgpack/_version.py b/pipenv/patched/notpip/_vendor/msgpack/_version.py index d28f0deb..1e73a00f 100644 --- a/pipenv/patched/notpip/_vendor/msgpack/_version.py +++ b/pipenv/patched/notpip/_vendor/msgpack/_version.py @@ -1 +1 @@ -version = (0, 5, 6) +version = (0, 6, 2) diff --git a/pipenv/patched/notpip/_vendor/msgpack/exceptions.py b/pipenv/patched/notpip/_vendor/msgpack/exceptions.py index 97668814..d6d2615c 100644 --- a/pipenv/patched/notpip/_vendor/msgpack/exceptions.py +++ b/pipenv/patched/notpip/_vendor/msgpack/exceptions.py @@ -1,5 +1,10 @@ class UnpackException(Exception): - """Deprecated. Use Exception instead to catch all exception during unpacking.""" + """Base class for some exceptions raised while unpacking. + + NOTE: unpack may raise exception other than subclass of + UnpackException. If you want to catch all error, catch + Exception instead. + """ class BufferFull(UnpackException): @@ -10,11 +15,25 @@ class OutOfData(UnpackException): pass -class UnpackValueError(UnpackException, ValueError): - """Deprecated. Use ValueError instead.""" +class FormatError(ValueError, UnpackException): + """Invalid msgpack format""" + + +class StackError(ValueError, UnpackException): + """Too nested""" + + +# Deprecated. Use ValueError instead +UnpackValueError = ValueError class ExtraData(UnpackValueError): + """ExtraData is raised when there is trailing data. + + This exception is raised while only one-shot (not streaming) + unpack. + """ + def __init__(self, unpacked, extra): self.unpacked = unpacked self.extra = extra @@ -23,19 +42,7 @@ class ExtraData(UnpackValueError): return "unpack(b) received extra data." -class PackException(Exception): - """Deprecated. Use Exception instead to catch all exception during packing.""" - - -class PackValueError(PackException, ValueError): - """PackValueError is raised when type of input data is supported but it's value is unsupported. - - Deprecated. Use ValueError instead. - """ - - -class PackOverflowError(PackValueError, OverflowError): - """PackOverflowError is raised when integer value is out of range of msgpack support [-2**31, 2**32). - - Deprecated. Use ValueError instead. - """ +# Deprecated. Use Exception instead to catch all exception during packing. +PackException = Exception +PackValueError = ValueError +PackOverflowError = OverflowError diff --git a/pipenv/patched/notpip/_vendor/msgpack/fallback.py b/pipenv/patched/notpip/_vendor/msgpack/fallback.py index d3a7d558..3836e830 100644 --- a/pipenv/patched/notpip/_vendor/msgpack/fallback.py +++ b/pipenv/patched/notpip/_vendor/msgpack/fallback.py @@ -4,20 +4,30 @@ import sys import struct import warnings -if sys.version_info[0] == 3: - PY3 = True + +if sys.version_info[0] == 2: + PY2 = True + int_types = (int, long) + def dict_iteritems(d): + return d.iteritems() +else: + PY2 = False int_types = int - Unicode = str + unicode = str xrange = range def dict_iteritems(d): return d.items() -else: - PY3 = False - int_types = (int, long) - Unicode = unicode - def dict_iteritems(d): - return d.iteritems() +if sys.version_info < (3, 5): + # Ugly hack... + RecursionError = RuntimeError + + def _is_recursionerror(e): + return len(e.args) == 1 and isinstance(e.args[0], str) and \ + e.args[0].startswith('maximum recursion depth exceeded') +else: + def _is_recursionerror(e): + return True if hasattr(sys, 'pypy_version_info'): # cStringIO is slow on PyPy, StringIO is faster. However: PyPy's own @@ -49,15 +59,15 @@ else: newlist_hint = lambda size: [] -from pipenv.patched.notpip._vendor.msgpack.exceptions import ( +from .exceptions import ( BufferFull, OutOfData, - UnpackValueError, - PackValueError, - PackOverflowError, - ExtraData) + ExtraData, + FormatError, + StackError, +) -from pipenv.patched.notpip._vendor.msgpack import ExtType +from . import ExtType EX_SKIP = 0 @@ -87,12 +97,12 @@ def _get_data_from_buffer(obj): view = memoryview(obj) except TypeError: # try to use legacy buffer protocol if 2.7, otherwise re-raise - if not PY3: + if PY2: view = memoryview(buffer(obj)) warnings.warn("using old buffer interface to unpack %s; " "this leads to unpacking errors if slicing is used and " "will be removed in a future version" % type(obj), - RuntimeWarning) + RuntimeWarning, stacklevel=3) else: raise if view.itemsize != 1: @@ -103,7 +113,7 @@ def _get_data_from_buffer(obj): def unpack(stream, **kwargs): warnings.warn( "Direct calling implementation's unpack() is deprecated, Use msgpack.unpack() or unpackb() instead.", - PendingDeprecationWarning) + DeprecationWarning, stacklevel=2) data = stream.read() return unpackb(data, **kwargs) @@ -112,20 +122,37 @@ def unpackb(packed, **kwargs): """ Unpack an object from `packed`. - Raises `ExtraData` when `packed` contains extra bytes. + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``ValueError`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. + See :class:`Unpacker` for options. """ - unpacker = Unpacker(None, **kwargs) + unpacker = Unpacker(None, max_buffer_size=len(packed), **kwargs) unpacker.feed(packed) try: ret = unpacker._unpack() except OutOfData: - raise UnpackValueError("Data is not enough.") + raise ValueError("Unpack failed: incomplete input") + except RecursionError as e: + if _is_recursionerror(e): + raise StackError + raise if unpacker._got_extradata(): raise ExtraData(ret, unpacker._get_extradata()) return ret +if sys.version_info < (2, 7, 6): + def _unpack_from(f, b, o=0): + """Explicit typcast for legacy struct.unpack_from""" + return struct.unpack_from(f, bytes(b), o) +else: + _unpack_from = struct.unpack_from + + class Unpacker(object): """Streaming unpacker. @@ -152,6 +179,11 @@ class Unpacker(object): *encoding* option which is deprecated overrides this option. + :param bool strict_map_key: + If true, only str or bytes are accepted for map (dict) keys. + It's False by default for backward-compatibility. + But it will be True from msgpack 1.0. + :param callable object_hook: When specified, it should be callable. Unpacker calls it with a dict argument after unpacking msgpack map. @@ -176,27 +208,34 @@ class Unpacker(object): You should set this parameter when unpacking data from untrusted source. :param int max_str_len: - Limits max length of str. (default: 2**31-1) + Deprecated, use *max_buffer_size* instead. + Limits max length of str. (default: max_buffer_size or 1024*1024) :param int max_bin_len: - Limits max length of bin. (default: 2**31-1) + Deprecated, use *max_buffer_size* instead. + Limits max length of bin. (default: max_buffer_size or 1024*1024) :param int max_array_len: - Limits max length of array. (default: 2**31-1) + Limits max length of array. + (default: max_buffer_size or 128*1024) :param int max_map_len: - Limits max length of map. (default: 2**31-1) + Limits max length of map. + (default: max_buffer_size//2 or 32*1024) + :param int max_ext_len: + Deprecated, use *max_buffer_size* instead. + Limits max size of ext type. (default: max_buffer_size or 1024*1024) - example of streaming deserialize from file-like object:: + Example of streaming deserialize from file-like object:: - unpacker = Unpacker(file_like, raw=False) + unpacker = Unpacker(file_like, raw=False, max_buffer_size=10*1024*1024) for o in unpacker: process(o) - example of streaming deserialize from socket:: + Example of streaming deserialize from socket:: - unpacker = Unpacker(raw=False) + unpacker = Unpacker(raw=False, max_buffer_size=10*1024*1024) while True: buf = sock.recv(1024**2) if not buf: @@ -204,22 +243,27 @@ class Unpacker(object): unpacker.feed(buf) for o in unpacker: process(o) + + Raises ``ExtraData`` when *packed* contains extra bytes. + Raises ``OutOfData`` when *packed* is incomplete. + Raises ``FormatError`` when *packed* is not valid msgpack. + Raises ``StackError`` when *packed* contains too nested. + Other exceptions can be raised during unpacking. """ - def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, + def __init__(self, file_like=None, read_size=0, use_list=True, raw=True, strict_map_key=False, object_hook=None, object_pairs_hook=None, list_hook=None, encoding=None, unicode_errors=None, max_buffer_size=0, ext_hook=ExtType, - max_str_len=2147483647, # 2**32-1 - max_bin_len=2147483647, - max_array_len=2147483647, - max_map_len=2147483647, - max_ext_len=2147483647): - + max_str_len=-1, + max_bin_len=-1, + max_array_len=-1, + max_map_len=-1, + max_ext_len=-1): if encoding is not None: warnings.warn( "encoding is deprecated, Use raw=False instead.", - PendingDeprecationWarning) + DeprecationWarning, stacklevel=2) if unicode_errors is None: unicode_errors = 'strict' @@ -234,12 +278,6 @@ class Unpacker(object): #: array of bytes fed. self._buffer = bytearray() - # Some very old pythons don't support `struct.unpack_from()` with a - # `bytearray`. So we wrap it in a `buffer()` there. - if sys.version_info < (2, 7, 6): - self._buffer_view = buffer(self._buffer) - else: - self._buffer_view = self._buffer #: Which position we currently reads self._buff_i = 0 @@ -252,11 +290,23 @@ class Unpacker(object): # state, which _buf_checkpoint records. self._buf_checkpoint = 0 + if max_str_len == -1: + max_str_len = max_buffer_size or 1024*1024 + if max_bin_len == -1: + max_bin_len = max_buffer_size or 1024*1024 + if max_array_len == -1: + max_array_len = max_buffer_size or 128*1024 + if max_map_len == -1: + max_map_len = max_buffer_size//2 or 32*1024 + if max_ext_len == -1: + max_ext_len = max_buffer_size or 1024*1024 + self._max_buffer_size = max_buffer_size or 2**31-1 if read_size > self._max_buffer_size: raise ValueError("read_size must be smaller than max_buffer_size") self._read_size = read_size or min(self._max_buffer_size, 16*1024) self._raw = bool(raw) + self._strict_map_key = bool(strict_map_key) self._encoding = encoding self._unicode_errors = unicode_errors self._use_list = use_list @@ -295,7 +345,8 @@ class Unpacker(object): self._buff_i -= self._buf_checkpoint self._buf_checkpoint = 0 - self._buffer += view + # Use extend here: INPLACE_ADD += doesn't reliably typecast memoryview in jython + self._buffer.extend(view) def _consume(self): """ Gets rid of the used parts of the buffer. """ @@ -365,18 +416,18 @@ class Unpacker(object): n = b & 0b00011111 typ = TYPE_RAW if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b & 0b11110000 == 0b10010000: n = b & 0b00001111 typ = TYPE_ARRAY if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b & 0b11110000 == 0b10000000: n = b & 0b00001111 typ = TYPE_MAP if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) elif b == 0xc0: obj = None elif b == 0xc2: @@ -389,55 +440,55 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc5: typ = TYPE_BIN self._reserve(2) - n = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + n = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc6: typ = TYPE_BIN self._reserve(4) - n = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + n = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 if n > self._max_bin_len: - raise UnpackValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) + raise ValueError("%s exceeds max_bin_len(%s)" % (n, self._max_bin_len)) obj = self._read(n) elif b == 0xc7: # ext 8 typ = TYPE_EXT self._reserve(2) - L, n = struct.unpack_from('Bb', self._buffer_view, self._buff_i) + L, n = _unpack_from('Bb', self._buffer, self._buff_i) self._buff_i += 2 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc8: # ext 16 typ = TYPE_EXT self._reserve(3) - L, n = struct.unpack_from('>Hb', self._buffer_view, self._buff_i) + L, n = _unpack_from('>Hb', self._buffer, self._buff_i) self._buff_i += 3 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xc9: # ext 32 typ = TYPE_EXT self._reserve(5) - L, n = struct.unpack_from('>Ib', self._buffer_view, self._buff_i) + L, n = _unpack_from('>Ib', self._buffer, self._buff_i) self._buff_i += 5 if L > self._max_ext_len: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (L, self._max_ext_len)) obj = self._read(L) elif b == 0xca: self._reserve(4) - obj = struct.unpack_from(">f", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">f", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcb: self._reserve(8) - obj = struct.unpack_from(">d", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">d", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xcc: self._reserve(1) @@ -445,66 +496,66 @@ class Unpacker(object): self._buff_i += 1 elif b == 0xcd: self._reserve(2) - obj = struct.unpack_from(">H", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">H", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xce: self._reserve(4) - obj = struct.unpack_from(">I", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">I", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xcf: self._reserve(8) - obj = struct.unpack_from(">Q", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">Q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd0: self._reserve(1) - obj = struct.unpack_from("b", self._buffer_view, self._buff_i)[0] + obj = _unpack_from("b", self._buffer, self._buff_i)[0] self._buff_i += 1 elif b == 0xd1: self._reserve(2) - obj = struct.unpack_from(">h", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">h", self._buffer, self._buff_i)[0] self._buff_i += 2 elif b == 0xd2: self._reserve(4) - obj = struct.unpack_from(">i", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">i", self._buffer, self._buff_i)[0] self._buff_i += 4 elif b == 0xd3: self._reserve(8) - obj = struct.unpack_from(">q", self._buffer_view, self._buff_i)[0] + obj = _unpack_from(">q", self._buffer, self._buff_i)[0] self._buff_i += 8 elif b == 0xd4: # fixext 1 typ = TYPE_EXT if self._max_ext_len < 1: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (1, self._max_ext_len)) self._reserve(2) - n, obj = struct.unpack_from("b1s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b1s", self._buffer, self._buff_i) self._buff_i += 2 elif b == 0xd5: # fixext 2 typ = TYPE_EXT if self._max_ext_len < 2: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (2, self._max_ext_len)) self._reserve(3) - n, obj = struct.unpack_from("b2s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b2s", self._buffer, self._buff_i) self._buff_i += 3 elif b == 0xd6: # fixext 4 typ = TYPE_EXT if self._max_ext_len < 4: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (4, self._max_ext_len)) self._reserve(5) - n, obj = struct.unpack_from("b4s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b4s", self._buffer, self._buff_i) self._buff_i += 5 elif b == 0xd7: # fixext 8 typ = TYPE_EXT if self._max_ext_len < 8: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (8, self._max_ext_len)) self._reserve(9) - n, obj = struct.unpack_from("b8s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b8s", self._buffer, self._buff_i) self._buff_i += 9 elif b == 0xd8: # fixext 16 typ = TYPE_EXT if self._max_ext_len < 16: - raise UnpackValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) + raise ValueError("%s exceeds max_ext_len(%s)" % (16, self._max_ext_len)) self._reserve(17) - n, obj = struct.unpack_from("b16s", self._buffer_view, self._buff_i) + n, obj = _unpack_from("b16s", self._buffer, self._buff_i) self._buff_i += 17 elif b == 0xd9: typ = TYPE_RAW @@ -512,54 +563,54 @@ class Unpacker(object): n = self._buffer[self._buff_i] self._buff_i += 1 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xda: typ = TYPE_RAW self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdb: typ = TYPE_RAW self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_str_len: - raise UnpackValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) + raise ValueError("%s exceeds max_str_len(%s)", n, self._max_str_len) obj = self._read(n) elif b == 0xdc: typ = TYPE_ARRAY self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xdd: typ = TYPE_ARRAY self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_array_len: - raise UnpackValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) + raise ValueError("%s exceeds max_array_len(%s)", n, self._max_array_len) elif b == 0xde: self._reserve(2) - n, = struct.unpack_from(">H", self._buffer_view, self._buff_i) + n, = _unpack_from(">H", self._buffer, self._buff_i) self._buff_i += 2 if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP elif b == 0xdf: self._reserve(4) - n, = struct.unpack_from(">I", self._buffer_view, self._buff_i) + n, = _unpack_from(">I", self._buffer, self._buff_i) self._buff_i += 4 if n > self._max_map_len: - raise UnpackValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) + raise ValueError("%s exceeds max_map_len(%s)", n, self._max_map_len) typ = TYPE_MAP else: - raise UnpackValueError("Unknown header: 0x%x" % b) + raise FormatError("Unknown header: 0x%x" % b) return typ, n, obj def _unpack(self, execute=EX_CONSTRUCT): @@ -567,11 +618,11 @@ class Unpacker(object): if execute == EX_READ_ARRAY_HEADER: if typ != TYPE_ARRAY: - raise UnpackValueError("Expected array") + raise ValueError("Expected array") return n if execute == EX_READ_MAP_HEADER: if typ != TYPE_MAP: - raise UnpackValueError("Expected map") + raise ValueError("Expected map") return n # TODO should we eliminate the recursion? if typ == TYPE_ARRAY: @@ -603,6 +654,8 @@ class Unpacker(object): ret = {} for _ in xrange(n): key = self._unpack(EX_CONSTRUCT) + if self._strict_map_key and type(key) not in (unicode, bytes): + raise ValueError("%s is not allowed for map key" % str(type(key))) ret[key] = self._unpack(EX_CONSTRUCT) if self._object_hook is not None: ret = self._object_hook(ret) @@ -635,37 +688,30 @@ class Unpacker(object): except OutOfData: self._consume() raise StopIteration + except RecursionError: + raise StackError next = __next__ - def skip(self, write_bytes=None): + def skip(self): self._unpack(EX_SKIP) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() - def unpack(self, write_bytes=None): - ret = self._unpack(EX_CONSTRUCT) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) + def unpack(self): + try: + ret = self._unpack(EX_CONSTRUCT) + except RecursionError: + raise StackError self._consume() return ret - def read_array_header(self, write_bytes=None): + def read_array_header(self): ret = self._unpack(EX_READ_ARRAY_HEADER) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret - def read_map_header(self, write_bytes=None): + def read_map_header(self): ret = self._unpack(EX_READ_MAP_HEADER) - if write_bytes is not None: - warnings.warn("`write_bytes` option is deprecated. Use `.tell()` instead.", DeprecationWarning) - write_bytes(self._buffer[self._buf_checkpoint:self._buff_i]) self._consume() return ret @@ -722,7 +768,7 @@ class Packer(object): else: warnings.warn( "encoding is deprecated, Use raw=False instead.", - PendingDeprecationWarning) + DeprecationWarning, stacklevel=2) if unicode_errors is None: unicode_errors = 'strict' @@ -749,7 +795,7 @@ class Packer(object): list_types = (list, tuple) while True: if nest_limit < 0: - raise PackValueError("recursion limit exceeded") + raise ValueError("recursion limit exceeded") if obj is None: return self._buffer.write(b"\xc0") if check(obj, bool): @@ -781,14 +827,14 @@ class Packer(object): obj = self._default(obj) default_used = True continue - raise PackOverflowError("Integer value out of range") + raise OverflowError("Integer value out of range") if check(obj, (bytes, bytearray)): n = len(obj) if n >= 2**32: - raise PackValueError("%s is too large" % type(obj).__name__) + raise ValueError("%s is too large" % type(obj).__name__) self._pack_bin_header(n) return self._buffer.write(obj) - if check(obj, Unicode): + if check(obj, unicode): if self._encoding is None: raise TypeError( "Can't encode unicode string: " @@ -796,13 +842,13 @@ class Packer(object): obj = obj.encode(self._encoding, self._unicode_errors) n = len(obj) if n >= 2**32: - raise PackValueError("String is too large") + raise ValueError("String is too large") self._pack_raw_header(n) return self._buffer.write(obj) if check(obj, memoryview): n = len(obj) * obj.itemsize if n >= 2**32: - raise PackValueError("Memoryview is too large") + raise ValueError("Memoryview is too large") self._pack_bin_header(n) return self._buffer.write(obj) if check(obj, float): @@ -855,43 +901,35 @@ class Packer(object): except: self._buffer = StringIO() # force reset raise - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_map_pairs(self, pairs): self._pack_map_pairs(len(pairs), pairs) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_array_header(self, n): if n >= 2**32: - raise PackValueError + raise ValueError self._pack_array_header(n) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_map_header(self, n): if n >= 2**32: - raise PackValueError + raise ValueError self._pack_map_header(n) - ret = self._buffer.getvalue() if self._autoreset: + ret = self._buffer.getvalue() self._buffer = StringIO() - elif USING_STRINGBUILDER: - self._buffer = StringIO(ret) - return ret + return ret def pack_ext_type(self, typecode, data): if not isinstance(typecode, int): @@ -902,7 +940,7 @@ class Packer(object): raise TypeError("data must have bytes type") L = len(data) if L > 0xffffffff: - raise PackValueError("Too large data") + raise ValueError("Too large data") if L == 1: self._buffer.write(b'\xd4') elif L == 2: @@ -929,7 +967,7 @@ class Packer(object): return self._buffer.write(struct.pack(">BH", 0xdc, n)) if n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xdd, n)) - raise PackValueError("Array is too large") + raise ValueError("Array is too large") def _pack_map_header(self, n): if n <= 0x0f: @@ -938,7 +976,7 @@ class Packer(object): return self._buffer.write(struct.pack(">BH", 0xde, n)) if n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xdf, n)) - raise PackValueError("Dict is too large") + raise ValueError("Dict is too large") def _pack_map_pairs(self, n, pairs, nest_limit=DEFAULT_RECURSE_LIMIT): self._pack_map_header(n) @@ -956,7 +994,7 @@ class Packer(object): elif n <= 0xffffffff: self._buffer.write(struct.pack(">BI", 0xdb, n)) else: - raise PackValueError('Raw is too large') + raise ValueError('Raw is too large') def _pack_bin_header(self, n): if not self._use_bin_type: @@ -968,10 +1006,22 @@ class Packer(object): elif n <= 0xffffffff: return self._buffer.write(struct.pack(">BI", 0xc6, n)) else: - raise PackValueError('Bin is too large') + raise ValueError('Bin is too large') def bytes(self): + """Return internal buffer contents as bytes object""" return self._buffer.getvalue() def reset(self): + """Reset internal buffer. + + This method is usaful only when autoreset=False. + """ self._buffer = StringIO() + + def getbuffer(self): + """Return view of internal buffer.""" + if USING_STRINGBUILDER or PY2: + return memoryview(self.bytes()) + else: + return self._buffer.getbuffer() diff --git a/pipenv/patched/notpip/_vendor/packaging/__about__.py b/pipenv/patched/notpip/_vendor/packaging/__about__.py index 21fc6ce3..dc95138d 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__about__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "19.2" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/patched/notpip/_vendor/packaging/__init__.py b/pipenv/patched/notpip/_vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/patched/notpip/_vendor/packaging/__init__.py +++ b/pipenv/patched/notpip/_vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/patched/notpip/_vendor/packaging/_compat.py b/pipenv/patched/notpip/_vendor/packaging/_compat.py index 210bb80b..25da473c 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_compat.py +++ b/pipenv/patched/notpip/_vendor/packaging/_compat.py @@ -12,9 +12,9 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): @@ -27,4 +27,5 @@ def with_metaclass(meta, *bases): class metaclass(meta): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/patched/notpip/_vendor/packaging/_structures.py b/pipenv/patched/notpip/_vendor/packaging/_structures.py index e9fc4a04..68dcca63 100644 --- a/pipenv/patched/notpip/_vendor/packaging/_structures.py +++ b/pipenv/patched/notpip/_vendor/packaging/_structures.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function class Infinity(object): - def __repr__(self): return "Infinity" @@ -38,7 +37,6 @@ Infinity = Infinity() class NegativeInfinity(object): - def __repr__(self): return "-Infinity" diff --git a/pipenv/patched/notpip/_vendor/packaging/markers.py b/pipenv/patched/notpip/_vendor/packaging/markers.py index dc3cef81..aef30331 100644 --- a/pipenv/patched/notpip/_vendor/packaging/markers.py +++ b/pipenv/patched/notpip/_vendor/packaging/markers.py @@ -17,8 +17,11 @@ from .specifiers import Specifier, InvalidSpecifier __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,7 +45,6 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): self.value = value @@ -57,62 +59,52 @@ class Node(object): class Variable(Node): - def serialize(self): return str(self) class Value(Node): - def serialize(self): return '"{0}"'.format(self) class Op(Node): - def serialize(self): return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # PEP-345 + | L("extra") # undocumented setuptools legacy ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -152,8 +144,11 @@ def _format_marker(marker, first=True): # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -239,20 +234,20 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): + if hasattr(sys, "implementation"): iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -264,19 +259,19 @@ def default_environment(): "platform_version": platform.version(), "python_full_version": platform.python_version(), "platform_python_implementation": platform.python_implementation(), - "python_version": platform.python_version()[:3], + "python_version": ".".join(platform.python_version_tuple()[:2]), "sys_platform": sys.platform, } class Marker(object): - def __init__(self, marker): try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): diff --git a/pipenv/patched/notpip/_vendor/packaging/requirements.py b/pipenv/patched/notpip/_vendor/packaging/requirements.py index 5ec5d74a..a3de7673 100644 --- a/pipenv/patched/notpip/_vendor/packaging/requirements.py +++ b/pipenv/patched/notpip/_vendor/packaging/requirements.py @@ -38,8 +38,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +48,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +67,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -92,15 +92,21 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -120,6 +126,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) diff --git a/pipenv/patched/notpip/_vendor/packaging/specifiers.py b/pipenv/patched/notpip/_vendor/packaging/specifiers.py index 4c798999..743576a0 100644 --- a/pipenv/patched/notpip/_vendor/packaging/specifiers.py +++ b/pipenv/patched/notpip/_vendor/packaging/specifiers.py @@ -19,7 +19,6 @@ class InvalidSpecifier(ValueError): class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - @abc.abstractmethod def __str__(self): """ @@ -84,10 +83,7 @@ class _IndividualSpecifier(BaseSpecifier): if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) - self._spec = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + self._spec = (match.group("operator").strip(), match.group("version").strip()) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -99,11 +95,7 @@ class _IndividualSpecifier(BaseSpecifier): else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): return "{0}{1}".format(*self._spec) @@ -194,8 +186,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +206,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P<operator>(==|!=|<=|>=|<|>)) \s* (?P<version> @@ -225,10 +217,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -269,13 +259,13 @@ def _require_version_compare(fn): if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P<operator>(~=|==|!=|<=|>=|<|>|===)) (?P<version> (?: @@ -367,10 +357,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -397,8 +385,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,8 +394,9 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): @@ -428,7 +416,7 @@ class Specifier(_IndividualSpecifier): # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + prospective = prospective[: len(spec)] # Pad out our two sides with zeros so that they both equal the same # length. @@ -567,27 +555,17 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): # Split on , to break each indidivual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. @@ -721,10 +699,7 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing diff --git a/pipenv/patched/notpip/_vendor/packaging/tags.py b/pipenv/patched/notpip/_vendor/packaging/tags.py new file mode 100644 index 00000000..ec9942f0 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/packaging/tags.py @@ -0,0 +1,404 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import + +import distutils.util + +try: + from importlib.machinery import EXTENSION_SUFFIXES +except ImportError: # pragma: no cover + import imp + + EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()] + del imp +import platform +import re +import sys +import sysconfig +import warnings + + +INTERPRETER_SHORT_NAMES = { + "python": "py", # Generic. + "cpython": "cp", + "pypy": "pp", + "ironpython": "ip", + "jython": "jy", +} + + +_32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32 + + +class Tag(object): + + __slots__ = ["_interpreter", "_abi", "_platform"] + + def __init__(self, interpreter, abi, platform): + self._interpreter = interpreter.lower() + self._abi = abi.lower() + self._platform = platform.lower() + + @property + def interpreter(self): + return self._interpreter + + @property + def abi(self): + return self._abi + + @property + def platform(self): + return self._platform + + def __eq__(self, other): + return ( + (self.platform == other.platform) + and (self.abi == other.abi) + and (self.interpreter == other.interpreter) + ) + + def __hash__(self): + return hash((self._interpreter, self._abi, self._platform)) + + def __str__(self): + return "{}-{}-{}".format(self._interpreter, self._abi, self._platform) + + def __repr__(self): + return "<{self} @ {self_id}>".format(self=self, self_id=id(self)) + + +def parse_tag(tag): + tags = set() + interpreters, abis, platforms = tag.split("-") + for interpreter in interpreters.split("."): + for abi in abis.split("."): + for platform_ in platforms.split("."): + tags.add(Tag(interpreter, abi, platform_)) + return frozenset(tags) + + +def _normalize_string(string): + return string.replace(".", "_").replace("-", "_") + + +def _cpython_interpreter(py_version): + # TODO: Is using py_version_nodot for interpreter version critical? + return "cp{major}{minor}".format(major=py_version[0], minor=py_version[1]) + + +def _cpython_abis(py_version): + abis = [] + version = "{}{}".format(*py_version[:2]) + debug = pymalloc = ucs4 = "" + with_debug = sysconfig.get_config_var("Py_DEBUG") + has_refcount = hasattr(sys, "gettotalrefcount") + # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled + # extension modules is the best option. + # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 + has_ext = "_d.pyd" in EXTENSION_SUFFIXES + if with_debug or (with_debug is None and (has_refcount or has_ext)): + debug = "d" + if py_version < (3, 8): + with_pymalloc = sysconfig.get_config_var("WITH_PYMALLOC") + if with_pymalloc or with_pymalloc is None: + pymalloc = "m" + if py_version < (3, 3): + unicode_size = sysconfig.get_config_var("Py_UNICODE_SIZE") + if unicode_size == 4 or ( + unicode_size is None and sys.maxunicode == 0x10FFFF + ): + ucs4 = "u" + elif debug: + # Debug builds can also load "normal" extension modules. + # We can also assume no UCS-4 or pymalloc requirement. + abis.append("cp{version}".format(version=version)) + abis.insert( + 0, + "cp{version}{debug}{pymalloc}{ucs4}".format( + version=version, debug=debug, pymalloc=pymalloc, ucs4=ucs4 + ), + ) + return abis + + +def _cpython_tags(py_version, interpreter, abis, platforms): + for abi in abis: + for platform_ in platforms: + yield Tag(interpreter, abi, platform_) + for tag in (Tag(interpreter, "abi3", platform_) for platform_ in platforms): + yield tag + for tag in (Tag(interpreter, "none", platform_) for platform_ in platforms): + yield tag + # PEP 384 was first implemented in Python 3.2. + for minor_version in range(py_version[1] - 1, 1, -1): + for platform_ in platforms: + interpreter = "cp{major}{minor}".format( + major=py_version[0], minor=minor_version + ) + yield Tag(interpreter, "abi3", platform_) + + +def _pypy_interpreter(): + return "pp{py_major}{pypy_major}{pypy_minor}".format( + py_major=sys.version_info[0], + pypy_major=sys.pypy_version_info.major, + pypy_minor=sys.pypy_version_info.minor, + ) + + +def _generic_abi(): + abi = sysconfig.get_config_var("SOABI") + if abi: + return _normalize_string(abi) + else: + return "none" + + +def _pypy_tags(py_version, interpreter, abi, platforms): + for tag in (Tag(interpreter, abi, platform) for platform in platforms): + yield tag + for tag in (Tag(interpreter, "none", platform) for platform in platforms): + yield tag + + +def _generic_tags(interpreter, py_version, abi, platforms): + for tag in (Tag(interpreter, abi, platform) for platform in platforms): + yield tag + if abi != "none": + tags = (Tag(interpreter, "none", platform_) for platform_ in platforms) + for tag in tags: + yield tag + + +def _py_interpreter_range(py_version): + """ + Yield Python versions in descending order. + + After the latest version, the major-only version will be yielded, and then + all following versions up to 'end'. + """ + yield "py{major}{minor}".format(major=py_version[0], minor=py_version[1]) + yield "py{major}".format(major=py_version[0]) + for minor in range(py_version[1] - 1, -1, -1): + yield "py{major}{minor}".format(major=py_version[0], minor=minor) + + +def _independent_tags(interpreter, py_version, platforms): + """ + Return the sequence of tags that are consistent across implementations. + + The tags consist of: + - py*-none-<platform> + - <interpreter>-none-any + - py*-none-any + """ + for version in _py_interpreter_range(py_version): + for platform_ in platforms: + yield Tag(version, "none", platform_) + yield Tag(interpreter, "none", "any") + for version in _py_interpreter_range(py_version): + yield Tag(version, "none", "any") + + +def _mac_arch(arch, is_32bit=_32_BIT_INTERPRETER): + if not is_32bit: + return arch + + if arch.startswith("ppc"): + return "ppc" + + return "i386" + + +def _mac_binary_formats(version, cpu_arch): + formats = [cpu_arch] + if cpu_arch == "x86_64": + if version < (10, 4): + return [] + formats.extend(["intel", "fat64", "fat32"]) + + elif cpu_arch == "i386": + if version < (10, 4): + return [] + formats.extend(["intel", "fat32", "fat"]) + + elif cpu_arch == "ppc64": + # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? + if version > (10, 5) or version < (10, 4): + return [] + formats.append("fat64") + + elif cpu_arch == "ppc": + if version > (10, 6): + return [] + formats.extend(["fat32", "fat"]) + + formats.append("universal") + return formats + + +def _mac_platforms(version=None, arch=None): + version_str, _, cpu_arch = platform.mac_ver() + if version is None: + version = tuple(map(int, version_str.split(".")[:2])) + if arch is None: + arch = _mac_arch(cpu_arch) + platforms = [] + for minor_version in range(version[1], -1, -1): + compat_version = version[0], minor_version + binary_formats = _mac_binary_formats(compat_version, arch) + for binary_format in binary_formats: + platforms.append( + "macosx_{major}_{minor}_{binary_format}".format( + major=compat_version[0], + minor=compat_version[1], + binary_format=binary_format, + ) + ) + return platforms + + +# From PEP 513. +def _is_manylinux_compatible(name, glibc_version): + # Check for presence of _manylinux module. + try: + import _manylinux + + return bool(getattr(_manylinux, name + "_compatible")) + except (ImportError, AttributeError): + # Fall through to heuristic check below. + pass + + return _have_compatible_glibc(*glibc_version) + + +def _glibc_version_string(): + # Returns glibc version string, or None if not using glibc. + import ctypes + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + process_namespace = ctypes.CDLL(None) + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +# Separated out from have_compatible_glibc for easier unit testing. +def _check_glibc_version(version_str, required_major, minimum_minor): + # Parse string and check against requested version. + # + # We use a regexp instead of str.split because we want to discard any + # random junk that might come after the minor version -- this might happen + # in patched/forked versions of glibc (e.g. Linaro's version of glibc + # uses version strings like "2.20-2014.11"). See gh-3588. + m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) + if not m: + warnings.warn( + "Expected glibc version with 2 components major.minor," + " got: %s" % version_str, + RuntimeWarning, + ) + return False + return ( + int(m.group("major")) == required_major + and int(m.group("minor")) >= minimum_minor + ) + + +def _have_compatible_glibc(required_major, minimum_minor): + version_str = _glibc_version_string() + if version_str is None: + return False + return _check_glibc_version(version_str, required_major, minimum_minor) + + +def _linux_platforms(is_32bit=_32_BIT_INTERPRETER): + linux = _normalize_string(distutils.util.get_platform()) + if linux == "linux_x86_64" and is_32bit: + linux = "linux_i686" + manylinux_support = ( + ("manylinux2014", (2, 17)), # CentOS 7 w/ glibc 2.17 (PEP 599) + ("manylinux2010", (2, 12)), # CentOS 6 w/ glibc 2.12 (PEP 571) + ("manylinux1", (2, 5)), # CentOS 5 w/ glibc 2.5 (PEP 513) + ) + manylinux_support_iter = iter(manylinux_support) + for name, glibc_version in manylinux_support_iter: + if _is_manylinux_compatible(name, glibc_version): + platforms = [linux.replace("linux", name)] + break + else: + platforms = [] + # Support for a later manylinux implies support for an earlier version. + platforms += [linux.replace("linux", name) for name, _ in manylinux_support_iter] + platforms.append(linux) + return platforms + + +def _generic_platforms(): + platform = _normalize_string(distutils.util.get_platform()) + return [platform] + + +def _interpreter_name(): + name = platform.python_implementation().lower() + return INTERPRETER_SHORT_NAMES.get(name) or name + + +def _generic_interpreter(name, py_version): + version = sysconfig.get_config_var("py_version_nodot") + if not version: + version = "".join(map(str, py_version[:2])) + return "{name}{version}".format(name=name, version=version) + + +def sys_tags(): + """ + Returns the sequence of tag triples for the running interpreter. + + The order of the sequence corresponds to priority order for the + interpreter, from most to least important. + """ + py_version = sys.version_info[:2] + interpreter_name = _interpreter_name() + if platform.system() == "Darwin": + platforms = _mac_platforms() + elif platform.system() == "Linux": + platforms = _linux_platforms() + else: + platforms = _generic_platforms() + + if interpreter_name == "cp": + interpreter = _cpython_interpreter(py_version) + abis = _cpython_abis(py_version) + for tag in _cpython_tags(py_version, interpreter, abis, platforms): + yield tag + elif interpreter_name == "pp": + interpreter = _pypy_interpreter() + abi = _generic_abi() + for tag in _pypy_tags(py_version, interpreter, abi, platforms): + yield tag + else: + interpreter = _generic_interpreter(interpreter_name, py_version) + abi = _generic_abi() + for tag in _generic_tags(interpreter, py_version, abi, platforms): + yield tag + for tag in _independent_tags(interpreter, py_version, platforms): + yield tag diff --git a/pipenv/patched/notpip/_vendor/packaging/utils.py b/pipenv/patched/notpip/_vendor/packaging/utils.py index 4b94a82f..88418786 100644 --- a/pipenv/patched/notpip/_vendor/packaging/utils.py +++ b/pipenv/patched/notpip/_vendor/packaging/utils.py @@ -36,13 +36,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/patched/notpip/_vendor/packaging/version.py b/pipenv/patched/notpip/_vendor/packaging/version.py index 6ed5cbbd..95157a1f 100644 --- a/pipenv/patched/notpip/_vendor/packaging/version.py +++ b/pipenv/patched/notpip/_vendor/packaging/version.py @@ -10,14 +10,11 @@ import re from ._structures import Infinity -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) @@ -40,7 +37,6 @@ class InvalidVersion(ValueError): class _BaseVersion(object): - def __hash__(self): return hash(self._key) @@ -70,7 +66,6 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): self._version = str(version) self._key = _legacy_cmpkey(self._version) @@ -126,12 +121,14 @@ class LegacyVersion(_BaseVersion): return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } @@ -215,10 +212,7 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): # Validate the version and parse it into pieces @@ -230,18 +224,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -395,12 +382,7 @@ def _cmpkey(epoch, release, pre, post, dev, local): # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -433,9 +415,6 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local - ) + local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local) return epoch, release, pre, post, dev, local diff --git a/pipenv/patched/notpip/_vendor/pep517/__init__.py b/pipenv/patched/notpip/_vendor/pep517/__init__.py index 8beedea4..38d8e63c 100644 --- a/pipenv/patched/notpip/_vendor/pep517/__init__.py +++ b/pipenv/patched/notpip/_vendor/pep517/__init__.py @@ -1,4 +1,4 @@ """Wrappers to build Python packages using PEP 517 hooks """ -__version__ = '0.2' +__version__ = '0.7.0' diff --git a/pipenv/patched/notpip/_vendor/pep517/_in_process.py b/pipenv/patched/notpip/_vendor/pep517/_in_process.py index baa14d38..1589a6ca 100644 --- a/pipenv/patched/notpip/_vendor/pep517/_in_process.py +++ b/pipenv/patched/notpip/_vendor/pep517/_in_process.py @@ -2,7 +2,9 @@ It expects: - Command line args: hook_name, control_dir -- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- Environment variables: + PEP517_BUILD_BACKEND=entry.point:spec + PEP517_BACKEND_PATH=paths (separated with os.pathsep) - control_dir/input.json: - {"kwargs": {...}} @@ -13,27 +15,71 @@ Results: from glob import glob from importlib import import_module import os +import os.path from os.path import join as pjoin import re import shutil import sys +import traceback # This is run as a script, not a module, so it can't do a relative import import compat + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + def __init__(self, traceback): + self.traceback = traceback + + +class BackendInvalid(Exception): + """Raised if the backend is invalid""" + def __init__(self, message): + self.message = message + + +class HookMissing(Exception): + """Raised if a hook is missing and we are not executing the fallback""" + + +def contained_in(filename, directory): + """Test if a file is located within the given directory.""" + filename = os.path.normcase(os.path.abspath(filename)) + directory = os.path.normcase(os.path.abspath(directory)) + return os.path.commonprefix([filename, directory]) == directory + + def _build_backend(): """Find and load the build backend""" + # Add in-tree backend directories to the front of sys.path. + backend_path = os.environ.get('PEP517_BACKEND_PATH') + if backend_path: + extra_pathitems = backend_path.split(os.pathsep) + sys.path[:0] = extra_pathitems + ep = os.environ['PEP517_BUILD_BACKEND'] mod_path, _, obj_path = ep.partition(':') - obj = import_module(mod_path) + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable(traceback.format_exc()) + + if backend_path: + if not any( + contained_in(obj.__file__, path) + for path in extra_pathitems + ): + raise BackendInvalid("Backend was not loaded from backend-path") + if obj_path: for path_part in obj_path.split('.'): obj = getattr(obj, path_part) return obj + def get_requires_for_build_wheel(config_settings): """Invoke the optional get_requires_for_build_wheel hook - + Returns [] if the hook is not defined. """ backend = _build_backend() @@ -44,22 +90,29 @@ def get_requires_for_build_wheel(config_settings): else: return hook(config_settings) -def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + +def prepare_metadata_for_build_wheel( + metadata_directory, config_settings, _allow_fallback): """Invoke optional prepare_metadata_for_build_wheel - - Implements a fallback by building a wheel if the hook isn't defined. + + Implements a fallback by building a wheel if the hook isn't defined, + unless _allow_fallback is False in which case HookMissing is raised. """ backend = _build_backend() try: hook = backend.prepare_metadata_for_build_wheel except AttributeError: + if not _allow_fallback: + raise HookMissing() return _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings) else: return hook(metadata_directory, config_settings) + WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + def _dist_info_files(whl_zip): """Identify the .dist-info folder inside a wheel ZipFile.""" res = [] @@ -71,11 +124,13 @@ def _dist_info_files(whl_zip): return res raise Exception("No .dist-info folder found in wheel") -def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings): + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): """Build a wheel and extract the metadata from it. - - Fallback for when the build backend does not define the 'get_wheel_metadata' - hook. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. """ from zipfile import ZipFile whl_basename = backend.build_wheel(metadata_directory, config_settings) @@ -88,6 +143,7 @@ def _get_wheel_metadata_from_wheel(backend, metadata_directory, config_settings) zipf.extractall(path=metadata_directory, members=dist_info) return dist_info[0].split('/')[0] + def _find_already_built_wheel(metadata_directory): """Check for a wheel already built during the get_wheel_metadata hook. """ @@ -105,14 +161,16 @@ def _find_already_built_wheel(metadata_directory): print('Found multiple .whl files; unspecified behaviour. ' 'Will call build_wheel.') return None - + # Exactly one .whl file return whl_files[0] + def build_wheel(wheel_directory, config_settings, metadata_directory=None): """Invoke the mandatory build_wheel hook. - - If a wheel was already built in the prepare_metadata_for_build_wheel fallback, this + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this will copy it rather than rebuilding the wheel. """ prebuilt_whl = _find_already_built_wheel(metadata_directory) @@ -137,11 +195,16 @@ def get_requires_for_build_sdist(config_settings): else: return hook(config_settings) + class _DummyException(Exception): """Nothing should ever raise this exception""" + class GotUnsupportedOperation(Exception): """For internal use when backend raises UnsupportedOperation""" + def __init__(self, traceback): + self.traceback = traceback + def build_sdist(sdist_directory, config_settings): """Invoke the mandatory build_sdist hook.""" @@ -149,7 +212,8 @@ def build_sdist(sdist_directory, config_settings): try: return backend.build_sdist(sdist_directory, config_settings) except getattr(backend, 'UnsupportedOperation', _DummyException): - raise GotUnsupportedOperation + raise GotUnsupportedOperation(traceback.format_exc()) + HOOK_NAMES = { 'get_requires_for_build_wheel', @@ -159,6 +223,7 @@ HOOK_NAMES = { 'build_sdist', } + def main(): if len(sys.argv) < 3: sys.exit("Needs args: hook_name, control_dir") @@ -173,10 +238,20 @@ def main(): json_out = {'unsupported': False, 'return_val': None} try: json_out['return_val'] = hook(**hook_input['kwargs']) - except GotUnsupportedOperation: + except BackendUnavailable as e: + json_out['no_backend'] = True + json_out['traceback'] = e.traceback + except BackendInvalid as e: + json_out['backend_invalid'] = True + json_out['backend_error'] = e.message + except GotUnsupportedOperation as e: json_out['unsupported'] = True - + json_out['traceback'] = e.traceback + except HookMissing: + json_out['hook_missing'] = True + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/build.py b/pipenv/patched/notpip/_vendor/pep517/build.py new file mode 100644 index 00000000..7618c78c --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/build.py @@ -0,0 +1,124 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import toml +import shutil + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller +from .dirtools import tempdir, mkdir_p +from .compat import FileNotFoundError + +log = logging.getLogger(__name__) + + +def validate_system(system): + """ + Ensure build system has the requisite fields. + """ + required = {'requires', 'build-backend'} + if not (required <= set(system)): + message = "Missing required fields: {missing}".format( + missing=required-set(system), + ) + raise ValueError(message) + + +def load_system(source_dir): + """ + Load the build system from a source dir (pyproject.toml). + """ + pyproject = os.path.join(source_dir, 'pyproject.toml') + with open(pyproject) as f: + pyproject_data = toml.load(f) + return pyproject_data['build-system'] + + +def compat_system(source_dir): + """ + Given a source dir, attempt to get a build system backend + and requirements from pyproject.toml. Fallback to + setuptools but only if the file was not found or a build + system was not indicated. + """ + try: + system = load_system(source_dir) + except (FileNotFoundError, KeyError): + system = {} + system.setdefault( + 'build-backend', + 'setuptools.build_meta:__legacy__', + ) + system.setdefault('requires', ['setuptools', 'wheel']) + return system + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def build(source_dir, dist, dest=None, system=None): + system = system or load_system(source_dir) + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + validate_system(system) + hooks = Pep517HookCaller( + source_dir, system['build-backend'], system.get('backend-path') + ) + + with BuildEnvironment() as env: + env.pip_install(system['requires']) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/patched/notpip/_vendor/pep517/check.py b/pipenv/patched/notpip/_vendor/pep517/check.py index 3dffd2e0..9e0c0682 100644 --- a/pipenv/patched/notpip/_vendor/pep517/check.py +++ b/pipenv/patched/notpip/_vendor/pep517/check.py @@ -4,7 +4,7 @@ import argparse import logging import os from os.path import isfile, join as pjoin -from pipenv.patched.notpip._vendor.pytoml import TomlError, load as toml_load +from toml import TomlDecodeError, load as toml_load import shutil from subprocess import CalledProcessError import sys @@ -18,10 +18,11 @@ from .wrappers import Pep517HookCaller log = logging.getLogger(__name__) -def check_build_sdist(hooks): + +def check_build_sdist(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -30,7 +31,7 @@ def check_build_sdist(hooks): try: reqs = hooks.get_requires_for_build_sdist({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -47,12 +48,13 @@ def check_build_sdist(hooks): try: filename = hooks.build_sdist(td, {}) log.info('build_sdist returned %r', filename) - except: + except Exception: log.info('Failure in build_sdist', exc_info=True) return False if not filename.endswith('.tar.gz'): - log.error("Filename %s doesn't have .tar.gz extension", filename) + log.error( + "Filename %s doesn't have .tar.gz extension", filename) return False path = pjoin(td, filename) @@ -73,10 +75,11 @@ def check_build_sdist(hooks): return True -def check_build_wheel(hooks): + +def check_build_wheel(hooks, build_sys_requires): with BuildEnvironment() as env: try: - env.pip_install(hooks.build_sys_requires) + env.pip_install(build_sys_requires) log.info('Installed static build dependencies') except CalledProcessError: log.error('Failed to install static build dependencies') @@ -85,7 +88,7 @@ def check_build_wheel(hooks): try: reqs = hooks.get_requires_for_build_wheel({}) log.info('Got build requires: %s', reqs) - except: + except Exception: log.error('Failure in get_requires_for_build_sdist', exc_info=True) return False @@ -102,7 +105,7 @@ def check_build_wheel(hooks): try: filename = hooks.build_wheel(td, {}) log.info('build_wheel returned %r', filename) - except: + except Exception: log.info('Failure in build_wheel', exc_info=True) return False @@ -144,15 +147,16 @@ def check(source_dir): buildsys = pyproject_data['build-system'] requires = buildsys['requires'] backend = buildsys['build-backend'] + backend_path = buildsys.get('backend-path') log.info('Loaded pyproject.toml') - except (TomlError, KeyError): + except (TomlDecodeError, KeyError): log.error("Invalid pyproject.toml", exc_info=True) return False - hooks = Pep517HookCaller(source_dir, backend) + hooks = Pep517HookCaller(source_dir, backend, backend_path) - sdist_ok = check_build_sdist(hooks) - wheel_ok = check_build_wheel(hooks) + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) if not sdist_ok: log.warning('Sdist checks failed; scroll up to see') @@ -164,7 +168,8 @@ def check(source_dir): def main(argv=None): ap = argparse.ArgumentParser() - ap.add_argument('source_dir', + ap.add_argument( + 'source_dir', help="A directory containing pyproject.toml") args = ap.parse_args(argv) @@ -178,17 +183,21 @@ def main(argv=None): print(ansi('Checks failed', 'red')) sys.exit(1) + ansi_codes = { 'reset': '\x1b[0m', 'bold': '\x1b[1m', 'red': '\x1b[31m', 'green': '\x1b[32m', } + + def ansi(s, attr): if os.name != 'nt' and sys.stdout.isatty(): return ansi_codes[attr] + str(s) + ansi_codes['reset'] else: return str(s) + if __name__ == '__main__': main() diff --git a/pipenv/patched/notpip/_vendor/pep517/colorlog.py b/pipenv/patched/notpip/_vendor/pep517/colorlog.py index 26cf7480..69c8a59d 100644 --- a/pipenv/patched/notpip/_vendor/pep517/colorlog.py +++ b/pipenv/patched/notpip/_vendor/pep517/colorlog.py @@ -24,6 +24,7 @@ try: except ImportError: curses = None + def _stderr_supports_color(): color = False if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): @@ -35,13 +36,14 @@ def _stderr_supports_color(): pass return color + class LogFormatter(logging.Formatter): """Log formatter with colour support """ DEFAULT_COLORS = { - logging.INFO: 2, # Green - logging.WARNING: 3, # Yellow - logging.ERROR: 1, # Red + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red logging.CRITICAL: 1, } @@ -75,7 +77,8 @@ class LogFormatter(logging.Formatter): fg_color = str(fg_color, "ascii") for levelno, code in self.DEFAULT_COLORS.items(): - self._colors[levelno] = str(curses.tparm(fg_color, code), "ascii") + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") self._normal = str(curses.tigetstr("sgr0"), "ascii") scr = curses.initscr() @@ -83,15 +86,16 @@ class LogFormatter(logging.Formatter): curses.endwin() else: self._normal = '' - # Default width is usually 80, but too wide is worse than too narrow + # Default width is usually 80, but too wide is + # worse than too narrow self.termwidth = 70 def formatMessage(self, record): - l = len(record.message) + mlen = len(record.message) right_text = '{initial}-{name}'.format(initial=record.levelname[0], name=record.name) - if l + len(right_text) < self.termwidth: - space = ' ' * (self.termwidth - (l + len(right_text))) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) else: space = ' ' @@ -103,6 +107,7 @@ class LogFormatter(logging.Formatter): return record.message + space + start_color + right_text + end_color + def enable_colourful_output(level=logging.INFO): handler = logging.StreamHandler() handler.setFormatter(LogFormatter()) diff --git a/pipenv/patched/notpip/_vendor/pep517/compat.py b/pipenv/patched/notpip/_vendor/pep517/compat.py index 01c66fc7..8432acb7 100644 --- a/pipenv/patched/notpip/_vendor/pep517/compat.py +++ b/pipenv/patched/notpip/_vendor/pep517/compat.py @@ -1,7 +1,10 @@ -"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +"""Python 2/3 compatibility""" import json import sys + +# Handle reading and writing JSON in UTF-8, on Python 3 and 2. + if sys.version_info[0] >= 3: # Python 3 def write_json(obj, path, **kwargs): @@ -21,3 +24,11 @@ else: def read_json(path): with open(path, 'rb') as f: return json.load(f) + + +# FileNotFoundError + +try: + FileNotFoundError = FileNotFoundError +except NameError: + FileNotFoundError = IOError diff --git a/pipenv/patched/notpip/_vendor/pep517/dirtools.py b/pipenv/patched/notpip/_vendor/pep517/dirtools.py new file mode 100644 index 00000000..58c6ca0c --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/dirtools.py @@ -0,0 +1,44 @@ +import os +import io +import contextlib +import tempfile +import shutil +import errno +import zipfile + + +@contextlib.contextmanager +def tempdir(): + """Create a temporary directory in a context manager.""" + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def dir_to_zipfile(root): + """Construct an in-memory zip file for a directory.""" + buffer = io.BytesIO() + zip_file = zipfile.ZipFile(buffer, 'w') + for root, dirs, files in os.walk(root): + for path in dirs: + fs_path = os.path.join(root, path) + rel_path = os.path.relpath(fs_path, root) + zip_file.writestr(rel_path + '/', '') + for path in files: + fs_path = os.path.join(root, path) + rel_path = os.path.relpath(fs_path, root) + zip_file.write(fs_path, rel_path) + return zip_file diff --git a/pipenv/patched/notpip/_vendor/pep517/envbuild.py b/pipenv/patched/notpip/_vendor/pep517/envbuild.py index c54d3585..cacd2b12 100644 --- a/pipenv/patched/notpip/_vendor/pep517/envbuild.py +++ b/pipenv/patched/notpip/_vendor/pep517/envbuild.py @@ -3,22 +3,27 @@ import os import logging -from pipenv.patched.notpip._vendor import pytoml +import toml import shutil from subprocess import check_call import sys from sysconfig import get_paths from tempfile import mkdtemp -from .wrappers import Pep517HookCaller +from .wrappers import Pep517HookCaller, LoggerWrapper log = logging.getLogger(__name__) + def _load_pyproject(source_dir): with open(os.path.join(source_dir, 'pyproject.toml')) as f: - pyproject_data = pytoml.load(f) + pyproject_data = toml.load(f) buildsys = pyproject_data['build-system'] - return buildsys['requires'], buildsys['build-backend'] + return ( + buildsys['requires'], + buildsys['build-backend'], + buildsys.get('backend-path'), + ) class BuildEnvironment(object): @@ -89,11 +94,22 @@ class BuildEnvironment(object): if not reqs: return log.info('Calling pip to install %s', reqs) - check_call([sys.executable, '-m', 'pip', 'install', '--ignore-installed', - '--prefix', self.path] + list(reqs)) + cmd = [ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs) + check_call( + cmd, + stdout=LoggerWrapper(log, logging.INFO), + stderr=LoggerWrapper(log, logging.ERROR), + ) def __exit__(self, exc_type, exc_val, exc_tb): - if self._cleanup and (self.path is not None) and os.path.isdir(self.path): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: shutil.rmtree(self.path) if self.save_path is None: @@ -106,6 +122,7 @@ class BuildEnvironment(object): else: os.environ['PYTHONPATH'] = self.save_pythonpath + def build_wheel(source_dir, wheel_dir, config_settings=None): """Build a wheel from a source directory using PEP 517 hooks. @@ -118,8 +135,8 @@ def build_wheel(source_dir, wheel_dir, config_settings=None): """ if config_settings is None: config_settings = {} - requires, backend = _load_pyproject(source_dir) - hooks = Pep517HookCaller(source_dir, backend) + requires, backend, backend_path = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend, backend_path) with BuildEnvironment() as env: env.pip_install(requires) @@ -140,8 +157,8 @@ def build_sdist(source_dir, sdist_dir, config_settings=None): """ if config_settings is None: config_settings = {} - requires, backend = _load_pyproject(source_dir) - hooks = Pep517HookCaller(source_dir, backend) + requires, backend, backend_path = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend, backend_path) with BuildEnvironment() as env: env.pip_install(requires) diff --git a/pipenv/patched/notpip/_vendor/pep517/meta.py b/pipenv/patched/notpip/_vendor/pep517/meta.py new file mode 100644 index 00000000..d525de5c --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pep517/meta.py @@ -0,0 +1,92 @@ +"""Build metadata for a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import shutil +import functools + +try: + import importlib.metadata as imp_meta +except ImportError: + import importlib_metadata as imp_meta + +try: + from zipfile import Path +except ImportError: + from zipp import Path + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller, quiet_subprocess_runner +from .dirtools import tempdir, mkdir_p, dir_to_zipfile +from .build import validate_system, load_system, compat_system + +log = logging.getLogger(__name__) + + +def _prep_meta(hooks, env, dest): + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build metadata in %s', td) + filename = hooks.prepare_metadata_for_build_wheel(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def build(source_dir='.', dest=None, system=None): + system = system or load_system(source_dir) + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + validate_system(system) + hooks = Pep517HookCaller( + source_dir, system['build-backend'], system.get('backend-path') + ) + + with hooks.subprocess_runner(quiet_subprocess_runner): + with BuildEnvironment() as env: + env.pip_install(system['requires']) + _prep_meta(hooks, env, dest) + + +def build_as_zip(builder=build): + with tempdir() as out_dir: + builder(dest=out_dir) + return dir_to_zipfile(out_dir) + + +def load(root): + """ + Given a source directory (root) of a package, + return an importlib.metadata.Distribution object + with metadata build from that package. + """ + root = os.path.expanduser(root) + system = compat_system(root) + builder = functools.partial(build, source_dir=root, system=system) + path = Path(build_as_zip(builder)) + return imp_meta.PathDistribution(path) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(): + args = parser.parse_args() + build(args.source_dir, args.out_dir) + + +if __name__ == '__main__': + main() diff --git a/pipenv/patched/notpip/_vendor/pep517/wrappers.py b/pipenv/patched/notpip/_vendor/pep517/wrappers.py index 28260f32..ad9a4f8c 100644 --- a/pipenv/patched/notpip/_vendor/pep517/wrappers.py +++ b/pipenv/patched/notpip/_vendor/pep517/wrappers.py @@ -1,8 +1,9 @@ +import threading from contextlib import contextmanager import os from os.path import dirname, abspath, join as pjoin import shutil -from subprocess import check_call +from subprocess import check_call, check_output, STDOUT import sys from tempfile import mkdtemp @@ -10,6 +11,7 @@ from . import compat _in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + @contextmanager def tempdir(): td = mkdtemp() @@ -18,18 +20,123 @@ def tempdir(): finally: shutil.rmtree(td) + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + def __init__(self, traceback): + self.traceback = traceback + + +class BackendInvalid(Exception): + """Will be raised if the backend is invalid.""" + def __init__(self, backend_name, backend_path, message): + self.backend_name = backend_name + self.backend_path = backend_path + self.message = message + + +class HookMissing(Exception): + """Will be raised on missing hooks.""" + def __init__(self, hook_name): + super(HookMissing, self).__init__(hook_name) + self.hook_name = hook_name + + class UnsupportedOperation(Exception): """May be raised by build_sdist if the backend indicates that it can't.""" + def __init__(self, traceback): + self.traceback = traceback + + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + +def quiet_subprocess_runner(cmd, cwd=None, extra_environ=None): + """A method of calling the wrapper subprocess while suppressing output.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_output(cmd, cwd=cwd, env=env, stderr=STDOUT) + + +def norm_and_check(source_tree, requested): + """Normalise and check a backend path. + + Ensure that the requested backend path is specified as a relative path, + and resolves to a location under the given source tree. + + Return an absolute version of the requested path. + """ + if os.path.isabs(requested): + raise ValueError("paths must be relative") + + abs_source = os.path.abspath(source_tree) + abs_requested = os.path.normpath(os.path.join(abs_source, requested)) + # We have to use commonprefix for Python 2.7 compatibility. So we + # normalise case to avoid problems because commonprefix is a character + # based comparison :-( + norm_source = os.path.normcase(abs_source) + norm_requested = os.path.normcase(abs_requested) + if os.path.commonprefix([norm_source, norm_requested]) != norm_source: + raise ValueError("paths must be inside source tree") + + return abs_requested + class Pep517HookCaller(object): """A wrapper around a source directory to be built with a PEP 517 backend. source_dir : The path to the source directory, containing pyproject.toml. - backend : The build backend spec, as per PEP 517, from pyproject.toml. + build_backend : The build backend spec, as per PEP 517, from + pyproject.toml. + backend_path : The backend path, as per PEP 517, from pyproject.toml. + runner : A callable that invokes the wrapper subprocess. + + The 'runner', if provided, must expect the following: + cmd : a list of strings representing the command and arguments to + execute, as would be passed to e.g. 'subprocess.check_call'. + cwd : a string representing the working directory that must be + used for the subprocess. Corresponds to the provided source_dir. + extra_environ : a dict mapping environment variable names to values + which must be set for the subprocess execution. """ - def __init__(self, source_dir, build_backend): + def __init__( + self, + source_dir, + build_backend, + backend_path=None, + runner=None, + ): + if runner is None: + runner = default_subprocess_runner + self.source_dir = abspath(source_dir) self.build_backend = build_backend + if backend_path: + backend_path = [ + norm_and_check(self.source_dir, p) for p in backend_path + ] + self.backend_path = backend_path + self._subprocess_runner = runner + + # TODO: Is this over-engineered? Maybe frontends only need to + # set this when creating the wrapper, not on every call. + @contextmanager + def subprocess_runner(self, runner): + """A context manager for temporarily overriding the default subprocess + runner. + """ + prev = self._subprocess_runner + self._subprocess_runner = runner + yield + self._subprocess_runner = prev def get_requires_for_build_wheel(self, config_settings=None): """Identify packages required for building a wheel @@ -45,21 +152,27 @@ class Pep517HookCaller(object): 'config_settings': config_settings }) - def prepare_metadata_for_build_wheel(self, metadata_directory, config_settings=None): + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None, + _allow_fallback=True): """Prepare a *.dist-info folder with metadata for this project. Returns the name of the newly created folder. If the build backend defines a hook with this name, it will be called in a subprocess. If not, the backend will be asked to build a wheel, - and the dist-info extracted from that. + and the dist-info extracted from that (unless _allow_fallback is + False). """ return self._call_hook('prepare_metadata_for_build_wheel', { 'metadata_directory': abspath(metadata_directory), 'config_settings': config_settings, + '_allow_fallback': _allow_fallback, }) - def build_wheel(self, wheel_directory, config_settings=None, metadata_directory=None): + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): """Build a wheel from this project. Returns the name of the newly created file. @@ -103,32 +216,83 @@ class Pep517HookCaller(object): 'config_settings': config_settings, }) - def _call_hook(self, hook_name, kwargs): - env = os.environ.copy() - # On Python 2, pytoml returns Unicode values (which is correct) but the # environment passed to check_call needs to contain string values. We # convert here by encoding using ASCII (the backend can only contain # letters, digits and _, . and : characters, and will be used as a # Python identifier, so non-ASCII content is wrong on Python 2 in # any case). + # For backend_path, we use sys.getfilesystemencoding. if sys.version_info[0] == 2: build_backend = self.build_backend.encode('ASCII') else: build_backend = self.build_backend + extra_environ = {'PEP517_BUILD_BACKEND': build_backend} + + if self.backend_path: + backend_path = os.pathsep.join(self.backend_path) + if sys.version_info[0] == 2: + backend_path = backend_path.encode(sys.getfilesystemencoding()) + extra_environ['PEP517_BACKEND_PATH'] = backend_path - env['PEP517_BUILD_BACKEND'] = build_backend with tempdir() as td: - compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + hook_input = {'kwargs': kwargs} + compat.write_json(hook_input, pjoin(td, 'input.json'), indent=2) # Run the hook in a subprocess - check_call([sys.executable, _in_proc_script, hook_name, td], - cwd=self.source_dir, env=env) + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, + extra_environ=extra_environ + ) data = compat.read_json(pjoin(td, 'output.json')) if data.get('unsupported'): - raise UnsupportedOperation + raise UnsupportedOperation(data.get('traceback', '')) + if data.get('no_backend'): + raise BackendUnavailable(data.get('traceback', '')) + if data.get('backend_invalid'): + raise BackendInvalid( + backend_name=self.build_backend, + backend_path=self.backend_path, + message=data.get('backend_error', '') + ) + if data.get('hook_missing'): + raise HookMissing(hook_name) return data['return_val'] + +class LoggerWrapper(threading.Thread): + """ + Read messages from a pipe and redirect them + to a logger (see python's logging module). + """ + + def __init__(self, logger, level): + threading.Thread.__init__(self) + self.daemon = True + + self.logger = logger + self.level = level + + # create the pipe and reader + self.fd_read, self.fd_write = os.pipe() + self.reader = os.fdopen(self.fd_read) + + self.start() + + def fileno(self): + return self.fd_write + + @staticmethod + def remove_newline(msg): + return msg[:-1] if msg.endswith(os.linesep) else msg + + def run(self): + for line in self.reader: + self._write(self.remove_newline(line)) + + def _write(self, message): + self.logger.log(self.level, message) diff --git a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py index ac893b66..c13e11c1 100644 --- a/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py +++ b/pipenv/patched/notpip/_vendor/pkg_resources/__init__.py @@ -39,6 +39,8 @@ import tempfile import textwrap import itertools import inspect +import ntpath +import posixpath from pkgutil import get_importer try: @@ -238,6 +240,9 @@ __all__ = [ 'register_finder', 'register_namespace_handler', 'register_loader_type', 'fixup_namespace_packages', 'get_importer', + # Warnings + 'PkgResourcesDeprecationWarning', + # Deprecated/backward compatibility only 'run_main', 'AvailableDistributions', ] @@ -1398,14 +1403,30 @@ class NullProvider: def has_resource(self, resource_name): return self._has(self._fn(self.module_path, resource_name)) + def _get_metadata_path(self, name): + return self._fn(self.egg_info, name) + def has_metadata(self, name): - return self.egg_info and self._has(self._fn(self.egg_info, name)) + if not self.egg_info: + return self.egg_info + + path = self._get_metadata_path(name) + return self._has(path) def get_metadata(self, name): if not self.egg_info: return "" - value = self._get(self._fn(self.egg_info, name)) - return value.decode('utf-8') if six.PY3 else value + path = self._get_metadata_path(name) + value = self._get(path) + if six.PY2: + return value + try: + return value.decode('utf-8') + except UnicodeDecodeError as exc: + # Include the path in the error message to simplify + # troubleshooting, and without changing the exception type. + exc.reason += ' in {} file at path: {}'.format(name, path) + raise def get_metadata_lines(self, name): return yield_lines(self.get_metadata(name)) @@ -1463,10 +1484,86 @@ class NullProvider: ) def _fn(self, base, resource_name): + self._validate_resource_path(resource_name) if resource_name: return os.path.join(base, *resource_name.split('/')) return base + @staticmethod + def _validate_resource_path(path): + """ + Validate the resource paths according to the docs. + https://setuptools.readthedocs.io/en/latest/pkg_resources.html#basic-resource-access + + >>> warned = getfixture('recwarn') + >>> warnings.simplefilter('always') + >>> vrp = NullProvider._validate_resource_path + >>> vrp('foo/bar.txt') + >>> bool(warned) + False + >>> vrp('../foo/bar.txt') + >>> bool(warned) + True + >>> warned.clear() + >>> vrp('/foo/bar.txt') + >>> bool(warned) + True + >>> vrp('foo/../../bar.txt') + >>> bool(warned) + True + >>> warned.clear() + >>> vrp('foo/f../bar.txt') + >>> bool(warned) + False + + Windows path separators are straight-up disallowed. + >>> vrp(r'\\foo/bar.txt') + Traceback (most recent call last): + ... + ValueError: Use of .. or absolute path in a resource path \ +is not allowed. + + >>> vrp(r'C:\\foo/bar.txt') + Traceback (most recent call last): + ... + ValueError: Use of .. or absolute path in a resource path \ +is not allowed. + + Blank values are allowed + + >>> vrp('') + >>> bool(warned) + False + + Non-string values are not. + + >>> vrp(None) + Traceback (most recent call last): + ... + AttributeError: ... + """ + invalid = ( + os.path.pardir in path.split(posixpath.sep) or + posixpath.isabs(path) or + ntpath.isabs(path) + ) + if not invalid: + return + + msg = "Use of .. or absolute path in a resource path is not allowed." + + # Aggressively disallow Windows absolute paths + if ntpath.isabs(path) and not posixpath.isabs(path): + raise ValueError(msg) + + # for compatibility, warn; in future + # raise ValueError(msg) + warnings.warn( + msg[:-1] + " and will raise exceptions in a future release.", + DeprecationWarning, + stacklevel=4, + ) + def _get(self, path): if hasattr(self.loader, 'get_data'): return self.loader.get_data(path) @@ -1787,6 +1884,9 @@ class FileMetadata(EmptyProvider): def __init__(self, path): self.path = path + def _get_metadata_path(self, name): + return self.path + def has_metadata(self, name): return name == 'PKG-INFO' and os.path.isfile(self.path) @@ -1885,7 +1985,7 @@ def find_eggs_in_zip(importer, path_item, only=False): if only: # don't yield nested distros return - for subitem in metadata.resource_listdir('/'): + for subitem in metadata.resource_listdir(''): if _is_egg_path(subitem): subpath = os.path.join(path_item, subitem) dists = find_eggs_in_zip(zipimport.zipimporter(subpath), subpath) @@ -2228,7 +2328,18 @@ register_namespace_handler(object, null_ns_handler) def normalize_path(filename): """Normalize a file/dir name for comparison purposes""" - return os.path.normcase(os.path.realpath(filename)) + return os.path.normcase(os.path.realpath(os.path.normpath(_cygwin_patch(filename)))) + + +def _cygwin_patch(filename): # pragma: nocover + """ + Contrary to POSIX 2008, on Cygwin, getcwd (3) contains + symlink components. Using + os.path.abspath() works around this limitation. A fix in os.getcwd() + would probably better, in Cygwin even more so, except + that this seems to be by design... + """ + return os.path.abspath(filename) if sys.platform == 'cygwin' else filename def _normalize_cached(filename, _cache={}): @@ -2324,7 +2435,7 @@ class EntryPoint: warnings.warn( "Parameters to load are deprecated. Call .resolve and " ".require separately.", - DeprecationWarning, + PkgResourcesDeprecationWarning, stacklevel=2, ) if require: @@ -2569,10 +2680,14 @@ class Distribution: try: return self._version except AttributeError: - version = _version_from_file(self._get_metadata(self.PKG_INFO)) + version = self._get_version() if version is None: - tmpl = "Missing 'Version:' header and/or %s file" - raise ValueError(tmpl % self.PKG_INFO, self) + path = self._get_metadata_path_for_display(self.PKG_INFO) + msg = ( + "Missing 'Version:' header and/or {} file at path: {}" + ).format(self.PKG_INFO, path) + raise ValueError(msg, self) + return version @property @@ -2630,11 +2745,34 @@ class Distribution: ) return deps + def _get_metadata_path_for_display(self, name): + """ + Return the path to the given metadata file, if available. + """ + try: + # We need to access _get_metadata_path() on the provider object + # directly rather than through this class's __getattr__() + # since _get_metadata_path() is marked private. + path = self._provider._get_metadata_path(name) + + # Handle exceptions e.g. in case the distribution's metadata + # provider doesn't support _get_metadata_path(). + except Exception: + return '[could not detect]' + + return path + def _get_metadata(self, name): if self.has_metadata(name): for line in self.get_metadata_lines(name): yield line + def _get_version(self): + lines = self._get_metadata(self.PKG_INFO) + version = _version_from_file(lines) + + return version + def activate(self, path=None, replace=False): """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: @@ -2853,7 +2991,7 @@ class EggInfoDistribution(Distribution): take an extra step and try to get the version number from the metadata file itself instead of the filename. """ - md_version = _version_from_file(self._get_metadata(self.PKG_INFO)) + md_version = self._get_version() if md_version: self._version = md_version return self @@ -3147,3 +3285,11 @@ def _initialize_master_working_set(): # match order list(map(working_set.add_entry, sys.path)) globals().update(locals()) + +class PkgResourcesDeprecationWarning(Warning): + """ + Base class for warning about deprecations in ``pkg_resources`` + + This class is not derived from ``DeprecationWarning``, and as such is + visible by default. + """ diff --git a/pipenv/patched/notpip/_vendor/progress/__init__.py b/pipenv/patched/notpip/_vendor/progress/__init__.py index a41f65dc..e434c257 100644 --- a/pipenv/patched/notpip/_vendor/progress/__init__.py +++ b/pipenv/patched/notpip/_vendor/progress/__init__.py @@ -12,31 +12,49 @@ # ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF # OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. -from __future__ import division +from __future__ import division, print_function from collections import deque from datetime import timedelta from math import ceil from sys import stderr -from time import time +try: + from time import monotonic +except ImportError: + from time import time as monotonic -__version__ = '1.4' +__version__ = '1.5' + +HIDE_CURSOR = '\x1b[?25l' +SHOW_CURSOR = '\x1b[?25h' class Infinite(object): file = stderr sma_window = 10 # Simple Moving Average window + check_tty = True + hide_cursor = True - def __init__(self, *args, **kwargs): + def __init__(self, message='', **kwargs): self.index = 0 - self.start_ts = time() + self.start_ts = monotonic() self.avg = 0 + self._avg_update_ts = self.start_ts self._ts = self.start_ts self._xput = deque(maxlen=self.sma_window) for key, val in kwargs.items(): setattr(self, key, val) + self._width = 0 + self.message = message + + if self.file and self.is_tty(): + if self.hide_cursor: + print(HIDE_CURSOR, end='', file=self.file) + print(self.message, end='', file=self.file) + self.file.flush() + def __getitem__(self, key): if key.startswith('_'): return None @@ -44,7 +62,7 @@ class Infinite(object): @property def elapsed(self): - return int(time() - self.start_ts) + return int(monotonic() - self.start_ts) @property def elapsed_td(self): @@ -52,8 +70,14 @@ class Infinite(object): def update_avg(self, n, dt): if n > 0: + xput_len = len(self._xput) self._xput.append(dt / n) - self.avg = sum(self._xput) / len(self._xput) + now = monotonic() + # update when we're still filling _xput, then after every second + if (xput_len < self.sma_window or + now - self._avg_update_ts > 1): + self.avg = sum(self._xput) / len(self._xput) + self._avg_update_ts = now def update(self): pass @@ -61,11 +85,34 @@ class Infinite(object): def start(self): pass + def clearln(self): + if self.file and self.is_tty(): + print('\r\x1b[K', end='', file=self.file) + + def write(self, s): + if self.file and self.is_tty(): + line = self.message + s.ljust(self._width) + print('\r' + line, end='', file=self.file) + self._width = max(self._width, len(s)) + self.file.flush() + + def writeln(self, line): + if self.file and self.is_tty(): + self.clearln() + print(line, end='', file=self.file) + self.file.flush() + def finish(self): - pass + if self.file and self.is_tty(): + print(file=self.file) + if self.hide_cursor: + print(SHOW_CURSOR, end='', file=self.file) + + def is_tty(self): + return self.file.isatty() if self.check_tty else True def next(self, n=1): - now = time() + now = monotonic() dt = now - self._ts self.update_avg(n, dt) self._ts = now @@ -73,12 +120,17 @@ class Infinite(object): self.update() def iter(self, it): - try: + with self: for x in it: yield x self.next() - finally: - self.finish() + + def __enter__(self): + self.start() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.finish() class Progress(Infinite): @@ -119,9 +171,7 @@ class Progress(Infinite): except TypeError: pass - try: + with self: for x in it: yield x self.next() - finally: - self.finish() diff --git a/pipenv/patched/notpip/_vendor/progress/bar.py b/pipenv/patched/notpip/_vendor/progress/bar.py index 025e61c4..8819efda 100644 --- a/pipenv/patched/notpip/_vendor/progress/bar.py +++ b/pipenv/patched/notpip/_vendor/progress/bar.py @@ -19,18 +19,15 @@ from __future__ import unicode_literals import sys from . import Progress -from .helpers import WritelnMixin -class Bar(WritelnMixin, Progress): +class Bar(Progress): width = 32 - message = '' suffix = '%(index)d/%(max)d' bar_prefix = ' |' bar_suffix = '| ' empty_fill = ' ' fill = '#' - hide_cursor = True def update(self): filled_length = int(self.width * self.progress) diff --git a/pipenv/patched/notpip/_vendor/progress/counter.py b/pipenv/patched/notpip/_vendor/progress/counter.py index 6b45a1ec..d955ca47 100644 --- a/pipenv/patched/notpip/_vendor/progress/counter.py +++ b/pipenv/patched/notpip/_vendor/progress/counter.py @@ -16,27 +16,20 @@ from __future__ import unicode_literals from . import Infinite, Progress -from .helpers import WriteMixin -class Counter(WriteMixin, Infinite): - message = '' - hide_cursor = True - +class Counter(Infinite): def update(self): self.write(str(self.index)) -class Countdown(WriteMixin, Progress): - hide_cursor = True - +class Countdown(Progress): def update(self): self.write(str(self.remaining)) -class Stack(WriteMixin, Progress): +class Stack(Progress): phases = (' ', '▁', '▂', '▃', '▄', '▅', '▆', '▇', '█') - hide_cursor = True def update(self): nphases = len(self.phases) diff --git a/pipenv/patched/notpip/_vendor/progress/helpers.py b/pipenv/patched/notpip/_vendor/progress/helpers.py deleted file mode 100644 index 0cde44ec..00000000 --- a/pipenv/patched/notpip/_vendor/progress/helpers.py +++ /dev/null @@ -1,91 +0,0 @@ -# Copyright (c) 2012 Giorgos Verigakis <verigak@gmail.com> -# -# Permission to use, copy, modify, and distribute this software for any -# purpose with or without fee is hereby granted, provided that the above -# copyright notice and this permission notice appear in all copies. -# -# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES -# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF -# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR -# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF -# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -from __future__ import print_function - - -HIDE_CURSOR = '\x1b[?25l' -SHOW_CURSOR = '\x1b[?25h' - - -class WriteMixin(object): - hide_cursor = False - - def __init__(self, message=None, **kwargs): - super(WriteMixin, self).__init__(**kwargs) - self._width = 0 - if message: - self.message = message - - if self.file and self.file.isatty(): - if self.hide_cursor: - print(HIDE_CURSOR, end='', file=self.file) - print(self.message, end='', file=self.file) - self.file.flush() - - def write(self, s): - if self.file and self.file.isatty(): - b = '\b' * self._width - c = s.ljust(self._width) - print(b + c, end='', file=self.file) - self._width = max(self._width, len(s)) - self.file.flush() - - def finish(self): - if self.file and self.file.isatty() and self.hide_cursor: - print(SHOW_CURSOR, end='', file=self.file) - - -class WritelnMixin(object): - hide_cursor = False - - def __init__(self, message=None, **kwargs): - super(WritelnMixin, self).__init__(**kwargs) - if message: - self.message = message - - if self.file and self.file.isatty() and self.hide_cursor: - print(HIDE_CURSOR, end='', file=self.file) - - def clearln(self): - if self.file and self.file.isatty(): - print('\r\x1b[K', end='', file=self.file) - - def writeln(self, line): - if self.file and self.file.isatty(): - self.clearln() - print(line, end='', file=self.file) - self.file.flush() - - def finish(self): - if self.file and self.file.isatty(): - print(file=self.file) - if self.hide_cursor: - print(SHOW_CURSOR, end='', file=self.file) - - -from signal import signal, SIGINT -from sys import exit - - -class SigIntMixin(object): - """Registers a signal handler that calls finish on SIGINT""" - - def __init__(self, *args, **kwargs): - super(SigIntMixin, self).__init__(*args, **kwargs) - signal(SIGINT, self._sigint_handler) - - def _sigint_handler(self, signum, frame): - self.finish() - exit(0) diff --git a/pipenv/patched/notpip/_vendor/progress/spinner.py b/pipenv/patched/notpip/_vendor/progress/spinner.py index 464c7b27..4e100cab 100644 --- a/pipenv/patched/notpip/_vendor/progress/spinner.py +++ b/pipenv/patched/notpip/_vendor/progress/spinner.py @@ -16,11 +16,9 @@ from __future__ import unicode_literals from . import Infinite -from .helpers import WriteMixin -class Spinner(WriteMixin, Infinite): - message = '' +class Spinner(Infinite): phases = ('-', '\\', '|', '/') hide_cursor = True @@ -40,5 +38,6 @@ class MoonSpinner(Spinner): class LineSpinner(Spinner): phases = ['⎺', '⎻', '⎼', '⎽', '⎼', '⎻'] + class PixelSpinner(Spinner): - phases = ['⣾','⣷', '⣯', '⣟', '⡿', '⢿', '⣻', '⣽'] + phases = ['⣾', '⣷', '⣯', '⣟', '⡿', '⢿', '⣻', '⣽'] diff --git a/pipenv/patched/notpip/_vendor/pyparsing.py b/pipenv/patched/notpip/_vendor/pyparsing.py index 455d1151..1d47c460 100644 --- a/pipenv/patched/notpip/_vendor/pyparsing.py +++ b/pipenv/patched/notpip/_vendor/pyparsing.py @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``"<salutation>, <addressee>!"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'<ParserElement.__add__>` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pipenv.patched.notpip._vendor.pyparsing import Word, alphas @@ -49,33 +53,50 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString<ParserElement.parseString>} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'<And>`, :class:`'|'<MatchFirst>`, :class:`'^'<Or>`, + and :class:`'&'<Each>` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - access the parsed data, which is returned as a :class:`ParseResults` + object + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.1" -__versionTime__ = "18 Sep 2018 00:49 UTC" +__version__ = "2.4.2" +__versionTime__ = "29 Jul 2019 02:58 UTC" __author__ = "Paul McGuire <ptmcg@users.sourceforge.net>" import string @@ -90,6 +111,15 @@ import pprint import traceback import types from datetime import datetime +from operator import itemgetter +import itertools +from functools import wraps + +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse try: from _thread import RLock @@ -99,11 +129,11 @@ except ImportError: try: # Python 3 from collections.abc import Iterable - from collections.abc import MutableMapping + from collections.abc import MutableMapping, Mapping except ImportError: # Python 2.7 from collections import Iterable - from collections import MutableMapping + from collections import MutableMapping, Mapping try: from collections import OrderedDict as _OrderedDict @@ -113,28 +143,71 @@ except ImportError: except ImportError: _OrderedDict = None -#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass -__all__ = [ -'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', -'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', -'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', -'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', -'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', -'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', -'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', -'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', -'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', -'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', -'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', -'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', -'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', -] +# version compatibility configuration +__compat__ = SimpleNamespace() +__compat__.__doc__ = """ + A cross-version compatibility configuration for pyparsing features that will be + released in a future version. By setting values in this configuration to True, + those features can be enabled in prior versions for compatibility development + and testing. + + - collect_all_And_tokens - flag to enable fix for Issue #63 that fixes erroneous grouping + of results names when an And expression is nested within an Or or MatchFirst; set to + True to enable bugfix released in pyparsing 2.3.0, or False to preserve + pre-2.3.0 handling of named results +""" +__compat__.collect_all_And_tokens = True + +__diag__ = SimpleNamespace() +__diag__.__doc__ = """ +Diagnostic configuration (all default to False) + - warn_multiple_tokens_in_named_alternation - flag to enable warnings when a results + name is defined on a MatchFirst or Or expression with one or more And subexpressions + (only warns if __compat__.collect_all_And_tokens is False) + - warn_ungrouped_named_tokens_in_collection - flag to enable warnings when a results + name is defined on a containing expression with ungrouped subexpressions that also + have results names + - warn_name_set_on_empty_Forward - flag to enable warnings whan a Forward is defined + with a results name, but has no contents defined + - warn_on_multiple_string_args_to_oneof - flag to enable warnings whan oneOf is + incorrectly called with multiple str arguments + - enable_debug_on_named_expressions - flag to auto-enable debug on all subsequent + calls to ParserElement.setName() +""" +__diag__.warn_multiple_tokens_in_named_alternation = False +__diag__.warn_ungrouped_named_tokens_in_collection = False +__diag__.warn_name_set_on_empty_Forward = False +__diag__.warn_on_multiple_string_args_to_oneof = False +__diag__.enable_debug_on_named_expressions = False + +# ~ sys.stderr.write("testing pyparsing module, version %s, %s\n" % (__version__, __versionTime__)) + +__all__ = ['__version__', '__versionTime__', '__author__', '__compat__', '__diag__', + 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', + 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', + 'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', + 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', + 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', + 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', + 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', + 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', + 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', + 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', + 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', + 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', + 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', + 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', + 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', + 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', + 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation', 'locatedExpr', 'withClass', + 'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', + 'conditionAsParseAction', + ] system_version = tuple(sys.version_info)[:3] PY_3 = system_version[0] == 3 @@ -142,6 +215,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,11 +226,13 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ - if isinstance(obj,unicode): + if isinstance(obj, unicode): return obj try: @@ -174,39 +250,50 @@ else: # build list of single arg builtins, tolerant of Python version, that can be used as parse actions singleArgBuiltins = [] import __builtin__ + for fname in "sum len sorted reversed list tuple set any all min max".split(): try: - singleArgBuiltins.append(getattr(__builtin__,fname)) + singleArgBuiltins.append(getattr(__builtin__, fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" # ampersand must be replaced first from_symbols = '&><"\'' - to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split()) - for from_,to_ in zip(from_symbols, to_symbols): + to_symbols = ('&' + s + ';' for s in "amp gt lt quot apos".split()) + for from_, to_ in zip(from_symbols, to_symbols): data = data.replace(from_, to_) return data -class _Constants(object): - pass - -alphas = string.ascii_uppercase + string.ascii_lowercase -nums = "0123456789" -hexnums = nums + "ABCDEFabcdef" -alphanums = alphas + nums -_bslash = chr(92) +alphas = string.ascii_uppercase + string.ascii_lowercase +nums = "0123456789" +hexnums = nums + "ABCDEFabcdef" +alphanums = alphas + nums +_bslash = chr(92) printables = "".join(c for c in string.printable if c not in string.whitespace) + +def conditionAsParseAction(fn, message=None, fatal=False): + msg = message if message is not None else "failed user-defined condition" + exc_type = ParseFatalException if fatal else ParseException + fn = _trim_arity(fn) + + @wraps(fn) + def pa(s, l, t): + if not bool(fn(s, l, t)): + raise exc_type(s, l, msg) + + return pa + class ParseBaseException(Exception): """base exception class for all parsing runtime exceptions""" # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible - def __init__( self, pstr, loc=0, msg=None, elem=None ): + def __init__(self, pstr, loc=0, msg=None, elem=None): self.loc = loc if msg is None: self.msg = pstr @@ -220,32 +307,39 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) - def __getattr__( self, aname ): + def __getattr__(self, aname): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ - if( aname == "lineno" ): - return lineno( self.loc, self.pstr ) - elif( aname in ("col", "column") ): - return col( self.loc, self.pstr ) - elif( aname == "line" ): - return line( self.loc, self.pstr ) + if aname == "lineno": + return lineno(self.loc, self.pstr) + elif aname in ("col", "column"): + return col(self.loc, self.pstr) + elif aname == "line": + return line(self.loc, self.pstr) else: raise AttributeError(aname) - def __str__( self ): - return "%s (at char %d), (line:%d, col:%d)" % \ - ( self.msg, self.loc, self.lineno, self.column ) - def __repr__( self ): + def __str__(self): + if self.pstr: + if self.loc >= len(self.pstr): + foundstr = ', found end of text' + else: + foundstr = (', found %r' % self.pstr[self.loc:self.loc + 1]).replace(r'\\', '\\') + else: + foundstr = '' + return ("%s%s (at char %d), (line:%d, col:%d)" % + (self.msg, foundstr, self.loc, self.lineno, self.column)) + def __repr__(self): return _ustr(self) - def markInputline( self, markerString = ">!<" ): + def markInputline(self, markerString=">!<"): """Extracts the exception line from the input string, and marks the location of the exception with a special symbol. """ @@ -262,22 +356,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff[0] + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', '<module>'): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +451,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop<And._ErrorStop>} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop<And._ErrorStop>` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,34 +472,38 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" - def __init__( self, parseElementList ): + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ + def __init__(self, parseElementList): self.parseElementTrace = parseElementList - def __str__( self ): + def __str__(self): return "RecursiveGrammarException: %s" % self.parseElementTrace class _ParseResultsWithOffset(object): - def __init__(self,p1,p2): - self.tup = (p1,p2) - def __getitem__(self,i): + def __init__(self, p1, p2): + self.tup = (p1, p2) + def __getitem__(self, i): return self.tup[i] def __repr__(self): return repr(self.tup[0]) - def setOffset(self,i): - self.tup = (self.tup[0],i) + def setOffset(self, i): + self.tup = (self.tup[0], i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.<resultsName>} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.<resultsName>`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +520,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -360,7 +534,7 @@ class ParseResults(object): - month: 12 - year: 1999 """ - def __new__(cls, toklist=None, name=None, asList=True, modal=True ): + def __new__(cls, toklist=None, name=None, asList=True, modal=True): if isinstance(toklist, cls): return toklist retobj = object.__new__(cls) @@ -369,7 +543,7 @@ class ParseResults(object): # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible - def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ): + def __init__(self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance): if self.__doinit: self.__doinit = False self.__name = None @@ -390,96 +564,104 @@ class ParseResults(object): if name is not None and name: if not modal: self.__accumNames[name] = 0 - if isinstance(name,int): - name = _ustr(name) # will always return a str, but use _ustr for consistency + if isinstance(name, int): + name = _ustr(name) # will always return a str, but use _ustr for consistency self.__name = name - if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])): - if isinstance(toklist,basestring): - toklist = [ toklist ] + if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None, '', [])): + if isinstance(toklist, basestring): + toklist = [toklist] if asList: - if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + if isinstance(toklist, ParseResults): + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: - self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]), 0) self[name].__name = name else: try: self[name] = toklist[0] - except (KeyError,TypeError,IndexError): + except (KeyError, TypeError, IndexError): self[name] = toklist - def __getitem__( self, i ): - if isinstance( i, (int,slice) ): + def __getitem__(self, i): + if isinstance(i, (int, slice)): return self.__toklist[i] else: if i not in self.__accumNames: return self.__tokdict[i][-1][0] else: - return ParseResults([ v[0] for v in self.__tokdict[i] ]) + return ParseResults([v[0] for v in self.__tokdict[i]]) - def __setitem__( self, k, v, isinstance=isinstance ): - if isinstance(v,_ParseResultsWithOffset): - self.__tokdict[k] = self.__tokdict.get(k,list()) + [v] + def __setitem__(self, k, v, isinstance=isinstance): + if isinstance(v, _ParseResultsWithOffset): + self.__tokdict[k] = self.__tokdict.get(k, list()) + [v] sub = v[0] - elif isinstance(k,(int,slice)): + elif isinstance(k, (int, slice)): self.__toklist[k] = v sub = v else: - self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)] + self.__tokdict[k] = self.__tokdict.get(k, list()) + [_ParseResultsWithOffset(v, 0)] sub = v - if isinstance(sub,ParseResults): + if isinstance(sub, ParseResults): sub.__parent = wkref(self) - def __delitem__( self, i ): - if isinstance(i,(int,slice)): - mylen = len( self.__toklist ) + def __delitem__(self, i): + if isinstance(i, (int, slice)): + mylen = len(self.__toklist) del self.__toklist[i] # convert int to slice if isinstance(i, int): if i < 0: i += mylen - i = slice(i, i+1) + i = slice(i, i + 1) # get removed indices removed = list(range(*i.indices(mylen))) removed.reverse() # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): + for name, occurrences in self.__tokdict.items(): for j in removed: for k, (value, position) in enumerate(occurrences): occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) else: del self.__tokdict[i] - def __contains__( self, k ): + def __contains__(self, k): return k in self.__tokdict - def __len__( self ): return len( self.__toklist ) - def __bool__(self): return ( not not self.__toklist ) + def __len__(self): + return len(self.__toklist) + + def __bool__(self): + return (not not self.__toklist) __nonzero__ = __bool__ - def __iter__( self ): return iter( self.__toklist ) - def __reversed__( self ): return iter( self.__toklist[::-1] ) - def _iterkeys( self ): + + def __iter__(self): + return iter(self.__toklist) + + def __reversed__(self): + return iter(self.__toklist[::-1]) + + def _iterkeys(self): if hasattr(self.__tokdict, "iterkeys"): return self.__tokdict.iterkeys() else: return iter(self.__tokdict) - def _itervalues( self ): + def _itervalues(self): return (self[k] for k in self._iterkeys()) - - def _iteritems( self ): + + def _iteritems(self): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -491,35 +673,36 @@ class ParseResults(object): iteritems = _iteritems """Returns an iterator of all named result key-value tuples (Python 2.x only).""" - def keys( self ): + def keys(self): """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.iterkeys()) - def values( self ): + def values(self): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - - def items( self ): + + def items(self): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) - def haskeys( self ): + def haskeys(self): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - - def pop( self, *args, **kwargs): + + def pop(self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +719,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -544,14 +729,14 @@ class ParseResults(object): """ if not args: args = [-1] - for k,v in kwargs.items(): + for k, v in kwargs.items(): if k == 'default': args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or - args[0] in self): + if (isinstance(args[0], int) + or len(args) == 1 + or args[0] in self): index = args[0] ret = self[index] del self[index] @@ -563,14 +748,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -582,13 +768,14 @@ class ParseResults(object): else: return defaultValue - def insert( self, index, insStr ): + def insert(self, index, insStr): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -598,17 +785,18 @@ class ParseResults(object): """ self.__toklist.insert(index, insStr) # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): + for name, occurrences in self.__tokdict.items(): for k, (value, position) in enumerate(occurrences): occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) - def append( self, item ): + def append(self, item): """ Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -616,13 +804,14 @@ class ParseResults(object): """ self.__toklist.append(item) - def extend( self, itemseq ): + def extend(self, itemseq): """ Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -630,104 +819,98 @@ class ParseResults(object): print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl' """ if isinstance(itemseq, ParseResults): - self += itemseq + self.__iadd__(itemseq) else: self.__toklist.extend(itemseq) - def clear( self ): + def clear(self): """ Clear all elements and results names. """ del self.__toklist[:] self.__tokdict.clear() - def __getattr__( self, name ): + def __getattr__(self, name): try: return self[name] except KeyError: return "" - - if name in self.__tokdict: - if name not in self.__accumNames: - return self.__tokdict[name][-1][0] - else: - return ParseResults([ v[0] for v in self.__tokdict[name] ]) - else: - return "" - def __add__( self, other ): + def __add__(self, other): ret = self.copy() ret += other return ret - def __iadd__( self, other ): + def __iadd__(self, other): if other.__tokdict: offset = len(self.__toklist) - addoffset = lambda a: offset if a<0 else a+offset + addoffset = lambda a: offset if a < 0 else a + offset otheritems = other.__tokdict.items() - otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) ) - for (k,vlist) in otheritems for v in vlist] - for k,v in otherdictitems: + otherdictitems = [(k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) + for k, vlist in otheritems for v in vlist] + for k, v in otherdictitems: self[k] = v - if isinstance(v[0],ParseResults): + if isinstance(v[0], ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist - self.__accumNames.update( other.__accumNames ) + self.__accumNames.update(other.__accumNames) return self def __radd__(self, other): - if isinstance(other,int) and other == 0: + if isinstance(other, int) and other == 0: # useful for merging many ParseResults using sum() builtin return self.copy() else: # this may raise a TypeError - so be it return other + self - - def __repr__( self ): - return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) - def __str__( self ): + def __repr__(self): + return "(%s, %s)" % (repr(self.__toklist), repr(self.__tokdict)) + + def __str__(self): return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']' - def _asStringList( self, sep='' ): + def _asStringList(self, sep=''): out = [] for item in self.__toklist: if out and sep: out.append(sep) - if isinstance( item, ParseResults ): + if isinstance(item, ParseResults): out += item._asStringList() else: - out.append( _ustr(item) ) + out.append(_ustr(item)) return out - def asList( self ): + def asList(self): """ Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> <class 'pyparsing.ParseResults'> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj'] """ - return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist] + return [res.asList() if isinstance(res, ParseResults) else res for res in self.__toklist] - def asDict( self ): + def asDict(self): """ Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> <class 'pyparsing.ParseResults'> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> <class 'dict'> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +923,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,28 +932,28 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - - return dict((k,toItem(v)) for k,v in item_fn()) - def copy( self ): + return dict((k, toItem(v)) for k, v in item_fn()) + + def copy(self): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ - ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret = ParseResults(self.__toklist) + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent - ret.__accumNames.update( self.__accumNames ) + ret.__accumNames.update(self.__accumNames) ret.__name = self.__name return ret - def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ): + def asXML(self, doctag=None, namedItemsOnly=False, indent="", formatted=True): """ (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names. """ nl = "\n" out = [] - namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items() - for v in vlist) + namedItems = dict((v[1], k) for (k, vlist) in self.__tokdict.items() + for v in vlist) nextLevelIndent = indent + " " # collapse out indents if formatting is not desired @@ -792,20 +975,20 @@ class ParseResults(object): else: selfTag = "ITEM" - out += [ nl, indent, "<", selfTag, ">" ] + out += [nl, indent, "<", selfTag, ">"] - for i,res in enumerate(self.__toklist): - if isinstance(res,ParseResults): + for i, res in enumerate(self.__toklist): + if isinstance(res, ParseResults): if i in namedItems: - out += [ res.asXML(namedItems[i], - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] + out += [res.asXML(namedItems[i], + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] else: - out += [ res.asXML(None, - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] + out += [res.asXML(None, + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] else: # individual token, see if there is a name for it resTag = None @@ -817,38 +1000,41 @@ class ParseResults(object): else: resTag = "ITEM" xmlBodyText = _xml_escape(_ustr(res)) - out += [ nl, nextLevelIndent, "<", resTag, ">", - xmlBodyText, - "</", resTag, ">" ] + out += [nl, nextLevelIndent, "<", resTag, ">", + xmlBodyText, + "</", resTag, ">"] - out += [ nl, indent, "</", selfTag, ">" ] + out += [nl, indent, "</", selfTag, ">"] return "".join(out) - def __lookup(self,sub): - for k,vlist in self.__tokdict.items(): - for v,loc in vlist: + def __lookup(self, sub): + for k, vlist in self.__tokdict.items(): + for v, loc in vlist: if sub is v: return k return None def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -861,26 +1047,29 @@ class ParseResults(object): return par.__lookup(self) else: return None - elif (len(self) == 1 and - len(self.__tokdict) == 1 and - next(iter(self.__tokdict.values()))[0][1] in (0,-1)): + elif (len(self) == 1 + and len(self.__tokdict) == 1 + and next(iter(self.__tokdict.values()))[0][1] in (0, -1)): return next(iter(self.__tokdict.keys())) else: return None - def dump(self, indent='', depth=0, full=True): + def dump(self, indent='', full=True, include_list=True, _depth=0): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -888,38 +1077,57 @@ class ParseResults(object): """ out = [] NL = '\n' - out.append( indent+_ustr(self.asList()) ) + if include_list: + out.append(indent + _ustr(self.asList())) + else: + out.append('') + if full: if self.haskeys(): - items = sorted((str(k), v) for k,v in self.items()) - for k,v in items: + items = sorted((str(k), v) for k, v in self.items()) + for k, v in items: if out: out.append(NL) - out.append( "%s%s- %s: " % (indent,(' '*depth), k) ) - if isinstance(v,ParseResults): + out.append("%s%s- %s: " % (indent, (' ' * _depth), k)) + if isinstance(v, ParseResults): if v: - out.append( v.dump(indent,depth+1) ) + out.append(v.dump(indent=indent, full=full, include_list=include_list, _depth=_depth + 1)) else: out.append(_ustr(v)) else: out.append(repr(v)) - elif any(isinstance(vv,ParseResults) for vv in self): + elif any(isinstance(vv, ParseResults) for vv in self): v = self - for i,vv in enumerate(v): - if isinstance(vv,ParseResults): - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) + for i, vv in enumerate(v): + if isinstance(vv, ParseResults): + out.append("\n%s%s[%d]:\n%s%s%s" % (indent, + (' ' * (_depth)), + i, + indent, + (' ' * (_depth + 1)), + vv.dump(indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1))) else: - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + out.append("\n%s%s[%d]:\n%s%s%s" % (indent, + (' ' * (_depth)), + i, + indent, + (' ' * (_depth + 1)), + _ustr(vv))) + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint <https://docs.python.org/3/library/pprint.html>`_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint <https://docs.python.org/3/library/pprint.html#pprint.pprint>`_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1135,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -938,18 +1148,15 @@ class ParseResults(object): # add support for pickle protocol def __getstate__(self): - return ( self.__toklist, - ( self.__tokdict.copy(), - self.__parent is not None and self.__parent() or None, - self.__accumNames, - self.__name ) ) + return (self.__toklist, + (self.__tokdict.copy(), + self.__parent is not None and self.__parent() or None, + self.__accumNames, + self.__name)) - def __setstate__(self,state): + def __setstate__(self, state): self.__toklist = state[0] - (self.__tokdict, - par, - inAccumNames, - self.__name) = state[1] + self.__tokdict, par, inAccumNames, self.__name = state[1] self.__accumNames = {} self.__accumNames.update(inAccumNames) if par is not None: @@ -961,53 +1168,82 @@ class ParseResults(object): return self.__toklist, self.__name, self.__asList, self.__modal def __dir__(self): - return (dir(type(self)) + list(self.keys())) + return dir(type(self)) + list(self.keys()) + + @classmethod + def from_dict(cls, other, name=None): + """ + Helper classmethod to construct a ParseResults from a dict, preserving the + name-value relations as results names. If an optional 'name' argument is + given, a nested ParseResults will be returned + """ + def is_iterable(obj): + try: + iter(obj) + except Exception: + return False + else: + if PY_3: + return not isinstance(obj, (str, bytes)) + else: + return not isinstance(obj, basestring) + + ret = cls([]) + for k, v in other.items(): + if isinstance(v, Mapping): + ret += cls.from_dict(v, name=k) + else: + ret += cls([v], name=k, asList=is_iterable(v)) + if name is not None: + ret = cls([ret], name=name) + return ret MutableMapping.register(ParseResults) -def col (loc,strg): +def col (loc, strg): """Returns current column within a string, counting newlines as line separators. The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ``<TAB>`` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg - return 1 if 0<loc<len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) + return 1 if 0 < loc < len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) -def lineno(loc,strg): +def lineno(loc, strg): """Returns current line number within a string, counting newlines as line separators. - The first line is number 1. + The first line is number 1. - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ - return strg.count("\n",0,loc) + 1 + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ``<TAB>`` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ + return strg.count("\n", 0, loc) + 1 -def line( loc, strg ): +def line(loc, strg): """Returns the line of text containing loc within a string, counting newlines as line separators. """ lastCR = strg.rfind("\n", 0, loc) nextCR = strg.find("\n", loc) if nextCR >= 0: - return strg[lastCR+1:nextCR] + return strg[lastCR + 1:nextCR] else: - return strg[lastCR+1:] + return strg[lastCR + 1:] -def _defaultStartDebugAction( instring, loc, expr ): - print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))) +def _defaultStartDebugAction(instring, loc, expr): + print(("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % (lineno(loc, instring), col(loc, instring)))) -def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ): - print ("Matched " + _ustr(expr) + " -> " + str(toks.asList())) +def _defaultSuccessDebugAction(instring, startloc, endloc, expr, toks): + print("Matched " + _ustr(expr) + " -> " + str(toks.asList())) -def _defaultExceptionDebugAction( instring, loc, expr, exc ): - print ("Exception raised:" + _ustr(exc)) +def _defaultExceptionDebugAction(instring, loc, expr, exc): + print("Exception raised:" + _ustr(exc)) def nullDebugAction(*args): """'Do-nothing' debug action, to suppress debugging output during parsing.""" @@ -1038,16 +1274,16 @@ def nullDebugAction(*args): 'decorator to trim function calls to match the arity of the target' def _trim_arity(func, maxargs=2): if func in singleArgBuiltins: - return lambda s,l,t: func(t) + return lambda s, l, t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples - if system_version[:2] >= (3,5): + if system_version[:2] >= (3, 5): def extract_stack(limit=0): # special handling for Python 3.5.0 - extra deep call stack by 1 - offset = -3 if system_version == (3,5,0) else -2 - frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] + offset = -3 if system_version == (3, 5, 0) else -2 + frame_summary = traceback.extract_stack(limit=-offset + limit - 1)[offset] return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) @@ -1056,15 +1292,15 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] - pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) + pa_call_line_synth = (this_line[0], this_line[1] + LINE_DIFF) def wrapper(*args): while 1: @@ -1082,7 +1318,10 @@ def _trim_arity(func, maxargs=2): if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth: raise finally: - del tb + try: + del tb + except NameError: + pass if limit[0] <= maxargs: limit[0] += 1 @@ -1092,7 +1331,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "<parse action>" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1100,20 +1339,22 @@ def _trim_arity(func, maxargs=2): return wrapper + class ParserElement(object): """Abstract base level parser element class.""" DEFAULT_WHITE_CHARS = " \n\t\r" verbose_stacktrace = False @staticmethod - def setDefaultWhitespaceChars( chars ): + def setDefaultWhitespaceChars(chars): r""" Overrides the default whitespace chars Example:: + # default whitespace chars are space, <TAB> and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,32 +1365,33 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ ParserElement._literalStringClass = cls - def __init__( self, savelist=False ): + def __init__(self, savelist=False): self.parseAction = list() self.failAction = None - #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall + # ~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall self.strRepr = None self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1159,116 +1401,134 @@ class ParserElement(object): self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index self.errmsg = "" self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) - self.debugActions = ( None, None, None ) #custom debug actions + self.debugActions = (None, None, None) # custom debug actions self.re = None self.callPreparse = True # used to avoid redundant calls to preParse self.callDuringTry = False - def copy( self ): + def copy(self): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") - integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + integerK = integer.copy().addParseAction(lambda toks: toks[0] * 1024) + Suppress("K") + integerM = integer.copy().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: - integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + + integerM = integer().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") """ - cpy = copy.copy( self ) + cpy = copy.copy(self) cpy.parseAction = self.parseAction[:] cpy.ignoreExprs = self.ignoreExprs[:] if self.copyDefaultWhiteChars: cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS return cpy - def setName( self, name ): + def setName(self, name): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ self.name = name self.errmsg = "Expected " + self.name - if hasattr(self,"exception"): - self.exception.msg = self.errmsg + if __diag__.enable_debug_on_named_expressions: + self.setDebug() return self - def setResultsName( self, name, listAllMatches=False ): + def setResultsName(self, name, listAllMatches=False): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: date_str = integer("year") + '/' + integer("month") + '/' + integer("day") """ + return self._setResultsName(name, listAllMatches) + + def _setResultsName(self, name, listAllMatches=False): newself = self.copy() if name.endswith("*"): name = name[:-1] - listAllMatches=True + listAllMatches = True newself.resultsName = name newself.modalResults = not listAllMatches return newself - def setBreak(self,breakFlag = True): + def setBreak(self, breakFlag=True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: _parseMethod = self._parse def breaker(instring, loc, doActions=True, callPreParse=True): import pdb + # this call to pdb.set_trace() is intentional, not a checkin error pdb.set_trace() - return _parseMethod( instring, loc, doActions, callPreParse ) + return _parseMethod(instring, loc, doActions, callPreParse) breaker._originalParseMethod = _parseMethod self._parse = breaker else: - if hasattr(self._parse,"_originalParseMethod"): + if hasattr(self._parse, "_originalParseMethod"): self._parse = self._parse._originalParseMethod return self - def setParseAction( self, *fns, **kwargs ): + def setParseAction(self, *fns, **kwargs): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s, loc, toks)`` , + ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. + If None is passed as the parse action, all previously added parse actions for this + expression are cleared. + Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False``) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}<parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ``<TAB>`` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1281,30 +1541,36 @@ class ParserElement(object): # note that integer fields are now ints, not strings date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31] """ - self.parseAction = list(map(_trim_arity, list(fns))) - self.callDuringTry = kwargs.get("callDuringTry", False) + if list(fns) == [None,]: + self.parseAction = [] + else: + if not all(callable(fn) for fn in fns): + raise TypeError("parse actions must be callable") + self.parseAction = list(map(_trim_arity, list(fns))) + self.callDuringTry = kwargs.get("callDuringTry", False) return self - def addParseAction( self, *fns, **kwargs ): + def addParseAction(self, *fns, **kwargs): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}. - - See examples in L{I{copy}<copy>}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}<setParseAction>} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1312,45 +1578,42 @@ class ParserElement(object): result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1) """ - msg = kwargs.get("message", "failed user-defined condition") - exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: - def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): - raise exc_type(s,l,msg) - self.parseAction.append(pa) + self.parseAction.append(conditionAsParseAction(fn, message=kwargs.get('message'), + fatal=kwargs.get('fatal', False))) + self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self - def setFailAction( self, fn ): + def setFailAction(self, fn): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s, loc, expr, err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self - def _skipIgnorables( self, instring, loc ): + def _skipIgnorables(self, instring, loc): exprsFound = True while exprsFound: exprsFound = False for e in self.ignoreExprs: try: while 1: - loc,dummy = e._parse( instring, loc ) + loc, dummy = e._parse(instring, loc) exprsFound = True except ParseException: pass return loc - def preParse( self, instring, loc ): + def preParse(self, instring, loc): if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) + loc = self._skipIgnorables(instring, loc) if self.skipWhitespace: wt = self.whiteChars @@ -1360,90 +1623,106 @@ class ParserElement(object): return loc - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): return loc, [] - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): return tokenlist - #~ @profile - def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): - debugging = ( self.debug ) #and doActions ) + # ~ @profile + def _parseNoCache(self, instring, loc, doActions=True, callPreParse=True): + TRY, MATCH, FAIL = 0, 1, 2 + debugging = (self.debug) # and doActions) if debugging or self.failAction: - #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) - if (self.debugActions[0] ): - self.debugActions[0]( instring, loc, self ) - if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) - else: - preloc = loc - tokensStart = preloc + # ~ print ("Match", self, "at loc", loc, "(%d, %d)" % (lineno(loc, instring), col(loc, instring))) + if self.debugActions[TRY]: + self.debugActions[TRY](instring, loc, self) try: - try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) - except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) - except ParseBaseException as err: - #~ print ("Exception raised:", err) - if self.debugActions[2]: - self.debugActions[2]( instring, tokensStart, self, err ) + if callPreParse and self.callPreparse: + preloc = self.preParse(instring, loc) + else: + preloc = loc + tokensStart = preloc + if self.mayIndexError or preloc >= len(instring): + try: + loc, tokens = self.parseImpl(instring, preloc, doActions) + except IndexError: + raise ParseException(instring, len(instring), self.errmsg, self) + else: + loc, tokens = self.parseImpl(instring, preloc, doActions) + except Exception as err: + # ~ print ("Exception raised:", err) + if self.debugActions[FAIL]: + self.debugActions[FAIL](instring, tokensStart, self, err) if self.failAction: - self.failAction( instring, tokensStart, self, err ) + self.failAction(instring, tokensStart, self, err) raise else: if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) + preloc = self.preParse(instring, loc) else: preloc = loc tokensStart = preloc if self.mayIndexError or preloc >= len(instring): try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) + loc, tokens = self.parseImpl(instring, preloc, doActions) except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) + raise ParseException(instring, len(instring), self.errmsg, self) else: - loc,tokens = self.parseImpl( instring, preloc, doActions ) + loc, tokens = self.parseImpl(instring, preloc, doActions) - tokens = self.postParse( instring, loc, tokens ) + tokens = self.postParse(instring, loc, tokens) - retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults ) + retTokens = ParseResults(tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults) if self.parseAction and (doActions or self.callDuringTry): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, + try: + tokens = fn(instring, tokensStart, retTokens) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: + retTokens = ParseResults(tokens, self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) - except ParseBaseException as err: - #~ print "Exception raised in user parse action:", err - if (self.debugActions[2] ): - self.debugActions[2]( instring, tokensStart, self, err ) + asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults) + except Exception as err: + # ~ print "Exception raised in user parse action:", err + if self.debugActions[FAIL]: + self.debugActions[FAIL](instring, tokensStart, self, err) raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, + try: + tokens = fn(instring, tokensStart, retTokens) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: + retTokens = ParseResults(tokens, self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) + asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults) if debugging: - #~ print ("Matched",self,"->",retTokens.asList()) - if (self.debugActions[1] ): - self.debugActions[1]( instring, tokensStart, loc, self, retTokens ) + # ~ print ("Matched", self, "->", retTokens.asList()) + if self.debugActions[MATCH]: + self.debugActions[MATCH](instring, tokensStart, loc, self, retTokens) return loc, retTokens - def tryParse( self, instring, loc ): + def tryParse(self, instring, loc): try: - return self._parse( instring, loc, doActions=False )[0] + return self._parse(instring, loc, doActions=False)[0] except ParseFatalException: - raise ParseException( instring, loc, self.errmsg, self) - + raise ParseException(instring, loc, self.errmsg, self) + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1744,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1539,7 +1818,7 @@ class ParserElement(object): # this method gets repeatedly called during backtracking with the same arguments - # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression - def _parseCache( self, instring, loc, doActions=True, callPreParse=True ): + def _parseCache(self, instring, loc, doActions=True, callPreParse=True): HIT, MISS = 0, 1 lookup = (self, instring, loc, callPreParse, doActions) with ParserElement.packrat_cache_lock: @@ -1560,7 +1839,7 @@ class ParserElement(object): ParserElement.packrat_cache_stats[HIT] += 1 if isinstance(value, Exception): raise value - return (value[0], value[1].copy()) + return value[0], value[1].copy() _parse = _parseNoCache @@ -1577,23 +1856,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + from pipenv.patched.notpip._vendor import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1605,47 +1884,53 @@ class ParserElement(object): ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit) ParserElement._parse = ParserElement._parseCache - def parseString( self, instring, parseAll=False ): + def parseString(self, instring, parseAll=False): """ Execute the parse expression with the given string. This is the main interface to the client code, once the complete expression has been built. - If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + Returns the parsed data as a :class:`ParseResults` object, which may be + accessed as a list, or as a dict or object with attributes if the given parser + includes results names. - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + If you want the grammar to require that the entire input string be + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). + + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}<parseWithTabs>}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s, loc, toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ ParserElement.resetCache() if not self.streamlined: self.streamline() - #~ self.saveAsList = True + # ~ self.saveAsList = True for e in self.ignoreExprs: e.streamline() if not self.keepTabs: instring = instring.expandtabs() try: - loc, tokens = self._parse( instring, 0 ) + loc, tokens = self._parse(instring, 0) if parseAll: - loc = self.preParse( instring, loc ) + loc = self.preParse(instring, loc) se = Empty() + StringEnd() - se._parse( instring, loc ) + se._parse(instring, loc) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise @@ -1655,26 +1940,27 @@ class ParserElement(object): else: return tokens - def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ): + def scanString(self, instring, maxMatches=_MAX_INT, overlap=False): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}<parseString>} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) - for tokens,start,end in Word(alphas).scanString(source): + for tokens, start, end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1701,16 +1987,16 @@ class ParserElement(object): try: while loc <= instrlen and matches < maxMatches: try: - preloc = preparseFn( instring, loc ) - nextLoc,tokens = parseFn( instring, preloc, callPreParse=False ) + preloc = preparseFn(instring, loc) + nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) except ParseException: - loc = preloc+1 + loc = preloc + 1 else: if nextLoc > loc: matches += 1 yield tokens, preloc, nextLoc if overlap: - nextloc = preparseFn( instring, loc ) + nextloc = preparseFn(instring, loc) if nextloc > loc: loc = nextLoc else: @@ -1718,7 +2004,7 @@ class ParserElement(object): else: loc = nextLoc else: - loc = preloc+1 + loc = preloc + 1 except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise @@ -1726,21 +2012,24 @@ class ParserElement(object): # catch and re-raise exception from here, clears out pyparsing internal stack trace raise exc - def transformString( self, instring ): + def transformString(self, instring): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1749,19 +2038,19 @@ class ParserElement(object): # keep string locs straight between transformString and scanString self.keepTabs = True try: - for t,s,e in self.scanString( instring ): - out.append( instring[lastE:s] ) + for t, s, e in self.scanString(instring): + out.append(instring[lastE:s]) if t: - if isinstance(t,ParseResults): + if isinstance(t, ParseResults): out += t.asList() - elif isinstance(t,list): + elif isinstance(t, list): out += t else: out.append(t) lastE = e out.append(instring[lastE:]) out = [o for o in out if o] - return "".join(map(_ustr,_flatten(out))) + return "".join(map(_ustr, _flatten(out))) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise @@ -1769,26 +2058,29 @@ class ParserElement(object): # catch and re-raise exception from here, clears out pyparsing internal stack trace raise exc - def searchString( self, instring, maxMatches=_MAX_INT ): + def searchString(self, instring, maxMatches=_MAX_INT): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ try: - return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ]) + return ParseResults([t for t, s, e in self.scanString(instring, maxMatches)]) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise @@ -1799,119 +2091,149 @@ class ParserElement(object): def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 last = 0 - for t,s,e in self.scanString(instring, maxMatches=maxsplit): + for t, s, e in self.scanString(instring, maxMatches=maxsplit): yield instring[last:s] if includeSeparators: yield t[0] last = e yield instring[last:] - def __add__(self, other ): + def __add__(self, other): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: - Hello, World! -> ['Hello', ',', 'World', '!'] - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return And( [ self, other ] ) - def __radd__(self, other ): + prints:: + + Hello, World! -> ['Hello', ',', 'World', '!'] + + ``...`` may be used as a parse expression as a short form of :class:`SkipTo`. + + Literal('start') + ... + Literal('end') + + is equivalent to: + + Literal('start') + SkipTo('end')("_skipped*") + Literal('end') + + Note that the skipped text is returned with '_skipped' as a results name, + and to support having multiple skips in the same parser, the value returned is + a list of all skipped text. """ - Implementation of + operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if other is Ellipsis: + return _PendingSkip(self) + + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) + return None + return And([self, other]) + + def __radd__(self, other): + """ + Implementation of + operator when left operand is not a :class:`ParserElement` + """ + if other is Ellipsis: + return SkipTo(self)("_skipped*") + self + + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) return None return other + self def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return self + And._ErrorStop() + other - def __rsub__(self, other ): + def __rsub__(self, other): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other - self - def __mul__(self,other): + def __mul__(self, other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None, n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None, n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None, n) + ~expr`` """ - if isinstance(other,int): - minElements, optElements = other,0 - elif isinstance(other,tuple): + if other is Ellipsis: + other = (0, None) + elif isinstance(other, tuple) and other[:1] == (Ellipsis,): + other = ((0, ) + other[1:] + (None,))[:2] + + if isinstance(other, int): + minElements, optElements = other, 0 + elif isinstance(other, tuple): + other = tuple(o if o is not Ellipsis else None for o in other) other = (other + (None, None))[:2] if other[0] is None: other = (0, other[1]) - if isinstance(other[0],int) and other[1] is None: + if isinstance(other[0], int) and other[1] is None: if other[0] == 0: return ZeroOrMore(self) if other[0] == 1: return OneOrMore(self) else: - return self*other[0] + ZeroOrMore(self) - elif isinstance(other[0],int) and isinstance(other[1],int): + return self * other[0] + ZeroOrMore(self) + elif isinstance(other[0], int) and isinstance(other[1], int): minElements, optElements = other optElements -= minElements else: - raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1])) + raise TypeError("cannot multiply 'ParserElement' and ('%s', '%s') objects", type(other[0]), type(other[1])) else: raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) @@ -1920,145 +2242,190 @@ class ParserElement(object): if optElements < 0: raise ValueError("second tuple value must be greater or equal to first tuple value") if minElements == optElements == 0: - raise ValueError("cannot multiply ParserElement by 0 or (0,0)") + raise ValueError("cannot multiply ParserElement by 0 or (0, 0)") - if (optElements): + if optElements: def makeOptionalList(n): - if n>1: - return Optional(self + makeOptionalList(n-1)) + if n > 1: + return Optional(self + makeOptionalList(n - 1)) else: return Optional(self) if minElements: if minElements == 1: ret = self + makeOptionalList(optElements) else: - ret = And([self]*minElements) + makeOptionalList(optElements) + ret = And([self] * minElements) + makeOptionalList(optElements) else: ret = makeOptionalList(optElements) else: if minElements == 1: ret = self else: - ret = And([self]*minElements) + ret = And([self] * minElements) return ret def __rmul__(self, other): return self.__mul__(other) - def __or__(self, other ): + def __or__(self, other): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return MatchFirst( [ self, other ] ) + if other is Ellipsis: + return _PendingSkip(self, must_skip=True) - def __ror__(self, other ): - """ - Implementation of | operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) + return None + return MatchFirst([self, other]) + + def __ror__(self, other): + """ + Implementation of | operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) return None return other | self - def __xor__(self, other ): + def __xor__(self, other): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None - return Or( [ self, other ] ) + return Or([self, other]) - def __rxor__(self, other ): + def __rxor__(self, other): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other ^ self - def __and__(self, other ): + def __and__(self, other): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None - return Each( [ self, other ] ) + return Each([self, other]) - def __rand__(self, other ): + def __rand__(self, other): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other & self - def __invert__( self ): + def __invert__(self): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ - return NotAny( self ) + return NotAny(self) + + def __iter__(self): + # must implement __iter__ to override legacy use of sequential access to __getitem__ to + # iterate over a sequence + raise TypeError('%r object is not iterable' % self.__class__.__name__) + + def __getitem__(self, key): + """ + use ``[]`` indexing notation as a short form for expression repetition: + - ``expr[n]`` is equivalent to ``expr*n`` + - ``expr[m, n]`` is equivalent to ``expr*(m, n)`` + - ``expr[n, ...]`` or ``expr[n,]`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr[..., n]`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)`` + - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)`` + ``None`` may be used in place of ``...``. + + Note that ``expr[..., n]`` and ``expr[m, n]``do not raise an exception + if more than ``n`` ``expr``s exist in the input stream. If this behavior is + desired, then write ``expr[..., n] + ~expr``. + """ + + # convert single arg keys to tuples + try: + if isinstance(key, str): + key = (key,) + iter(key) + except TypeError: + key = (key, key) + + if len(key) > 2: + warnings.warn("only 1 or 2 index arguments supported ({0}{1})".format(key[:5], + '... [{0}]'.format(len(key)) + if len(key) > 5 else '')) + + # clip to 2 elements + ret = self * tuple(key[:2]) + return ret def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent - userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas).setResultsName("name") + Word(nums + "-").setResultsName("socsecno") + userdata = Word(alphas)("name") + Word(nums + "-")("socsecno") """ if name is not None: - return self.setResultsName(name) + return self._setResultsName(name) else: return self.copy() - def suppress( self ): + def suppress(self): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ - return Suppress( self ) + return Suppress(self) - def leaveWhitespace( self ): + def leaveWhitespace(self): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False return self - def setWhitespaceChars( self, chars ): + def setWhitespaceChars(self, chars): """ Overrides the default whitespace chars """ @@ -2067,39 +2434,40 @@ class ParserElement(object): self.copyDefaultWhiteChars = False return self - def parseWithTabs( self ): + def parseWithTabs(self): """ - Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{<TAB>} characters. + Overrides default behavior to expand ``<TAB>``s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ``<TAB>`` characters. """ self.keepTabs = True return self - def ignore( self, other ): + def ignore(self, other): """ Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ if isinstance(other, basestring): other = Suppress(other) - if isinstance( other, Suppress ): + if isinstance(other, Suppress): if other not in self.ignoreExprs: self.ignoreExprs.append(other) else: - self.ignoreExprs.append( Suppress( other.copy() ) ) + self.ignoreExprs.append(Suppress(other.copy())) return self - def setDebugActions( self, startAction, successAction, exceptionAction ): + def setDebugActions(self, startAction, successAction, exceptionAction): """ Enable display of debugging messages while doing pattern matching. """ @@ -2109,22 +2477,24 @@ class ParserElement(object): self.debug = True return self - def setDebug( self, flag=True ): + def setDebug(self, flag=True): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,40 +2507,40 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match <exprname> at loc <n>(<line>,<col>)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match <exprname> at loc <n>(<line>,<col>)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: - self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) + self.setDebugActions(_defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction) else: self.debug = False return self - def __str__( self ): + def __str__(self): return self.name - def __repr__( self ): + def __repr__(self): return _ustr(self) - def streamline( self ): + def streamline(self): self.streamlined = True self.strRepr = None return self - def checkRecursion( self, parseElementList ): + def checkRecursion(self, parseElementList): pass - def validate( self, validateTrace=[] ): + def validate(self, validateTrace=None): """ Check defined expressions for valid structure, check for infinite recursive definitions. """ - self.checkRecursion( [] ) + self.checkRecursion([]) - def parseFile( self, file_or_filename, parseAll=False ): + def parseFile(self, file_or_filename, parseAll=False): """ Execute the parse expression on the given file or filename. If a filename is specified (instead of a file object), @@ -2190,36 +2560,40 @@ class ParserElement(object): # catch and re-raise exception from here, clears out pyparsing internal stack trace raise exc - def __eq__(self,other): + def __eq__(self, other): if isinstance(other, ParserElement): - return self is other or vars(self) == vars(other) + if PY_3: + self is other or super(ParserElement, self).__eq__(other) + else: + return self is other or vars(self) == vars(other) elif isinstance(other, basestring): return self.matches(other) else: - return super(ParserElement,self)==other + return super(ParserElement, self) == other - def __ne__(self,other): + def __ne__(self, other): return not (self == other) def __hash__(self): - return hash(id(self)) + return id(self) - def __req__(self,other): + def __req__(self, other): return self == other - def __rne__(self,other): + def __rne__(self, other): return not (self == other) def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2602,35 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None, + file=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output + - file - (default=``None``) optional file-like object to which test output will be written; + if None, will default to ``sys.stdout`` Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2654,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2674,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,16 +2696,22 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): tests = list(map(str.strip, tests.rstrip().splitlines())) if isinstance(comment, basestring): comment = Literal(comment) + if file is None: + file = sys.stdout + print_ = file.write + allResults = [] comments = [] success = True + NL = Literal(r'\n').addParseAction(replaceWith('\n')).ignore(quotedString) + BOM = u'\ufeff' for t in tests: if comment is not None and comment.matches(t, False) or comments and not t: comments.append(t) @@ -2332,17 +2721,16 @@ class ParserElement(object): out = ['\n'.join(comments), t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = NL.transformString(t.lstrip(BOM)) result = self.parseString(t, parseAll=parseAll) - out.append(result.dump(full=fullDump)) - success = success and not failureTests except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: out.append(line(pe.loc, t)) - out.append(' '*(col(pe.loc,t)-1) + '^' + fatal) + out.append(' ' * (col(pe.loc, t) - 1) + '^' + fatal) else: - out.append(' '*pe.loc + '^' + fatal) + out.append(' ' * pe.loc + '^' + fatal) out.append("FAIL: " + str(pe)) success = success and failureTests result = pe @@ -2350,67 +2738,115 @@ class ParserElement(object): out.append("FAIL-EXCEPTION: " + str(exc)) success = success and failureTests result = exc + else: + success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + if isinstance(pp_value, ParseResults): + out.append(pp_value.dump()) + else: + out.append(str(pp_value)) + else: + out.append(result.dump()) + except Exception as e: + out.append(result.dump(full=fullDump)) + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) + else: + out.append(result.dump(full=fullDump)) if printResults: if fullDump: out.append('') - print('\n'.join(out)) + print_('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + +class _PendingSkip(ParserElement): + # internal placeholder class to hold a place were '...' is added to a parser element, + # once another ParserElement is added, this placeholder will be replaced with a SkipTo + def __init__(self, expr, must_skip=False): + super(_PendingSkip, self).__init__() + self.strRepr = str(expr + Empty()).replace('Empty', '...') + self.name = self.strRepr + self.anchor = expr + self.must_skip = must_skip + + def __add__(self, other): + skipper = SkipTo(other).setName("...")("_skipped*") + if self.must_skip: + def must_skip(t): + if not t._skipped or t._skipped.asList() == ['']: + del t[0] + t.pop("_skipped", None) + def show_skip(t): + if t._skipped.asList()[-1:] == ['']: + skipped = t.pop('_skipped') + t['_skipped'] = 'missing <' + repr(self.anchor) + '>' + return (self.anchor + skipper().addParseAction(must_skip) + | skipper().addParseAction(show_skip)) + other + + return self.anchor + skipper + other + + def __repr__(self): + return self.strRepr + + def parseImpl(self, *args): + raise Exception("use of `...` expression without following SkipTo target expression") + + class Token(ParserElement): + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. - """ - def __init__( self ): - super(Token,self).__init__( savelist=False ) + def __init__(self): + super(Token, self).__init__(savelist=False) class Empty(Token): + """An empty token, will always match. """ - An empty token, will always match. - """ - def __init__( self ): - super(Empty,self).__init__() + def __init__(self): + super(Empty, self).__init__() self.name = "Empty" self.mayReturnEmpty = True self.mayIndexError = False class NoMatch(Token): + """A token that will never match. """ - A token that will never match. - """ - def __init__( self ): - super(NoMatch,self).__init__() + def __init__(self): + super(NoMatch, self).__init__() self.name = "NoMatch" self.mayReturnEmpty = True self.mayIndexError = False self.errmsg = "Unmatchable token" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): raise ParseException(instring, loc, self.errmsg, self) class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ - def __init__( self, matchString ): - super(Literal,self).__init__() + def __init__(self, matchString): + super(Literal, self).__init__() self.match = matchString self.matchLen = len(matchString) try: @@ -2424,39 +2860,54 @@ class Literal(Token): self.mayReturnEmpty = False self.mayIndexError = False - # Performance tuning: this routine gets called a *lot* - # if this is a single character match string and the first character matches, - # short-circuit as quickly as possible, and avoid calling startswith - #~ @profile - def parseImpl( self, instring, loc, doActions=True ): - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) ): - return loc+self.matchLen, self.match + # Performance tuning: modify __class__ to select + # a parseImpl optimized for single-character check + if self.matchLen == 1 and type(self) is Literal: + self.__class__ = _SingleCharLiteral + + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] == self.firstMatchChar and instring.startswith(self.match, loc): + return loc + self.matchLen, self.match raise ParseException(instring, loc, self.errmsg, self) + +class _SingleCharLiteral(Literal): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] == self.firstMatchChar: + return loc + 1, self.match + raise ParseException(instring, loc, self.errmsg, self) + _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ - DEFAULT_KEYWORD_CHARS = alphanums+"_$" + DEFAULT_KEYWORD_CHARS = alphanums + "_$" - def __init__( self, matchString, identChars=None, caseless=False ): - super(Keyword,self).__init__() + def __init__(self, matchString, identChars=None, caseless=False): + super(Keyword, self).__init__() if identChars is None: identChars = Keyword.DEFAULT_KEYWORD_CHARS self.match = matchString @@ -2465,7 +2916,7 @@ class Keyword(Token): self.firstMatchChar = matchString[0] except IndexError: warnings.warn("null string passed to Keyword; use Empty() instead", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) self.name = '"%s"' % self.match self.errmsg = "Expected " + self.name self.mayReturnEmpty = False @@ -2476,86 +2927,94 @@ class Keyword(Token): identChars = identChars.upper() self.identChars = set(identChars) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.caseless: - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and - (loc == 0 or instring[loc-1].upper() not in self.identChars) ): - return loc+self.matchLen, self.match + if ((instring[loc:loc + self.matchLen].upper() == self.caselessmatch) + and (loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen].upper() not in self.identChars) + and (loc == 0 + or instring[loc - 1].upper() not in self.identChars)): + return loc + self.matchLen, self.match + else: - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and - (loc == 0 or instring[loc-1] not in self.identChars) ): - return loc+self.matchLen, self.match + if instring[loc] == self.firstMatchChar: + if ((self.matchLen == 1 or instring.startswith(self.match, loc)) + and (loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen] not in self.identChars) + and (loc == 0 or instring[loc - 1] not in self.identChars)): + return loc + self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) def copy(self): - c = super(Keyword,self).copy() + c = super(Keyword, self).copy() c.identChars = Keyword.DEFAULT_KEYWORD_CHARS return c @staticmethod - def setDefaultKeywordChars( chars ): + def setDefaultKeywordChars(chars): """Overrides the default Keyword chars """ Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ - def __init__( self, matchString ): - super(CaselessLiteral,self).__init__( matchString.upper() ) + def __init__(self, matchString): + super(CaselessLiteral, self).__init__(matchString.upper()) # Preserve the defining literal. self.returnString = matchString self.name = "'%s'" % self.returnString self.errmsg = "Expected " + self.name - def parseImpl( self, instring, loc, doActions=True ): - if instring[ loc:loc+self.matchLen ].upper() == self.match: - return loc+self.matchLen, self.returnString + def parseImpl(self, instring, loc, doActions=True): + if instring[loc:loc + self.matchLen].upper() == self.match: + return loc + self.matchLen, self.returnString raise ParseException(instring, loc, self.errmsg, self) class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: - OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) - """ - def __init__( self, matchString, identChars=None ): - super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] + + (Contrast with example for :class:`CaselessLiteral`.) + """ + def __init__(self, matchString, identChars=None): + super(CaselessKeyword, self).__init__(matchString, identChars, caseless=True) class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2568,7 +3027,7 @@ class CloseMatch(Token): patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']}) """ def __init__(self, match_string, maxMismatches=1): - super(CloseMatch,self).__init__() + super(CloseMatch, self).__init__() self.name = match_string self.match_string = match_string self.maxMismatches = maxMismatches @@ -2576,7 +3035,7 @@ class CloseMatch(Token): self.mayIndexError = False self.mayReturnEmpty = False - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): start = loc instrlen = len(instring) maxloc = start + len(self.match_string) @@ -2587,8 +3046,8 @@ class CloseMatch(Token): mismatches = [] maxMismatches = self.maxMismatches - for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)): - src,mat = s_m + for match_stringloc, s_m in enumerate(zip(instring[loc:maxloc], match_string)): + src, mat = s_m if src != mat: mismatches.append(match_stringloc) if len(mismatches) > maxMismatches: @@ -2596,7 +3055,7 @@ class CloseMatch(Token): else: loc = match_stringloc + 1 results = ParseResults([instring[start:loc]]) - results['original'] = self.match_string + results['original'] = match_string results['mismatches'] = mismatches return loc, results @@ -2604,61 +3063,68 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' - hostname = Word(alphas, alphanums+'-') - + hostname = Word(alphas, alphanums + '-') + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ - def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ): - super(Word,self).__init__() + def __init__(self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None): + super(Word, self).__init__() if excludeChars: + excludeChars = set(excludeChars) initChars = ''.join(c for c in initChars if c not in excludeChars) if bodyChars: bodyChars = ''.join(c for c in bodyChars if c not in excludeChars) self.initCharsOrig = initChars self.initChars = set(initChars) - if bodyChars : + if bodyChars: self.bodyCharsOrig = bodyChars self.bodyChars = set(bodyChars) else: @@ -2686,34 +3152,28 @@ class Word(Token): self.mayIndexError = False self.asKeyword = asKeyword - if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0): + if ' ' not in self.initCharsOrig + self.bodyCharsOrig and (min == 1 and max == 0 and exact == 0): if self.bodyCharsOrig == self.initCharsOrig: self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) elif len(self.initCharsOrig) == 1: - self.reString = "%s[%s]*" % \ - (re.escape(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "%s[%s]*" % (re.escape(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) else: - self.reString = "[%s][%s]*" % \ - (_escapeRegexRangeChars(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "[%s][%s]*" % (_escapeRegexRangeChars(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) if self.asKeyword: - self.reString = r"\b"+self.reString+r"\b" + self.reString = r"\b" + self.reString + r"\b" + try: - self.re = re.compile( self.reString ) + self.re = re.compile(self.reString) except Exception: self.re = None + else: + self.re_match = self.re.match + self.__class__ = _WordRegex - def parseImpl( self, instring, loc, doActions=True ): - if self.re: - result = self.re.match(instring,loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - return loc, result.group() - - if not(instring[ loc ] in self.initChars): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] not in self.initChars: raise ParseException(instring, loc, self.errmsg, self) start = loc @@ -2721,17 +3181,18 @@ class Word(Token): instrlen = len(instring) bodychars = self.bodyChars maxloc = start + self.maxLen - maxloc = min( maxloc, instrlen ) + maxloc = min(maxloc, instrlen) while loc < maxloc and instring[loc] in bodychars: loc += 1 throwException = False if loc - start < self.minLen: throwException = True - if self.maxSpecified and loc < instrlen and instring[loc] in bodychars: + elif self.maxSpecified and loc < instrlen and instring[loc] in bodychars: throwException = True - if self.asKeyword: - if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars): + elif self.asKeyword: + if (start > 0 and instring[start - 1] in bodychars + or loc < instrlen and instring[loc] in bodychars): throwException = True if throwException: @@ -2739,51 +3200,78 @@ class Word(Token): return loc, instring[start:loc] - def __str__( self ): + def __str__(self): try: - return super(Word,self).__str__() + return super(Word, self).__str__() except Exception: pass - if self.strRepr is None: def charsAsStr(s): - if len(s)>4: - return s[:4]+"..." + if len(s) > 4: + return s[:4] + "..." else: return s - if ( self.initCharsOrig != self.bodyCharsOrig ): - self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) ) + if self.initCharsOrig != self.bodyCharsOrig: + self.strRepr = "W:(%s, %s)" % (charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig)) else: self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) return self.strRepr +class _WordRegex(Word): + def parseImpl(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + return loc, result.group() + + +class Char(_WordRegex): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset, asKeyword=False, excludeChars=None): + super(Char, self).__init__(charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars) + self.reString = "[%s]" % _escapeRegexRangeChars(''.join(self.initChars)) + if asKeyword: + self.reString = r"\b%s\b" % self.reString + self.re = re.compile(self.reString) + self.re_match = self.re.match + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module <https://docs.python.org/3/library/re.html>`_. + If the given regex contains named groups (defined using ``(?P<name>...)``), + these will be preserved as named parse results. Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") """ compiledREtype = type(re.compile("[A-Z]")) - def __init__( self, pattern, flags=0): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" - super(Regex,self).__init__() + def __init__(self, pattern, flags=0, asGroupList=False, asMatch=False): + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module <https://docs.python.org/3/library/re.html>`_ module for an + explanation of the acceptable patterns and flags. + """ + super(Regex, self).__init__() if isinstance(pattern, basestring): if not pattern: warnings.warn("null string passed to Regex; use Empty() instead", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) self.pattern = pattern self.flags = flags @@ -2793,39 +3281,64 @@ class Regex(Token): self.reString = self.pattern except sre_constants.error: warnings.warn("invalid pattern (%s) passed to Regex" % pattern, - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) raise elif isinstance(pattern, Regex.compiledREtype): self.re = pattern - self.pattern = \ - self.reString = str(pattern) + self.pattern = self.reString = str(pattern) self.flags = flags - + else: raise ValueError("Regex may only be constructed with a string or a compiled RE object") + self.re_match = self.re.match + self.name = _ustr(self) self.errmsg = "Expected " + self.name self.mayIndexError = False self.mayReturnEmpty = True + self.asGroupList = asGroupList + self.asMatch = asMatch + if self.asGroupList: + self.parseImpl = self.parseImplAsGroupList + if self.asMatch: + self.parseImpl = self.parseImplAsMatch - def parseImpl( self, instring, loc, doActions=True ): - result = self.re.match(instring,loc) + def parseImpl(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) loc = result.end() - d = result.groupdict() ret = ParseResults(result.group()) + d = result.groupdict() if d: - for k in d: - ret[k] = d[k] - return loc,ret + for k, v in d.items(): + ret[k] = v + return loc, ret - def __str__( self ): + def parseImplAsGroupList(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result.groups() + return loc, ret + + def parseImplAsMatch(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result + return loc, ret + + def __str__(self): try: - return super(Regex,self).__str__() + return super(Regex, self).__str__() except Exception: pass @@ -2834,39 +3347,82 @@ class Regex(Token): return self.strRepr + def sub(self, repl): + r""" + Return Regex with an attached parse action to transform the parsed + result as if called using `re.sub(expr, repl, string) <https://docs.python.org/3/library/re.html#re.sub>`_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2</\1>") + print(make_html.transformString("h1:main title:")) + # prints "<h1>main title</h1>" + """ + if self.asGroupList: + warnings.warn("cannot use sub() with Regex(asGroupList=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch and callable(repl): + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() + + if self.asMatch: + def pa(tokens): + return tokens[0].expand(repl) + else: + def pa(tokens): + return self.re.sub(repl, tokens[0]) + return self.addParseAction(pa) class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None``) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None``) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False``) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True``) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True``) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] """ - def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): - super(QuotedString,self).__init__() + def __init__(self, quoteChar, escChar=None, escQuote=None, multiline=False, + unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): + super(QuotedString, self).__init__() # remove white space from quote chars - wont work anyway quoteChar = quoteChar.strip() if not quoteChar: - warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + warnings.warn("quoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) raise SyntaxError() if endQuoteChar is None: @@ -2874,7 +3430,7 @@ class QuotedString(Token): else: endQuoteChar = endQuoteChar.strip() if not endQuoteChar: - warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + warnings.warn("endQuoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) raise SyntaxError() self.quoteChar = quoteChar @@ -2889,35 +3445,34 @@ class QuotedString(Token): if multiline: self.flags = re.MULTILINE | re.DOTALL - self.pattern = r'%s(?:[^%s%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + self.pattern = r'%s(?:[^%s%s]' % (re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '')) else: self.flags = 0 - self.pattern = r'%s(?:[^%s\n\r%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + self.pattern = r'%s(?:[^%s\n\r%s]' % (re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '')) if len(self.endQuoteChar) > 1: self.pattern += ( '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]), - _escapeRegexRangeChars(self.endQuoteChar[i])) - for i in range(len(self.endQuoteChar)-1,0,-1)) + ')' - ) + _escapeRegexRangeChars(self.endQuoteChar[i])) + for i in range(len(self.endQuoteChar) - 1, 0, -1)) + ')') + if escQuote: self.pattern += (r'|(?:%s)' % re.escape(escQuote)) if escChar: self.pattern += (r'|(?:%s.)' % re.escape(escChar)) - self.escCharReplacePattern = re.escape(self.escChar)+"(.)" + self.escCharReplacePattern = re.escape(self.escChar) + "(.)" self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) try: self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern + self.re_match = self.re.match except sre_constants.error: warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) raise self.name = _ustr(self) @@ -2925,8 +3480,8 @@ class QuotedString(Token): self.mayIndexError = False self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): - result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None + def parseImpl(self, instring, loc, doActions=True): + result = instring[loc] == self.firstQuoteChar and self.re_match(instring, loc) or None if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -2936,18 +3491,18 @@ class QuotedString(Token): if self.unquoteResults: # strip off quotes - ret = ret[self.quoteCharLen:-self.endQuoteCharLen] + ret = ret[self.quoteCharLen: -self.endQuoteCharLen] - if isinstance(ret,basestring): + if isinstance(ret, basestring): # replace escaped whitespace if '\\' in ret and self.convertWhitespaceEscapes: ws_map = { - r'\t' : '\t', - r'\n' : '\n', - r'\f' : '\f', - r'\r' : '\r', + r'\t': '\t', + r'\n': '\n', + r'\f': '\f', + r'\r': '\r', } - for wslit,wschar in ws_map.items(): + for wslit, wschar in ws_map.items(): ret = ret.replace(wslit, wschar) # replace escaped characters @@ -2960,9 +3515,9 @@ class QuotedString(Token): return loc, ret - def __str__( self ): + def __str__(self): try: - return super(QuotedString,self).__str__() + return super(QuotedString, self).__str__() except Exception: pass @@ -2973,28 +3528,33 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ - def __init__( self, notChars, min=1, max=0, exact=0 ): - super(CharsNotIn,self).__init__() + def __init__(self, notChars, min=1, max=0, exact=0): + super(CharsNotIn, self).__init__() self.skipWhitespace = False self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError("cannot specify a minimum length < 1; use " + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3009,19 +3569,18 @@ class CharsNotIn(Token): self.name = _ustr(self) self.errmsg = "Expected " + self.name - self.mayReturnEmpty = ( self.minLen == 0 ) + self.mayReturnEmpty = (self.minLen == 0) self.mayIndexError = False - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if instring[loc] in self.notChars: raise ParseException(instring, loc, self.errmsg, self) start = loc loc += 1 notchars = self.notChars - maxlen = min( start+self.maxLen, len(instring) ) - while loc < maxlen and \ - (instring[loc] not in notchars): + maxlen = min(start + self.maxLen, len(instring)) + while loc < maxlen and instring[loc] not in notchars: loc += 1 if loc - start < self.minLen: @@ -3029,7 +3588,7 @@ class CharsNotIn(Token): return loc, instring[start:loc] - def __str__( self ): + def __str__(self): try: return super(CharsNotIn, self).__str__() except Exception: @@ -3044,25 +3603,44 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "<SPC>", - "\t": "<TAB>", - "\n": "<LF>", - "\r": "<CR>", - "\f": "<FF>", + ' ' : '<SP>', + '\t': '<TAB>', + '\n': '<LF>', + '\r': '<CR>', + '\f': '<FF>', + 'u\00A0': '<NBSP>', + 'u\1680': '<OGHAM_SPACE_MARK>', + 'u\180E': '<MONGOLIAN_VOWEL_SEPARATOR>', + 'u\2000': '<EN_QUAD>', + 'u\2001': '<EM_QUAD>', + 'u\2002': '<EN_SPACE>', + 'u\2003': '<EM_SPACE>', + 'u\2004': '<THREE-PER-EM_SPACE>', + 'u\2005': '<FOUR-PER-EM_SPACE>', + 'u\2006': '<SIX-PER-EM_SPACE>', + 'u\2007': '<FIGURE_SPACE>', + 'u\2008': '<PUNCTUATION_SPACE>', + 'u\2009': '<THIN_SPACE>', + 'u\200A': '<HAIR_SPACE>', + 'u\200B': '<ZERO_WIDTH_SPACE>', + 'u\202F': '<NNBSP>', + 'u\205F': '<MMSP>', + 'u\3000': '<IDEOGRAPHIC_SPACE>', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): - super(White,self).__init__() + super(White, self).__init__() self.matchWhite = ws - self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) ) - #~ self.leaveWhitespace() + self.setWhitespaceChars("".join(c for c in self.whiteChars if c not in self.matchWhite)) + # ~ self.leaveWhitespace() self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite)) self.mayReturnEmpty = True self.errmsg = "Expected " + self.name @@ -3078,13 +3656,13 @@ class White(Token): self.maxLen = exact self.minLen = exact - def parseImpl( self, instring, loc, doActions=True ): - if not(instring[ loc ] in self.matchWhite): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] not in self.matchWhite: raise ParseException(instring, loc, self.errmsg, self) start = loc loc += 1 maxloc = start + self.maxLen - maxloc = min( maxloc, len(instring) ) + maxloc = min(maxloc, len(instring)) while loc < maxloc and instring[loc] in self.matchWhite: loc += 1 @@ -3095,44 +3673,44 @@ class White(Token): class _PositionToken(Token): - def __init__( self ): - super(_PositionToken,self).__init__() - self.name=self.__class__.__name__ + def __init__(self): + super(_PositionToken, self).__init__() + self.name = self.__class__.__name__ self.mayReturnEmpty = True self.mayIndexError = False class GoToColumn(_PositionToken): + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ - Token to advance to a specific column of input text; useful for tabular report scraping. - """ - def __init__( self, colno ): - super(GoToColumn,self).__init__() + def __init__(self, colno): + super(GoToColumn, self).__init__() self.col = colno - def preParse( self, instring, loc ): - if col(loc,instring) != self.col: + def preParse(self, instring, loc): + if col(loc, instring) != self.col: instrlen = len(instring) if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) - while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col : + loc = self._skipIgnorables(instring, loc) + while loc < instrlen and instring[loc].isspace() and col(loc, instring) != self.col: loc += 1 return loc - def parseImpl( self, instring, loc, doActions=True ): - thiscol = col( loc, instring ) + def parseImpl(self, instring, loc, doActions=True): + thiscol = col(loc, instring) if thiscol > self.col: - raise ParseException( instring, loc, "Text not in expected column", self ) + raise ParseException(instring, loc, "Text not in expected column", self) newloc = loc + self.col - thiscol - ret = instring[ loc: newloc ] + ret = instring[loc: newloc] return newloc, ret class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + r"""Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3142,262 +3720,304 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ - def __init__( self ): - super(LineStart,self).__init__() + def __init__(self): + super(LineStart, self).__init__() self.errmsg = "Expected start of line" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if col(loc, instring) == 1: return loc, [] raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): + """Matches if current position is at the end of a line within the + parse string """ - Matches if current position is at the end of a line within the parse string - """ - def __init__( self ): - super(LineEnd,self).__init__() - self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + def __init__(self): + super(LineEnd, self).__init__() + self.setWhitespaceChars(ParserElement.DEFAULT_WHITE_CHARS.replace("\n", "")) self.errmsg = "Expected end of line" - def parseImpl( self, instring, loc, doActions=True ): - if loc<len(instring): + def parseImpl(self, instring, loc, doActions=True): + if loc < len(instring): if instring[loc] == "\n": - return loc+1, "\n" + return loc + 1, "\n" else: raise ParseException(instring, loc, self.errmsg, self) elif loc == len(instring): - return loc+1, [] + return loc + 1, [] else: raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): + """Matches if current position is at the beginning of the parse + string """ - Matches if current position is at the beginning of the parse string - """ - def __init__( self ): - super(StringStart,self).__init__() + def __init__(self): + super(StringStart, self).__init__() self.errmsg = "Expected start of text" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc != 0: # see if entire string up to here is just whitespace and ignoreables - if loc != self.preParse( instring, 0 ): + if loc != self.preParse(instring, 0): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class StringEnd(_PositionToken): + """Matches if current position is at the end of the parse string """ - Matches if current position is at the end of the parse string - """ - def __init__( self ): - super(StringEnd,self).__init__() + def __init__(self): + super(StringEnd, self).__init__() self.errmsg = "Expected end of text" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc < len(instring): raise ParseException(instring, loc, self.errmsg, self) elif loc == len(instring): - return loc+1, [] + return loc + 1, [] elif loc > len(instring): return loc, [] else: raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. - """ - def __init__(self, wordChars = printables): - super(WordStart,self).__init__() + def __init__(self, wordChars=printables): + super(WordStart, self).__init__() self.wordChars = set(wordChars) self.errmsg = "Not at the start of a word" - def parseImpl(self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc != 0: - if (instring[loc-1] in self.wordChars or - instring[loc] not in self.wordChars): + if (instring[loc - 1] in self.wordChars + or instring[loc] not in self.wordChars): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class WordEnd(_PositionToken): + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. - """ - def __init__(self, wordChars = printables): - super(WordEnd,self).__init__() + def __init__(self, wordChars=printables): + super(WordEnd, self).__init__() self.wordChars = set(wordChars) self.skipWhitespace = False self.errmsg = "Not at the end of a word" - def parseImpl(self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): instrlen = len(instring) - if instrlen>0 and loc<instrlen: + if instrlen > 0 and loc < instrlen: if (instring[loc] in self.wordChars or - instring[loc-1] not in self.wordChars): + instring[loc - 1] not in self.wordChars): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class ParseExpression(ParserElement): + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. - """ - def __init__( self, exprs, savelist = False ): - super(ParseExpression,self).__init__(savelist) - if isinstance( exprs, _generatorType ): + def __init__(self, exprs, savelist=False): + super(ParseExpression, self).__init__(savelist) + if isinstance(exprs, _generatorType): exprs = list(exprs) - if isinstance( exprs, basestring ): - self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, Iterable ): + if isinstance(exprs, basestring): + self.exprs = [self._literalStringClass(exprs)] + elif isinstance(exprs, ParserElement): + self.exprs = [exprs] + elif isinstance(exprs, Iterable): exprs = list(exprs) # if sequence of strings provided, wrap with Literal - if all(isinstance(expr, basestring) for expr in exprs): - exprs = map(ParserElement._literalStringClass, exprs) + if any(isinstance(expr, basestring) for expr in exprs): + exprs = (self._literalStringClass(e) if isinstance(e, basestring) else e for e in exprs) self.exprs = list(exprs) else: try: - self.exprs = list( exprs ) + self.exprs = list(exprs) except TypeError: - self.exprs = [ exprs ] + self.exprs = [exprs] self.callPreparse = False - def __getitem__( self, i ): - return self.exprs[i] - - def append( self, other ): - self.exprs.append( other ) + def append(self, other): + self.exprs.append(other) self.strRepr = None return self - def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + def leaveWhitespace(self): + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False - self.exprs = [ e.copy() for e in self.exprs ] + self.exprs = [e.copy() for e in self.exprs] for e in self.exprs: e.leaveWhitespace() return self - def ignore( self, other ): - if isinstance( other, Suppress ): + def ignore(self, other): + if isinstance(other, Suppress): if other not in self.ignoreExprs: - super( ParseExpression, self).ignore( other ) + super(ParseExpression, self).ignore(other) for e in self.exprs: - e.ignore( self.ignoreExprs[-1] ) + e.ignore(self.ignoreExprs[-1]) else: - super( ParseExpression, self).ignore( other ) + super(ParseExpression, self).ignore(other) for e in self.exprs: - e.ignore( self.ignoreExprs[-1] ) + e.ignore(self.ignoreExprs[-1]) return self - def __str__( self ): + def __str__(self): try: - return super(ParseExpression,self).__str__() + return super(ParseExpression, self).__str__() except Exception: pass if self.strRepr is None: - self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) ) + self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.exprs)) return self.strRepr - def streamline( self ): - super(ParseExpression,self).streamline() + def streamline(self): + super(ParseExpression, self).streamline() for e in self.exprs: e.streamline() - # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d ) + # collapse nested And's of the form And(And(And(a, b), c), d) to And(a, b, c, d) # but only if there are no parse actions or resultsNames on the nested And's # (likewise for Or's and MatchFirst's) - if ( len(self.exprs) == 2 ): + if len(self.exprs) == 2: other = self.exprs[0] - if ( isinstance( other, self.__class__ ) and - not(other.parseAction) and - other.resultsName is None and - not other.debug ): - self.exprs = other.exprs[:] + [ self.exprs[1] ] + if (isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug): + self.exprs = other.exprs[:] + [self.exprs[1]] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError other = self.exprs[-1] - if ( isinstance( other, self.__class__ ) and - not(other.parseAction) and - other.resultsName is None and - not other.debug ): + if (isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug): self.exprs = self.exprs[:-1] + other.exprs[:] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self - def setResultsName( self, name, listAllMatches=False ): - ret = super(ParseExpression,self).setResultsName(name,listAllMatches) - return ret - - def validate( self, validateTrace=[] ): - tmp = validateTrace[:]+[self] + def validate(self, validateTrace=None): + tmp = (validateTrace if validateTrace is not None else [])[:] + [self] for e in self.exprs: e.validate(tmp) - self.checkRecursion( [] ) - + self.checkRecursion([]) + def copy(self): - ret = super(ParseExpression,self).copy() + ret = super(ParseExpression, self).copy() ret.exprs = [e.copy() for e in self.exprs] return ret + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_ungrouped_named_tokens_in_collection: + for e in self.exprs: + if isinstance(e, ParserElement) and e.resultsName: + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName), + stacklevel=3) + + return super(ParseExpression, self)._setResultsName(name, listAllMatches) + + class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) - expr = And([integer("id"),name_expr("name"),integer("age")]) + expr = And([integer("id"), name_expr("name"), integer("age")]) # more easily written as: expr = integer("id") + name_expr("name") + integer("age") """ class _ErrorStop(Empty): def __init__(self, *args, **kwargs): - super(And._ErrorStop,self).__init__(*args, **kwargs) + super(And._ErrorStop, self).__init__(*args, **kwargs) self.name = '-' self.leaveWhitespace() - def __init__( self, exprs, savelist = True ): - super(And,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=True): + if exprs and Ellipsis in exprs: + tmp = [] + for i, expr in enumerate(exprs): + if expr is Ellipsis: + if i < len(exprs) - 1: + skipto_arg = (Empty() + exprs[i + 1]).exprs[-1] + tmp.append(SkipTo(skipto_arg)("_skipped*")) + else: + raise Exception("cannot construct And with sequence ending in ...") + else: + tmp.append(expr) + exprs[:] = tmp + super(And, self).__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.setWhitespaceChars( self.exprs[0].whiteChars ) + self.setWhitespaceChars(self.exprs[0].whiteChars) self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + # collapse any _PendingSkip's + if self.exprs: + if any(isinstance(e, ParseExpression) and e.exprs and isinstance(e.exprs[-1], _PendingSkip) + for e in self.exprs[:-1]): + for i, e in enumerate(self.exprs[:-1]): + if e is None: + continue + if (isinstance(e, ParseExpression) + and e.exprs and isinstance(e.exprs[-1], _PendingSkip)): + e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] + self.exprs[i + 1] = None + self.exprs = [e for e in self.exprs if e is not None] + + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing - loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False ) + loc, resultlist = self.exprs[0]._parse(instring, loc, doActions, callPreParse=False) errorStop = False for e in self.exprs[1:]: if isinstance(e, And._ErrorStop): @@ -3405,7 +4025,7 @@ class And(ParseExpression): continue if errorStop: try: - loc, exprtokens = e._parse( instring, loc, doActions ) + loc, exprtokens = e._parse(instring, loc, doActions) except ParseSyntaxException: raise except ParseBaseException as pe: @@ -3414,25 +4034,25 @@ class And(ParseExpression): except IndexError: raise ParseSyntaxException(instring, len(instring), self.errmsg, self) else: - loc, exprtokens = e._parse( instring, loc, doActions ) + loc, exprtokens = e._parse(instring, loc, doActions) if exprtokens or exprtokens.haskeys(): resultlist += exprtokens return loc, resultlist - def __iadd__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #And( [ self, other ] ) + def __iadd__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # And([self, other]) - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) if not e.mayReturnEmpty: break - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3442,33 +4062,42 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ - def __init__( self, exprs, savelist = False ): - super(Or,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=False): + super(Or, self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) else: self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(Or, self).streamline() + if __compat__.collect_all_And_tokens: + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): maxExcLoc = -1 maxException = None matches = [] for e in self.exprs: try: - loc2 = e.tryParse( instring, loc ) + loc2 = e.tryParse(instring, loc) except ParseException as err: err.__traceback__ = None if err.loc > maxExcLoc: @@ -3476,22 +4105,45 @@ class Or(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) + maxException = ParseException(instring, len(instring), e.errmsg, self) maxExcLoc = len(instring) else: # save match among all matches, to retry longest to shortest matches.append((loc2, e)) if matches: - matches.sort(key=lambda x: -x[0]) - for _,e in matches: + # re-evaluate all matches in descending order of length of match, in case attached actions + # might change whether or how much they match of the input. + matches.sort(key=itemgetter(0), reverse=True) + + if not doActions: + # no further conditions or parse actions to change the selection of + # alternative, so the first match will be the best match + best_expr = matches[0][1] + return best_expr._parse(instring, loc, doActions) + + longest = -1, None + for loc1, expr1 in matches: + if loc1 <= longest[0]: + # already have a longer match than this one will deliver, we are done + return longest + try: - return e._parse( instring, loc, doActions ) + loc2, toks = expr1._parse(instring, loc, doActions) except ParseException as err: err.__traceback__ = None if err.loc > maxExcLoc: maxException = err maxExcLoc = err.loc + else: + if loc2 >= loc1: + return loc2, toks + # didn't match as much as before + elif loc2 > longest[0]: + longest = loc2, toks + + if longest != (-1, None): + return longest if maxException is not None: maxException.msg = self.errmsg @@ -3500,13 +4152,13 @@ class Or(ParseExpression): raise ParseException(instring, loc, "no defined alternatives to match", self) - def __ixor__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #Or( [ self, other ] ) + def __ixor__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # Or([self, other]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3514,21 +4166,33 @@ class Or(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) + + def _setResultsName(self, name, listAllMatches=False): + if (not __compat__.collect_all_And_tokens + and __diag__.warn_multiple_tokens_in_named_alternation): + if any(isinstance(e, And) for e in self.exprs): + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "may only return a single token for an And alternative, " + "in future will return the full list of tokens".format( + "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), + stacklevel=3) + + return super(Or, self)._setResultsName(name, listAllMatches) class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3537,19 +4201,25 @@ class MatchFirst(ParseExpression): number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] """ - def __init__( self, exprs, savelist = False ): - super(MatchFirst,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=False): + super(MatchFirst, self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) else: self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(MatchFirst, self).streamline() + if __compat__.collect_all_And_tokens: + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): maxExcLoc = -1 maxException = None for e in self.exprs: try: - ret = e._parse( instring, loc, doActions ) + ret = e._parse(instring, loc, doActions) return ret except ParseException as err: if err.loc > maxExcLoc: @@ -3557,7 +4227,7 @@ class MatchFirst(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) + maxException = ParseException(instring, len(instring), e.errmsg, self) maxExcLoc = len(instring) # only got here if no expression matched, raise exception for match that made it the furthest @@ -3568,13 +4238,13 @@ class MatchFirst(ParseExpression): else: raise ParseException(instring, loc, "no defined alternatives to match", self) - def __ior__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #MatchFirst( [ self, other ] ) + def __ior__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # MatchFirst([self, other]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3582,19 +4252,32 @@ class MatchFirst(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) + + def _setResultsName(self, name, listAllMatches=False): + if (not __compat__.collect_all_And_tokens + and __diag__.warn_multiple_tokens_in_named_alternation): + if any(isinstance(e, And) for e in self.exprs): + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "may only return a single token for an And alternative, " + "in future will return the full list of tokens".format( + "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), + stacklevel=3) + + return super(MatchFirst, self)._setResultsName(name, listAllMatches) class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3603,7 +4286,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3613,7 +4296,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3642,21 +4327,27 @@ class Each(ParseExpression): - shape: TRIANGLE - size: 20 """ - def __init__( self, exprs, savelist = True ): - super(Each,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=True): + super(Each, self).__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): if self.initExprGroups: - self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional)) - opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ] - opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)] + self.opt1map = dict((id(e.expr), e) for e in self.exprs if isinstance(e, Optional)) + opt1 = [e.expr for e in self.exprs if isinstance(e, Optional)] + opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, Optional)] self.optionals = opt1 + opt2 - self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ] - self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ] - self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ] + self.multioptionals = [e.expr for e in self.exprs if isinstance(e, ZeroOrMore)] + self.multirequired = [e.expr for e in self.exprs if isinstance(e, OneOrMore)] + self.required = [e for e in self.exprs if not isinstance(e, (Optional, ZeroOrMore, OneOrMore))] self.required += self.multirequired self.initExprGroups = False tmpLoc = loc @@ -3670,11 +4361,11 @@ class Each(ParseExpression): failed = [] for e in tmpExprs: try: - tmpLoc = e.tryParse( instring, tmpLoc ) + tmpLoc = e.tryParse(instring, tmpLoc) except ParseException: failed.append(e) else: - matchOrder.append(self.opt1map.get(id(e),e)) + matchOrder.append(self.opt1map.get(id(e), e)) if e in tmpReqd: tmpReqd.remove(e) elif e in tmpOpt: @@ -3684,21 +4375,21 @@ class Each(ParseExpression): if tmpReqd: missing = ", ".join(_ustr(e) for e in tmpReqd) - raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing ) + raise ParseException(instring, loc, "Missing one or more required elements (%s)" % missing) # add any unmatched Optionals, in case they have default values defined - matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt] + matchOrder += [e for e in self.exprs if isinstance(e, Optional) and e.expr in tmpOpt] resultlist = [] for e in matchOrder: - loc,results = e._parse(instring,loc,doActions) + loc, results = e._parse(instring, loc, doActions) resultlist.append(results) finalResults = sum(resultlist, ParseResults([])) return loc, finalResults - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3706,140 +4397,238 @@ class Each(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) class ParseElementEnhance(ParserElement): + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. - """ - def __init__( self, expr, savelist=False ): - super(ParseElementEnhance,self).__init__(savelist) - if isinstance( expr, basestring ): - if issubclass(ParserElement._literalStringClass, Token): - expr = ParserElement._literalStringClass(expr) + def __init__(self, expr, savelist=False): + super(ParseElementEnhance, self).__init__(savelist) + if isinstance(expr, basestring): + if issubclass(self._literalStringClass, Token): + expr = self._literalStringClass(expr) else: - expr = ParserElement._literalStringClass(Literal(expr)) + expr = self._literalStringClass(Literal(expr)) self.expr = expr self.strRepr = None if expr is not None: self.mayIndexError = expr.mayIndexError self.mayReturnEmpty = expr.mayReturnEmpty - self.setWhitespaceChars( expr.whiteChars ) + self.setWhitespaceChars(expr.whiteChars) self.skipWhitespace = expr.skipWhitespace self.saveAsList = expr.saveAsList self.callPreparse = expr.callPreparse self.ignoreExprs.extend(expr.ignoreExprs) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.expr is not None: - return self.expr._parse( instring, loc, doActions, callPreParse=False ) + return self.expr._parse(instring, loc, doActions, callPreParse=False) else: - raise ParseException("",loc,self.errmsg,self) + raise ParseException("", loc, self.errmsg, self) - def leaveWhitespace( self ): + def leaveWhitespace(self): self.skipWhitespace = False self.expr = self.expr.copy() if self.expr is not None: self.expr.leaveWhitespace() return self - def ignore( self, other ): - if isinstance( other, Suppress ): + def ignore(self, other): + if isinstance(other, Suppress): if other not in self.ignoreExprs: - super( ParseElementEnhance, self).ignore( other ) + super(ParseElementEnhance, self).ignore(other) if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) + self.expr.ignore(self.ignoreExprs[-1]) else: - super( ParseElementEnhance, self).ignore( other ) + super(ParseElementEnhance, self).ignore(other) if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) + self.expr.ignore(self.ignoreExprs[-1]) return self - def streamline( self ): - super(ParseElementEnhance,self).streamline() + def streamline(self): + super(ParseElementEnhance, self).streamline() if self.expr is not None: self.expr.streamline() return self - def checkRecursion( self, parseElementList ): + def checkRecursion(self, parseElementList): if self in parseElementList: - raise RecursiveGrammarException( parseElementList+[self] ) - subRecCheckList = parseElementList[:] + [ self ] + raise RecursiveGrammarException(parseElementList + [self]) + subRecCheckList = parseElementList[:] + [self] if self.expr is not None: - self.expr.checkRecursion( subRecCheckList ) + self.expr.checkRecursion(subRecCheckList) - def validate( self, validateTrace=[] ): - tmp = validateTrace[:]+[self] + def validate(self, validateTrace=None): + if validateTrace is None: + validateTrace = [] + tmp = validateTrace[:] + [self] if self.expr is not None: self.expr.validate(tmp) - self.checkRecursion( [] ) + self.checkRecursion([]) - def __str__( self ): + def __str__(self): try: - return super(ParseElementEnhance,self).__str__() + return super(ParseElementEnhance, self).__str__() except Exception: pass if self.strRepr is None and self.expr is not None: - self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) ) + self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.expr)) return self.strRepr class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ - def __init__( self, expr ): - super(FollowedBy,self).__init__(expr) + def __init__(self, expr): + super(FollowedBy, self).__init__(expr) self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + def parseImpl(self, instring, loc, doActions=True): + # by using self._expr.parse and deleting the contents of the returned ParseResults list + # we keep any named results that were defined in the FollowedBy expression + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[:loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat + 1)): + try: + _, ret = test_expr._parse(instring_slice, loc - offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + # return empty list of tokens, but preserve any defined results names + del ret[:] + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ - def __init__( self, expr ): - super(NotAny,self).__init__(expr) - #~ self.leaveWhitespace() + def __init__(self, expr): + super(NotAny, self).__init__(expr) + # ~ self.leaveWhitespace() self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs self.mayReturnEmpty = True - self.errmsg = "Found unwanted token, "+_ustr(self.expr) + self.errmsg = "Found unwanted token, " + _ustr(self.expr) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.expr.canParseNext(instring, loc): raise ParseException(instring, loc, self.errmsg, self) return loc, [] - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3848,54 +4637,74 @@ class NotAny(ParseElementEnhance): return self.strRepr class _MultipleMatch(ParseElementEnhance): - def __init__( self, expr, stopOn=None): + def __init__(self, expr, stopOn=None): super(_MultipleMatch, self).__init__(expr) self.saveAsList = True ender = stopOn if isinstance(ender, basestring): - ender = ParserElement._literalStringClass(ender) - self.not_ender = ~ender if ender is not None else None + ender = self._literalStringClass(ender) + self.stopOn(ender) - def parseImpl( self, instring, loc, doActions=True ): + def stopOn(self, ender): + if isinstance(ender, basestring): + ender = self._literalStringClass(ender) + self.not_ender = ~ender if ender is not None else None + return self + + def parseImpl(self, instring, loc, doActions=True): self_expr_parse = self.expr._parse self_skip_ignorables = self._skipIgnorables check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: try_not_ender(instring, loc) - loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False ) + loc, tokens = self_expr_parse(instring, loc, doActions, callPreParse=False) try: hasIgnoreExprs = (not not self.ignoreExprs) while 1: if check_ender: try_not_ender(instring, loc) if hasIgnoreExprs: - preloc = self_skip_ignorables( instring, loc ) + preloc = self_skip_ignorables(instring, loc) else: preloc = loc - loc, tmptokens = self_expr_parse( instring, preloc, doActions ) + loc, tmptokens = self_expr_parse(instring, preloc, doActions) if tmptokens or tmptokens.haskeys(): tokens += tmptokens - except (ParseException,IndexError): + except (ParseException, IndexError): pass return loc, tokens - + + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_ungrouped_named_tokens_in_collection: + for e in [self.expr] + getattr(self.expr, 'exprs', []): + if isinstance(e, ParserElement) and e.resultsName: + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName), + stacklevel=3) + + return super(_MultipleMatch, self)._setResultsName(name, listAllMatches) + + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3906,13 +4715,13 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3921,29 +4730,28 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ - def __init__( self, expr, stopOn=None): - super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) + def __init__(self, expr, stopOn=None): + super(ZeroOrMore, self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): + + def parseImpl(self, instring, loc, doActions=True): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) - except (ParseException,IndexError): + except (ParseException, IndexError): return loc, [] - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3951,6 +4759,7 @@ class ZeroOrMore(_MultipleMatch): return self.strRepr + class _NullToken(object): def __bool__(self): return False @@ -3958,29 +4767,30 @@ class _NullToken(object): def __str__(self): return "" -_optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -3994,28 +4804,30 @@ class Optional(ParseElementEnhance): ^ FAIL: Expected end of text (at char 5), (line:1, col:6) """ - def __init__( self, expr, default=_optionalNotMatched ): - super(Optional,self).__init__( expr, savelist=False ) + __optionalNotMatched = _NullToken() + + def __init__(self, expr, default=__optionalNotMatched): + super(Optional, self).__init__(expr, savelist=False) self.saveAsList = self.expr.saveAsList self.defaultValue = default self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): try: - loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) - except (ParseException,IndexError): - if self.defaultValue is not _optionalNotMatched: + loc, tokens = self.expr._parse(instring, loc, doActions, callPreParse=False) + except (ParseException, IndexError): + if self.defaultValue is not self.__optionalNotMatched: if self.expr.resultsName: - tokens = ParseResults([ self.defaultValue ]) + tokens = ParseResults([self.defaultValue]) tokens[self.expr.resultsName] = self.defaultValue else: - tokens = [ self.defaultValue ] + tokens = [self.defaultValue] else: tokens = [] return loc, tokens - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -4024,20 +4836,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4054,14 +4867,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4078,34 +4893,34 @@ class SkipTo(ParseElementEnhance): - issue_num: 79 - sev: Minor """ - def __init__( self, other, include=False, ignore=None, failOn=None ): - super( SkipTo, self ).__init__( other ) + def __init__(self, other, include=False, ignore=None, failOn=None): + super(SkipTo, self).__init__(other) self.ignoreExpr = ignore self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include - self.asList = False + self.saveAsList = False if isinstance(failOn, basestring): - self.failOn = ParserElement._literalStringClass(failOn) + self.failOn = self._literalStringClass(failOn) else: self.failOn = failOn - self.errmsg = "No match found for "+_ustr(self.expr) + self.errmsg = "No match found for " + _ustr(self.expr) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): startloc = loc instrlen = len(instring) expr = self.expr expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4113,7 +4928,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4131,113 +4946,135 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: - loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) + loc, mat = expr_parse(instring, loc, doActions, callPreParse=False) skipresult += mat return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ - def __init__( self, other=None ): - super(Forward,self).__init__( other, savelist=False ) + def __init__(self, other=None): + super(Forward, self).__init__(other, savelist=False) - def __lshift__( self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass(other) + def __lshift__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) self.expr = other self.strRepr = None self.mayIndexError = self.expr.mayIndexError self.mayReturnEmpty = self.expr.mayReturnEmpty - self.setWhitespaceChars( self.expr.whiteChars ) + self.setWhitespaceChars(self.expr.whiteChars) self.skipWhitespace = self.expr.skipWhitespace self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - - def leaveWhitespace( self ): + + def leaveWhitespace(self): self.skipWhitespace = False return self - def streamline( self ): + def streamline(self): if not self.streamlined: self.streamlined = True if self.expr is not None: self.expr.streamline() return self - def validate( self, validateTrace=[] ): + def validate(self, validateTrace=None): + if validateTrace is None: + validateTrace = [] + if self not in validateTrace: - tmp = validateTrace[:]+[self] + tmp = validateTrace[:] + [self] if self.expr is not None: self.expr.validate(tmp) self.checkRecursion([]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name - return self.__class__.__name__ + ": ..." + if self.strRepr is not None: + return self.strRepr - # stubbed out for now - creates awful memory and perf issues - self._revertClass = self.__class__ - self.__class__ = _ForwardNoRecurse + # Avoid infinite recursion by setting a temporary strRepr + self.strRepr = ": ..." + + # Use the string representation of main expression. + retString = '...' try: if self.expr is not None: - retString = _ustr(self.expr) + retString = _ustr(self.expr)[:1000] else: retString = "None" finally: - self.__class__ = self._revertClass - return self.__class__.__name__ + ": " + retString + self.strRepr = self.__class__.__name__ + ": " + retString + return self.strRepr def copy(self): if self.expr is not None: - return super(Forward,self).copy() + return super(Forward, self).copy() else: ret = Forward() ret <<= self return ret -class _ForwardNoRecurse(Forward): - def __str__( self ): - return "..." + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_name_set_on_empty_Forward: + if self.expr is None: + warnings.warn("{0}: setting results name {0!r} on {1} expression " + "that has no contained expression".format("warn_name_set_on_empty_Forward", + name, + type(self).__name__), + stacklevel=3) + + return super(Forward, self)._setResultsName(name, listAllMatches) class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ - def __init__( self, expr, savelist=False ): - super(TokenConverter,self).__init__( expr )#, savelist ) + def __init__(self, expr, savelist=False): + super(TokenConverter, self).__init__(expr) # , savelist) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4248,8 +5085,8 @@ class Combine(TokenConverter): # no match when there are internal spaces print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...) """ - def __init__( self, expr, joinString="", adjacent=True ): - super(Combine,self).__init__( expr ) + def __init__(self, expr, joinString="", adjacent=True): + super(Combine, self).__init__(expr) # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself if adjacent: self.leaveWhitespace() @@ -4258,71 +5095,74 @@ class Combine(TokenConverter): self.joinString = joinString self.callPreparse = True - def ignore( self, other ): + def ignore(self, other): if self.adjacent: ParserElement.ignore(self, other) else: - super( Combine, self).ignore( other ) + super(Combine, self).ignore(other) return self - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): retToks = tokenlist.copy() del retToks[:] - retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults) + retToks += ParseResults(["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults) if self.resultsName and retToks.haskeys(): - return [ retToks ] + return [retToks] else: return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num func = ident + Optional(delimitedList(term)) - print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100'] + print(func.parseString("fn a, b, 100")) # -> ['fn', 'a', 'b', '100'] func = ident + Group(Optional(delimitedList(term))) - print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']] + print(func.parseString("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] """ - def __init__( self, expr ): - super(Group,self).__init__( expr ) + def __init__(self, expr): + super(Group, self).__init__(expr) self.saveAsList = True - def postParse( self, instring, loc, tokenlist ): - return [ tokenlist ] + def postParse(self, instring, loc, tokenlist): + return [tokenlist] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4330,42 +5170,43 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ - def __init__( self, expr ): - super(Dict,self).__init__( expr ) + def __init__(self, expr): + super(Dict, self).__init__(expr) self.saveAsList = True - def postParse( self, instring, loc, tokenlist ): - for i,tok in enumerate(tokenlist): + def postParse(self, instring, loc, tokenlist): + for i, tok in enumerate(tokenlist): if len(tok) == 0: continue ikey = tok[0] - if isinstance(ikey,int): + if isinstance(ikey, int): ikey = _ustr(tok[0]).strip() - if len(tok)==1: - tokenlist[ikey] = _ParseResultsWithOffset("",i) - elif len(tok)==2 and not isinstance(tok[1],ParseResults): - tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i) + if len(tok) == 1: + tokenlist[ikey] = _ParseResultsWithOffset("", i) + elif len(tok) == 2 and not isinstance(tok[1], ParseResults): + tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i) else: - dictvalue = tok.copy() #ParseResults(i) + dictvalue = tok.copy() # ParseResults(i) del dictvalue[0] - if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()): - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i) + if len(dictvalue) != 1 or (isinstance(dictvalue, ParseResults) and dictvalue.haskeys()): + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i) else: - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i) + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i) if self.resultsName: - return [ tokenlist ] + return [tokenlist] else: return tokenlist class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4375,42 +5216,46 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): return [] - def suppress( self ): + def suppress(self): return self class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) self.called = False - def __call__(self,s,l,t): + def __call__(self, s, l, t): if not self.called: - results = self.callable(s,l,t) + results = self.callable(s, l, t) self.called = True return results - raise ParseException(s,l,"") + raise ParseException(s, l, "") def reset(self): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:<current_source_line>, <parse_location>, <matched_tokens>)"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4419,7 +5264,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) <<leaving remove_duplicate_chars (ret: 'dfjkls') ['dfjkls'] @@ -4427,16 +5274,16 @@ def traceParseAction(f): f = _trim_arity(f) def z(*paArgs): thisFunc = f.__name__ - s,l,t = paArgs[-3:] - if len(paArgs)>3: + s, l, t = paArgs[-3:] + if len(paArgs) > 3: thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc - sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) ) + sys.stderr.write(">>entering %s(line: '%s', %d, %r)\n" % (thisFunc, line(l, s), l, t)) try: ret = f(*paArgs) except Exception as exc: - sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) ) + sys.stderr.write("<<leaving %s (exception: %s)\n" % (thisFunc, exc)) raise - sys.stderr.write( "<<leaving %s (ret: %r)\n" % (thisFunc,ret) ) + sys.stderr.write("<<leaving %s (ret: %r)\n" % (thisFunc, ret)) return ret try: z.__name__ = f.__name__ @@ -4447,36 +5294,43 @@ def traceParseAction(f): # # global helpers # -def delimitedList( expr, delim=",", combine=False ): - """ - Helper to define a delimited list of expressions - the delimiter defaults to ','. - By default, the list elements and delimiters can have intervening whitespace, and - comments, but this can be overridden by passing C{combine=True} in the constructor. - If C{combine} is set to C{True}, the matching tokens are returned as a single token - string, with the delimiters included; otherwise, the matching tokens are returned - as a list of tokens, with the delimiters suppressed. +def delimitedList(expr, delim=",", combine=False): + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. Example:: + delimitedList(Word(alphas)).parseString("aa,bb,cc") # -> ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ - dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..." + dlName = _ustr(expr) + " [" + _ustr(delim) + " " + _ustr(expr) + "]..." if combine: - return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName) + return Combine(expr + ZeroOrMore(delim + expr)).setName(dlName) else: - return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) + return (expr + ZeroOrMore(Suppress(delim) + expr)).setName(dlName) + +def countedArray(expr, intExpr=None): + """Helper to define a counted list of expressions. -def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4485,42 +5339,44 @@ def countedArray( expr, intExpr=None ): countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd'] """ arrayExpr = Forward() - def countFieldParseAction(s,l,t): + def countFieldParseAction(s, l, t): n = t[0] - arrayExpr << (n and Group(And([expr]*n)) or Group(empty)) + arrayExpr << (n and Group(And([expr] * n)) or Group(empty)) return [] if intExpr is None: - intExpr = Word(nums).setParseAction(lambda t:int(t[0])) + intExpr = Word(nums).setParseAction(lambda t: int(t[0])) else: intExpr = intExpr.copy() intExpr.setName("arrayLen") intExpr.addParseAction(countFieldParseAction, callDuringTry=True) - return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...') + return (intExpr + arrayExpr).setName('(len) ' + _ustr(expr) + '...') def _flatten(L): ret = [] for i in L: - if isinstance(i,list): + if isinstance(i, list): ret.extend(_flatten(i)) else: ret.append(i) return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() - def copyTokenToRepeater(s,l,t): + def copyTokenToRepeater(s, l, t): if t: if len(t) == 1: rep << t[0] @@ -4535,128 +5391,145 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() rep <<= e2 - def copyTokenToRepeater(s,l,t): + def copyTokenToRepeater(s, l, t): matchTokens = _flatten(t.asList()) - def mustMatchTheseTokens(s,l,t): + def mustMatchTheseTokens(s, l, t): theseTokens = _flatten(t.asList()) - if theseTokens != matchTokens: - raise ParseException("",0,"") - rep.setParseAction( mustMatchTheseTokens, callDuringTry=True ) + if theseTokens != matchTokens: + raise ParseException('', 0, '') + rep.setParseAction(mustMatchTheseTokens, callDuringTry=True) expr.addParseAction(copyTokenToRepeater, callDuringTry=True) rep.setName('(prev) ' + _ustr(expr)) return rep def _escapeRegexRangeChars(s): - #~ escape these chars: ^-] + # ~ escape these chars: ^-] for c in r"\^-]": - s = s.replace(c,_bslash+c) - s = s.replace("\n",r"\n") - s = s.replace("\t",r"\t") + s = s.replace(c, _bslash + c) + s = s.replace("\n", r"\n") + s = s.replace("\t", r"\t") return _ustr(s) -def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. +def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True`` or ``asKeyword=True``, or if + creating a :class:`Regex` raises an exception) + - asKeyword - (default=``False``) - enforce Keyword-style matching on the + generated expressions Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ + if isinstance(caseless, basestring): + warnings.warn("More than one string argument passed to oneOf, pass " + "choices as a list or space-delimited string", stacklevel=2) + if caseless: - isequal = ( lambda a,b: a.upper() == b.upper() ) - masks = ( lambda a,b: b.upper().startswith(a.upper()) ) - parseElementClass = CaselessLiteral + isequal = (lambda a, b: a.upper() == b.upper()) + masks = (lambda a, b: b.upper().startswith(a.upper())) + parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral else: - isequal = ( lambda a,b: a == b ) - masks = ( lambda a,b: b.startswith(a) ) - parseElementClass = Literal + isequal = (lambda a, b: a == b) + masks = (lambda a, b: b.startswith(a)) + parseElementClass = Keyword if asKeyword else Literal symbols = [] - if isinstance(strs,basestring): + if isinstance(strs, basestring): symbols = strs.split() elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) if not symbols: return NoMatch() - i = 0 - while i < len(symbols)-1: - cur = symbols[i] - for j,other in enumerate(symbols[i+1:]): - if ( isequal(other, cur) ): - del symbols[i+j+1] - break - elif ( masks(cur, other) ): - del symbols[i+j+1] - symbols.insert(i,other) - cur = other - break - else: - i += 1 - - if not caseless and useRegex: - #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] )) - try: - if len(symbols)==len("".join(symbols)): - return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols)) + if not asKeyword: + # if not producing keywords, need to reorder to take care to avoid masking + # longer choices with shorter ones + i = 0 + while i < len(symbols) - 1: + cur = symbols[i] + for j, other in enumerate(symbols[i + 1:]): + if isequal(other, cur): + del symbols[i + j + 1] + break + elif masks(cur, other): + del symbols[i + j + 1] + symbols.insert(i, other) + break else: - return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols)) + i += 1 + + if not (caseless or asKeyword) and useRegex: + # ~ print (strs, "->", "|".join([_escapeRegexChars(sym) for sym in symbols])) + try: + if len(symbols) == len("".join(symbols)): + return Regex("[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols)).setName(' | '.join(symbols)) + else: + return Regex("|".join(re.escape(sym) for sym in symbols)).setName(' | '.join(symbols)) except Exception: warnings.warn("Exception creating Regex for oneOf, building MatchFirst", SyntaxWarning, stacklevel=2) - # last resort, just use MatchFirst return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) -def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. +def dictOf(key, value): + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4666,7 +5539,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4676,73 +5551,82 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test <b> bold <i>text</i> </b> normal text " - for tag in ("b","i"): - opener,closer = makeHTMLTags(tag) + for tag in ("b", "i"): + opener, closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + ['<b> bold <i>text</i> </b>'] ['<i>text</i>'] """ - locMarker = Empty().setParseAction(lambda s,loc,t: loc) + locMarker = Empty().setParseAction(lambda s, loc, t: loc) endlocMarker = locMarker.copy() endlocMarker.callPreparse = False matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") if asString: - extractText = lambda s,l,t: s[t._original_start:t._original_end] + extractText = lambda s, l, t: s[t._original_start: t._original_end] else: - def extractText(s,l,t): + def extractText(s, l, t): t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]] matchExpr.setParseAction(extractText) matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. - """ - return TokenConverter(expr).setParseAction(lambda t:t[0]) + return TokenConverter(expr).addParseAction(lambda t: t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{<TAB>} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ``<TAB>`` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] """ - locator = Empty().setParseAction(lambda s,l,t: l) + locator = Empty().setParseAction(lambda s, l, t: l) return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end")) @@ -4753,66 +5637,75 @@ lineEnd = LineEnd().setName("lineEnd") stringStart = StringStart().setName("stringStart") stringEnd = StringEnd().setName("stringEnd") -_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) -_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) -_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) +_escapedPunc = Word(_bslash, r"\[]-*.$+^?()~ ", exact=2).setParseAction(lambda s, l, t: t[0][1]) +_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s, l, t: unichr(int(t[0].lstrip(r'\0x'), 16))) +_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s, l, t: unichr(int(t[0][1:], 8))) _singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) -_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" +_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group(OneOrMore(_charRange | _singleChar)).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ - _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) + _expanded = lambda p: p if not isinstance(p, ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) try: return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body) except Exception: return "" def matchOnlyAtCol(n): + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ - Helper method for defining parse actions that require matching at a specific - column in the input text. - """ - def verifyCol(strg,locn,toks): - if col(locn,strg) != n: - raise ParseException(strg,locn,"matched token not at column %d" % n) + def verifyCol(strg, locn, toks): + if col(locn, strg) != n: + raise ParseException(strg, locn, "matched token not at column %d" % n) return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString<ParserElement.transformString>}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString<ParserElement.transformString>` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ - return lambda s,l,t: [replStr] + return lambda s, l, t: [replStr] -def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. +def removeQuotes(s, l, t): + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4823,18 +5716,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4844,7 +5739,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4854,11 +5751,11 @@ def tokenMap(func, *args): now is the winter of our discontent made glorious summer by this sun of york ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] """ - def pa(s,l,t): + def pa(s, l, t): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4867,90 +5764,108 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - -def _makeTags(tagStr, xml): +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + +def _makeTags(tagStr, xml, + suppress_LT=Suppress("<"), + suppress_GT=Suppress(">")): """Internal helper to construct opening and closing tag expressions, given a tag name""" - if isinstance(tagStr,basestring): + if isinstance(tagStr, basestring): resname = tagStr tagStr = Keyword(tagStr, caseless=not xml) else: resname = tagStr.name - tagAttrName = Word(alphas,alphanums+"_-:") - if (xml): - tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes ) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + tagAttrName = Word(alphas, alphanums + "_-:") + if xml: + tagAttrValue = dblQuotedString.copy().setParseAction(removeQuotes) + openTag = (suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) + + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') + + suppress_GT) else: - printablesLessRAbrack = "".join(c for c in printables if c not in ">") - tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \ - Optional( Suppress("=") + tagAttrValue ) ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") - closeTag = Combine(_L("</") + tagStr + ">") + tagAttrValue = quotedString.copy().setParseAction(removeQuotes) | Word(printables, excludeChars=">") + openTag = (suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName.setParseAction(downcaseTokens) + + Optional(Suppress("=") + tagAttrValue)))) + + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') + + suppress_GT) + closeTag = Combine(_L("</") + tagStr + ">", adjacent=False) - openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname) - closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % resname) + openTag.setName("<%s>" % resname) + # add start<tagname> results name in parse action now that ungrouped names are not reported at two levels + openTag.addParseAction(lambda t: t.__setitem__("start" + "".join(resname.replace(":", " ").title().split()), t.copy())) + closeTag = closeTag("end" + "".join(resname.replace(":", " ").title().split())).setName("</%s>" % resname) openTag.tag = resname closeTag.tag = resname + openTag.tag_body = SkipTo(closeTag()) return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple - a,a_end = makeHTMLTags("A") + + text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple + a, a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the <A> tag (like "href" shown here) are also accessible as named results + # attributes in the <A> tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ - return _makeTags( tagStr, False ) + return _makeTags(tagStr, False) def makeXMLTags(tagStr): + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. + + Example: similar to :class:`makeHTMLTags` """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + return _makeTags(tagStr, True) - Example: similar to L{makeHTMLTags} - """ - return _makeTags( tagStr, True ) +def withAttribute(*args, **attrDict): + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ``<TD>`` or ``<DIV>``. -def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{<TD>} or C{<DIV>}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align", "right"))`` - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = ''' <div> Some text @@ -4958,7 +5873,7 @@ def withAttribute(*args,**attrDict): <div type="graph">1,3 2,3 1,1</div> <div>this has no type</div> </div> - + ''' div,div_end = makeHTMLTags("div") @@ -4967,13 +5882,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -4983,23 +5900,24 @@ def withAttribute(*args,**attrDict): attrs = args[:] else: attrs = attrDict.items() - attrs = [(k,v) for k,v in attrs] - def pa(s,l,tokens): - for attrName,attrValue in attrs: + attrs = [(k, v) for k, v in attrs] + def pa(s, l, tokens): + for attrName, attrValue in attrs: if attrName not in tokens: - raise ParseException(s,l,"no matching attribute " + attrName) + raise ParseException(s, l, "no matching attribute " + attrName) if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: - raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" % + raise ParseException(s, l, "attribute '%s' has value '%s', must be '%s'" % (attrName, tokens[attrName], attrValue)) return pa withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = ''' <div> Some text @@ -5007,84 +5925,96 @@ def withClass(classname, namespace=''): <div class="graph">1,3 2,3 1,1</div> <div>this <div> has no class</div> </div> - + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr: classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() -def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. +def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5094,27 +6024,34 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() - lastExpr = baseExpr | ( lpar + ret + rpar ) - for i,operDef in enumerate(opList): - opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4] + lastExpr = baseExpr | (lpar + ret + rpar) + for i, operDef in enumerate(opList): + opExpr, arity, rightLeftAssoc, pa = (operDef + (None, ))[:4] termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group(lastExpr + OneOrMore(opExpr)) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group(lastExpr + OneOrMore(opExpr + lastExpr)) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr + OneOrMore(lastExpr)) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ - Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) + matchExpr = (_FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + + Group(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr)) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") elif rightLeftAssoc == opAssoc.RIGHT: @@ -5122,15 +6059,15 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group(opExpr + thisExpr) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group(lastExpr + OneOrMore(opExpr + thisExpr)) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group(lastExpr + OneOrMore(thisExpr)) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ - Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) + matchExpr = (_FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr)) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") else: @@ -5140,128 +6077,144 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): matchExpr.setParseAction(*pa) else: matchExpr.setParseAction(pa) - thisExpr <<= ( matchExpr.setName(termName) | lastExpr ) + thisExpr <<= (matchExpr.setName(termName) | lastExpr) lastExpr = thisExpr ret <<= lastExpr return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" -dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") -sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") -quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'| - Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes") +dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("string enclosed in double quotes") +sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("string enclosed in single quotes") +quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' + | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("quotedString using single or double quotes") unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') number = pyparsing_common.number arg = Group(decl_data_type + ident) - LPAR,RPAR = map(Suppress, "()") + LPAR, RPAR = map(Suppress, "()") code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ if opener == closer: raise ValueError("opening and closing strings cannot be the same") if content is None: - if isinstance(opener,basestring) and isinstance(closer,basestring): - if len(opener) == 1 and len(closer)==1: + if isinstance(opener, basestring) and isinstance(closer, basestring): + if len(opener) == 1 and len(closer) == 1: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~ignoreExpr + + CharsNotIn(opener + + closer + + ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).setParseAction(lambda t: t[0].strip())) else: - content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS - ).setParseAction(lambda t:t[0].strip())) + content = (empty.copy() + CharsNotIn(opener + + closer + + ParserElement.DEFAULT_WHITE_CHARS + ).setParseAction(lambda t: t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - ~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~ignoreExpr + + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) + ).setParseAction(lambda t: t[0].strip())) else: - content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) + ).setParseAction(lambda t: t[0].strip())) else: raise ValueError("opening and closing arguments must be strings if no content expression is given") ret = Forward() if ignoreExpr is not None: - ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) ) + ret <<= Group(Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer)) else: - ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) ) - ret.setName('nested %s%s expression' % (opener,closer)) + ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) + ret.setName('nested %s%s expression' % (opener, closer)) return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5288,21 +6241,23 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): stmt = Forward() identifier = Word(alphas, alphanums) - funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":") + funcDecl = ("def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":") func_body = indentedBlock(stmt, indentStack) - funcDef = Group( funcDecl + func_body ) + funcDef = Group(funcDecl + func_body) rvalue = Forward() funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") rvalue << (funcCall | identifier | Word(nums)) assignment = Group(identifier + "=" + rvalue) - stmt << ( funcDef | assignment | identifier ) + stmt << (funcDef | assignment | identifier) module_body = OneOrMore(stmt) parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5320,49 +6275,58 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ - def checkPeerIndent(s,l,t): + backup_stack = indentStack[:] + + def reset_stack(): + indentStack[:] = backup_stack + + def checkPeerIndent(s, l, t): if l >= len(s): return - curCol = col(l,s) + curCol = col(l, s) if curCol != indentStack[-1]: if curCol > indentStack[-1]: - raise ParseFatalException(s,l,"illegal nesting") - raise ParseException(s,l,"not a peer entry") + raise ParseException(s, l, "illegal nesting") + raise ParseException(s, l, "not a peer entry") - def checkSubIndent(s,l,t): - curCol = col(l,s) + def checkSubIndent(s, l, t): + curCol = col(l, s) if curCol > indentStack[-1]: - indentStack.append( curCol ) + indentStack.append(curCol) else: - raise ParseException(s,l,"not a subentry") + raise ParseException(s, l, "not a subentry") - def checkUnindent(s,l,t): + def checkUnindent(s, l, t): if l >= len(s): return - curCol = col(l,s) - if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]): - raise ParseException(s,l,"not an unindent") - indentStack.pop() + curCol = col(l, s) + if not(indentStack and curCol in indentStack): + raise ParseException(s, l, "not an unindent") + if curCol < indentStack[-1]: + indentStack.pop() NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT') PEER = Empty().setParseAction(checkPeerIndent).setName('') UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT') if indent: - smExpr = Group( Optional(NL) + - #~ FollowedBy(blockStatementExpr) + - INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT) + smExpr = Group(Optional(NL) + + INDENT + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) + + UNDENT) else: - smExpr = Group( Optional(NL) + - (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) ) + smExpr = Group(Optional(NL) + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL)) + + UNDENT) + smExpr.setFailAction(lambda a, b, c, d: reset_stack()) blockStatementExpr.ignore(_bslash + LineEnd()) return smExpr.setName('indented block') alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") -anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag')) -_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\'')) +anyOpenTag, anyCloseTag = makeHTMLTags(Word(alphas, alphanums + "_:").setName('any tag')) +_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(), '><& "\'')) commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity") def replaceHTMLEntity(t): """Helper parser action to replace common HTML entities with their special characters""" @@ -5370,51 +6334,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment") -"Comment of the form C{<!-- ... -->}" +"Comment of the form ``<!-- ... -->``" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" -cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/' | dblSlashComment).setName("C++ style comment") +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" -_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + - Optional( Word(" \t") + - ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") -commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + + Optional(Word(" \t") + + ~Literal(",") + ~LineEnd()))).streamline().setName("commaItem") +commaSeparatedList = delimitedList(Optional(quotedString.copy() | _commasepitem, default="")).setName("commaSeparatedList") +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers<integer>}, L{reals<real>}, L{scientific notation<sci_real>}) - - common L{programming identifiers<identifier>} - - network addresses (L{MAC<mac_address>}, L{IPv4<ipv4_address>}, L{IPv6<ipv6_address>}) - - ISO8601 L{dates<iso8601_date>} and L{datetime<iso8601_datetime>} - - L{UUID<uuid>} - - L{comma-separated list<comma_separated_list>} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers<integer>`, :class:`reals<real>`, + :class:`scientific notation<sci_real>`) + - common :class:`programming identifiers<identifier>` + - network addresses (:class:`MAC<mac_address>`, + :class:`IPv4<ipv4_address>`, :class:`IPv6<ipv6_address>`) + - ISO8601 :class:`dates<iso8601_date>` and + :class:`datetime<iso8601_datetime>` + - :class:`UUID<uuid>` + - :class:`comma-separated list<comma_separated_list>` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5461,7 +6435,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5545,7 +6521,7 @@ class pyparsing_common: integer = Word(nums).setName("integer").setParseAction(convertToInteger) """expression that parses an unsigned integer, returns an int""" - hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16)) + hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int, 16)) """expression that parses a hexadecimal integer, returns an int""" signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger) @@ -5559,11 +6535,12 @@ class pyparsing_common: """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" mixed_integer.addParseAction(sum) - real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat) + real = Regex(r'[+-]?(:?\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) """expression that parses a floating point number and returns a float""" - sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + sci_real = Regex(r'[+-]?(:?\d+(:?[eE][+-]?\d+)|(:?\d+\.\d*|\.\d+)(:?[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5571,21 +6548,24 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - - identifier = Word(alphas+'_', alphanums+'_').setName("identifier") + + identifier = Word(alphas + '_', alphanums + '_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") - _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") - _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address") + _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part) * 7).setName("full IPv6 address") + _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) + + "::" + + Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) + ).setName("short IPv6 address") _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8) _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5595,16 +6575,19 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ - def cvt_fn(s,l,t): + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt).date() except ValueError as ve: @@ -5613,20 +6596,23 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ - def cvt_fn(s,l,t): + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt) except ValueError as ve: @@ -5634,33 +6620,40 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>' - td,td_end = makeHTMLTags("TD") + + # strip HTML links from normal text + text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>' + td, td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') - + Optional( White(" \t") ) ) ).streamline().setName("commaItem") - comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") + _commasepitem = Combine(OneOrMore(~Literal(",") + + ~LineEnd() + + Word(printables, excludeChars=',') + + Optional(White(" \t")))).streamline().setName("commaItem") + comma_separated_list = delimitedList(Optional(quotedString.copy() + | _commasepitem, default='') + ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper())) @@ -5670,6 +6663,165 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) + for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1] + 1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f),] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f),] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f),] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff),] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f),] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b),] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f),] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff),] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, u"العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, u"中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, u"кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, u"Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, u"עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, u"日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, u"漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, u"カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, u"ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, u"한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, u"ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, u"देवनागरी", pyparsing_unicode.Devanagari) + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5683,7 +6835,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/patched/notpip/_vendor/pytoml/__init__.py b/pipenv/patched/notpip/_vendor/pytoml/__init__.py index 8dc73155..8ed060ff 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/__init__.py +++ b/pipenv/patched/notpip/_vendor/pytoml/__init__.py @@ -1,3 +1,4 @@ from .core import TomlError from .parser import load, loads -from .writer import dump, dumps +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/patched/notpip/_vendor/pytoml/parser.py b/pipenv/patched/notpip/_vendor/pytoml/parser.py index 9f94e923..f074317f 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/parser.py +++ b/pipenv/patched/notpip/_vendor/pytoml/parser.py @@ -1,5 +1,6 @@ -import string, re, sys, datetime +import re, sys from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re if sys.version_info[0] == 2: _chr = unichr @@ -27,8 +28,6 @@ def loads(s, filename='<string>', translate=lambda t, x, v: v, object_pairs_hook def process_value(v, object_pairs_hook): kind, text, value, pos = v - if kind == 'str' and value.startswith('\n'): - value = value[1:] if kind == 'array': if value and any(k != value[0][0] for k, t, v, p in value[1:]): error('array-type-mismatch') @@ -179,13 +178,13 @@ _ws_re = re.compile(r'[ \t]*') def _p_ws(s): s.expect_re(_ws_re) -_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', '\'': '\'', - '\\': '\\', '/': '/', 'f': '\f' } +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } _basicstr_re = re.compile(r'[^"\\\000-\037]*') _short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') _long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') -_escapes_re = re.compile('[bnrt"\'\\\\/f]') +_escapes_re = re.compile(r'[btnfr\"\\]') _newline_esc_re = re.compile('\n[ \t\n]*') def _p_basicstr_content(s, content=_basicstr_re): res = [] @@ -196,7 +195,10 @@ def _p_basicstr_content(s, content=_basicstr_re): if s.consume_re(_newline_esc_re): pass elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): - res.append(_chr(int(s.last().group(1), 16))) + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) else: s.expect_re(_escapes_re) res.append(_escapes[s.last().group(0)]) @@ -211,6 +213,7 @@ def _p_key(s): return r if s.consume('\''): if s.consume('\'\''): + s.consume('\n') r = s.expect_re(_litstr_ml_re).group(0) s.expect('\'\'\'') else: @@ -220,9 +223,8 @@ def _p_key(s): return s.expect_re(_key_re).group(0) _float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') -_datetime_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') -_basicstr_ml_re = re.compile(r'(?:(?:|"|"")[^"\\\000-\011\013-\037])*') +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') _litstr_re = re.compile(r"[^'\000\010\012-\037]*") _litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") def _p_value(s, object_pairs_hook): @@ -235,6 +237,7 @@ def _p_value(s, object_pairs_hook): if s.consume('"'): if s.consume('""'): + s.consume('\n') r = _p_basicstr_content(s, _basicstr_ml_re) s.expect('"""') else: @@ -244,6 +247,7 @@ def _p_value(s, object_pairs_hook): if s.consume('\''): if s.consume('\'\''): + s.consume('\n') r = s.expect_re(_litstr_ml_re).group(0) s.expect('\'\'\'') else: @@ -251,24 +255,9 @@ def _p_value(s, object_pairs_hook): s.expect('\'') return 'str', r, r, pos - if s.consume_re(_datetime_re): + if s.consume_re(rfc3339_re): m = s.last() - s0 = m.group(0) - r = map(int, m.groups()[:6]) - if m.group(7): - micro = float(m.group(7)) - else: - micro = 0 - - if m.group(8): - g = int(m.group(8), 10) * 60 + int(m.group(9), 10) - tz = _TimeZone(datetime.timedelta(0, g * 60)) - else: - tz = _TimeZone(datetime.timedelta(0, 0)) - - y, m, d, H, M, S = r - dt = datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) - return 'datetime', s0, dt, pos + return 'datetime', m.group(0), parse_rfc3339_re(m), pos if s.consume_re(_float_re): m = s.last().group(0) @@ -351,24 +340,3 @@ def _p_toml(s, object_pairs_hook): _p_ews(s) s.expect_eof() return stmts - -class _TimeZone(datetime.tzinfo): - def __init__(self, offset): - self._offset = offset - - def utcoffset(self, dt): - return self._offset - - def dst(self, dt): - return None - - def tzname(self, dt): - m = self._offset.total_seconds() // 60 - if m < 0: - res = '-' - m = -m - else: - res = '+' - h = m // 60 - m = m - h * 60 - return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/test.py b/pipenv/patched/notpip/_vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/patched/notpip/_vendor/pytoml/utils.py b/pipenv/patched/notpip/_vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/patched/notpip/_vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/patched/notpip/_vendor/pytoml/writer.py b/pipenv/patched/notpip/_vendor/pytoml/writer.py index 6eaf5d76..d2e849f6 100644 --- a/pipenv/patched/notpip/_vendor/pytoml/writer.py +++ b/pipenv/patched/notpip/_vendor/pytoml/writer.py @@ -1,5 +1,13 @@ from __future__ import unicode_literals -import io, datetime, math, sys +import io, datetime, math, string, sys + +from .utils import format_rfc3339 + +try: + from pathlib import PurePath as _path_types +except ImportError: + _path_types = () + if sys.version_info[0] == 3: long = int @@ -39,22 +47,13 @@ def _escape_string(s): return '"' + ''.join(res) + '"' +_key_chars = string.digits + string.ascii_letters + '-_' def _escape_id(s): - if any(not c.isalnum() and c not in '-_' for c in s): + if any(c not in _key_chars for c in s): return _escape_string(s) return s -def _format_list(v): - return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) - -# Formula from: -# https://docs.python.org/2/library/datetime.html#datetime.timedelta.total_seconds -# Once support for py26 is dropped, this can be replaced by td.total_seconds() -def _total_seconds(td): - return ((td.microseconds - + (td.seconds + td.days * 24 * 3600) * 10**6) / 10.0**6) - def _format_value(v): if isinstance(v, bool): return 'true' if v else 'false' @@ -68,25 +67,13 @@ def _format_value(v): elif isinstance(v, unicode) or isinstance(v, bytes): return _escape_string(v) elif isinstance(v, datetime.datetime): - offs = v.utcoffset() - offs = _total_seconds(offs) // 60 if offs is not None else 0 - - if offs == 0: - suffix = 'Z' - else: - if offs > 0: - suffix = '+' - else: - suffix = '-' - offs = -offs - suffix = '{0}{1:.02}{2:.02}'.format(suffix, offs // 60, offs % 60) - - if v.microsecond: - return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix - else: - return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + return format_rfc3339(v) elif isinstance(v, list): - return _format_list(v) + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) + elif isinstance(v, _path_types): + return _escape_string(str(v)) else: raise RuntimeError(v) diff --git a/pipenv/patched/notpip/_vendor/requests/LICENSE b/pipenv/patched/notpip/_vendor/requests/LICENSE index 2e68b82e..841c6023 100644 --- a/pipenv/patched/notpip/_vendor/requests/LICENSE +++ b/pipenv/patched/notpip/_vendor/requests/LICENSE @@ -4,7 +4,7 @@ Copyright 2018 Kenneth Reitz you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + https://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pipenv/patched/notpip/_vendor/requests/__init__.py b/pipenv/patched/notpip/_vendor/requests/__init__.py index af3d4b1d..bcb74535 100644 --- a/pipenv/patched/notpip/_vendor/requests/__init__.py +++ b/pipenv/patched/notpip/_vendor/requests/__init__.py @@ -22,7 +22,7 @@ usage: ... or POST: >>> payload = dict(key1='value1', key2='value2') - >>> r = requests.post('http://httpbin.org/post', data=payload) + >>> r = requests.post('https://httpbin.org/post', data=payload) >>> print(r.text) { ... @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.23 + # urllib3 >= 1.21.1, <= 1.25 assert major == 1 assert minor >= 21 - assert minor <= 23 + assert minor <= 25 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] @@ -79,14 +79,14 @@ def _check_cryptography(cryptography_version): return if cryptography_version < [1, 3, 4]: - warning = 'Old version of cryptography ({0}) may cause slowdown.'.format(cryptography_version) + warning = 'Old version of cryptography ({}) may cause slowdown.'.format(cryptography_version) warnings.warn(warning, RequestsDependencyWarning) # Check imported dependencies for compatibility. try: check_compatibility(urllib3.__version__, chardet.__version__) except (AssertionError, ValueError): - warnings.warn("urllib3 ({0}) or chardet ({1}) doesn't match a supported " + warnings.warn("urllib3 ({}) or chardet ({}) doesn't match a supported " "version!".format(urllib3.__version__, chardet.__version__), RequestsDependencyWarning) @@ -125,12 +125,7 @@ from .exceptions import ( # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler logging.getLogger(__name__).addHandler(NullHandler()) diff --git a/pipenv/patched/notpip/_vendor/requests/__version__.py b/pipenv/patched/notpip/_vendor/requests/__version__.py index ef61ec0f..9844f740 100644 --- a/pipenv/patched/notpip/_vendor/requests/__version__.py +++ b/pipenv/patched/notpip/_vendor/requests/__version__.py @@ -5,10 +5,10 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.19.1' -__build__ = 0x021901 +__version__ = '2.22.0' +__build__ = 0x022200 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2018 Kenneth Reitz' +__copyright__ = 'Copyright 2019 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/pipenv/patched/notpip/_vendor/requests/adapters.py b/pipenv/patched/notpip/_vendor/requests/adapters.py index 014c2675..abae6d6d 100644 --- a/pipenv/patched/notpip/_vendor/requests/adapters.py +++ b/pipenv/patched/notpip/_vendor/requests/adapters.py @@ -26,6 +26,7 @@ from pipenv.patched.notpip._vendor.urllib3.exceptions import ProtocolError from pipenv.patched.notpip._vendor.urllib3.exceptions import ReadTimeoutError from pipenv.patched.notpip._vendor.urllib3.exceptions import SSLError as _SSLError from pipenv.patched.notpip._vendor.urllib3.exceptions import ResponseError +from pipenv.patched.notpip._vendor.urllib3.exceptions import LocationValueError from .models import Response from .compat import urlparse, basestring @@ -35,7 +36,8 @@ from .utils import (DEFAULT_CA_BUNDLE_PATH, extract_zipped_paths, from .structures import CaseInsensitiveDict from .cookies import extract_cookies_to_jar from .exceptions import (ConnectionError, ConnectTimeout, ReadTimeout, SSLError, - ProxyError, RetryError, InvalidSchema, InvalidProxyURL) + ProxyError, RetryError, InvalidSchema, InvalidProxyURL, + InvalidURL) from .auth import _basic_auth_str try: @@ -127,8 +129,7 @@ class HTTPAdapter(BaseAdapter): self.init_poolmanager(pool_connections, pool_maxsize, block=pool_block) def __getstate__(self): - return dict((attr, getattr(self, attr, None)) for attr in - self.__attrs__) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): # Can't handle by adding 'proxy_manager' to self.__attrs__ because @@ -224,7 +225,7 @@ class HTTPAdapter(BaseAdapter): if not cert_loc or not os.path.exists(cert_loc): raise IOError("Could not find a suitable TLS CA certificate bundle, " - "invalid path: {0}".format(cert_loc)) + "invalid path: {}".format(cert_loc)) conn.cert_reqs = 'CERT_REQUIRED' @@ -246,10 +247,10 @@ class HTTPAdapter(BaseAdapter): conn.key_file = None if conn.cert_file and not os.path.exists(conn.cert_file): raise IOError("Could not find the TLS certificate file, " - "invalid path: {0}".format(conn.cert_file)) + "invalid path: {}".format(conn.cert_file)) if conn.key_file and not os.path.exists(conn.key_file): raise IOError("Could not find the TLS key file, " - "invalid path: {0}".format(conn.key_file)) + "invalid path: {}".format(conn.key_file)) def build_response(self, req, resp): """Builds a :class:`Response <requests.Response>` object from a urllib3 @@ -378,7 +379,7 @@ class HTTPAdapter(BaseAdapter): when subclassing the :class:`HTTPAdapter <requests.adapters.HTTPAdapter>`. - :param proxies: The url of the proxy being used for this request. + :param proxy: The url of the proxy being used for this request. :rtype: dict """ headers = {} @@ -407,7 +408,10 @@ class HTTPAdapter(BaseAdapter): :rtype: requests.Response """ - conn = self.get_connection(request.url, proxies) + try: + conn = self.get_connection(request.url, proxies) + except LocationValueError as e: + raise InvalidURL(e, request=request) self.cert_verify(conn, request.url, verify, cert) url = self.request_url(request, proxies) @@ -421,7 +425,7 @@ class HTTPAdapter(BaseAdapter): timeout = TimeoutSauce(connect=connect, read=read) except ValueError as e: # this may raise a string formatting error. - err = ("Invalid timeout {0}. Pass a (connect, read) " + err = ("Invalid timeout {}. Pass a (connect, read) " "timeout tuple, or a single float to set " "both timeouts to the same value".format(timeout)) raise ValueError(err) @@ -471,11 +475,10 @@ class HTTPAdapter(BaseAdapter): # Receive the response from the server try: - # For Python 2.7+ versions, use buffering of HTTP - # responses + # For Python 2.7, use buffering of HTTP responses r = low_conn.getresponse(buffering=True) except TypeError: - # For compatibility with Python 2.6 versions and back + # For compatibility with Python 3.3+ r = low_conn.getresponse() resp = HTTPResponse.from_httplib( diff --git a/pipenv/patched/notpip/_vendor/requests/api.py b/pipenv/patched/notpip/_vendor/requests/api.py index a2cc84d7..ef71d075 100644 --- a/pipenv/patched/notpip/_vendor/requests/api.py +++ b/pipenv/patched/notpip/_vendor/requests/api.py @@ -18,8 +18,10 @@ def request(method, url, **kwargs): :param method: method for the new :class:`Request` object. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary or list of tuples ``[(key, value)]`` (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the :class:`Request`. :param cookies: (optional) Dict or CookieJar object to send with the :class:`Request`. @@ -47,7 +49,7 @@ def request(method, url, **kwargs): Usage:: >>> import requests - >>> req = requests.request('GET', 'http://httpbin.org/get') + >>> req = requests.request('GET', 'https://httpbin.org/get') <Response [200]> """ @@ -62,7 +64,8 @@ def get(url, params=None, **kwargs): r"""Sends a GET request. :param url: URL for the new :class:`Request` object. - :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. + :param params: (optional) Dictionary, list of tuples or bytes to send + in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object :rtype: requests.Response @@ -102,7 +105,8 @@ def post(url, data=None, json=None, **kwargs): r"""Sends a POST request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object @@ -116,7 +120,8 @@ def put(url, data=None, **kwargs): r"""Sends a PUT request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object @@ -130,7 +135,8 @@ def patch(url, data=None, **kwargs): r"""Sends a PATCH request. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary (will be form-encoded), bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json data to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object diff --git a/pipenv/patched/notpip/_vendor/requests/auth.py b/pipenv/patched/notpip/_vendor/requests/auth.py index 4ae45947..bdde51c7 100644 --- a/pipenv/patched/notpip/_vendor/requests/auth.py +++ b/pipenv/patched/notpip/_vendor/requests/auth.py @@ -38,7 +38,7 @@ def _basic_auth_str(username, password): if not isinstance(username, basestring): warnings.warn( "Non-string usernames will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(username), category=DeprecationWarning, @@ -48,7 +48,7 @@ def _basic_auth_str(username, password): if not isinstance(password, basestring): warnings.warn( "Non-string passwords will no longer be supported in Requests " - "3.0.0. Please convert the object you've passed in ({0!r}) to " + "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " "problems.".format(password), category=DeprecationWarning, diff --git a/pipenv/patched/notpip/_vendor/requests/compat.py b/pipenv/patched/notpip/_vendor/requests/compat.py index 4fbd6231..7c143940 100644 --- a/pipenv/patched/notpip/_vendor/requests/compat.py +++ b/pipenv/patched/notpip/_vendor/requests/compat.py @@ -47,9 +47,8 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO - from collections import Callable, Mapping, MutableMapping + from collections import Callable, Mapping, MutableMapping, OrderedDict - from pipenv.patched.notpip._vendor.urllib3.packages.ordered_dict import OrderedDict builtin_str = str bytes = str diff --git a/pipenv/patched/notpip/_vendor/requests/cookies.py b/pipenv/patched/notpip/_vendor/requests/cookies.py index 50883a84..56fccd9c 100644 --- a/pipenv/patched/notpip/_vendor/requests/cookies.py +++ b/pipenv/patched/notpip/_vendor/requests/cookies.py @@ -444,20 +444,21 @@ def create_cookie(name, value, **kwargs): By default, the pair of `name` and `value` will be set for the domain '' and sent on every request (this is sometimes called a "supercookie"). """ - result = dict( - version=0, - name=name, - value=value, - port=None, - domain='', - path='/', - secure=False, - expires=None, - discard=True, - comment=None, - comment_url=None, - rest={'HttpOnly': None}, - rfc2109=False,) + result = { + 'version': 0, + 'name': name, + 'value': value, + 'port': None, + 'domain': '', + 'path': '/', + 'secure': False, + 'expires': None, + 'discard': True, + 'comment': None, + 'comment_url': None, + 'rest': {'HttpOnly': None}, + 'rfc2109': False, + } badargs = set(kwargs) - set(result) if badargs: @@ -511,6 +512,7 @@ def cookiejar_from_dict(cookie_dict, cookiejar=None, overwrite=True): :param cookiejar: (optional) A cookiejar to add the cookies to. :param overwrite: (optional) If False, will not replace cookies already in the jar with new ones. + :rtype: CookieJar """ if cookiejar is None: cookiejar = RequestsCookieJar() @@ -529,6 +531,7 @@ def merge_cookies(cookiejar, cookies): :param cookiejar: CookieJar object to add the cookies to. :param cookies: Dictionary or CookieJar object to be added. + :rtype: CookieJar """ if not isinstance(cookiejar, cookielib.CookieJar): raise ValueError('You can only merge into CookieJar') diff --git a/pipenv/patched/notpip/_vendor/requests/help.py b/pipenv/patched/notpip/_vendor/requests/help.py index eba69edb..72d72160 100644 --- a/pipenv/patched/notpip/_vendor/requests/help.py +++ b/pipenv/patched/notpip/_vendor/requests/help.py @@ -89,8 +89,7 @@ def info(): 'version': getattr(idna, '__version__', ''), } - # OPENSSL_VERSION_NUMBER doesn't exist in the Python 2.6 ssl module. - system_ssl = getattr(ssl, 'OPENSSL_VERSION_NUMBER', None) + system_ssl = ssl.OPENSSL_VERSION_NUMBER system_ssl_info = { 'version': '%x' % system_ssl if system_ssl is not None else '' } diff --git a/pipenv/patched/notpip/_vendor/requests/hooks.py b/pipenv/patched/notpip/_vendor/requests/hooks.py index 32b32de7..7a51f212 100644 --- a/pipenv/patched/notpip/_vendor/requests/hooks.py +++ b/pipenv/patched/notpip/_vendor/requests/hooks.py @@ -15,14 +15,14 @@ HOOKS = ['response'] def default_hooks(): - return dict((event, []) for event in HOOKS) + return {event: [] for event in HOOKS} # TODO: response is the only one def dispatch_hook(key, hooks, hook_data, **kwargs): """Dispatches a hook dictionary on a given piece of data.""" - hooks = hooks or dict() + hooks = hooks or {} hooks = hooks.get(key) if hooks: if hasattr(hooks, '__call__'): diff --git a/pipenv/patched/notpip/_vendor/requests/models.py b/pipenv/patched/notpip/_vendor/requests/models.py index 6708f09b..5f899c42 100644 --- a/pipenv/patched/notpip/_vendor/requests/models.py +++ b/pipenv/patched/notpip/_vendor/requests/models.py @@ -204,9 +204,13 @@ class Request(RequestHooksMixin): :param url: URL to send. :param headers: dictionary of headers to send. :param files: dictionary of {filename: fileobject} files to multipart upload. - :param data: the body to attach to the request. If a dictionary is provided, form-encoding will take place. + :param data: the body to attach to the request. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param json: json for the body to attach to the request (if files or data is not specified). - :param params: dictionary of URL parameters to append to the URL. + :param params: URL parameters to append to the URL. If a dictionary or + list of tuples ``[(key, value)]`` is provided, form-encoding will + take place. :param auth: Auth handler or (user, pass) tuple. :param cookies: dictionary or CookieJar of cookies to attach to this request. :param hooks: dictionary of callback hooks, for internal usage. @@ -214,7 +218,7 @@ class Request(RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> req.prepare() <PreparedRequest [GET]> """ @@ -274,7 +278,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): Usage:: >>> import requests - >>> req = requests.Request('GET', 'http://httpbin.org/get') + >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() <PreparedRequest [GET]> @@ -648,10 +652,7 @@ class Response(object): if not self._content_consumed: self.content - return dict( - (attr, getattr(self, attr, None)) - for attr in self.__attrs__ - ) + return {attr: getattr(self, attr, None) for attr in self.__attrs__} def __setstate__(self, state): for name, value in state.items(): @@ -780,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/patched/notpip/_vendor/requests/packages.py b/pipenv/patched/notpip/_vendor/requests/packages.py index 258c89ed..5fb6f07d 100644 --- a/pipenv/patched/notpip/_vendor/requests/packages.py +++ b/pipenv/patched/notpip/_vendor/requests/packages.py @@ -4,13 +4,13 @@ import sys # I don't like it either. Just look the other way. :) for package in ('urllib3', 'idna', 'chardet'): - vendored_package = "notpip._vendor." + package + vendored_package = "pipenv.patched.notpip._vendor." + package locals()[package] = __import__(vendored_package) # This traversal is apparently necessary such that the identities are # preserved (requests.packages.urllib3.* is urllib3.*) for mod in list(sys.modules): if mod == vendored_package or mod.startswith(vendored_package + '.'): - unprefixed_mod = mod[len("notpip._vendor."):] - sys.modules['notpip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] + unprefixed_mod = mod[len("pipenv.patched.notpip._vendor."):] + sys.modules['pipenv.patched.notpip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] # Kinda cool, though, right? diff --git a/pipenv/patched/notpip/_vendor/requests/sessions.py b/pipenv/patched/notpip/_vendor/requests/sessions.py index ba135268..d73d700f 100644 --- a/pipenv/patched/notpip/_vendor/requests/sessions.py +++ b/pipenv/patched/notpip/_vendor/requests/sessions.py @@ -19,7 +19,7 @@ from .cookies import ( from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT from .hooks import default_hooks, dispatch_hook from ._internal_utils import to_native_string -from .utils import to_key_val_list, default_headers +from .utils import to_key_val_list, default_headers, DEFAULT_PORTS from .exceptions import ( TooManyRedirects, InvalidSchema, ChunkedEncodingError, ContentDecodingError) @@ -115,6 +115,31 @@ class SessionRedirectMixin(object): return to_native_string(location, 'utf8') return None + def should_strip_auth(self, old_url, new_url): + """Decide whether Authorization header should be removed when redirecting""" + old_parsed = urlparse(old_url) + new_parsed = urlparse(new_url) + if old_parsed.hostname != new_parsed.hostname: + return True + # Special case: allow http -> https redirect when using the standard + # ports. This isn't specified by RFC 7235, but is kept to avoid + # breaking backwards compatibility with older versions of requests + # that allowed any redirects on the same host. + if (old_parsed.scheme == 'http' and old_parsed.port in (80, None) + and new_parsed.scheme == 'https' and new_parsed.port in (443, None)): + return False + + # Handle default port usage corresponding to scheme. + changed_port = old_parsed.port != new_parsed.port + changed_scheme = old_parsed.scheme != new_parsed.scheme + default_port = (DEFAULT_PORTS.get(old_parsed.scheme, None), None) + if (not changed_scheme and old_parsed.port in default_port + and new_parsed.port in default_port): + return False + + # Standard case: root URI must match + return changed_port or changed_scheme + def resolve_redirects(self, resp, req, stream=False, timeout=None, verify=True, cert=None, proxies=None, yield_requests=False, **adapter_kwargs): """Receives a Response. Returns a generator of Responses or Requests.""" @@ -236,14 +261,10 @@ class SessionRedirectMixin(object): headers = prepared_request.headers url = prepared_request.url - if 'Authorization' in headers: + if 'Authorization' in headers and self.should_strip_auth(response.request.url, url): # If we get redirected to a new host, we should strip out any # authentication headers. - original_parsed = urlparse(response.request.url) - redirect_parsed = urlparse(url) - - if (original_parsed.hostname != redirect_parsed.hostname): - del headers['Authorization'] + del headers['Authorization'] # .netrc might have more auth for us on our new host. new_auth = get_netrc_auth(url) if self.trust_env else None @@ -299,7 +320,7 @@ class SessionRedirectMixin(object): """ method = prepared_request.method - # http://tools.ietf.org/html/rfc7231#section-6.4.4 + # https://tools.ietf.org/html/rfc7231#section-6.4.4 if response.status_code == codes.see_other and method != 'HEAD': method = 'GET' @@ -325,13 +346,13 @@ class Session(SessionRedirectMixin): >>> import requests >>> s = requests.Session() - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') <Response [200]> Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('http://httpbin.org/get') + >>> s.get('https://httpbin.org/get') <Response [200]> """ @@ -453,8 +474,8 @@ class Session(SessionRedirectMixin): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary or bytes to be sent in the query string for the :class:`Request`. - :param data: (optional) Dictionary, bytes, or file-like object to send - in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param headers: (optional) Dictionary of HTTP Headers to send with the @@ -550,7 +571,8 @@ class Session(SessionRedirectMixin): r"""Sends a POST request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response @@ -562,7 +584,8 @@ class Session(SessionRedirectMixin): r"""Sends a PUT request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -573,7 +596,8 @@ class Session(SessionRedirectMixin): r"""Sends a PATCH request. Returns :class:`Response` object. :param url: URL for the new :class:`Request` object. - :param data: (optional) Dictionary, bytes, or file-like object to send in the body of the :class:`Request`. + :param data: (optional) Dictionary, list of tuples, bytes, or file-like + object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: requests.Response """ @@ -723,7 +747,7 @@ class Session(SessionRedirectMixin): self.adapters[key] = self.adapters.pop(key) def __getstate__(self): - state = dict((attr, getattr(self, attr, None)) for attr in self.__attrs__) + state = {attr: getattr(self, attr, None) for attr in self.__attrs__} return state def __setstate__(self, state): @@ -735,7 +759,12 @@ def session(): """ Returns a :class:`Session` for context-management. + .. deprecated:: 1.0.0 + + This method has been deprecated since version 1.0.0 and is only kept for + backwards compatibility. New code should use :class:`~requests.sessions.Session` + to create a session. This may be removed at a future date. + :rtype: Session """ - return Session() diff --git a/pipenv/patched/notpip/_vendor/requests/status_codes.py b/pipenv/patched/notpip/_vendor/requests/status_codes.py index ff462c6c..813e8c4e 100644 --- a/pipenv/patched/notpip/_vendor/requests/status_codes.py +++ b/pipenv/patched/notpip/_vendor/requests/status_codes.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -""" +r""" The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. diff --git a/pipenv/patched/notpip/_vendor/requests/utils.py b/pipenv/patched/notpip/_vendor/requests/utils.py index 431f6be0..8170a8d2 100644 --- a/pipenv/patched/notpip/_vendor/requests/utils.py +++ b/pipenv/patched/notpip/_vendor/requests/utils.py @@ -38,6 +38,8 @@ NETRC_FILES = ('.netrc', '_netrc') DEFAULT_CA_BUNDLE_PATH = certs.where() +DEFAULT_PORTS = {'http': 80, 'https': 443} + if sys.platform == 'win32': # provide a proxy_bypass version on Windows without DNS lookups @@ -173,10 +175,10 @@ def get_netrc_auth(url, raise_errors=False): for f in NETRC_FILES: try: - loc = os.path.expanduser('~/{0}'.format(f)) + loc = os.path.expanduser('~/{}'.format(f)) except KeyError: # os.path.expanduser can fail when $HOME is undefined and - # getpwuid fails. See http://bugs.python.org/issue20164 & + # getpwuid fails. See https://bugs.python.org/issue20164 & # https://github.com/requests/requests/issues/1846 return @@ -264,7 +266,7 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') - ValueError: need more than 1 value to unpack + ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -466,7 +468,7 @@ def _parse_content_type_header(header): if index_of_equals != -1: key = param[:index_of_equals].strip(items_to_strip) value = param[index_of_equals + 1:].strip(items_to_strip) - params_dict[key] = value + params_dict[key.lower()] = value return content_type, params_dict @@ -706,6 +708,10 @@ def should_bypass_proxies(url, no_proxy): no_proxy = get_proxy('no_proxy') parsed = urlparse(url) + if parsed.hostname is None: + # URLs don't always have hostnames, e.g. file:/// urls. + return True + if no_proxy: # We need to check whether we match here. We need to see if we match # the end of the hostname, both with and without the port. @@ -725,7 +731,7 @@ def should_bypass_proxies(url, no_proxy): else: host_with_port = parsed.hostname if parsed.port: - host_with_port += ':{0}'.format(parsed.port) + host_with_port += ':{}'.format(parsed.port) for host in no_proxy: if parsed.hostname.endswith(host) or host_with_port.endswith(host): @@ -733,13 +739,8 @@ def should_bypass_proxies(url, no_proxy): # to apply the proxies on this URL. return True - # If the system proxy settings indicate that this URL should be bypassed, - # don't proxy. - # The proxy_bypass function is incredibly buggy on OS X in early versions - # of Python 2.6, so allow this call to fail. Only catch the specific - # exceptions we've seen, though: this call failing in other ways can reveal - # legitimate problems. with set_environ('no_proxy', no_proxy_arg): + # parsed.hostname can be `None` in cases such as a file URI. try: bypass = proxy_bypass(parsed.hostname) except (TypeError, socket.gaierror): diff --git a/pipenv/patched/notpip/_vendor/six.LICENSE b/pipenv/patched/notpip/_vendor/six.LICENSE index f3068bfd..365d1074 100644 --- a/pipenv/patched/notpip/_vendor/six.LICENSE +++ b/pipenv/patched/notpip/_vendor/six.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Benjamin Peterson +Copyright (c) 2010-2018 Benjamin Peterson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/pipenv/patched/notpip/_vendor/six.py b/pipenv/patched/notpip/_vendor/six.py index 6bf4fd38..89b2188f 100644 --- a/pipenv/patched/notpip/_vendor/six.py +++ b/pipenv/patched/notpip/_vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson <benjamin@python.org>" -__version__ = "1.11.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -844,10 +844,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt b/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt index 1c3283ee..c89cf27b 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt +++ b/pipenv/patched/notpip/_vendor/urllib3/LICENSE.txt @@ -1,19 +1,21 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php +MIT License -Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or -substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/notpip/_vendor/urllib3/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/__init__.py index 4bd533b5..8f5a21f3 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/__init__.py @@ -1,15 +1,10 @@ """ urllib3 - Thread-safe connection pooling and re-using. """ - from __future__ import absolute_import import warnings -from .connectionpool import ( - HTTPConnectionPool, - HTTPSConnectionPool, - connection_from_url -) +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url from . import exceptions from .filepost import encode_multipart_formdata @@ -23,32 +18,27 @@ from .util.retry import Retry # Set default logging handler to avoid "No handler found" warnings. import logging -try: # Python 2.7+ - from logging import NullHandler -except ImportError: - class NullHandler(logging.Handler): - def emit(self, record): - pass +from logging import NullHandler -__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' -__license__ = 'MIT' -__version__ = '1.23' +__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" +__license__ = "MIT" +__version__ = "1.25.6" __all__ = ( - 'HTTPConnectionPool', - 'HTTPSConnectionPool', - 'PoolManager', - 'ProxyManager', - 'HTTPResponse', - 'Retry', - 'Timeout', - 'add_stderr_logger', - 'connection_from_url', - 'disable_warnings', - 'encode_multipart_formdata', - 'get_host', - 'make_headers', - 'proxy_from_url', + "HTTPConnectionPool", + "HTTPSConnectionPool", + "PoolManager", + "ProxyManager", + "HTTPResponse", + "Retry", + "Timeout", + "add_stderr_logger", + "connection_from_url", + "disable_warnings", + "encode_multipart_formdata", + "get_host", + "make_headers", + "proxy_from_url", ) logging.getLogger(__name__).addHandler(NullHandler()) @@ -65,10 +55,10 @@ def add_stderr_logger(level=logging.DEBUG): # even if urllib3 is vendored within another package. logger = logging.getLogger(__name__) handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s', __name__) + logger.debug("Added a stderr logging handler to logger: %s", __name__) return handler @@ -80,18 +70,17 @@ del NullHandler # shouldn't be: otherwise, it's very hard for users to use most Python # mechanisms to silence them. # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +warnings.simplefilter("always", exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) +warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning, - append=True) +warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) +warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): """ Helper for quickly disabling all urllib3 warnings. """ - warnings.simplefilter('ignore', category) + warnings.simplefilter("ignore", category) diff --git a/pipenv/patched/notpip/_vendor/urllib3/_collections.py b/pipenv/patched/notpip/_vendor/urllib3/_collections.py index 6e36b84e..019d1511 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/_collections.py +++ b/pipenv/patched/notpip/_vendor/urllib3/_collections.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + try: from collections.abc import Mapping, MutableMapping except ImportError: @@ -6,6 +7,7 @@ except ImportError: try: from threading import RLock except ImportError: # Platform-specific: No threads available + class RLock: def __enter__(self): pass @@ -14,15 +16,12 @@ except ImportError: # Platform-specific: No threads available pass -try: # Python 2.7+ - from collections import OrderedDict -except ImportError: - from .packages.ordered_dict import OrderedDict +from collections import OrderedDict from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 -__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] +__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] _Null = object() @@ -85,7 +84,9 @@ class RecentlyUsedContainer(MutableMapping): return len(self._container) def __iter__(self): - raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') + raise NotImplementedError( + "Iteration over this class is unlikely to be threadsafe." + ) def clear(self): with self.lock: @@ -153,7 +154,7 @@ class HTTPHeaderDict(MutableMapping): def __getitem__(self, key): val = self._container[key.lower()] - return ', '.join(val[1:]) + return ", ".join(val[1:]) def __delitem__(self, key): del self._container[key.lower()] @@ -162,12 +163,13 @@ class HTTPHeaderDict(MutableMapping): return key.lower() in self._container def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, 'keys'): + if not isinstance(other, Mapping) and not hasattr(other, "keys"): return False if not isinstance(other, type(self)): other = type(self)(other) - return (dict((k.lower(), v) for k, v in self.itermerged()) == - dict((k.lower(), v) for k, v in other.itermerged())) + return dict((k.lower(), v) for k, v in self.itermerged()) == dict( + (k.lower(), v) for k, v in other.itermerged() + ) def __ne__(self, other): return not self.__eq__(other) @@ -187,9 +189,9 @@ class HTTPHeaderDict(MutableMapping): yield vals[0] def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. - ''' + """ # Using the MutableMapping function directly fails due to the private marker. # Using ordinary dict.pop would expose the internal structures. # So let's reinvent the wheel. @@ -231,8 +233,10 @@ class HTTPHeaderDict(MutableMapping): with self.add instead of self.__setitem__ """ if len(args) > 1: - raise TypeError("extend() takes at most 1 positional " - "arguments ({0} given)".format(len(args))) + raise TypeError( + "extend() takes at most 1 positional " + "arguments ({0} given)".format(len(args)) + ) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): @@ -298,7 +302,7 @@ class HTTPHeaderDict(MutableMapping): """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] - yield val[0], ', '.join(val[1:]) + yield val[0], ", ".join(val[1:]) def items(self): return list(self.iteritems()) @@ -309,7 +313,7 @@ class HTTPHeaderDict(MutableMapping): # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. - obs_fold_continued_leaders = (' ', '\t') + obs_fold_continued_leaders = (" ", "\t") headers = [] for line in message.headers: @@ -319,14 +323,14 @@ class HTTPHeaderDict(MutableMapping): # in RFC-7230 S3.2.4. This indicates a multiline header, but # there exists no previous header to which we can attach it. raise InvalidHeader( - 'Header continuation with no previous header: %s' % line + "Header continuation with no previous header: %s" % line ) else: key, value = headers[-1] - headers[-1] = (key, value + ' ' + line.strip()) + headers[-1] = (key, value + " " + line.strip()) continue - key, value = line.split(':', 1) + key, value = line.split(":", 1) headers.append((key, value.strip())) return cls(headers) diff --git a/pipenv/patched/notpip/_vendor/urllib3/connection.py b/pipenv/patched/notpip/_vendor/urllib3/connection.py index a03b573f..3eeb1af5 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connection.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import datetime import logging import os -import sys import socket from socket import error as SocketError, timeout as SocketTimeout import warnings @@ -12,6 +11,7 @@ from .packages.six.moves.http_client import HTTPException # noqa: F401 try: # Compiled with SSL? import ssl + BaseSSLError = ssl.SSLError except (ImportError, AttributeError): # Platform-specific: No SSL. ssl = None @@ -20,10 +20,11 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass -try: # Python 3: - # Not a no-op, we're adding this to the namespace so it can be imported. +try: + # Python 3: not a no-op, we're adding this to the namespace so it can be imported. ConnectionError = ConnectionError -except NameError: # Python 2: +except NameError: + # Python 2 class ConnectionError(Exception): pass @@ -41,7 +42,7 @@ from .util.ssl_ import ( resolve_ssl_version, assert_fingerprint, create_urllib3_context, - ssl_wrap_socket + ssl_wrap_socket, ) @@ -51,20 +52,16 @@ from ._collections import HTTPHeaderDict log = logging.getLogger(__name__) -port_by_scheme = { - 'http': 80, - 'https': 443, -} +port_by_scheme = {"http": 80, "https": 443} -# When updating RECENT_DATE, move it to within two years of the current date, -# and not less than 6 months ago. -# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or -# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) -RECENT_DATE = datetime.date(2017, 6, 30) +# When it comes time to update this value as a part of regular maintenance +# (ie test_recent_date is failing) update it to ~6 months before the current date. +RECENT_DATE = datetime.date(2019, 1, 1) class DummyConnection(object): """Used to detect a failed ConnectionCls import.""" + pass @@ -78,9 +75,6 @@ class HTTPConnection(_HTTPConnection, object): - ``strict``: See the documentation on :class:`urllib3.connectionpool.HTTPConnectionPool` - ``source_address``: Set the source address for the current connection. - - .. note:: This is ignored for Python 2.6. It is only applied for 2.7 and 3.x - - ``socket_options``: Set specific options on the underlying socket. If not specified, then defaults are loaded from ``HTTPConnection.default_socket_options`` which includes disabling Nagle's algorithm (sets TCP_NODELAY to 1) unless the connection is behind a proxy. @@ -95,7 +89,7 @@ class HTTPConnection(_HTTPConnection, object): Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ - default_port = port_by_scheme['http'] + default_port = port_by_scheme["http"] #: Disable Nagle's algorithm by default. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` @@ -105,24 +99,16 @@ class HTTPConnection(_HTTPConnection, object): is_verified = False def __init__(self, *args, **kw): - if six.PY3: # Python 3 - kw.pop('strict', None) + if not six.PY2: + kw.pop("strict", None) - # Pre-set source_address in case we have an older Python like 2.6. - self.source_address = kw.get('source_address') - - if sys.version_info < (2, 7): # Python 2.6 - # _HTTPConnection on Python 2.6 will balk at this keyword arg, but - # not newer versions. We can still use it when creating a - # connection though, so we pop it *after* we have saved it as - # self.source_address. - kw.pop('source_address', None) + # Pre-set source_address. + self.source_address = kw.get("source_address") #: The socket options provided by the user. If no options are #: provided, we use the default options. - self.socket_options = kw.pop('socket_options', self.default_socket_options) + self.socket_options = kw.pop("socket_options", self.default_socket_options) - # Superclass also sets self.source_address in Python 2.7+. _HTTPConnection.__init__(self, *args, **kw) @property @@ -142,7 +128,7 @@ class HTTPConnection(_HTTPConnection, object): those cases where it's appropriate (i.e., when doing DNS lookup to establish the actual TCP connection across which we're going to send HTTP requests). """ - return self._dns_host.rstrip('.') + return self._dns_host.rstrip(".") @host.setter def host(self, value): @@ -161,32 +147,34 @@ class HTTPConnection(_HTTPConnection, object): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = connection.create_connection( - (self._dns_host, self.port), self.timeout, **extra_kw) + (self._dns_host, self.port), self.timeout, **extra_kw + ) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except SocketError as e: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn def _prepare_conn(self, conn): self.sock = conn - # the _tunnel_host attribute was added in python 2.6.3 (via - # http://hg.python.org/cpython/rev/0f57b30a152f) so pythons 2.6(0-2) do - # not have them. - if getattr(self, '_tunnel_host', None): + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, "_tunnel_host", None): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -202,74 +190,99 @@ class HTTPConnection(_HTTPConnection, object): body with chunked encoding and not as one block """ headers = HTTPHeaderDict(headers if headers is not None else {}) - skip_accept_encoding = 'accept-encoding' in headers - skip_host = 'host' in headers + skip_accept_encoding = "accept-encoding" in headers + skip_host = "host" in headers self.putrequest( - method, - url, - skip_accept_encoding=skip_accept_encoding, - skip_host=skip_host + method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host ) for header, value in headers.items(): self.putheader(header, value) - if 'transfer-encoding' not in headers: - self.putheader('Transfer-Encoding', 'chunked') + if "transfer-encoding" not in headers: + self.putheader("Transfer-Encoding", "chunked") self.endheaders() if body is not None: - stringish_types = six.string_types + (six.binary_type,) + stringish_types = six.string_types + (bytes,) if isinstance(body, stringish_types): body = (body,) for chunk in body: if not chunk: continue - if not isinstance(chunk, six.binary_type): - chunk = chunk.encode('utf8') + if not isinstance(chunk, bytes): + chunk = chunk.encode("utf8") len_str = hex(len(chunk))[2:] - self.send(len_str.encode('utf-8')) - self.send(b'\r\n') + self.send(len_str.encode("utf-8")) + self.send(b"\r\n") self.send(chunk) - self.send(b'\r\n') + self.send(b"\r\n") # After the if clause, to always have a closed body - self.send(b'0\r\n\r\n') + self.send(b"0\r\n\r\n") class HTTPSConnection(HTTPConnection): - default_port = port_by_scheme['https'] + default_port = port_by_scheme["https"] ssl_version = None - def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, **kw): + def __init__( + self, + host, + port=None, + key_file=None, + cert_file=None, + key_password=None, + strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, + server_hostname=None, + **kw + ): - HTTPConnection.__init__(self, host, port, strict=strict, - timeout=timeout, **kw) + HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) self.key_file = key_file self.cert_file = cert_file + self.key_password = key_password self.ssl_context = ssl_context + self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = 'https' + self._protocol = "https" def connect(self): conn = self._new_conn() self._prepare_conn(conn) + # Wrap socket using verification with the root certs in + # trusted_root_certs + default_ssl_context = False if self.ssl_context is None: + default_ssl_context = True self.ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(None), - cert_reqs=resolve_cert_reqs(None), + ssl_version=resolve_ssl_version(self.ssl_version), + cert_reqs=resolve_cert_reqs(self.cert_reqs), ) + # Try to load OS default certs if none are given. + # Works well on Windows (requires Python3.4+) + context = self.ssl_context + if ( + not self.ca_certs + and not self.ca_cert_dir + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + self.sock = ssl_wrap_socket( sock=conn, keyfile=self.key_file, certfile=self.cert_file, + key_password=self.key_password, ssl_context=self.ssl_context, + server_hostname=self.server_hostname, ) @@ -278,32 +291,39 @@ class VerifiedHTTPSConnection(HTTPSConnection): Based on httplib.HTTPSConnection but wraps the socket with SSL certification. """ + cert_reqs = None ca_certs = None ca_cert_dir = None ssl_version = None assert_fingerprint = None - def set_cert(self, key_file=None, cert_file=None, - cert_reqs=None, ca_certs=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None): + def set_cert( + self, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ): """ This method should only be called once, before the connection is used. """ - # If cert_reqs is not provided, we can try to guess. If the user gave - # us a cert database, we assume they want to use it: otherwise, if - # they gave us an SSL Context object we should use whatever is set for - # it. + # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also + # have an SSLContext object in which case we'll use its verify_mode. if cert_reqs is None: - if ca_certs or ca_cert_dir: - cert_reqs = 'CERT_REQUIRED' - elif self.ssl_context is not None: + if self.ssl_context is not None: cert_reqs = self.ssl_context.verify_mode + else: + cert_reqs = resolve_cert_reqs(None) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint self.ca_certs = ca_certs and os.path.expanduser(ca_certs) @@ -312,12 +332,10 @@ class VerifiedHTTPSConnection(HTTPSConnection): def connect(self): # Add certificate verification conn = self._new_conn() - hostname = self.host - if getattr(self, '_tunnel_host', None): - # _tunnel_host was added in Python 2.6.3 - # (See: http://hg.python.org/cpython/rev/0f57b30a152f) + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, "_tunnel_host", None): self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -328,17 +346,25 @@ class VerifiedHTTPSConnection(HTTPSConnection): # Override the host with the one we're requesting data from. hostname = self._tunnel_host + server_hostname = hostname + if self.server_hostname is not None: + server_hostname = self.server_hostname + is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: - warnings.warn(( - 'System time is way off (before {0}). This will probably ' - 'lead to SSL verification errors').format(RECENT_DATE), - SystemTimeWarning + warnings.warn( + ( + "System time is way off (before {0}). This will probably " + "lead to SSL verification errors" + ).format(RECENT_DATE), + SystemTimeWarning, ) # Wrap socket using verification with the root certs in # trusted_root_certs + default_ssl_context = False if self.ssl_context is None: + default_ssl_context = True self.ssl_context = create_urllib3_context( ssl_version=resolve_ssl_version(self.ssl_version), cert_reqs=resolve_cert_reqs(self.cert_reqs), @@ -346,38 +372,56 @@ class VerifiedHTTPSConnection(HTTPSConnection): context = self.ssl_context context.verify_mode = resolve_cert_reqs(self.cert_reqs) + + # Try to load OS default certs if none are given. + # Works well on Windows (requires Python3.4+) + if ( + not self.ca_certs + and not self.ca_cert_dir + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + self.sock = ssl_wrap_socket( sock=conn, keyfile=self.key_file, certfile=self.cert_file, + key_password=self.key_password, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, - server_hostname=hostname, - ssl_context=context) + server_hostname=server_hostname, + ssl_context=context, + ) if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif context.verify_mode != ssl.CERT_NONE \ - and not getattr(context, 'check_hostname', False) \ - and self.assert_hostname is not False: + assert_fingerprint( + self.sock.getpeercert(binary_form=True), self.assert_fingerprint + ) + elif ( + context.verify_mode != ssl.CERT_NONE + and not getattr(context, "check_hostname", False) + and self.assert_hostname is not False + ): # While urllib3 attempts to always turn off hostname matching from # the TLS library, this cannot always be done. So we check whether # the TLS Library still thinks it's matching hostnames. cert = self.sock.getpeercert() - if not cert.get('subjectAltName', ()): - warnings.warn(( - 'Certificate for {0} has no `subjectAltName`, falling back to check for a ' - '`commonName` for now. This feature is being removed by major browsers and ' - 'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 ' - 'for details.)'.format(hostname)), - SubjectAltNameWarning + if not cert.get("subjectAltName", ()): + warnings.warn( + ( + "Certificate for {0} has no `subjectAltName`, falling back to check for a " + "`commonName` for now. This feature is being removed by major browsers and " + "deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 " + "for details.)".format(hostname) + ), + SubjectAltNameWarning, ) - _match_hostname(cert, self.assert_hostname or hostname) + _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( - context.verify_mode == ssl.CERT_REQUIRED or - self.assert_fingerprint is not None + context.verify_mode == ssl.CERT_REQUIRED + or self.assert_fingerprint is not None ) @@ -385,9 +429,10 @@ def _match_hostname(cert, asserted_hostname): try: match_hostname(cert, asserted_hostname) except CertificateError as e: - log.error( - 'Certificate did not match expected hostname: %s. ' - 'Certificate: %s', asserted_hostname, cert + log.warning( + "Certificate did not match expected hostname: %s. " "Certificate: %s", + asserted_hostname, + cert, ) # Add cert to exception and reraise so client code can inspect # the cert when catching the exception, if they want to diff --git a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py index 8fcb0bce..e73fa57a 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/connectionpool.py @@ -29,8 +29,11 @@ from .packages.six.moves import queue from .connection import ( port_by_scheme, DummyConnection, - HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPConnection, + HTTPSConnection, + VerifiedHTTPSConnection, + HTTPException, + BaseSSLError, ) from .request import RequestMethods from .response import HTTPResponse @@ -40,7 +43,13 @@ from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host, Url, NORMALIZABLE_SCHEMES +from .util.url import ( + get_host, + parse_url, + Url, + _normalize_host as normalize_host, + _encode_target, +) from .util.queue import LifoQueue @@ -65,13 +74,12 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - self.host = _ipv6_host(host, self.scheme) + self.host = _normalize_host(host, scheme=self.scheme) self._proxy_host = host.lower() self.port = port def __str__(self): - return '%s(host=%r, port=%r)' % (type(self).__name__, - self.host, self.port) + return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) def __enter__(self): return self @@ -89,7 +97,7 @@ class ConnectionPool(object): # This is taken from http://hg.python.org/cpython/file/7aaba721ebc0/Lib/socket.py#l252 -_blocking_errnos = set([errno.EAGAIN, errno.EWOULDBLOCK]) +_blocking_errnos = {errno.EAGAIN, errno.EWOULDBLOCK} class HTTPConnectionPool(ConnectionPool, RequestMethods): @@ -152,15 +160,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :class:`urllib3.connection.HTTPSConnection` instances. """ - scheme = 'http' + scheme = "http" ConnectionCls = HTTPConnection ResponseCls = HTTPResponse - def __init__(self, host, port=None, strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, - headers=None, retries=None, - _proxy=None, _proxy_headers=None, - **conn_kw): + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + **conn_kw + ): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) @@ -194,19 +211,27 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. # We cannot know if the user has added default socket options, so we cannot replace the # list. - self.conn_kw.setdefault('socket_options', []) + self.conn_kw.setdefault("socket_options", []) def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTP connection (%d): %s:%s", - self.num_connections, self.host, self.port or "80") + log.debug( + "Starting new HTTP connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "80", + ) - conn = self.ConnectionCls(host=self.host, port=self.port, - timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + conn = self.ConnectionCls( + host=self.host, + port=self.port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + **self.conn_kw + ) return conn def _get_conn(self, timeout=None): @@ -230,16 +255,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except queue.Empty: if self.block: - raise EmptyPoolError(self, - "Pool reached maximum size and no more " - "connections are allowed.") + raise EmptyPoolError( + self, + "Pool reached maximum size and no more " "connections are allowed.", + ) pass # Oh well, we'll create a new connection then # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) conn.close() - if getattr(conn, 'auto_open', 1) == 0: + if getattr(conn, "auto_open", 1) == 0: # This is a proxied connection that has been mutated by # httplib._tunnel() and cannot be reused (since it would # attempt to bypass the proxy) @@ -269,9 +295,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass except queue.Full: # This should never happen if self.block == True - log.warning( - "Connection pool is full, discarding connection: %s", - self.host) + log.warning("Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. if conn: @@ -303,21 +327,30 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # See the above comment about EAGAIN in Python 3. In Python 2 we have # to specifically catch it and throw the timeout error - if hasattr(err, 'errno') and err.errno in _blocking_errnos: - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + if hasattr(err, "errno") and err.errno in _blocking_errnos: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python 2.6 - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + if "timed out" in str(err) or "did not complete (read)" in str( + err + ): # Python < 2.7.4 + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) - def _make_request(self, conn, method, url, timeout=_Default, chunked=False, - **httplib_request_kw): + def _make_request( + self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw + ): """ Perform a request on a given urllib connection object taken from our pool. @@ -357,7 +390,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): read_timeout = timeout_obj.read_timeout # App Engine doesn't have a sock attr - if getattr(conn, 'sock', None): + if getattr(conn, "sock", None): # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -365,7 +398,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) + self, url, "Read timed out. (read timeout=%s)" % read_timeout + ) if read_timeout is Timeout.DEFAULT_TIMEOUT: conn.sock.settimeout(socket.getdefaulttimeout()) else: # None or a value @@ -373,31 +407,45 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Receive the response from the server try: - try: # Python 2.7, use buffering of HTTP responses + try: + # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 2.6 and older, Python 3 + except TypeError: + # Python 3 try: httplib_response = conn.getresponse() - except Exception as e: - # Remove the TypeError from the exception chain in Python 3; - # otherwise it looks like a programming error was the cause. + except BaseException as e: + # Remove the TypeError from the exception chain in + # Python 3 (including for exceptions like SystemExit). + # Otherwise it looks like a bug in the code. six.raise_from(e, None) except (SocketTimeout, BaseSSLError, SocketError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. - http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, - method, url, http_version, httplib_response.status, - httplib_response.length) + http_version = getattr(conn, "_http_vsn_str", "HTTP/?") + log.debug( + '%s://%s:%s "%s %s %s" %s %s', + self.scheme, + self.host, + self.port, + method, + url, + http_version, + httplib_response.status, + httplib_response.length, + ) try: assert_header_parsing(httplib_response.msg) except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 log.warning( - 'Failed to parse headers (url=%s): %s', - self._absolute_url(url), hpe, exc_info=True) + "Failed to parse headers (url=%s): %s", + self._absolute_url(url), + hpe, + exc_info=True, + ) return httplib_response @@ -427,13 +475,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Check if the given ``url`` is a member of the same host as this connection pool. """ - if url.startswith('/'): + if url.startswith("/"): return True # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - - host = _ipv6_host(host, self.scheme) + if host is not None: + host = _normalize_host(host, scheme=scheme) # Use explicit default port for comparison when none is given if self.port and not port: @@ -443,10 +491,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen(self, method, url, body=None, headers=None, retries=None, - redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, chunked=False, - body_pos=None, **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + assert_same_host=True, + timeout=_Default, + pool_timeout=None, + release_conn=None, + chunked=False, + body_pos=None, + **response_kw + ): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -544,12 +604,18 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: - release_conn = response_kw.get('preload_content', True) + release_conn = response_kw.get("preload_content", True) # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) + # Ensure that the URL we're connecting to is properly encoded + if url.startswith("/"): + url = six.ensure_str(_encode_target(url)) + else: + url = six.ensure_str(parse_url(url).url) + conn = None # Track whether `conn` needs to be released before @@ -566,7 +632,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Merge the proxy headers. Only do this in HTTP. We have to copy the # headers dict so we can safely change it without those changes being # reflected in anyone else's copy. - if self.scheme == 'http': + if self.scheme == "http": headers = headers.copy() headers.update(self.proxy_headers) @@ -589,15 +655,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout - is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) + is_new_proxy_conn = self.proxy is not None and not getattr( + conn, "sock", None + ) if is_new_proxy_conn: self._prepare_proxy(conn) # Make the request on the httplib connection object. - httplib_response = self._make_request(conn, method, url, - timeout=timeout_obj, - body=body, headers=headers, - chunked=chunked) + httplib_response = self._make_request( + conn, + method, + url, + timeout=timeout_obj, + body=body, + headers=headers, + chunked=chunked, + ) # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise @@ -606,14 +679,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response_conn = conn if not release_conn else None # Pass method to Response for length checking - response_kw['request_method'] = method + response_kw["request_method"] = method # Import httplib's response into our own wrapper object - response = self.ResponseCls.from_httplib(httplib_response, - pool=self, - connection=response_conn, - retries=retries, - **response_kw) + response = self.ResponseCls.from_httplib( + httplib_response, + pool=self, + connection=response_conn, + retries=retries, + **response_kw + ) # Everything went great! clean_exit = True @@ -622,20 +697,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Timed out by queue. raise EmptyPoolError(self, "No pool connections are available.") - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError, CertificateError) as e: + except ( + TimeoutError, + HTTPException, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + CertificateError, + ) as e: # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False if isinstance(e, (BaseSSLError, CertificateError)): e = SSLError(e) elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: - e = ProxyError('Cannot connect to proxy.', e) + e = ProxyError("Cannot connect to proxy.", e) elif isinstance(e, (SocketError, HTTPException)): - e = ProtocolError('Connection aborted.', e) + e = ProtocolError("Connection aborted.", e) - retries = retries.increment(method, url, error=e, _pool=self, - _stacktrace=sys.exc_info()[2]) + retries = retries.increment( + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + ) retries.sleep() # Keep track of the error for the retry warning. @@ -658,28 +741,47 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again - log.warning("Retrying (%r) after connection " - "broken by '%r': %s", retries, err, url) - return self.urlopen(method, url, body, headers, retries, - redirect, assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) + log.warning( + "Retrying (%r) after connection " "broken by '%r': %s", + retries, + err, + url, + ) + return self.urlopen( + method, + url, + body, + headers, + retries, + redirect, + assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + body_pos=body_pos, + **response_kw + ) def drain_and_release_conn(response): try: # discard any remaining response body, the connection will be # released back to the pool once the entire response is read response.read() - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError) as e: + except ( + TimeoutError, + HTTPException, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + ): pass # Handle redirect? redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: - method = 'GET' + method = "GET" try: retries = retries.increment(method, url, response=response, _pool=self) @@ -697,15 +799,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries.sleep_for_retry(response) log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( - method, redirect_location, body, headers, - retries=retries, redirect=redirect, + method, + redirect_location, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + body_pos=body_pos, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(response.getheader('Retry-After')) + has_retry_after = bool(response.getheader("Retry-After")) if retries.is_retry(method, response.status, has_retry_after): try: retries = retries.increment(method, url, response=response, _pool=self) @@ -723,12 +832,19 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries.sleep(response) log.debug("Retry: %s", url) return self.urlopen( - method, url, body, headers, - retries=retries, redirect=redirect, + method, + url, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, + timeout=timeout, + pool_timeout=pool_timeout, release_conn=release_conn, - body_pos=body_pos, **response_kw) + body_pos=body_pos, + **response_kw + ) return response @@ -746,33 +862,57 @@ class HTTPSConnectionPool(HTTPConnectionPool): If ``assert_hostname`` is False, no verification is done. The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, - ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is - available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` + is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket. """ - scheme = 'https' + scheme = "https" ConnectionCls = HTTPSConnection - def __init__(self, host, port=None, - strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, - block=False, headers=None, retries=None, - _proxy=None, _proxy_headers=None, - key_file=None, cert_file=None, cert_reqs=None, - ca_certs=None, ssl_version=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None, **conn_kw): + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + ssl_version=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + **conn_kw + ): - HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, - block, headers, retries, _proxy, _proxy_headers, - **conn_kw) - - if ca_certs and cert_reqs is None: - cert_reqs = 'CERT_REQUIRED' + HTTPConnectionPool.__init__( + self, + host, + port, + strict, + timeout, + maxsize, + block, + headers, + retries, + _proxy, + _proxy_headers, + **conn_kw + ) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version @@ -786,13 +926,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ if isinstance(conn, VerifiedHTTPSConnection): - conn.set_cert(key_file=self.key_file, - cert_file=self.cert_file, - cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint) + conn.set_cert( + key_file=self.key_file, + key_password=self.key_password, + cert_file=self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ) conn.ssl_version = self.ssl_version return conn @@ -801,17 +944,7 @@ class HTTPSConnectionPool(HTTPConnectionPool): Establish tunnel connection early, because otherwise httplib would improperly set Host: header to proxy's IP:port. """ - # Python 2.7+ - try: - set_tunnel = conn.set_tunnel - except AttributeError: # Platform-specific: Python 2.6 - set_tunnel = conn._set_tunnel - - if sys.version_info <= (2, 6, 4) and not self.proxy_headers: # Python 2.6.4 and older - set_tunnel(self._proxy_host, self.port) - else: - set_tunnel(self._proxy_host, self.port, self.proxy_headers) - + conn.set_tunnel(self._proxy_host, self.port, self.proxy_headers) conn.connect() def _new_conn(self): @@ -819,12 +952,17 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTPS connection (%d): %s:%s", - self.num_connections, self.host, self.port or "443") + log.debug( + "Starting new HTTPS connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "443", + ) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - raise SSLError("Can't connect to HTTPS URL because the SSL " - "module is not available.") + raise SSLError( + "Can't connect to HTTPS URL because the SSL " "module is not available." + ) actual_host = self.host actual_port = self.port @@ -832,9 +970,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): actual_host = self.proxy.host actual_port = self.proxy.port - conn = self.ConnectionCls(host=actual_host, port=actual_port, - timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + conn = self.ConnectionCls( + host=actual_host, + port=actual_port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + cert_file=self.cert_file, + key_file=self.key_file, + key_password=self.key_password, + **self.conn_kw + ) return self._prepare_conn(conn) @@ -845,16 +990,19 @@ class HTTPSConnectionPool(HTTPConnectionPool): super(HTTPSConnectionPool, self)._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` + if not getattr(conn, "sock", None): # AppEngine might not have `.sock` conn.connect() if not conn.is_verified: - warnings.warn(( - 'Unverified HTTPS request is being made. ' - 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings'), - InsecureRequestWarning) + warnings.warn( + ( + "Unverified HTTPS request is being made. " + "Adding certificate verification is strongly advised. See: " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings" + ), + InsecureRequestWarning, + ) def connection_from_url(url, **kw): @@ -879,28 +1027,25 @@ def connection_from_url(url, **kw): """ scheme, host, port = get_host(url) port = port or port_by_scheme.get(scheme, 80) - if scheme == 'https': + if scheme == "https": return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) -def _ipv6_host(host, scheme): +def _normalize_host(host, scheme): """ - Process IPv6 address literals + Normalize hosts for comparisons and use with sockets. """ + host = normalize_host(host, scheme) + # httplib doesn't like it when we include brackets in IPv6 addresses # Specifically, if we include brackets but also pass the port then # httplib crazily doubles up the square brackets on the Host header. # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - # - # Also if an IPv6 address literal has a zone identifier, the - # percent sign might be URIencoded, convert it back into ASCII - if host.startswith('[') and host.endswith(']'): - host = host.replace('%25', '%').strip('[]') - if scheme in NORMALIZABLE_SCHEMES: - host = host.lower() + if host.startswith("[") and host.endswith("]"): + host = host[1:-1] return host diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py new file mode 100644 index 00000000..c909010b --- /dev/null +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/_appengine_environ.py @@ -0,0 +1,32 @@ +""" +This module provides means to detect the App Engine environment. +""" + +import os + + +def is_appengine(): + return is_local_appengine() or is_prod_appengine() or is_prod_appengine_mvms() + + +def is_appengine_sandbox(): + return is_appengine() and not is_prod_appengine_mvms() + + +def is_local_appengine(): + return ( + "APPENGINE_RUNTIME" in os.environ + and "Development/" in os.environ["SERVER_SOFTWARE"] + ) + + +def is_prod_appengine(): + return ( + "APPENGINE_RUNTIME" in os.environ + and "Google App Engine/" in os.environ["SERVER_SOFTWARE"] + and not is_prod_appengine_mvms() + ) + + +def is_prod_appengine_mvms(): + return os.environ.get("GAE_VM", False) == "true" diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/bindings.py index bcf41c02..b46e1e3b 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/bindings.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/bindings.py @@ -34,29 +34,35 @@ from __future__ import absolute_import import platform from ctypes.util import find_library from ctypes import ( - c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long, - c_bool + c_void_p, + c_int32, + c_char_p, + c_size_t, + c_byte, + c_uint32, + c_ulong, + c_long, + c_bool, ) from ctypes import CDLL, POINTER, CFUNCTYPE -security_path = find_library('Security') +security_path = find_library("Security") if not security_path: - raise ImportError('The library Security could not be found') + raise ImportError("The library Security could not be found") -core_foundation_path = find_library('CoreFoundation') +core_foundation_path = find_library("CoreFoundation") if not core_foundation_path: - raise ImportError('The library CoreFoundation could not be found') + raise ImportError("The library CoreFoundation could not be found") version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split('.'))) +version_info = tuple(map(int, version.split("."))) if version_info < (10, 8): raise OSError( - 'Only OS X 10.8 and newer are supported, not %s.%s' % ( - version_info[0], version_info[1] - ) + "Only OS X 10.8 and newer are supported, not %s.%s" + % (version_info[0], version_info[1]) ) Security = CDLL(security_path, use_errno=True) @@ -129,27 +135,19 @@ try: Security.SecKeyGetTypeID.argtypes = [] Security.SecKeyGetTypeID.restype = CFTypeID - Security.SecCertificateCreateWithData.argtypes = [ - CFAllocatorRef, - CFDataRef - ] + Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] Security.SecCertificateCreateWithData.restype = SecCertificateRef - Security.SecCertificateCopyData.argtypes = [ - SecCertificateRef - ] + Security.SecCertificateCopyData.argtypes = [SecCertificateRef] Security.SecCertificateCopyData.restype = CFDataRef - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SecIdentityCreateWithCertificate.argtypes = [ CFTypeRef, SecCertificateRef, - POINTER(SecIdentityRef) + POINTER(SecIdentityRef), ] Security.SecIdentityCreateWithCertificate.restype = OSStatus @@ -159,201 +157,126 @@ try: c_void_p, Boolean, c_void_p, - POINTER(SecKeychainRef) + POINTER(SecKeychainRef), ] Security.SecKeychainCreate.restype = OSStatus - Security.SecKeychainDelete.argtypes = [ - SecKeychainRef - ] + Security.SecKeychainDelete.argtypes = [SecKeychainRef] Security.SecKeychainDelete.restype = OSStatus Security.SecPKCS12Import.argtypes = [ CFDataRef, CFDictionaryRef, - POINTER(CFArrayRef) + POINTER(CFArrayRef), ] Security.SecPKCS12Import.restype = OSStatus SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)) + SSLWriteFunc = CFUNCTYPE( + OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) + ) - Security.SSLSetIOFuncs.argtypes = [ - SSLContextRef, - SSLReadFunc, - SSLWriteFunc - ] + Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] Security.SSLSetIOFuncs.restype = OSStatus - Security.SSLSetPeerID.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerID.restype = OSStatus - Security.SSLSetCertificate.argtypes = [ - SSLContextRef, - CFArrayRef - ] + Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] Security.SSLSetCertificate.restype = OSStatus - Security.SSLSetCertificateAuthorities.argtypes = [ - SSLContextRef, - CFTypeRef, - Boolean - ] + Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] Security.SSLSetCertificateAuthorities.restype = OSStatus - Security.SSLSetConnection.argtypes = [ - SSLContextRef, - SSLConnectionRef - ] + Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] Security.SSLSetConnection.restype = OSStatus - Security.SSLSetPeerDomainName.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerDomainName.restype = OSStatus - Security.SSLHandshake.argtypes = [ - SSLContextRef - ] + Security.SSLHandshake.argtypes = [SSLContextRef] Security.SSLHandshake.restype = OSStatus - Security.SSLRead.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLRead.restype = OSStatus - Security.SSLWrite.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLWrite.restype = OSStatus - Security.SSLClose.argtypes = [ - SSLContextRef - ] + Security.SSLClose.argtypes = [SSLContextRef] Security.SSLClose.restype = OSStatus - Security.SSLGetNumberSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberSupportedCiphers.restype = OSStatus Security.SSLGetSupportedCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetSupportedCiphers.restype = OSStatus Security.SSLSetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - c_size_t + c_size_t, ] Security.SSLSetEnabledCiphers.restype = OSStatus - Security.SSLGetNumberEnabledCiphers.argtype = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberEnabledCiphers.restype = OSStatus Security.SSLGetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetEnabledCiphers.restype = OSStatus - Security.SSLGetNegotiatedCipher.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite) - ] + Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] Security.SSLGetNegotiatedCipher.restype = OSStatus Security.SSLGetNegotiatedProtocolVersion.argtypes = [ SSLContextRef, - POINTER(SSLProtocol) + POINTER(SSLProtocol), ] Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - Security.SSLCopyPeerTrust.argtypes = [ - SSLContextRef, - POINTER(SecTrustRef) - ] + Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] Security.SSLCopyPeerTrust.restype = OSStatus - Security.SecTrustSetAnchorCertificates.argtypes = [ - SecTrustRef, - CFArrayRef - ] + Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] Security.SecTrustSetAnchorCertificates.restype = OSStatus - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [ - SecTrustRef, - Boolean - ] + Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - Security.SecTrustEvaluate.argtypes = [ - SecTrustRef, - POINTER(SecTrustResultType) - ] + Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] Security.SecTrustEvaluate.restype = OSStatus - Security.SecTrustGetCertificateCount.argtypes = [ - SecTrustRef - ] + Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] Security.SecTrustGetCertificateCount.restype = CFIndex - Security.SecTrustGetCertificateAtIndex.argtypes = [ - SecTrustRef, - CFIndex - ] + Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef Security.SSLCreateContext.argtypes = [ CFAllocatorRef, SSLProtocolSide, - SSLConnectionType + SSLConnectionType, ] Security.SSLCreateContext.restype = SSLContextRef - Security.SSLSetSessionOption.argtypes = [ - SSLContextRef, - SSLSessionOption, - Boolean - ] + Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] Security.SSLSetSessionOption.restype = OSStatus - Security.SSLSetProtocolVersionMin.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMin.restype = OSStatus - Security.SSLSetProtocolVersionMax.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMax.restype = OSStatus - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SSLReadFunc = SSLReadFunc @@ -369,64 +292,47 @@ try: Security.OSStatus = OSStatus Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, 'kSecImportExportPassphrase' + Security, "kSecImportExportPassphrase" ) Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, 'kSecImportItemIdentity' + Security, "kSecImportItemIdentity" ) # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRetain.argtypes = [CFTypeRef] CoreFoundation.CFRetain.restype = CFTypeRef - CoreFoundation.CFRelease.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRelease.argtypes = [CFTypeRef] CoreFoundation.CFRelease.restype = None - CoreFoundation.CFGetTypeID.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] CoreFoundation.CFGetTypeID.restype = CFTypeID CoreFoundation.CFStringCreateWithCString.argtypes = [ CFAllocatorRef, c_char_p, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - CoreFoundation.CFStringGetCStringPtr.argtypes = [ - CFStringRef, - CFStringEncoding - ] + CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] CoreFoundation.CFStringGetCStringPtr.restype = c_char_p CoreFoundation.CFStringGetCString.argtypes = [ CFStringRef, c_char_p, CFIndex, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringGetCString.restype = c_bool - CoreFoundation.CFDataCreate.argtypes = [ - CFAllocatorRef, - c_char_p, - CFIndex - ] + CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] CoreFoundation.CFDataCreate.restype = CFDataRef - CoreFoundation.CFDataGetLength.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] CoreFoundation.CFDataGetLength.restype = CFIndex - CoreFoundation.CFDataGetBytePtr.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] CoreFoundation.CFDataGetBytePtr.restype = c_void_p CoreFoundation.CFDictionaryCreate.argtypes = [ @@ -435,14 +341,11 @@ try: POINTER(CFTypeRef), CFIndex, CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks + CFDictionaryValueCallBacks, ] CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - CoreFoundation.CFDictionaryGetValue.argtypes = [ - CFDictionaryRef, - CFTypeRef - ] + CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef CoreFoundation.CFArrayCreate.argtypes = [ @@ -456,36 +359,30 @@ try: CoreFoundation.CFArrayCreateMutable.argtypes = [ CFAllocatorRef, CFIndex, - CFArrayCallBacks + CFArrayCallBacks, ] CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - CoreFoundation.CFArrayAppendValue.argtypes = [ - CFMutableArrayRef, - c_void_p - ] + CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] CoreFoundation.CFArrayAppendValue.restype = None - CoreFoundation.CFArrayGetCount.argtypes = [ - CFArrayRef - ] + CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] CoreFoundation.CFArrayGetCount.restype = CFIndex - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [ - CFArrayRef, - CFIndex - ] + CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, 'kCFAllocatorDefault' + CoreFoundation, "kCFAllocatorDefault" + ) + CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( + CoreFoundation, "kCFTypeArrayCallBacks" ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks') CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryKeyCallBacks' + CoreFoundation, "kCFTypeDictionaryKeyCallBacks" ) CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryValueCallBacks' + CoreFoundation, "kCFTypeDictionaryValueCallBacks" ) CoreFoundation.CFTypeRef = CFTypeRef @@ -494,7 +391,7 @@ try: CoreFoundation.CFDictionaryRef = CFDictionaryRef except (AttributeError): - raise ImportError('Error initializing ctypes') + raise ImportError("Error initializing ctypes") class CFConst(object): @@ -502,6 +399,7 @@ class CFConst(object): A class object that acts as essentially a namespace for CoreFoundation constants. """ + kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) @@ -509,6 +407,7 @@ class SecurityConst(object): """ A class object that acts as essentially a namespace for Security constants. """ + kSSLSessionOptionBreakOnServerAuth = 0 kSSLProtocol2 = 1 @@ -516,6 +415,8 @@ class SecurityConst(object): kTLSProtocol1 = 4 kTLSProtocol11 = 7 kTLSProtocol12 = 8 + kTLSProtocol13 = 10 + kTLSProtocolMaxSupported = 999 kSSLClientSide = 1 kSSLStreamType = 0 @@ -558,30 +459,27 @@ class SecurityConst(object): errSecInvalidTrustSettings = -25262 # Cipher suites. We only pick the ones our default cipher string allows. + # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D @@ -590,4 +488,5 @@ class SecurityConst(object): TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F TLS_AES_128_GCM_SHA256 = 0x1301 TLS_AES_256_GCM_SHA384 = 0x1302 - TLS_CHACHA20_POLY1305_SHA256 = 0x1303 + TLS_AES_128_CCM_8_SHA256 = 0x1305 + TLS_AES_128_CCM_SHA256 = 0x1304 diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/low_level.py index b13cd9e7..e60168ca 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/low_level.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/_securetransport/low_level.py @@ -66,22 +66,18 @@ def _cf_string_to_unicode(value): value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, CFConst.kCFStringEncodingUTF8 ) if string is None: buffer = ctypes.create_string_buffer(1024) result = CoreFoundation.CFStringGetCString( - value_as_void_p, - buffer, - 1024, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 ) if not result: - raise OSError('Error copying C string from CFStringRef') + raise OSError("Error copying C string from CFStringRef") string = buffer.value if string is not None: - string = string.decode('utf-8') + string = string.decode("utf-8") return string @@ -97,8 +93,8 @@ def _assert_no_error(error, exception_class=None): output = _cf_string_to_unicode(cf_error_string) CoreFoundation.CFRelease(cf_error_string) - if output is None or output == u'': - output = u'OSStatus %s' % error + if output is None or output == u"": + output = u"OSStatus %s" % error if exception_class is None: exception_class = ssl.SSLError @@ -115,8 +111,7 @@ def _cert_array_from_pem(pem_bundle): pem_bundle = pem_bundle.replace(b"\r\n", b"\n") der_certs = [ - base64.b64decode(match.group(1)) - for match in _PEM_CERTS_RE.finditer(pem_bundle) + base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) ] if not der_certs: raise ssl.SSLError("No root certificates specified") @@ -124,7 +119,7 @@ def _cert_array_from_pem(pem_bundle): cert_array = CoreFoundation.CFArrayCreateMutable( CoreFoundation.kCFAllocatorDefault, 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks) + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), ) if not cert_array: raise ssl.SSLError("Unable to allocate memory!") @@ -186,21 +181,16 @@ def _temporary_keychain(): # some random bytes to password-protect the keychain we're creating, so we # ask for 40 random bytes. random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode('utf-8') + filename = base64.b16encode(random_bytes[:8]).decode("utf-8") password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 tempdirectory = tempfile.mkdtemp() - keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') + keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") # We now want to create the keychain itself. keychain = Security.SecKeychainRef() status = Security.SecKeychainCreate( - keychain_path, - len(password), - password, - False, - None, - ctypes.byref(keychain) + keychain_path, len(password), password, False, None, ctypes.byref(keychain) ) _assert_no_error(status) @@ -219,14 +209,12 @@ def _load_items_from_file(keychain, path): identities = [] result_array = None - with open(path, 'rb') as f: + with open(path, "rb") as f: raw_filedata = f.read() try: filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, - raw_filedata, - len(raw_filedata) + CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) ) result_array = CoreFoundation.CFArrayRef() result = Security.SecItemImport( @@ -237,7 +225,7 @@ def _load_items_from_file(keychain, path): 0, # import flags None, # key params, can include passphrase in the future keychain, # The keychain to insert into - ctypes.byref(result_array) # Results + ctypes.byref(result_array), # Results ) _assert_no_error(result) @@ -247,9 +235,7 @@ def _load_items_from_file(keychain, path): # keychain already has them! result_count = CoreFoundation.CFArrayGetCount(result_array) for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex( - result_array, index - ) + item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) item = ctypes.cast(item, CoreFoundation.CFTypeRef) if _is_cert(item): @@ -307,9 +293,7 @@ def _load_client_cert_chain(keychain, *paths): try: for file_path in paths: - new_identities, new_certs = _load_items_from_file( - keychain, file_path - ) + new_identities, new_certs = _load_items_from_file(keychain, file_path) identities.extend(new_identities) certificates.extend(new_certs) @@ -318,9 +302,7 @@ def _load_client_cert_chain(keychain, *paths): if not identities: new_identity = Security.SecIdentityRef() status = Security.SecIdentityCreateWithCertificate( - keychain, - certificates[0], - ctypes.byref(new_identity) + keychain, certificates[0], ctypes.byref(new_identity) ) _assert_no_error(status) identities.append(new_identity) diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py index 06586352..d8716b9f 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/appengine.py @@ -39,8 +39,8 @@ urllib3 on Google App Engine: """ from __future__ import absolute_import +import io import logging -import os import warnings from ..packages.six.moves.urllib.parse import urljoin @@ -50,14 +50,14 @@ from ..exceptions import ( MaxRetryError, ProtocolError, TimeoutError, - SSLError + SSLError, ) -from ..packages.six import BytesIO from ..request import RequestMethods from ..response import HTTPResponse from ..util.timeout import Timeout from ..util.retry import Retry +from . import _appengine_environ try: from google.appengine.api import urlfetch @@ -96,23 +96,31 @@ class AppEngineManager(RequestMethods): Beyond those cases, it will raise normal urllib3 errors. """ - def __init__(self, headers=None, retries=None, validate_certificate=True, - urlfetch_retries=True): + def __init__( + self, + headers=None, + retries=None, + validate_certificate=True, + urlfetch_retries=True, + ): if not urlfetch: raise AppEnginePlatformError( - "URLFetch is not available in this environment.") + "URLFetch is not available in this environment." + ) if is_prod_appengine_mvms(): raise AppEnginePlatformError( "Use normal urllib3.PoolManager instead of AppEngineManager" "on Managed VMs, as using URLFetch is not necessary in " - "this environment.") + "this environment." + ) warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) RequestMethods.__init__(self, headers) self.validate_certificate = validate_certificate @@ -127,17 +135,22 @@ class AppEngineManager(RequestMethods): # Return False to re-raise any potential exceptions return False - def urlopen(self, method, url, body=None, headers=None, - retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, - **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + timeout=Timeout.DEFAULT_TIMEOUT, + **response_kw + ): retries = self._get_retries(retries, redirect) try: - follow_redirects = ( - redirect and - retries.redirect != 0 and - retries.total) + follow_redirects = redirect and retries.redirect != 0 and retries.total response = urlfetch.fetch( url, payload=body, @@ -152,44 +165,52 @@ class AppEngineManager(RequestMethods): raise TimeoutError(self, e) except urlfetch.InvalidURLError as e: - if 'too large' in str(e): + if "too large" in str(e): raise AppEnginePlatformError( "URLFetch request too large, URLFetch only " - "supports requests up to 10mb in size.", e) + "supports requests up to 10mb in size.", + e, + ) raise ProtocolError(e) except urlfetch.DownloadError as e: - if 'Too many redirects' in str(e): + if "Too many redirects" in str(e): raise MaxRetryError(self, url, reason=e) raise ProtocolError(e) except urlfetch.ResponseTooLargeError as e: raise AppEnginePlatformError( "URLFetch response too large, URLFetch only supports" - "responses up to 32mb in size.", e) + "responses up to 32mb in size.", + e, + ) except urlfetch.SSLCertificateError as e: raise SSLError(e) except urlfetch.InvalidMethodError as e: raise AppEnginePlatformError( - "URLFetch does not support method: %s" % method, e) + "URLFetch does not support method: %s" % method, e + ) http_response = self._urlfetch_response_to_http_response( - response, retries=retries, **response_kw) + response, retries=retries, **response_kw + ) # Handle redirect? redirect_location = redirect and http_response.get_redirect_location() if redirect_location: # Check for redirect response - if (self.urlfetch_retries and retries.raise_on_redirect): + if self.urlfetch_retries and retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") else: if http_response.status == 303: - method = 'GET' + method = "GET" try: - retries = retries.increment(method, url, response=http_response, _pool=self) + retries = retries.increment( + method, url, response=http_response, _pool=self + ) except MaxRetryError: if retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") @@ -199,22 +220,32 @@ class AppEngineManager(RequestMethods): log.debug("Redirecting %s -> %s", url, redirect_location) redirect_url = urljoin(url, redirect_location) return self.urlopen( - method, redirect_url, body, headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + redirect_url, + body, + headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(http_response.getheader('Retry-After')) + has_retry_after = bool(http_response.getheader("Retry-After")) if retries.is_retry(method, http_response.status, has_retry_after): - retries = retries.increment( - method, url, response=http_response, _pool=self) + retries = retries.increment(method, url, response=http_response, _pool=self) log.debug("Retry: %s", url) retries.sleep(http_response) return self.urlopen( - method, url, - body=body, headers=headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + url, + body=body, + headers=headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) return http_response @@ -223,23 +254,23 @@ class AppEngineManager(RequestMethods): if is_prod_appengine(): # Production GAE handles deflate encoding automatically, but does # not remove the encoding header. - content_encoding = urlfetch_resp.headers.get('content-encoding') + content_encoding = urlfetch_resp.headers.get("content-encoding") - if content_encoding == 'deflate': - del urlfetch_resp.headers['content-encoding'] + if content_encoding == "deflate": + del urlfetch_resp.headers["content-encoding"] - transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") # We have a full response's content, # so let's make sure we don't report ourselves as chunked data. - if transfer_encoding == 'chunked': + if transfer_encoding == "chunked": encodings = transfer_encoding.split(",") - encodings.remove('chunked') - urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + encodings.remove("chunked") + urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) original_response = HTTPResponse( # In order for decoding to work, we must present the content as # a file-like object. - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), msg=urlfetch_resp.header_msg, headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, @@ -247,7 +278,7 @@ class AppEngineManager(RequestMethods): ) return HTTPResponse( - body=BytesIO(urlfetch_resp.content), + body=io.BytesIO(urlfetch_resp.content), headers=urlfetch_resp.headers, status=urlfetch_resp.status_code, original_response=original_response, @@ -262,44 +293,29 @@ class AppEngineManager(RequestMethods): warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total or default URLFetch timeout.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return timeout.total return timeout def _get_retries(self, retries, redirect): if not isinstance(retries, Retry): - retries = Retry.from_int( - retries, redirect=redirect, default=self.retries) + retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if retries.connect or retries.read or retries.redirect: warnings.warn( "URLFetch only supports total retries and does not " "recognize connect, read, or redirect retry parameters.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return retries -def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) +# Alias methods from _appengine_environ to maintain public API interface. - -def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() - - -def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) - - -def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) - - -def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' +is_appengine = _appengine_environ.is_appengine +is_appengine_sandbox = _appengine_environ.is_appengine_sandbox +is_local_appengine = _appengine_environ.is_local_appengine +is_prod_appengine = _appengine_environ.is_prod_appengine +is_prod_appengine_mvms = _appengine_environ.is_prod_appengine_mvms diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py index 642e99ed..9c96be29 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/ntlmpool.py @@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): Implements an NTLM authentication version of an urllib3 connection pool """ - scheme = 'https' + scheme = "https" def __init__(self, user, pw, authurl, *args, **kwargs): """ @@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): super(NTLMConnectionPool, self).__init__(*args, **kwargs) self.authurl = authurl self.rawuser = user - user_parts = user.split('\\', 1) + user_parts = user.split("\\", 1) self.domain = user_parts[0].upper() self.user = user_parts[1] self.pw = pw @@ -40,73 +40,84 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', - self.num_connections, self.host, self.authurl) + log.debug( + "Starting NTLM HTTPS connection no. %d: https://%s%s", + self.num_connections, + self.host, + self.authurl, + ) - headers = {} - headers['Connection'] = 'Keep-Alive' - req_header = 'Authorization' - resp_header = 'www-authenticate' + headers = {"Connection": "Keep-Alive"} + req_header = "Authorization" + resp_header = "www-authenticate" conn = HTTPSConnection(host=self.host, port=self.port) # Send negotiation message - headers[req_header] = ( - 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( + self.rawuser + ) + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', reshdr) - log.debug('Response data: %s [...]', res.read(100)) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", reshdr) + log.debug("Response data: %s [...]", res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) res.fp = None # Server should respond with a challenge message - auth_header_values = reshdr[resp_header].split(', ') + auth_header_values = reshdr[resp_header].split(", ") auth_header_value = None for s in auth_header_values: - if s[:5] == 'NTLM ': + if s[:5] == "NTLM ": auth_header_value = s[5:] if auth_header_value is None: - raise Exception('Unexpected %s response header: %s' % - (resp_header, reshdr[resp_header])) + raise Exception( + "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) + ) # Send authentication message - ServerChallenge, NegotiateFlags = \ - ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) - auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, - self.user, - self.domain, - self.pw, - NegotiateFlags) - headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( + auth_header_value + ) + auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( + ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags + ) + headers[req_header] = "NTLM %s" % auth_msg + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', dict(res.getheaders())) - log.debug('Response data: %s [...]', res.read()[:100]) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", dict(res.getheaders())) + log.debug("Response data: %s [...]", res.read()[:100]) if res.status != 200: if res.status == 401: - raise Exception('Server rejected request: wrong ' - 'username or password') - raise Exception('Wrong server response: %s %s' % - (res.status, res.reason)) + raise Exception( + "Server rejected request: wrong " "username or password" + ) + raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) res.fp = None - log.debug('Connection established') + log.debug("Connection established") return conn - def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=3, + redirect=True, + assert_same_host=True, + ): if headers is None: headers = {} - headers['Connection'] = 'Keep-Alive' - return super(NTLMConnectionPool, self).urlopen(method, url, body, - headers, retries, - redirect, - assert_same_host) + headers["Connection"] = "Keep-Alive" + return super(NTLMConnectionPool, self).urlopen( + method, url, body, headers, retries, redirect, assert_same_host + ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py index 7787d4e4..e533512d 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/pyopenssl.py @@ -47,6 +47,7 @@ import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate + try: from cryptography.x509 import UnsupportedExtension except ImportError: @@ -54,6 +55,7 @@ except ImportError: class UnsupportedExtension(Exception): pass + from socket import timeout, error as SocketError from io import BytesIO @@ -70,37 +72,35 @@ import sys from .. import util -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] + +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works. HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { - ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, + util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } -if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): +if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): + _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD + +if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD -if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): +if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD -try: - _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) -except AttributeError: - pass _stdlib_to_openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, - ssl.CERT_REQUIRED: - OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -_openssl_to_stdlib_verify = dict( - (v, k) for k, v in _stdlib_to_openssl_verify.items() -) +_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -113,10 +113,11 @@ log = logging.getLogger(__name__) def inject_into_urllib3(): - 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." _validate_dependencies_met() + util.SSLContext = PyOpenSSLContext util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -125,8 +126,9 @@ def inject_into_urllib3(): def extract_from_urllib3(): - 'Undo monkey-patching by :func:`inject_into_urllib3`.' + "Undo monkey-patching by :func:`inject_into_urllib3`." + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -141,17 +143,23 @@ def _validate_dependencies_met(): """ # Method added in `cryptography==1.1`; not available in older versions from cryptography.x509.extensions import Extensions + if getattr(Extensions, "get_extension_for_class", None) is None: - raise ImportError("'cryptography' module missing required functionality. " - "Try upgrading to v1.3.4 or newer.") + raise ImportError( + "'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer." + ) # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 # attribute is only present on those versions. from OpenSSL.crypto import X509 + x509 = X509() if getattr(x509, "_x509", None) is None: - raise ImportError("'pyOpenSSL' module missing required functionality. " - "Try upgrading to v0.14 or newer.") + raise ImportError( + "'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer." + ) def _dnsname_to_stdlib(name): @@ -163,7 +171,11 @@ def _dnsname_to_stdlib(name): from ASCII bytes. We need to idna-encode that string to get it back, and then on Python 3 we also need to convert to unicode via UTF-8 (the stdlib uses PyUnicode_FromStringAndSize on it, which decodes via UTF-8). + + If the name cannot be idna-encoded then we return None signalling that + the name given should be skipped. """ + def idna_encode(name): """ Borrowed wholesale from the Python Cryptography Project. It turns out @@ -172,15 +184,24 @@ def _dnsname_to_stdlib(name): """ from pipenv.patched.notpip._vendor import idna - for prefix in [u'*.', u'.']: - if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) - return idna.encode(name) + try: + for prefix in [u"*.", u"."]: + if name.startswith(prefix): + name = name[len(prefix) :] + return prefix.encode("ascii") + idna.encode(name) + return idna.encode(name) + except idna.core.IDNAError: + return None + + # Don't send IPv6 addresses through the IDNA encoder. + if ":" in name: + return name name = idna_encode(name) - if sys.version_info >= (3, 0): - name = name.decode('utf-8') + if name is None: + return None + elif sys.version_info >= (3, 0): + name = name.decode("utf-8") return name @@ -199,14 +220,16 @@ def get_subj_alt_name(peer_cert): # We want to find the SAN extension. Ask Cryptography to locate it (it's # faster than looping in Python) try: - ext = cert.extensions.get_extension_for_class( - x509.SubjectAlternativeName - ).value + ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value except x509.ExtensionNotFound: # No such extension, return the empty list. return [] - except (x509.DuplicateExtension, UnsupportedExtension, - x509.UnsupportedGeneralNameType, UnicodeError) as e: + except ( + x509.DuplicateExtension, + UnsupportedExtension, + x509.UnsupportedGeneralNameType, + UnicodeError, + ) as e: # A problem has been found with the quality of the certificate. Assume # no SAN field is present. log.warning( @@ -223,24 +246,25 @@ def get_subj_alt_name(peer_cert): # Sadly the DNS names need to be idna encoded and then, on Python 3, UTF-8 # decoded. This is pretty frustrating, but that's what the standard library # does with certificates, and so we need to attempt to do the same. + # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', _dnsname_to_stdlib(name)) - for name in ext.get_values_for_type(x509.DNSName) + ("DNS", name) + for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + if name is not None ] names.extend( - ('IP Address', str(name)) - for name in ext.get_values_for_type(x509.IPAddress) + ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress) ) return names class WrappedSocket(object): - '''API-compatibility wrapper for Python OpenSSL's Connection-class. + """API-compatibility wrapper for Python OpenSSL's Connection-class. Note: _makefile_refs, _drop() and _reuse() are needed for the garbage collector of pypy. - ''' + """ def __init__(self, connection, socket, suppress_ragged_eofs=True): self.connection = connection @@ -263,20 +287,24 @@ class WrappedSocket(object): try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): - return b'' + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): + return b"" else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: - return b'' + return b"" else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv(*args, **kwargs) + + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) else: return data @@ -284,21 +312,25 @@ class WrappedSocket(object): try: return self.connection.recv_into(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return 0 else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return 0 else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv_into(*args, **kwargs) + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -316,7 +348,9 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self._send_until_done( + data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE] + ) total_sent += sent def shutdown(self): @@ -340,17 +374,16 @@ class WrappedSocket(object): return x509 if binary_form: - return OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - x509) + return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) return { - 'subject': ( - (('commonName', x509.get_subject().CN),), - ), - 'subjectAltName': get_subj_alt_name(x509) + "subject": ((("commonName", x509.get_subject().CN),),), + "subjectAltName": get_subj_alt_name(x509), } + def version(self): + return self.connection.get_protocol_version_name() + def _reuse(self): self._makefile_refs += 1 @@ -362,9 +395,12 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 makefile = backport_makefile @@ -377,6 +413,7 @@ class PyOpenSSLContext(object): for translating the interface of the standard library ``SSLContext`` object to calls into PyOpenSSL. """ + def __init__(self, protocol): self.protocol = _openssl_versions[protocol] self._ctx = OpenSSL.SSL.Context(self.protocol) @@ -398,24 +435,21 @@ class PyOpenSSLContext(object): @verify_mode.setter def verify_mode(self, value): - self._ctx.set_verify( - _stdlib_to_openssl_verify[value], - _verify_callback - ) + self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) def set_default_verify_paths(self): self._ctx.set_default_verify_paths() def set_ciphers(self, ciphers): if isinstance(ciphers, six.text_type): - ciphers = ciphers.encode('utf-8') + ciphers = ciphers.encode("utf-8") self._ctx.set_cipher_list(ciphers) def load_verify_locations(self, cafile=None, capath=None, cadata=None): if cafile is not None: - cafile = cafile.encode('utf-8') + cafile = cafile.encode("utf-8") if capath is not None: - capath = capath.encode('utf-8') + capath = capath.encode("utf-8") self._ctx.load_verify_locations(cafile, capath) if cadata is not None: self._ctx.load_verify_locations(BytesIO(cadata)) @@ -423,16 +457,23 @@ class PyOpenSSLContext(object): def load_cert_chain(self, certfile, keyfile=None, password=None): self._ctx.use_certificate_chain_file(certfile) if password is not None: - self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) + if not isinstance(password, six.binary_type): + password = password.encode("utf-8") + self._ctx.set_passwd_cb(lambda *_: password) self._ctx.use_privatekey_file(keyfile or certfile) - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): cnx = OpenSSL.SSL.Connection(self._ctx, sock) if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") if server_hostname is not None: cnx.set_tlsext_host_name(server_hostname) @@ -444,10 +485,10 @@ class PyOpenSSLContext(object): cnx.do_handshake() except OpenSSL.SSL.WantReadError: if not util.wait_for_read(sock, sock.gettimeout()): - raise timeout('select timed out') + raise timeout("select timed out") continue except OpenSSL.SSL.Error as e: - raise ssl.SSLError('bad handshake: %r' % e) + raise ssl.SSLError("bad handshake: %r" % e) break return WrappedSocket(cnx, sock) diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/securetransport.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/securetransport.py index 77cb59ed..24e6b5c4 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/securetransport.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/securetransport.py @@ -23,6 +23,31 @@ To use this module, simply import and inject it:: urllib3.contrib.securetransport.inject_into_urllib3() Happy TLSing! + +This code is a bastardised version of the code found in Will Bond's oscrypto +library. An enormous debt is owed to him for blazing this trail for us. For +that reason, this code should be considered to be covered both by urllib3's +license and by oscrypto's: + + Copyright (c) 2015-2016 Will Bond <will@wbond.net> + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. """ from __future__ import absolute_import @@ -37,12 +62,12 @@ import threading import weakref from .. import util -from ._securetransport.bindings import ( - Security, SecurityConst, CoreFoundation -) +from ._securetransport.bindings import Security, SecurityConst, CoreFoundation from ._securetransport.low_level import ( - _assert_no_error, _cert_array_from_pem, _temporary_keychain, - _load_client_cert_chain + _assert_no_error, + _cert_array_from_pem, + _temporary_keychain, + _load_client_cert_chain, ) try: # Platform-specific: Python 2 @@ -51,7 +76,7 @@ except ImportError: # Platform-specific: Python 3 _fileobject = None from ..packages.backports.makefile import backport_makefile -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works HAS_SNI = True @@ -86,35 +111,32 @@ SSL_WRITE_BLOCKSIZE = 16384 # individual cipher suites. We need to do this because this is how # SecureTransport wants them. CIPHER_SUITES = [ - SecurityConst.TLS_AES_256_GCM_SHA384, - SecurityConst.TLS_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_AES_256_GCM_SHA384, + SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_AES_128_CCM_8_SHA256, + SecurityConst.TLS_AES_128_CCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, @@ -122,39 +144,47 @@ CIPHER_SUITES = [ ] # Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of -# TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. +# TLSv1 and a high of TLSv1.3. For everything else, we pin to that version. +# TLSv1 to 1.2 are supported on macOS 10.8+ and TLSv1.3 is macOS 10.13+ _protocol_to_min_max = { - ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + util.PROTOCOL_TLS: ( + SecurityConst.kTLSProtocol1, + SecurityConst.kTLSProtocolMaxSupported, + ) } if hasattr(ssl, "PROTOCOL_SSLv2"): _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2 + SecurityConst.kSSLProtocol2, + SecurityConst.kSSLProtocol2, ) if hasattr(ssl, "PROTOCOL_SSLv3"): _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3 + SecurityConst.kSSLProtocol3, + SecurityConst.kSSLProtocol3, ) if hasattr(ssl, "PROTOCOL_TLSv1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1 + SecurityConst.kTLSProtocol1, + SecurityConst.kTLSProtocol1, ) if hasattr(ssl, "PROTOCOL_TLSv1_1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11 + SecurityConst.kTLSProtocol11, + SecurityConst.kTLSProtocol11, ) if hasattr(ssl, "PROTOCOL_TLSv1_2"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 + SecurityConst.kTLSProtocol12, + SecurityConst.kTLSProtocol12, ) -if hasattr(ssl, "PROTOCOL_TLS"): - _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23] def inject_into_urllib3(): """ Monkey-patch urllib3 with SecureTransport-backed SSL-support. """ + util.SSLContext = SecureTransportContext util.ssl_.SSLContext = SecureTransportContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -166,6 +196,7 @@ def extract_from_urllib3(): """ Undo monkey-patching by :func:`inject_into_urllib3`. """ + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -195,7 +226,7 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): while read_count < requested_length: if timeout is None or timeout >= 0: if not util.wait_for_read(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") remaining = requested_length - read_count buffer = (ctypes.c_char * remaining).from_address( @@ -251,7 +282,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): while sent < bytes_to_write: if timeout is None or timeout >= 0: if not util.wait_for_write(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") chunk_sent = base_socket.send(data) sent += chunk_sent @@ -293,6 +324,7 @@ class WrappedSocket(object): Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage collector of PyPy. """ + def __init__(self, socket): self.socket = socket self.context = None @@ -357,7 +389,7 @@ class WrappedSocket(object): # We want data in memory, so load it up. if os.path.isfile(trust_bundle): - with open(trust_bundle, 'rb') as f: + with open(trust_bundle, "rb") as f: trust_bundle = f.read() cert_array = None @@ -371,9 +403,7 @@ class WrappedSocket(object): # created for this connection, shove our CAs into it, tell ST to # ignore everything else it knows, and then ask if it can build a # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: raise ssl.SSLError("Failed to copy trust reference") @@ -385,9 +415,7 @@ class WrappedSocket(object): _assert_no_error(result) trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate( - trust, ctypes.byref(trust_result) - ) + result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) _assert_no_error(result) finally: if trust: @@ -399,23 +427,24 @@ class WrappedSocket(object): # Ok, now we can look at what the result was. successes = ( SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed + SecurityConst.kSecTrustResultProceed, ) if trust_result.value not in successes: raise ssl.SSLError( - "certificate verify failed, error code: %d" % - trust_result.value + "certificate verify failed, error code: %d" % trust_result.value ) - def handshake(self, - server_hostname, - verify, - trust_bundle, - min_version, - max_version, - client_cert, - client_key, - client_key_passphrase): + def handshake( + self, + server_hostname, + verify, + trust_bundle, + min_version, + max_version, + client_cert, + client_key, + client_key_passphrase, + ): """ Actually performs the TLS handshake. This is run automatically by wrapped socket, and shouldn't be needed in user code. @@ -445,7 +474,7 @@ class WrappedSocket(object): # If we have a server hostname, we should set that too. if server_hostname: if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") result = Security.SSLSetPeerDomainName( self.context, server_hostname, len(server_hostname) @@ -458,7 +487,16 @@ class WrappedSocket(object): # Set the minimum and maximum TLS versions. result = Security.SSLSetProtocolVersionMin(self.context, min_version) _assert_no_error(result) + + # TLS 1.3 isn't necessarily enabled by the OS + # so we have to detect when we error out and try + # setting TLS 1.3 if it's allowed. kTLSProtocolMaxSupported + # was added in macOS 10.13 along with kTLSProtocol13. result = Security.SSLSetProtocolVersionMax(self.context, max_version) + if result != 0 and max_version == SecurityConst.kTLSProtocolMaxSupported: + result = Security.SSLSetProtocolVersionMax( + self.context, SecurityConst.kTLSProtocol12 + ) _assert_no_error(result) # If there's a trust DB, we need to use it. We do that by telling @@ -467,9 +505,7 @@ class WrappedSocket(object): # authing in that case. if not verify or trust_bundle is not None: result = Security.SSLSetSessionOption( - self.context, - SecurityConst.kSSLSessionOptionBreakOnServerAuth, - True + self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True ) _assert_no_error(result) @@ -479,9 +515,7 @@ class WrappedSocket(object): self._client_cert_chain = _load_client_cert_chain( self._keychain, client_cert, client_key ) - result = Security.SSLSetCertificate( - self.context, self._client_cert_chain - ) + result = Security.SSLSetCertificate(self.context, self._client_cert_chain) _assert_no_error(result) while True: @@ -532,7 +566,7 @@ class WrappedSocket(object): # There are some result codes that we want to treat as "not always # errors". Specifically, those are errSSLWouldBlock, # errSSLClosedGraceful, and errSSLClosedNoNotify. - if (result == SecurityConst.errSSLWouldBlock): + if result == SecurityConst.errSSLWouldBlock: # If we didn't process any bytes, then this was just a time out. # However, we can get errSSLWouldBlock in situations when we *did* # read some data, and in those cases we should just read "short" @@ -540,7 +574,10 @@ class WrappedSocket(object): if processed_bytes.value == 0: # Timed out, no data read. raise socket.timeout("recv timed out") - elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify): + elif result in ( + SecurityConst.errSSLClosedGraceful, + SecurityConst.errSSLClosedNoNotify, + ): # The remote peer has closed this connection. We should do so as # well. Note that we don't actually return here because in # principle this could actually be fired along with return data. @@ -579,7 +616,7 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) total_sent += sent def shutdown(self): @@ -626,18 +663,14 @@ class WrappedSocket(object): # instead to just flag to urllib3 that it shouldn't do its own hostname # validation when using SecureTransport. if not binary_form: - raise ValueError( - "SecureTransport only supports dumping binary certs" - ) + raise ValueError("SecureTransport only supports dumping binary certs") trust = Security.SecTrustRef() certdata = None der_bytes = None try: # Grab the trust store. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: # Probably we haven't done the handshake yet. No biggie. @@ -667,6 +700,27 @@ class WrappedSocket(object): return der_bytes + def version(self): + protocol = Security.SSLProtocol() + result = Security.SSLGetNegotiatedProtocolVersion( + self.context, ctypes.byref(protocol) + ) + _assert_no_error(result) + if protocol.value == SecurityConst.kTLSProtocol13: + return "TLSv1.3" + elif protocol.value == SecurityConst.kTLSProtocol12: + return "TLSv1.2" + elif protocol.value == SecurityConst.kTLSProtocol11: + return "TLSv1.1" + elif protocol.value == SecurityConst.kTLSProtocol1: + return "TLSv1" + elif protocol.value == SecurityConst.kSSLProtocol3: + return "SSLv3" + elif protocol.value == SecurityConst.kSSLProtocol2: + return "SSLv2" + else: + raise ssl.SSLError("Unknown TLS version: %r" % protocol) + def _reuse(self): self._makefile_refs += 1 @@ -678,16 +732,21 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 + def makefile(self, mode="r", buffering=None, *args, **kwargs): # We disable buffering with SecureTransport because it conflicts with # the buffering that ST does internally (see issue #1153 for more). buffering = 0 return backport_makefile(self, mode, buffering, *args, **kwargs) + WrappedSocket.makefile = makefile @@ -697,6 +756,7 @@ class SecureTransportContext(object): interface of the standard library ``SSLContext`` object to calls into SecureTransport. """ + def __init__(self, protocol): self._min_version, self._max_version = _protocol_to_min_max[protocol] self._options = 0 @@ -763,16 +823,12 @@ class SecureTransportContext(object): def set_ciphers(self, ciphers): # For now, we just require the default cipher string. if ciphers != util.ssl_.DEFAULT_CIPHERS: - raise ValueError( - "SecureTransport doesn't support custom cipher strings" - ) + raise ValueError("SecureTransport doesn't support custom cipher strings") def load_verify_locations(self, cafile=None, capath=None, cadata=None): # OK, we only really support cadata and cafile. if capath is not None: - raise ValueError( - "SecureTransport does not support cert directories" - ) + raise ValueError("SecureTransport does not support cert directories") self._trust_bundle = cafile or cadata @@ -781,9 +837,14 @@ class SecureTransportContext(object): self._client_key = keyfile self._client_cert_passphrase = password - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): # So, what do we do here? Firstly, we assert some properties. This is a # stripped down shim, so there is some functionality we don't support. # See PEP 543 for the real deal. @@ -797,8 +858,13 @@ class SecureTransportContext(object): # Now we can handshake wrapped_socket.handshake( - server_hostname, self._verify, self._trust_bundle, - self._min_version, self._max_version, self._client_cert, - self._client_key, self._client_key_passphrase + server_hostname, + self._verify, + self._trust_bundle, + self._min_version, + self._max_version, + self._client_cert, + self._client_key, + self._client_key_passphrase, ) return wrapped_socket diff --git a/pipenv/patched/notpip/_vendor/urllib3/contrib/socks.py b/pipenv/patched/notpip/_vendor/urllib3/contrib/socks.py index 811e312e..9e97f7aa 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/contrib/socks.py +++ b/pipenv/patched/notpip/_vendor/urllib3/contrib/socks.py @@ -1,25 +1,38 @@ # -*- coding: utf-8 -*- """ This module contains provisional support for SOCKS proxies from within -urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and SOCKS5. To enable its functionality, either install PySocks or install this module with the ``socks`` extra. The SOCKS implementation supports the full range of urllib3 features. It also supports the following SOCKS features: -- SOCKS4 -- SOCKS4a -- SOCKS5 +- SOCKS4A (``proxy_url='socks4a://...``) +- SOCKS4 (``proxy_url='socks4://...``) +- SOCKS5 with remote DNS (``proxy_url='socks5h://...``) +- SOCKS5 with local DNS (``proxy_url='socks5://...``) - Usernames and passwords for the SOCKS proxy -Known Limitations: + .. note:: + It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in + your ``proxy_url`` to ensure that DNS resolution is done from the remote + server instead of client-side when connecting to a domain name. + +SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5 +supports IPv4, IPv6, and domain names. + +When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url`` +will be sent as the ``userid`` section of the SOCKS request:: + + proxy_url="socks4a://<userid>@proxy-host" + +When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion +of the ``proxy_url`` will be sent as the username/password to authenticate +with the proxy:: + + proxy_url="socks5h://<username>:<password>@proxy-host" -- Currently PySocks does not support contacting remote websites via literal - IPv6 addresses. Any such connection attempt will fail. You must use a domain - name. -- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any - such connection attempt will fail. """ from __future__ import absolute_import @@ -29,23 +42,20 @@ except ImportError: import warnings from ..exceptions import DependencyWarning - warnings.warn(( - 'SOCKS support in urllib3 requires the installation of optional ' - 'dependencies: specifically, PySocks. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' + warnings.warn( + ( + "SOCKS support in urllib3 requires the installation of optional " + "dependencies: specifically, PySocks. For more information, see " + "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies" ), - DependencyWarning + DependencyWarning, ) raise from socket import error as SocketError, timeout as SocketTimeout -from ..connection import ( - HTTPConnection, HTTPSConnection -) -from ..connectionpool import ( - HTTPConnectionPool, HTTPSConnectionPool -) +from ..connection import HTTPConnection, HTTPSConnection +from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool from ..exceptions import ConnectTimeoutError, NewConnectionError from ..poolmanager import PoolManager from ..util.url import parse_url @@ -60,8 +70,9 @@ class SOCKSConnection(HTTPConnection): """ A plain-text HTTP connection that connects via a SOCKS proxy. """ + def __init__(self, *args, **kwargs): - self._socks_options = kwargs.pop('_socks_options') + self._socks_options = kwargs.pop("_socks_options") super(SOCKSConnection, self).__init__(*args, **kwargs) def _new_conn(self): @@ -70,28 +81,30 @@ class SOCKSConnection(HTTPConnection): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = socks.create_connection( (self.host, self.port), - proxy_type=self._socks_options['socks_version'], - proxy_addr=self._socks_options['proxy_host'], - proxy_port=self._socks_options['proxy_port'], - proxy_username=self._socks_options['username'], - proxy_password=self._socks_options['password'], - proxy_rdns=self._socks_options['rdns'], + proxy_type=self._socks_options["socks_version"], + proxy_addr=self._socks_options["proxy_host"], + proxy_port=self._socks_options["proxy_port"], + proxy_username=self._socks_options["username"], + proxy_password=self._socks_options["password"], + proxy_rdns=self._socks_options["rdns"], timeout=self.timeout, **extra_kw ) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except socks.ProxyError as e: # This is fragile as hell, but it seems to be the only way to raise @@ -101,23 +114,22 @@ class SOCKSConnection(HTTPConnection): if isinstance(error, SocketTimeout): raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout) + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % error + self, "Failed to establish a new connection: %s" % error ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % e + self, "Failed to establish a new connection: %s" % e ) except SocketError as e: # Defensive: PySocks should catch all these. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn @@ -143,47 +155,53 @@ class SOCKSProxyManager(PoolManager): A version of the urllib3 ProxyManager that routes connections via the defined SOCKS proxy. """ + pool_classes_by_scheme = { - 'http': SOCKSHTTPConnectionPool, - 'https': SOCKSHTTPSConnectionPool, + "http": SOCKSHTTPConnectionPool, + "https": SOCKSHTTPSConnectionPool, } - def __init__(self, proxy_url, username=None, password=None, - num_pools=10, headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + username=None, + password=None, + num_pools=10, + headers=None, + **connection_pool_kw + ): parsed = parse_url(proxy_url) if username is None and password is None and parsed.auth is not None: - split = parsed.auth.split(':') + split = parsed.auth.split(":") if len(split) == 2: username, password = split - if parsed.scheme == 'socks5': + if parsed.scheme == "socks5": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = False - elif parsed.scheme == 'socks5h': + elif parsed.scheme == "socks5h": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = True - elif parsed.scheme == 'socks4': + elif parsed.scheme == "socks4": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = False - elif parsed.scheme == 'socks4a': + elif parsed.scheme == "socks4a": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = True else: - raise ValueError( - "Unable to determine SOCKS version from %s" % proxy_url - ) + raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) self.proxy_url = proxy_url socks_options = { - 'socks_version': socks_version, - 'proxy_host': parsed.host, - 'proxy_port': parsed.port, - 'username': username, - 'password': password, - 'rdns': rdns + "socks_version": socks_version, + "proxy_host": parsed.host, + "proxy_port": parsed.port, + "username": username, + "password": password, + "rdns": rdns, } - connection_pool_kw['_socks_options'] = socks_options + connection_pool_kw["_socks_options"] = socks_options super(SOCKSProxyManager, self).__init__( num_pools, headers, **connection_pool_kw diff --git a/pipenv/patched/notpip/_vendor/urllib3/exceptions.py b/pipenv/patched/notpip/_vendor/urllib3/exceptions.py index 7bbaa987..93d93fba 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/exceptions.py +++ b/pipenv/patched/notpip/_vendor/urllib3/exceptions.py @@ -1,7 +1,6 @@ from __future__ import absolute_import -from .packages.six.moves.http_client import ( - IncompleteRead as httplib_IncompleteRead -) +from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead + # Base Exceptions @@ -17,6 +16,7 @@ class HTTPWarning(Warning): class PoolError(HTTPError): "Base exception for errors caused within a pool." + def __init__(self, pool, message): self.pool = pool HTTPError.__init__(self, "%s: %s" % (pool, message)) @@ -28,6 +28,7 @@ class PoolError(HTTPError): class RequestError(PoolError): "Base exception for PoolErrors that have associated URLs." + def __init__(self, pool, url, message): self.url = url PoolError.__init__(self, pool, message) @@ -63,6 +64,7 @@ ConnectionError = ProtocolError # Leaf Exceptions + class MaxRetryError(RequestError): """Raised when the maximum number of retries is exceeded. @@ -76,8 +78,7 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s (Caused by %r)" % ( - url, reason) + message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) RequestError.__init__(self, pool, url, message) @@ -93,6 +94,7 @@ class HostChangedError(RequestError): class TimeoutStateError(HTTPError): """ Raised when passing an invalid state to a timeout """ + pass @@ -102,6 +104,7 @@ class TimeoutError(HTTPError): Catching this error will catch both :exc:`ReadTimeoutErrors <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`. """ + pass @@ -149,8 +152,8 @@ class LocationParseError(LocationValueError): class ResponseError(HTTPError): "Used as a container for an error reason supplied in a MaxRetryError." - GENERIC_ERROR = 'too many error responses' - SPECIFIC_ERROR = 'too many {status_code} error responses' + GENERIC_ERROR = "too many error responses" + SPECIFIC_ERROR = "too many {status_code} error responses" class SecurityWarning(HTTPWarning): @@ -188,6 +191,7 @@ class DependencyWarning(HTTPWarning): Warned when an attempt is made to import a module with missing optional dependencies. """ + pass @@ -201,6 +205,7 @@ class BodyNotHttplibCompatible(HTTPError): Body should be httplib.HTTPResponse like (have an fp attribute which returns raw chunks) for read_chunked(). """ + pass @@ -212,12 +217,15 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for `partial` to avoid creating large objects on streamed reads. """ + def __init__(self, partial, expected): super(IncompleteRead, self).__init__(partial, expected) def __repr__(self): - return ('IncompleteRead(%i bytes read, ' - '%i more expected)' % (self.partial, self.expected)) + return "IncompleteRead(%i bytes read, " "%i more expected)" % ( + self.partial, + self.expected, + ) class InvalidHeader(HTTPError): @@ -236,8 +244,9 @@ class ProxySchemeUnknown(AssertionError, ValueError): class HeaderParsingError(HTTPError): "Raised by assert_header_parsing, but we convert it to a log.warning statement." + def __init__(self, defects, unparsed_data): - message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) + message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) super(HeaderParsingError, self).__init__(message) diff --git a/pipenv/patched/notpip/_vendor/urllib3/fields.py b/pipenv/patched/notpip/_vendor/urllib3/fields.py index 37fe64a3..8715b220 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/fields.py +++ b/pipenv/patched/notpip/_vendor/urllib3/fields.py @@ -1,11 +1,12 @@ from __future__ import absolute_import import email.utils import mimetypes +import re from .packages import six -def guess_content_type(filename, default='application/octet-stream'): +def guess_content_type(filename, default="application/octet-stream"): """ Guess the "Content-Type" of a file. @@ -19,57 +20,143 @@ def guess_content_type(filename, default='application/octet-stream'): return default -def format_header_param(name, value): +def format_header_param_rfc2231(name, value): """ - Helper function to format and quote a single header parameter. + Helper function to format and quote a single header parameter using the + strategy defined in RFC 2231. Particularly useful for header parameters which might contain - non-ASCII values, like file names. This follows RFC 2231, as - suggested by RFC 2388 Section 4.4. + non-ASCII values, like file names. This follows RFC 2388 Section 4.4. :param name: The name of the parameter, a string expected to be ASCII only. :param value: - The value of the parameter, provided as a unicode string. + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + An RFC-2231-formatted unicode string. """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + if not any(ch in value for ch in '"\\\r\n'): - result = '%s="%s"' % (name, value) + result = u'%s="%s"' % (name, value) try: - result.encode('ascii') + result.encode("ascii") except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3 and isinstance(value, six.text_type): # Python 2: - value = value.encode('utf-8') - value = email.utils.encode_rfc2231(value, 'utf-8') - value = '%s*=%s' % (name, value) + + if six.PY2: # Python 2: + value = value.encode("utf-8") + + # encode_rfc2231 accepts an encoded string and returns an ascii-encoded + # string in Python 2 but accepts and returns unicode strings in Python 3 + value = email.utils.encode_rfc2231(value, "utf-8") + value = "%s*=%s" % (name, value) + + if six.PY2: # Python 2: + value = value.decode("utf-8") + return value +_HTML5_REPLACEMENTS = { + u"\u0022": u"%22", + # Replace "\" with "\\". + u"\u005C": u"\u005C\u005C", + u"\u005C": u"\u005C\u005C", +} + +# All control characters from 0x00 to 0x1F *except* 0x1B. +_HTML5_REPLACEMENTS.update( + { + six.unichr(cc): u"%{:02X}".format(cc) + for cc in range(0x00, 0x1F + 1) + if cc not in (0x1B,) + } +) + + +def _replace_multiple(value, needles_and_replacements): + def replacer(match): + return needles_and_replacements[match.group(0)] + + pattern = re.compile( + r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) + ) + + result = pattern.sub(replacer, value) + + return result + + +def format_header_param_html5(name, value): + """ + Helper function to format and quote a single header parameter using the + HTML5 strategy. + + Particularly useful for header parameters which might contain + non-ASCII values, like file names. This follows the `HTML5 Working Draft + Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. + + .. _HTML5 Working Draft Section 4.10.22.7: + https://w3c.github.io/html/sec-forms.html#multipart-form-data + + :param name: + The name of the parameter, a string expected to be ASCII only. + :param value: + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + A unicode string, stripped of troublesome characters. + """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + + value = _replace_multiple(value, _HTML5_REPLACEMENTS) + + return u'%s="%s"' % (name, value) + + +# For backwards-compatibility. +format_header_param = format_header_param_html5 + + class RequestField(object): """ A data container for request body parameters. :param name: - The name of this request field. + The name of this request field. Must be unicode. :param data: The data/value body. :param filename: - An optional filename of the request field. + An optional filename of the request field. Must be unicode. :param headers: An optional dict-like object of headers to initially use for the field. + :param header_formatter: + An optional callable that is used to encode and format the headers. By + default, this is :func:`format_header_param_html5`. """ - def __init__(self, name, data, filename=None, headers=None): + + def __init__( + self, + name, + data, + filename=None, + headers=None, + header_formatter=format_header_param_html5, + ): self._name = name self._filename = filename self.data = data self.headers = {} if headers: self.headers = dict(headers) + self.header_formatter = header_formatter @classmethod - def from_tuples(cls, fieldname, value): + def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -97,21 +184,25 @@ class RequestField(object): content_type = None data = value - request_param = cls(fieldname, data, filename=filename) + request_param = cls( + fieldname, data, filename=filename, header_formatter=header_formatter + ) request_param.make_multipart(content_type=content_type) return request_param def _render_part(self, name, value): """ - Overridable helper function to format a single header parameter. + Overridable helper function to format a single header parameter. By + default, this calls ``self.header_formatter``. :param name: The name of the parameter, a string expected to be ASCII only. :param value: The value of the parameter, provided as a unicode string. """ - return format_header_param(name, value) + + return self.header_formatter(name, value) def _render_parts(self, header_parts): """ @@ -133,7 +224,7 @@ class RequestField(object): if value is not None: parts.append(self._render_part(name, value)) - return '; '.join(parts) + return u"; ".join(parts) def render_headers(self): """ @@ -141,21 +232,22 @@ class RequestField(object): """ lines = [] - sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] + sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append('%s: %s' % (sort_key, self.headers[sort_key])) + lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append('%s: %s' % (header_name, header_value)) + lines.append(u"%s: %s" % (header_name, header_value)) - lines.append('\r\n') - return '\r\n'.join(lines) + lines.append(u"\r\n") + return u"\r\n".join(lines) - def make_multipart(self, content_disposition=None, content_type=None, - content_location=None): + def make_multipart( + self, content_disposition=None, content_type=None, content_location=None + ): """ Makes this request field into a multipart request field. @@ -168,11 +260,14 @@ class RequestField(object): The 'Content-Location' of the request body. """ - self.headers['Content-Disposition'] = content_disposition or 'form-data' - self.headers['Content-Disposition'] += '; '.join([ - '', self._render_parts( - (('name', self._name), ('filename', self._filename)) - ) - ]) - self.headers['Content-Type'] = content_type - self.headers['Content-Location'] = content_location + self.headers["Content-Disposition"] = content_disposition or u"form-data" + self.headers["Content-Disposition"] += u"; ".join( + [ + u"", + self._render_parts( + ((u"name", self._name), (u"filename", self._filename)) + ), + ] + ) + self.headers["Content-Type"] = content_type + self.headers["Content-Location"] = content_location diff --git a/pipenv/patched/notpip/_vendor/urllib3/filepost.py b/pipenv/patched/notpip/_vendor/urllib3/filepost.py index 78f1e19b..b7b00992 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/filepost.py +++ b/pipenv/patched/notpip/_vendor/urllib3/filepost.py @@ -9,7 +9,7 @@ from .packages import six from .packages.six import b from .fields import RequestField -writer = codecs.lookup('utf-8')[3] +writer = codecs.lookup("utf-8")[3] def choose_boundary(): @@ -17,8 +17,8 @@ def choose_boundary(): Our embarrassingly-simple replacement for mimetools.choose_boundary. """ boundary = binascii.hexlify(os.urandom(16)) - if six.PY3: - boundary = boundary.decode('ascii') + if not six.PY2: + boundary = boundary.decode("ascii") return boundary @@ -76,7 +76,7 @@ def encode_multipart_formdata(fields, boundary=None): boundary = choose_boundary() for field in iter_field_objects(fields): - body.write(b('--%s\r\n' % (boundary))) + body.write(b("--%s\r\n" % (boundary))) writer(body).write(field.render_headers()) data = field.data @@ -89,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None): else: body.write(data) - body.write(b'\r\n') + body.write(b"\r\n") - body.write(b('--%s--\r\n' % (boundary))) + body.write(b("--%s--\r\n" % (boundary))) - content_type = str('multipart/form-data; boundary=%s' % boundary) + content_type = str("multipart/form-data; boundary=%s" % boundary) return body.getvalue(), content_type diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/packages/__init__.py index 170e974c..fce4caa6 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/__init__.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from . import ssl_match_hostname -__all__ = ('ssl_match_hostname', ) +__all__ = ("ssl_match_hostname",) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py index 75b80dcf..a3156a69 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/backports/makefile.py @@ -11,15 +11,14 @@ import io from socket import SocketIO -def backport_makefile(self, mode="r", buffering=None, encoding=None, - errors=None, newline=None): +def backport_makefile( + self, mode="r", buffering=None, encoding=None, errors=None, newline=None +): """ Backport of ``socket.makefile`` from Python 3.5. """ - if not set(mode) <= set(["r", "w", "b"]): - raise ValueError( - "invalid mode %r (only r, w, b allowed)" % (mode,) - ) + if not set(mode) <= {"r", "w", "b"}: + raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) writing = "w" in mode reading = "r" in mode or not writing assert reading or writing diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py deleted file mode 100644 index 4479363c..00000000 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ordered_dict.py +++ /dev/null @@ -1,259 +0,0 @@ -# Backport of OrderedDict() class that runs on Python 2.4, 2.5, 2.6, 2.7 and pypy. -# Passes Python2.7's test suite and incorporates all the latest updates. -# Copyright 2009 Raymond Hettinger, released under the MIT License. -# http://code.activestate.com/recipes/576693/ -try: - from thread import get_ident as _get_ident -except ImportError: - from dummy_thread import get_ident as _get_ident - -try: - from _abcoll import KeysView, ValuesView, ItemsView -except ImportError: - pass - - -class OrderedDict(dict): - 'Dictionary that remembers insertion order' - # An inherited dict maps keys to values. - # The inherited dict provides __getitem__, __len__, __contains__, and get. - # The remaining methods are order-aware. - # Big-O running times for all methods are the same as for regular dictionaries. - - # The internal self.__map dictionary maps keys to links in a doubly linked list. - # The circular doubly linked list starts and ends with a sentinel element. - # The sentinel element never gets deleted (this simplifies the algorithm). - # Each link is stored as a list of length three: [PREV, NEXT, KEY]. - - def __init__(self, *args, **kwds): - '''Initialize an ordered dictionary. Signature is the same as for - regular dictionaries, but keyword arguments are not recommended - because their insertion order is arbitrary. - - ''' - if len(args) > 1: - raise TypeError('expected at most 1 arguments, got %d' % len(args)) - try: - self.__root - except AttributeError: - self.__root = root = [] # sentinel node - root[:] = [root, root, None] - self.__map = {} - self.__update(*args, **kwds) - - def __setitem__(self, key, value, dict_setitem=dict.__setitem__): - 'od.__setitem__(i, y) <==> od[i]=y' - # Setting a new item creates a new link which goes at the end of the linked - # list, and the inherited dictionary is updated with the new key/value pair. - if key not in self: - root = self.__root - last = root[0] - last[1] = root[0] = self.__map[key] = [last, root, key] - dict_setitem(self, key, value) - - def __delitem__(self, key, dict_delitem=dict.__delitem__): - 'od.__delitem__(y) <==> del od[y]' - # Deleting an existing item uses self.__map to find the link which is - # then removed by updating the links in the predecessor and successor nodes. - dict_delitem(self, key) - link_prev, link_next, key = self.__map.pop(key) - link_prev[1] = link_next - link_next[0] = link_prev - - def __iter__(self): - 'od.__iter__() <==> iter(od)' - root = self.__root - curr = root[1] - while curr is not root: - yield curr[2] - curr = curr[1] - - def __reversed__(self): - 'od.__reversed__() <==> reversed(od)' - root = self.__root - curr = root[0] - while curr is not root: - yield curr[2] - curr = curr[0] - - def clear(self): - 'od.clear() -> None. Remove all items from od.' - try: - for node in self.__map.itervalues(): - del node[:] - root = self.__root - root[:] = [root, root, None] - self.__map.clear() - except AttributeError: - pass - dict.clear(self) - - def popitem(self, last=True): - '''od.popitem() -> (k, v), return and remove a (key, value) pair. - Pairs are returned in LIFO order if last is true or FIFO order if false. - - ''' - if not self: - raise KeyError('dictionary is empty') - root = self.__root - if last: - link = root[0] - link_prev = link[0] - link_prev[1] = root - root[0] = link_prev - else: - link = root[1] - link_next = link[1] - root[1] = link_next - link_next[0] = root - key = link[2] - del self.__map[key] - value = dict.pop(self, key) - return key, value - - # -- the following methods do not depend on the internal structure -- - - def keys(self): - 'od.keys() -> list of keys in od' - return list(self) - - def values(self): - 'od.values() -> list of values in od' - return [self[key] for key in self] - - def items(self): - 'od.items() -> list of (key, value) pairs in od' - return [(key, self[key]) for key in self] - - def iterkeys(self): - 'od.iterkeys() -> an iterator over the keys in od' - return iter(self) - - def itervalues(self): - 'od.itervalues -> an iterator over the values in od' - for k in self: - yield self[k] - - def iteritems(self): - 'od.iteritems -> an iterator over the (key, value) items in od' - for k in self: - yield (k, self[k]) - - def update(*args, **kwds): - '''od.update(E, **F) -> None. Update od from dict/iterable E and F. - - If E is a dict instance, does: for k in E: od[k] = E[k] - If E has a .keys() method, does: for k in E.keys(): od[k] = E[k] - Or if E is an iterable of items, does: for k, v in E: od[k] = v - In either case, this is followed by: for k, v in F.items(): od[k] = v - - ''' - if len(args) > 2: - raise TypeError('update() takes at most 2 positional ' - 'arguments (%d given)' % (len(args),)) - elif not args: - raise TypeError('update() takes at least 1 argument (0 given)') - self = args[0] - # Make progressively weaker assumptions about "other" - other = () - if len(args) == 2: - other = args[1] - if isinstance(other, dict): - for key in other: - self[key] = other[key] - elif hasattr(other, 'keys'): - for key in other.keys(): - self[key] = other[key] - else: - for key, value in other: - self[key] = value - for key, value in kwds.items(): - self[key] = value - - __update = update # let subclasses override update without breaking __init__ - - __marker = object() - - def pop(self, key, default=__marker): - '''od.pop(k[,d]) -> v, remove specified key and return the corresponding value. - If key is not found, d is returned if given, otherwise KeyError is raised. - - ''' - if key in self: - result = self[key] - del self[key] - return result - if default is self.__marker: - raise KeyError(key) - return default - - def setdefault(self, key, default=None): - 'od.setdefault(k[,d]) -> od.get(k,d), also set od[k]=d if k not in od' - if key in self: - return self[key] - self[key] = default - return default - - def __repr__(self, _repr_running={}): - 'od.__repr__() <==> repr(od)' - call_key = id(self), _get_ident() - if call_key in _repr_running: - return '...' - _repr_running[call_key] = 1 - try: - if not self: - return '%s()' % (self.__class__.__name__,) - return '%s(%r)' % (self.__class__.__name__, self.items()) - finally: - del _repr_running[call_key] - - def __reduce__(self): - 'Return state information for pickling' - items = [[k, self[k]] for k in self] - inst_dict = vars(self).copy() - for k in vars(OrderedDict()): - inst_dict.pop(k, None) - if inst_dict: - return (self.__class__, (items,), inst_dict) - return self.__class__, (items,) - - def copy(self): - 'od.copy() -> a shallow copy of od' - return self.__class__(self) - - @classmethod - def fromkeys(cls, iterable, value=None): - '''OD.fromkeys(S[, v]) -> New ordered dictionary with keys from S - and values equal to v (which defaults to None). - - ''' - d = cls() - for key in iterable: - d[key] = value - return d - - def __eq__(self, other): - '''od.__eq__(y) <==> od==y. Comparison to another OD is order-sensitive - while comparison to a regular mapping is order-insensitive. - - ''' - if isinstance(other, OrderedDict): - return len(self)==len(other) and self.items() == other.items() - return dict.__eq__(self, other) - - def __ne__(self, other): - return not self == other - - # -- the following methods are only used in Python 2.7 -- - - def viewkeys(self): - "od.viewkeys() -> a set-like object providing a view on od's keys" - return KeysView(self) - - def viewvalues(self): - "od.viewvalues() -> an object providing a view on od's values" - return ValuesView(self) - - def viewitems(self): - "od.viewitems() -> a set-like object providing a view on od's items" - return ItemsView(self) diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/six.py b/pipenv/patched/notpip/_vendor/urllib3/packages/six.py index 190c0239..31442409 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/six.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/six.py @@ -1,6 +1,4 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2019 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Utilities for writing code that runs on Python 2 and 3""" + from __future__ import absolute_import import functools @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson <benjamin@python.org>" -__version__ = "1.10.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3 PY34 = sys.version_info[0:2] >= (3, 4) if PY3: - string_types = str, - integer_types = int, - class_types = type, + string_types = (str,) + integer_types = (int,) + class_types = (type,) text_type = str binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, + string_types = (basestring,) integer_types = (int, long) class_types = (type, types.ClassType) text_type = unicode @@ -58,9 +58,9 @@ else: else: # It's possible to have sizeof(long) != sizeof(Py_ssize_t). class X(object): - def __len__(self): return 1 << 31 + try: len(X()) except OverflowError: @@ -84,7 +84,6 @@ def _import_module(name): class _LazyDescr(object): - def __init__(self, name): self.name = name @@ -101,7 +100,6 @@ class _LazyDescr(object): class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): super(MovedModule, self).__init__(name) if PY3: @@ -122,7 +120,6 @@ class MovedModule(_LazyDescr): class _LazyModule(types.ModuleType): - def __init__(self, name): super(_LazyModule, self).__init__(name) self.__doc__ = self.__class__.__doc__ @@ -137,7 +134,6 @@ class _LazyModule(types.ModuleType): class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): super(MovedAttribute, self).__init__(name) if PY3: @@ -221,28 +217,36 @@ class _SixMetaPathImporter(object): Required, if is_package is implemented""" self.__get_module(fullname) # eventually raises ImportError return None + get_source = get_code # same as get_code + _importer = _SixMetaPathImporter(__name__) class _MovedItems(_LazyModule): """Lazy loading of moved objects""" + __path__ = [] # mark as package _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute( + "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" + ), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute( + "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" + ), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), @@ -251,7 +255,9 @@ _moved_attributes = [ MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedAttribute( + "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" + ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), @@ -262,10 +268,13 @@ _moved_attributes = [ MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule( + "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" + ), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -283,15 +292,12 @@ _moved_attributes = [ MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), + MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), MovedModule("tkinter_font", "tkFont", "tkinter.font"), MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), @@ -301,9 +307,7 @@ _moved_attributes = [ ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] + _moved_attributes += [MovedModule("winreg", "_winreg")] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -337,10 +341,14 @@ _urllib_parse_moved_attributes = [ MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute( + "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" + ), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -353,8 +361,11 @@ del attr Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") +_importer._add_module( + Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", + "moves.urllib.parse", +) class Module_six_moves_urllib_error(_LazyModule): @@ -373,8 +384,11 @@ del attr Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") +_importer._add_module( + Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", + "moves.urllib.error", +) class Module_six_moves_urllib_request(_LazyModule): @@ -416,6 +430,8 @@ _urllib_request_moved_attributes = [ MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -423,8 +439,11 @@ del attr Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") +_importer._add_module( + Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", + "moves.urllib.request", +) class Module_six_moves_urllib_response(_LazyModule): @@ -444,8 +463,11 @@ del attr Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") +_importer._add_module( + Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", + "moves.urllib.response", +) class Module_six_moves_urllib_robotparser(_LazyModule): @@ -454,21 +476,27 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) del attr -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes +Module_six_moves_urllib_robotparser._moved_attributes = ( + _urllib_robotparser_moved_attributes +) -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") +_importer._add_module( + Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", + "moves.urllib.robotparser", +) class Module_six_moves_urllib(types.ModuleType): """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package parse = _importer._get_module("moves.urllib_parse") error = _importer._get_module("moves.urllib_error") @@ -477,10 +505,12 @@ class Module_six_moves_urllib(types.ModuleType): robotparser = _importer._get_module("moves.urllib_robotparser") def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] + return ["parse", "error", "request", "response", "robotparser"] -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") + +_importer._add_module( + Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" +) def add_move(move): @@ -520,19 +550,24 @@ else: try: advance_iterator = next except NameError: + def advance_iterator(it): return it.next() + + next = advance_iterator try: callable = callable except NameError: + def callable(obj): return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) if PY3: + def get_unbound_function(unbound): return unbound @@ -543,6 +578,7 @@ if PY3: Iterator = object else: + def get_unbound_function(unbound): return unbound.im_func @@ -553,13 +589,13 @@ else: return types.MethodType(func, None, cls) class Iterator(object): - def next(self): return type(self).__next__(self) callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") +_add_doc( + get_unbound_function, """Get the function out of a possibly unbound function""" +) get_method_function = operator.attrgetter(_meth_func) @@ -571,6 +607,7 @@ get_function_globals = operator.attrgetter(_func_globals) if PY3: + def iterkeys(d, **kw): return iter(d.keys(**kw)) @@ -589,6 +626,7 @@ if PY3: viewitems = operator.methodcaller("items") else: + def iterkeys(d, **kw): return d.iterkeys(**kw) @@ -609,28 +647,33 @@ else: _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") +_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc( + iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." +) if PY3: + def b(s): return s.encode("latin-1") def u(s): return s + unichr = chr import struct + int2byte = struct.Struct(">B").pack del struct byte2int = operator.itemgetter(0) indexbytes = operator.getitem iterbytes = iter import io + StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" @@ -639,12 +682,15 @@ if PY3: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" else: + def b(s): return s + # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") + unichr = unichr int2byte = chr @@ -653,8 +699,10 @@ else: def indexbytes(buf, i): return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) import StringIO + StringIO = BytesIO = StringIO.StringIO _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" @@ -679,13 +727,19 @@ if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + else: + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" if _globs_ is None: @@ -698,28 +752,45 @@ else: _locs_ = _globs_ exec("""exec _code_ in _globs_, _locs_""") - exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb -""") + exec_( + """def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""" + ) if sys.version_info[:2] == (3, 2): - exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value -""") + exec_( + """def raise_from(value, from_value): + try: + if from_value is None: + raise value + raise value from from_value + finally: + value = None +""" + ) elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): - raise value from from_value -""") + exec_( + """def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""" + ) else: + def raise_from(value, from_value): raise value print_ = getattr(moves.builtins, "print", None) if print_ is None: + def print_(*args, **kwargs): """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) @@ -730,14 +801,17 @@ if print_ is None: if not isinstance(data, basestring): data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): + if ( + isinstance(fp, file) + and isinstance(data, unicode) + and fp.encoding is not None + ): errors = getattr(fp, "errors", None) if errors is None: errors = "strict" data = data.encode(fp.encoding, errors) fp.write(data) + want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: @@ -773,6 +847,8 @@ if print_ is None: write(sep) write(arg) write(end) + + if sys.version_info[:2] < (3, 3): _print = print_ @@ -783,16 +859,24 @@ if sys.version_info[:2] < (3, 3): if flush and fp is not None: fp.flush() + _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): + + def wraps( + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): def wrapper(f): f = functools.wraps(wrapped, assigned, updated)(f) f.__wrapped__ = wrapped return f + return wrapper + + else: wraps = functools.wraps @@ -802,29 +886,95 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): - + class metaclass(type): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + + return type.__new__(metaclass, "temporary_class", (), {}) def add_metaclass(metaclass): """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') + slots = orig_vars.get("__slots__") if slots is not None: if isinstance(slots, str): slots = [slots] for slots_var in slots: orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) + orig_vars.pop("__dict__", None) + orig_vars.pop("__weakref__", None) + if hasattr(cls, "__qualname__"): + orig_vars["__qualname__"] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper +def ensure_binary(s, encoding="utf-8", errors="strict"): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding="utf-8", errors="strict"): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding="utf-8", errors="strict"): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. @@ -834,12 +984,13 @@ def python_2_unicode_compatible(klass): returning text and apply this decorator to the class. """ if PY2: - if '__str__' not in klass.__dict__: - raise ValueError("@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) + if "__str__" not in klass.__dict__: + raise ValueError( + "@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % klass.__name__ + ) klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + klass.__str__ = lambda self: self.__unicode__().encode("utf-8") return klass @@ -859,8 +1010,10 @@ if sys.meta_path: # be floating around. Therefore, we can't use isinstance() to check for # the six meta path importer, since the other six instance will have # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): + if ( + type(importer).__name__ == "_SixMetaPathImporter" + and importer.name == __name__ + ): del sys.meta_path[i] break del i, importer diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py index d6594eb2..75b6bb1c 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/__init__.py @@ -16,4 +16,4 @@ except ImportError: from ._implementation import CertificateError, match_hostname # Not needed, but documenting what we provide. -__all__ = ('CertificateError', 'match_hostname') +__all__ = ("CertificateError", "match_hostname") diff --git a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py index 62a177a7..507c655d 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/patched/notpip/_vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -9,14 +9,13 @@ import sys # ipaddress has been backported to 2.6+ in pypi. If it is installed on the # system, use it to handle IPAddress ServerAltnames (this was added in # python-3.5) otherwise only do DNS matching. This allows -# backports.ssl_match_hostname to continue to be used all the way back to -# python-2.4. +# backports.ssl_match_hostname to continue to be used in Python 2.7. try: from pipenv.patched.notpip._vendor import ipaddress except ImportError: ipaddress = None -__version__ = '3.5.0.1' +__version__ = "3.5.0.1" class CertificateError(ValueError): @@ -34,18 +33,19 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # Ported from python3-syntax: # leftmost, *remainder = dn.split(r'.') - parts = dn.split(r'.') + parts = dn.split(r".") leftmost = parts[0] remainder = parts[1:] - wildcards = leftmost.count('*') + wildcards = leftmost.count("*") if wildcards > max_wildcards: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a # reasonable choice. raise CertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) + "too many wildcards in certificate DNS name: " + repr(dn) + ) # speed up common case w/o wildcards if not wildcards: @@ -54,11 +54,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which # the wildcard character comprises a label other than the left-most label. - if leftmost == '*': + if leftmost == "*": # When '*' is a fragment by itself, it matches a non-empty dotless # fragment. - pats.append('[^.]+') - elif leftmost.startswith('xn--') or hostname.startswith('xn--'): + pats.append("[^.]+") + elif leftmost.startswith("xn--") or hostname.startswith("xn--"): # RFC 6125, section 6.4.3, subitem 3. # The client SHOULD NOT attempt to match a presented identifier # where the wildcard character is embedded within an A-label or @@ -66,21 +66,22 @@ def _dnsname_match(dn, hostname, max_wildcards=1): pats.append(re.escape(leftmost)) else: # Otherwise, '*' matches any dotless string, e.g. www* - pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) + pats.append(re.escape(leftmost).replace(r"\*", "[^.]*")) # add the remaining fragments, ignore any wildcards for frag in remainder: pats.append(re.escape(frag)) - pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE) return pat.match(hostname) def _to_unicode(obj): if isinstance(obj, str) and sys.version_info < (3,): - obj = unicode(obj, encoding='ascii', errors='strict') + obj = unicode(obj, encoding="ascii", errors="strict") return obj + def _ipaddress_match(ipname, host_ip): """Exact matching of IP addresses. @@ -102,9 +103,11 @@ def match_hostname(cert, hostname): returns nothing. """ if not cert: - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") + raise ValueError( + "empty or no certificate, match_hostname needs a " + "SSL socket or SSL context with either " + "CERT_OPTIONAL or CERT_REQUIRED" + ) try: # Divergence from upstream: ipaddress can't handle byte str host_ip = ipaddress.ip_address(_to_unicode(hostname)) @@ -123,35 +126,37 @@ def match_hostname(cert, hostname): else: raise dnsnames = [] - san = cert.get('subjectAltName', ()) + san = cert.get("subjectAltName", ()) for key, value in san: - if key == 'DNS': + if key == "DNS": if host_ip is None and _dnsname_match(value, hostname): return dnsnames.append(value) - elif key == 'IP Address': + elif key == "IP Address": if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName - for sub in cert.get('subject', ()): + for sub in cert.get("subject", ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. - if key == 'commonName': + if key == "commonName": if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) + raise CertificateError( + "hostname %r " + "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) + ) elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) + raise CertificateError( + "hostname %r " "doesn't match %r" % (hostname, dnsnames[0]) + ) else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") + raise CertificateError( + "no appropriate commonName or " "subjectAltName fields were found" + ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py index 506a3c9b..242a2f82 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py +++ b/pipenv/patched/notpip/_vendor/urllib3/poolmanager.py @@ -7,51 +7,62 @@ from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry -__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] +__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] log = logging.getLogger(__name__) -SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir', 'ssl_context') +SSL_KEYWORDS = ( + "key_file", + "cert_file", + "cert_reqs", + "ca_certs", + "ssl_version", + "ca_cert_dir", + "ssl_context", + "key_password", +) # All known keyword arguments that could be provided to the pool manager, its # pools, or the underlying connections. This is used to construct a pool key. _key_fields = ( - 'key_scheme', # str - 'key_host', # str - 'key_port', # int - 'key_timeout', # int or float or Timeout - 'key_retries', # int or Retry - 'key_strict', # bool - 'key_block', # bool - 'key_source_address', # str - 'key_key_file', # str - 'key_cert_file', # str - 'key_cert_reqs', # str - 'key_ca_certs', # str - 'key_ssl_version', # str - 'key_ca_cert_dir', # str - 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext - 'key_maxsize', # int - 'key_headers', # dict - 'key__proxy', # parsed proxy url - 'key__proxy_headers', # dict - 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples - 'key__socks_options', # dict - 'key_assert_hostname', # bool or string - 'key_assert_fingerprint', # str + "key_scheme", # str + "key_host", # str + "key_port", # int + "key_timeout", # int or float or Timeout + "key_retries", # int or Retry + "key_strict", # bool + "key_block", # bool + "key_source_address", # str + "key_key_file", # str + "key_key_password", # str + "key_cert_file", # str + "key_cert_reqs", # str + "key_ca_certs", # str + "key_ssl_version", # str + "key_ca_cert_dir", # str + "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext + "key_maxsize", # int + "key_headers", # dict + "key__proxy", # parsed proxy url + "key__proxy_headers", # dict + "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples + "key__socks_options", # dict + "key_assert_hostname", # bool or string + "key_assert_fingerprint", # str + "key_server_hostname", # str ) #: The namedtuple class used to construct keys for the connection pool. #: All custom key schemes should include the fields in this key at a minimum. -PoolKey = collections.namedtuple('PoolKey', _key_fields) +PoolKey = collections.namedtuple("PoolKey", _key_fields) def _default_key_normalizer(key_class, request_context): @@ -76,24 +87,24 @@ def _default_key_normalizer(key_class, request_context): """ # Since we mutate the dictionary, make a copy first context = request_context.copy() - context['scheme'] = context['scheme'].lower() - context['host'] = context['host'].lower() + context["scheme"] = context["scheme"].lower() + context["host"] = context["host"].lower() # These are both dictionaries and need to be transformed into frozensets - for key in ('headers', '_proxy_headers', '_socks_options'): + for key in ("headers", "_proxy_headers", "_socks_options"): if key in context and context[key] is not None: context[key] = frozenset(context[key].items()) # The socket_options key may be a list and needs to be transformed into a # tuple. - socket_opts = context.get('socket_options') + socket_opts = context.get("socket_options") if socket_opts is not None: - context['socket_options'] = tuple(socket_opts) + context["socket_options"] = tuple(socket_opts) # Map the kwargs to the names in the namedtuple - this is necessary since # namedtuples can't have fields starting with '_'. for key in list(context.keys()): - context['key_' + key] = context.pop(key) + context["key_" + key] = context.pop(key) # Default to ``None`` for keys missing from the context for field in key_class._fields: @@ -108,14 +119,11 @@ def _default_key_normalizer(key_class, request_context): #: Each PoolManager makes a copy of this dictionary so they can be configured #: globally here, or individually on the instance. key_fn_by_scheme = { - 'http': functools.partial(_default_key_normalizer, PoolKey), - 'https': functools.partial(_default_key_normalizer, PoolKey), + "http": functools.partial(_default_key_normalizer, PoolKey), + "https": functools.partial(_default_key_normalizer, PoolKey), } -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} +pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool} class PoolManager(RequestMethods): @@ -151,8 +159,7 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools, - dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) # Locally set the pool classes and keys so other PoolManagers can # override them. @@ -185,10 +192,10 @@ class PoolManager(RequestMethods): # this function has historically only used the scheme, host, and port # in the positional args. When an API change is acceptable these can # be removed. - for key in ('scheme', 'host', 'port'): + for key in ("scheme", "host", "port"): request_context.pop(key, None) - if scheme == 'http': + if scheme == "http": for kw in SSL_KEYWORDS: request_context.pop(kw, None) @@ -203,7 +210,7 @@ class PoolManager(RequestMethods): """ self.pools.clear() - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): """ Get a :class:`ConnectionPool` based on the host, port, and scheme. @@ -218,11 +225,11 @@ class PoolManager(RequestMethods): raise LocationValueError("No host specified.") request_context = self._merge_pool_kwargs(pool_kwargs) - request_context['scheme'] = scheme or 'http' + request_context["scheme"] = scheme or "http" if not port: - port = port_by_scheme.get(request_context['scheme'].lower(), 80) - request_context['port'] = port - request_context['host'] = host + port = port_by_scheme.get(request_context["scheme"].lower(), 80) + request_context["port"] = port + request_context["host"] = host return self.connection_from_context(request_context) @@ -233,7 +240,7 @@ class PoolManager(RequestMethods): ``request_context`` must at least contain the ``scheme`` key and its value must be a key in ``key_fn_by_scheme`` instance variable. """ - scheme = request_context['scheme'].lower() + scheme = request_context["scheme"].lower() pool_key_constructor = self.key_fn_by_scheme[scheme] pool_key = pool_key_constructor(request_context) @@ -255,9 +262,9 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - scheme = request_context['scheme'] - host = request_context['host'] - port = request_context['port'] + scheme = request_context["scheme"] + host = request_context["host"] + port = request_context["port"] pool = self._new_pool(scheme, host, port, request_context=request_context) self.pools[pool_key] = pool @@ -275,8 +282,9 @@ class PoolManager(RequestMethods): not used. """ u = parse_url(url) - return self.connection_from_host(u.host, port=u.port, scheme=u.scheme, - pool_kwargs=pool_kwargs) + return self.connection_from_host( + u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs + ) def _merge_pool_kwargs(self, override): """ @@ -310,11 +318,11 @@ class PoolManager(RequestMethods): u = parse_url(url) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) - kw['assert_same_host'] = False - kw['redirect'] = False + kw["assert_same_host"] = False + kw["redirect"] = False - if 'headers' not in kw: - kw['headers'] = self.headers.copy() + if "headers" not in kw: + kw["headers"] = self.headers.copy() if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) @@ -330,19 +338,22 @@ class PoolManager(RequestMethods): # RFC 7231, Section 6.4.4 if response.status == 303: - method = 'GET' + method = "GET" - retries = kw.get('retries') + retries = kw.get("retries") if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) # Strip headers marked as unsafe to forward to the redirected location. # Check remove_headers_on_redirect to avoid a potential network call within # conn.is_same_host() which may use socket.gethostbyname() in the future. - if (retries.remove_headers_on_redirect - and not conn.is_same_host(redirect_location)): - for header in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + if retries.remove_headers_on_redirect and not conn.is_same_host( + redirect_location + ): + headers = list(six.iterkeys(kw["headers"])) + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + kw["headers"].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) @@ -351,8 +362,8 @@ class PoolManager(RequestMethods): raise return response - kw['retries'] = retries - kw['redirect'] = redirect + kw["retries"] = retries + kw["redirect"] = redirect log.info("Redirecting %s -> %s", url, redirect_location) return self.urlopen(method, redirect_location, **kw) @@ -385,12 +396,21 @@ class ProxyManager(PoolManager): """ - def __init__(self, proxy_url, num_pools=10, headers=None, - proxy_headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + num_pools=10, + headers=None, + proxy_headers=None, + **connection_pool_kw + ): if isinstance(proxy_url, HTTPConnectionPool): - proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host, - proxy_url.port) + proxy_url = "%s://%s:%i" % ( + proxy_url.scheme, + proxy_url.host, + proxy_url.port, + ) proxy = parse_url(proxy_url) if not proxy.port: port = port_by_scheme.get(proxy.scheme, 80) @@ -402,30 +422,31 @@ class ProxyManager(PoolManager): self.proxy = proxy self.proxy_headers = proxy_headers or {} - connection_pool_kw['_proxy'] = self.proxy - connection_pool_kw['_proxy_headers'] = self.proxy_headers + connection_pool_kw["_proxy"] = self.proxy + connection_pool_kw["_proxy_headers"] = self.proxy_headers - super(ProxyManager, self).__init__( - num_pools, headers, **connection_pool_kw) + super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): if scheme == "https": return super(ProxyManager, self).connection_from_host( - host, port, scheme, pool_kwargs=pool_kwargs) + host, port, scheme, pool_kwargs=pool_kwargs + ) return super(ProxyManager, self).connection_from_host( - self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs) + self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs + ) def _set_proxy_headers(self, url, headers=None): """ Sets headers needed by proxies: specifically, the Accept and Host headers. Only sets headers not provided by the user. """ - headers_ = {'Accept': '*/*'} + headers_ = {"Accept": "*/*"} netloc = parse_url(url).netloc if netloc: - headers_['Host'] = netloc + headers_["Host"] = netloc if headers: headers_.update(headers) @@ -439,8 +460,8 @@ class ProxyManager(PoolManager): # For proxied HTTPS requests, httplib sets the necessary headers # on the CONNECT to the proxy. For HTTP, we'll definitely # need to set 'Host' at the very least. - headers = kw.get('headers', self.headers) - kw['headers'] = self._set_proxy_headers(url, headers) + headers = kw.get("headers", self.headers) + kw["headers"] = self._set_proxy_headers(url, headers) return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) diff --git a/pipenv/patched/notpip/_vendor/urllib3/request.py b/pipenv/patched/notpip/_vendor/urllib3/request.py index 1be33341..55f160bb 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/request.py +++ b/pipenv/patched/notpip/_vendor/urllib3/request.py @@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata from .packages.six.moves.urllib.parse import urlencode -__all__ = ['RequestMethods'] +__all__ = ["RequestMethods"] class RequestMethods(object): @@ -36,16 +36,25 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = set(['DELETE', 'GET', 'HEAD', 'OPTIONS']) + _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} def __init__(self, headers=None): self.headers = headers or {} - def urlopen(self, method, url, body=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **kw): # Abstract - raise NotImplementedError("Classes extending RequestMethods must implement " - "their own ``urlopen`` method.") + def urlopen( + self, + method, + url, + body=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **kw + ): # Abstract + raise NotImplementedError( + "Classes extending RequestMethods must implement " + "their own ``urlopen`` method." + ) def request(self, method, url, fields=None, headers=None, **urlopen_kw): """ @@ -60,19 +69,18 @@ class RequestMethods(object): """ method = method.upper() - urlopen_kw['request_url'] = url + urlopen_kw["request_url"] = url if method in self._encode_url_methods: - return self.request_encode_url(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_url( + method, url, fields=fields, headers=headers, **urlopen_kw + ) else: - return self.request_encode_body(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_body( + method, url, fields=fields, headers=headers, **urlopen_kw + ) - def request_encode_url(self, method, url, fields=None, headers=None, - **urlopen_kw): + def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. @@ -80,17 +88,24 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': headers} + extra_kw = {"headers": headers} extra_kw.update(urlopen_kw) if fields: - url += '?' + urlencode(fields) + url += "?" + urlencode(fields) return self.urlopen(method, url, **extra_kw) - def request_encode_body(self, method, url, fields=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **urlopen_kw): + def request_encode_body( + self, + method, + url, + fields=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **urlopen_kw + ): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -129,22 +144,28 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': {}} + extra_kw = {"headers": {}} if fields: - if 'body' in urlopen_kw: + if "body" in urlopen_kw: raise TypeError( - "request got values for both 'fields' and 'body', can only specify one.") + "request got values for both 'fields' and 'body', can only specify one." + ) if encode_multipart: - body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + body, content_type = encode_multipart_formdata( + fields, boundary=multipart_boundary + ) else: - body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + body, content_type = ( + urlencode(fields), + "application/x-www-form-urlencoded", + ) - extra_kw['body'] = body - extra_kw['headers'] = {'Content-Type': content_type} + extra_kw["body"] = body + extra_kw["headers"] = {"Content-Type": content_type} - extra_kw['headers'].update(headers) + extra_kw["headers"].update(headers) extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) diff --git a/pipenv/patched/notpip/_vendor/urllib3/response.py b/pipenv/patched/notpip/_vendor/urllib3/response.py index 9873cb94..adc321e7 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/response.py @@ -6,12 +6,22 @@ import logging from socket import timeout as SocketTimeout from socket import error as SocketError +try: + import brotli +except ImportError: + brotli = None + from ._collections import HTTPHeaderDict from .exceptions import ( - BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, - ResponseNotChunked, IncompleteRead, InvalidHeader + BodyNotHttplibCompatible, + ProtocolError, + DecodeError, + ReadTimeoutError, + ResponseNotChunked, + IncompleteRead, + InvalidHeader, ) -from .packages.six import string_types as basestring, binary_type, PY3 +from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib from .connection import HTTPException, BaseSSLError from .util.response import is_fp_closed, is_response_to_head @@ -20,10 +30,9 @@ log = logging.getLogger(__name__) class DeflateDecoder(object): - def __init__(self): self._first_try = True - self._data = binary_type() + self._data = b"" self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -60,7 +69,6 @@ class GzipDecoderState(object): class GzipDecoder(object): - def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER @@ -69,9 +77,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = binary_type() + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,19 +89,66 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +if brotli is not None: + + class BrotliDecoder(object): + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__(self): + self._obj = brotli.Decompressor() + + def decompress(self, data): + if hasattr(self._obj, "decompress"): + return self._obj.decompress(data) + return self._obj.process(data) + + def flush(self): + if hasattr(self._obj, "flush"): + return self._obj.flush() + return b"" + + +class MultiDecoder(object): + """ + From RFC7231: + If one or more encodings have been applied to a representation, the + sender that applied the encodings MUST generate a Content-Encoding + header field that lists the content codings in the order in which + they were applied. + """ + + def __init__(self, modes): + self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] + + def flush(self): + return self._decoders[0].flush() + + def decompress(self, data): + for d in reversed(self._decoders): + data = d.decompress(data) + return data + + def _get_decoder(mode): - if mode == 'gzip': + if "," in mode: + return MultiDecoder(mode) + + if mode == "gzip": return GzipDecoder() + if brotli is not None and mode == "br": + return BrotliDecoder() + return DeflateDecoder() @@ -130,14 +185,31 @@ class HTTPResponse(io.IOBase): value of Content-Length header, if present. Otherwise, raise error. """ - CONTENT_DECODERS = ['gzip', 'deflate'] + CONTENT_DECODERS = ["gzip", "deflate"] + if brotli is not None: + CONTENT_DECODERS += ["br"] REDIRECT_STATUSES = [301, 302, 303, 307, 308] - def __init__(self, body='', headers=None, status=0, version=0, reason=None, - strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None, msg=None, - retries=None, enforce_content_length=False, - request_method=None, request_url=None): + def __init__( + self, + body="", + headers=None, + status=0, + version=0, + reason=None, + strict=0, + preload_content=True, + decode_content=True, + original_response=None, + pool=None, + connection=None, + msg=None, + retries=None, + enforce_content_length=False, + request_method=None, + request_url=None, + auto_close=True, + ): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -150,6 +222,7 @@ class HTTPResponse(io.IOBase): self.decode_content = decode_content self.retries = retries self.enforce_content_length = enforce_content_length + self.auto_close = auto_close self._decoder = None self._body = None @@ -159,19 +232,19 @@ class HTTPResponse(io.IOBase): self.msg = msg self._request_url = request_url - if body and isinstance(body, (basestring, binary_type)): + if body and isinstance(body, (basestring, bytes)): self._body = body self._pool = pool self._connection = connection - if hasattr(body, 'read'): + if hasattr(body, "read"): self._fp = body # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '').lower() + tr_enc = self.headers.get("transfer-encoding", "").lower() # Don't incur the penalty of creating a list and then discarding it encodings = (enc.strip() for enc in tr_enc.split(",")) if "chunked" in encodings: @@ -193,7 +266,7 @@ class HTTPResponse(io.IOBase): location. ``False`` if not a redirect status code. """ if self.status in self.REDIRECT_STATUSES: - return self.headers.get('location') + return self.headers.get("location") return False @@ -232,18 +305,20 @@ class HTTPResponse(io.IOBase): """ Set initial length value for Response content if available. """ - length = self.headers.get('content-length') + length = self.headers.get("content-length") if length is not None: if self.chunked: # This Response will fail with an IncompleteRead if it can't be # received as chunked. This method falls back to attempt reading # the response before raising an exception. - log.warning("Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked.") + log.warning( + "Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked." + ) return None try: @@ -252,10 +327,12 @@ class HTTPResponse(io.IOBase): # (e.g. Content-Length: 42, 42). This line ensures the values # are all valid ints and that as long as the `set` length is 1, # all values are the same. Otherwise, the header is invalid. - lengths = set([int(val) for val in length.split(',')]) + lengths = set([int(val) for val in length.split(",")]) if len(lengths) > 1: - raise InvalidHeader("Content-Length contained multiple " - "unmatching values (%s)" % length) + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % length + ) length = lengths.pop() except ValueError: length = None @@ -271,7 +348,7 @@ class HTTPResponse(io.IOBase): status = 0 # Check for responses that shouldn't include a body - if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': + if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": length = 0 return length @@ -282,24 +359,41 @@ class HTTPResponse(io.IOBase): """ # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() - if self._decoder is None and content_encoding in self.CONTENT_DECODERS: - self._decoder = _get_decoder(content_encoding) + content_encoding = self.headers.get("content-encoding", "").lower() + if self._decoder is None: + if content_encoding in self.CONTENT_DECODERS: + self._decoder = _get_decoder(content_encoding) + elif "," in content_encoding: + encodings = [ + e.strip() + for e in content_encoding.split(",") + if e.strip() in self.CONTENT_DECODERS + ] + if len(encodings): + self._decoder = _get_decoder(content_encoding) + + DECODER_ERROR_CLASSES = (IOError, zlib.error) + if brotli is not None: + DECODER_ERROR_CLASSES += (brotli.error,) def _decode(self, data, decode_content, flush_decoder): """ Decode the data passed in and potentially flush the decoder. """ + if not decode_content: + return data + try: - if decode_content and self._decoder: + if self._decoder: data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - content_encoding = self.headers.get('content-encoding', '').lower() + except self.DECODER_ERROR_CLASSES as e: + content_encoding = self.headers.get("content-encoding", "").lower() raise DecodeError( "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content: + "failed to decode it." % content_encoding, + e, + ) + if flush_decoder: data += self._flush_decoder() return data @@ -310,10 +404,10 @@ class HTTPResponse(io.IOBase): being used. """ if self._decoder: - buf = self._decoder.decompress(b'') + buf = self._decoder.decompress(b"") return buf + self._decoder.flush() - return b'' + return b"" @contextmanager def _error_catcher(self): @@ -333,20 +427,20 @@ class HTTPResponse(io.IOBase): except SocketTimeout: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? - if 'read operation timed out' not in str(e): # Defensive: + if "read operation timed out" not in str(e): # Defensive: # This shouldn't happen but just in case we're missing an edge # case, let's avoid swallowing SSL errors. raise - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except (HTTPException, SocketError) as e: # This includes IncompleteRead. - raise ProtocolError('Connection broken: %r' % e, e) + raise ProtocolError("Connection broken: %r" % e, e) # If no exception is thrown, we should avoid cleaning up # unnecessarily. @@ -401,17 +495,19 @@ class HTTPResponse(io.IOBase): return flush_decoder = False - data = None + fp_closed = getattr(self._fp, "closed", False) with self._error_catcher(): if amt is None: # cStringIO doesn't like amt=None - data = self._fp.read() + data = self._fp.read() if not fp_closed else b"" flush_decoder = True else: cache_content = False - data = self._fp.read(amt) - if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + data = self._fp.read(amt) if not fp_closed else b"" + if ( + amt != 0 and not data + ): # Platform-specific: Buggy versions of Python. # Close the connection when no data is returned # # This is redundant to what httplib/http.client _should_ @@ -421,7 +517,10 @@ class HTTPResponse(io.IOBase): # no harm in redundantly calling close. self._fp.close() flush_decoder = True - if self.enforce_content_length and self.length_remaining not in (0, None): + if self.enforce_content_length and self.length_remaining not in ( + 0, + None, + ): # This is an edge case that httplib failed to cover due # to concerns of backward compatibility. We're # addressing it here to make sure IncompleteRead is @@ -441,7 +540,7 @@ class HTTPResponse(io.IOBase): return data - def stream(self, amt=2**16, decode_content=None): + def stream(self, amt=2 ** 16, decode_content=None): """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -479,21 +578,24 @@ class HTTPResponse(io.IOBase): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: # Python 3 + if PY3: headers = HTTPHeaderDict(headers.items()) - else: # Python 2 + else: + # Python 2.7 headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute - strict = getattr(r, 'strict', 0) - resp = ResponseCls(body=r, - headers=headers, - status=r.status, - version=r.version, - reason=r.reason, - strict=strict, - original_response=r, - **response_kw) + strict = getattr(r, "strict", 0) + resp = ResponseCls( + body=r, + headers=headers, + status=r.status, + version=r.version, + reason=r.reason, + strict=strict, + original_response=r, + **response_kw + ) return resp # Backwards-compatibility methods for httplib.HTTPResponse @@ -515,13 +617,18 @@ class HTTPResponse(io.IOBase): if self._connection: self._connection.close() + if not self.auto_close: + io.IOBase.close(self) + @property def closed(self): - if self._fp is None: + if not self.auto_close: + return io.IOBase.closed.__get__(self) + elif self._fp is None: return True - elif hasattr(self._fp, 'isclosed'): + elif hasattr(self._fp, "isclosed"): return self._fp.isclosed() - elif hasattr(self._fp, 'closed'): + elif hasattr(self._fp, "closed"): return self._fp.closed else: return True @@ -532,11 +639,17 @@ class HTTPResponse(io.IOBase): elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError("The file-like object this HTTPResponse is wrapped " - "around has no file descriptor") + raise IOError( + "The file-like object this HTTPResponse is wrapped " + "around has no file descriptor" + ) def flush(self): - if self._fp is not None and hasattr(self._fp, 'flush'): + if ( + self._fp is not None + and hasattr(self._fp, "flush") + and not getattr(self._fp, "closed", False) + ): return self._fp.flush() def readable(self): @@ -549,7 +662,7 @@ class HTTPResponse(io.IOBase): if len(temp) == 0: return 0 else: - b[:len(temp)] = temp + b[: len(temp)] = temp return len(temp) def supports_chunked_reads(self): @@ -559,7 +672,7 @@ class HTTPResponse(io.IOBase): attribute. If it is present we assume it returns raw chunks as processed by read_chunked(). """ - return hasattr(self._fp, 'fp') + return hasattr(self._fp, "fp") def _update_chunk_length(self): # First, we'll figure out length of a chunk and then @@ -567,7 +680,7 @@ class HTTPResponse(io.IOBase): if self.chunk_left is not None: return line = self._fp.fp.readline() - line = line.split(b';', 1)[0] + line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) except ValueError: @@ -616,11 +729,13 @@ class HTTPResponse(io.IOBase): if not self.chunked: raise ResponseNotChunked( "Response is not chunked. " - "Header 'transfer-encoding: chunked' is missing.") + "Header 'transfer-encoding: chunked' is missing." + ) if not self.supports_chunked_reads(): raise BodyNotHttplibCompatible( "Body should be httplib.HTTPResponse like. " - "It should have have an fp attribute which returns raw chunks.") + "It should have have an fp attribute which returns raw chunks." + ) with self._error_catcher(): # Don't bother reading the body of a HEAD request. @@ -638,8 +753,9 @@ class HTTPResponse(io.IOBase): if self.chunk_left == 0: break chunk = self._handle_chunk(amt) - decoded = self._decode(chunk, decode_content=decode_content, - flush_decoder=False) + decoded = self._decode( + chunk, decode_content=decode_content, flush_decoder=False + ) if decoded: yield decoded @@ -657,7 +773,7 @@ class HTTPResponse(io.IOBase): if not line: # Some sites may not end with '\r\n'. break - if line == b'\r\n': + if line == b"\r\n": break # We read everything; close the "file". @@ -674,3 +790,20 @@ class HTTPResponse(io.IOBase): return self.retries.history[-1].redirect_location else: return self._request_url + + def __iter__(self): + buffer = [b""] + for chunk in self.stream(decode_content=True): + if b"\n" in chunk: + chunk = chunk.split(b"\n") + yield b"".join(buffer) + chunk[0] + b"\n" + for x in chunk[1:-1]: + yield x + b"\n" + if chunk[-1]: + buffer = [chunk[-1]] + else: + buffer = [] + else: + buffer.append(chunk) + if buffer: + yield b"".join(buffer) diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/__init__.py b/pipenv/patched/notpip/_vendor/urllib3/util/__init__.py index 2f2770b6..a96c73a9 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/__init__.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/__init__.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers @@ -12,43 +13,34 @@ from .ssl_ import ( resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, + PROTOCOL_TLS, ) -from .timeout import ( - current_time, - Timeout, -) +from .timeout import current_time, Timeout from .retry import Retry -from .url import ( - get_host, - parse_url, - split_first, - Url, -) -from .wait import ( - wait_for_read, - wait_for_write -) +from .url import get_host, parse_url, split_first, Url +from .wait import wait_for_read, wait_for_write __all__ = ( - 'HAS_SNI', - 'IS_PYOPENSSL', - 'IS_SECURETRANSPORT', - 'SSLContext', - 'Retry', - 'Timeout', - 'Url', - 'assert_fingerprint', - 'current_time', - 'is_connection_dropped', - 'is_fp_closed', - 'get_host', - 'parse_url', - 'make_headers', - 'resolve_cert_reqs', - 'resolve_ssl_version', - 'split_first', - 'ssl_wrap_socket', - 'wait_for_read', - 'wait_for_write' + "HAS_SNI", + "IS_PYOPENSSL", + "IS_SECURETRANSPORT", + "SSLContext", + "PROTOCOL_TLS", + "Retry", + "Timeout", + "Url", + "assert_fingerprint", + "current_time", + "is_connection_dropped", + "is_fp_closed", + "get_host", + "parse_url", + "make_headers", + "resolve_cert_reqs", + "resolve_ssl_version", + "split_first", + "ssl_wrap_socket", + "wait_for_read", + "wait_for_write", ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py index 5cf488f4..0e111262 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/connection.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/connection.py @@ -1,6 +1,7 @@ from __future__ import absolute_import import socket from .wait import NoWayToWaitForSocketError, wait_for_read +from ..contrib import _appengine_environ def is_connection_dropped(conn): # Platform-specific @@ -13,7 +14,7 @@ def is_connection_dropped(conn): # Platform-specific Note: For platforms like AppEngine, this will always return ``False`` to let the platform handle connection recycling transparently for us. """ - sock = getattr(conn, 'sock', False) + sock = getattr(conn, "sock", False) if sock is False: # Platform-specific: AppEngine return False if sock is None: # Connection already closed (such as by httplib). @@ -29,8 +30,12 @@ def is_connection_dropped(conn): # Platform-specific # library test suite. Added to its signature is only `socket_options`. # One additional modification is that we avoid binding to IPv6 servers # discovered in DNS if the system doesn't have IPv6 functionality. -def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, socket_options=None): +def create_connection( + address, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + socket_options=None, +): """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -44,8 +49,8 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, """ host, port = address - if host.startswith('['): - host = host.strip('[]') + if host.startswith("["): + host = host.strip("[]") err = None # Using the value from allowed_gai_family() in the context of getaddrinfo lets @@ -105,6 +110,13 @@ def _has_ipv6(host): sock = None has_ipv6 = False + # App Engine doesn't support IPV6 sockets and actually has a quota on the + # number of sockets that can be used, so just early out here instead of + # creating a socket needlessly. + # See https://github.com/urllib3/urllib3/issues/1446 + if _appengine_environ.is_appengine_sandbox(): + return False + if socket.has_ipv6: # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To @@ -123,4 +135,4 @@ def _has_ipv6(host): return has_ipv6 -HAS_IPV6 = _has_ipv6('::1') +HAS_IPV6 = _has_ipv6("::1") diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/request.py b/pipenv/patched/notpip/_vendor/urllib3/util/request.py index 3ddfcd55..262a6d61 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/request.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/request.py @@ -4,12 +4,25 @@ from base64 import b64encode from ..packages.six import b, integer_types from ..exceptions import UnrewindableBodyError -ACCEPT_ENCODING = 'gzip,deflate' +ACCEPT_ENCODING = "gzip,deflate" +try: + import brotli as _unused_module_brotli # noqa: F401 +except ImportError: + pass +else: + ACCEPT_ENCODING += ",br" + _FAILEDTELL = object() -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None, proxy_basic_auth=None, disable_cache=None): +def make_headers( + keep_alive=None, + accept_encoding=None, + user_agent=None, + basic_auth=None, + proxy_basic_auth=None, + disable_cache=None, +): """ Shortcuts for generating request headers. @@ -49,27 +62,27 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, if isinstance(accept_encoding, str): pass elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) + accept_encoding = ",".join(accept_encoding) else: accept_encoding = ACCEPT_ENCODING - headers['accept-encoding'] = accept_encoding + headers["accept-encoding"] = accept_encoding if user_agent: - headers['user-agent'] = user_agent + headers["user-agent"] = user_agent if keep_alive: - headers['connection'] = 'keep-alive' + headers["connection"] = "keep-alive" if basic_auth: - headers['authorization'] = 'Basic ' + \ - b64encode(b(basic_auth)).decode('utf-8') + headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") if proxy_basic_auth: - headers['proxy-authorization'] = 'Basic ' + \ - b64encode(b(proxy_basic_auth)).decode('utf-8') + headers["proxy-authorization"] = "Basic " + b64encode( + b(proxy_basic_auth) + ).decode("utf-8") if disable_cache: - headers['cache-control'] = 'no-cache' + headers["cache-control"] = "no-cache" return headers @@ -81,7 +94,7 @@ def set_file_position(body, pos): """ if pos is not None: rewind_body(body, pos) - elif getattr(body, 'tell', None) is not None: + elif getattr(body, "tell", None) is not None: try: pos = body.tell() except (IOError, OSError): @@ -103,16 +116,20 @@ def rewind_body(body, body_pos): :param int pos: Position to seek to in file. """ - body_seek = getattr(body, 'seek', None) + body_seek = getattr(body, "seek", None) if body_seek is not None and isinstance(body_pos, integer_types): try: body_seek(body_pos) except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect/retry.") + raise UnrewindableBodyError( + "An error occurred when rewinding request " "body for redirect/retry." + ) elif body_pos is _FAILEDTELL: - raise UnrewindableBodyError("Unable to record file position for rewinding " - "request body during a redirect/retry.") + raise UnrewindableBodyError( + "Unable to record file position for rewinding " + "request body during a redirect/retry." + ) else: - raise ValueError("body_pos must be of type integer, " - "instead it was %s." % type(body_pos)) + raise ValueError( + "body_pos must be of type integer, " "instead it was %s." % type(body_pos) + ) diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/response.py b/pipenv/patched/notpip/_vendor/urllib3/util/response.py index 67cf730a..715868dd 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/response.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/response.py @@ -52,15 +52,20 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError('expected httplib.Message, got {0}.'.format( - type(headers))) + raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) - defects = getattr(headers, 'defects', None) - get_payload = getattr(headers, 'get_payload', None) + defects = getattr(headers, "defects", None) + get_payload = getattr(headers, "get_payload", None) unparsed_data = None - if get_payload: # Platform-specific: Python 3. - unparsed_data = get_payload() + if get_payload: + # get_payload is actually email.message.Message.get_payload; + # we're only interested in the result if it's not a multipart message + if not headers.is_multipart(): + payload = get_payload() + + if isinstance(payload, (bytes, str)): + unparsed_data = payload if defects or unparsed_data: raise HeaderParsingError(defects=defects, unparsed_data=unparsed_data) @@ -78,4 +83,4 @@ def is_response_to_head(response): method = response._method if isinstance(method, int): # Platform-specific: Appengine return method == 3 - return method.upper() == 'HEAD' + return method.upper() == "HEAD" diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py index 7ad3dc66..5a049fe6 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/retry.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/retry.py @@ -21,8 +21,9 @@ log = logging.getLogger(__name__) # Data structure for representing the metadata of requests that result in a retry. -RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", - "status", "redirect_location"]) +RequestHistory = namedtuple( + "RequestHistory", ["method", "url", "error", "status", "redirect_location"] +) class Retry(object): @@ -115,7 +116,7 @@ class Retry(object): (most errors are resolved immediately by a second try without a delay). urllib3 will sleep for:: - {backoff factor} * (2 ^ ({number of total retries} - 1)) + {backoff factor} * (2 ** ({number of total retries} - 1)) seconds. If the backoff_factor is 0.1, then :func:`.sleep` will sleep for [0.0s, 0.2s, 0.4s, ...] between retries. It will never be longer @@ -146,21 +147,33 @@ class Retry(object): request. """ - DEFAULT_METHOD_WHITELIST = frozenset([ - 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + DEFAULT_METHOD_WHITELIST = frozenset( + ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] + ) RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) - DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) + DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"]) #: Maximum backoff time. BACKOFF_MAX = 120 - def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, - method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - history=None, respect_retry_after_header=True, - remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): + def __init__( + self, + total=10, + connect=None, + read=None, + redirect=None, + status=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, + status_forcelist=None, + backoff_factor=0, + raise_on_redirect=True, + raise_on_status=True, + history=None, + respect_retry_after_header=True, + remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST, + ): self.total = total self.connect = connect @@ -179,19 +192,25 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = remove_headers_on_redirect + self.remove_headers_on_redirect = frozenset( + [h.lower() for h in remove_headers_on_redirect] + ) def new(self, **kw): params = dict( total=self.total, - connect=self.connect, read=self.read, redirect=self.redirect, status=self.status, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, method_whitelist=self.method_whitelist, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, - remove_headers_on_redirect=self.remove_headers_on_redirect + remove_headers_on_redirect=self.remove_headers_on_redirect, + respect_retry_after_header=self.respect_retry_after_header, ) params.update(kw) return type(self)(**params) @@ -216,8 +235,11 @@ class Retry(object): :rtype: float """ # We want to consider only the last consecutive errors sequence (Ignore redirects). - consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, - reversed(self.history)))) + consecutive_errors_len = len( + list( + takewhile(lambda x: x.redirect_location is None, reversed(self.history)) + ) + ) if consecutive_errors_len <= 1: return 0 @@ -273,7 +295,7 @@ class Retry(object): this method will return immediately. """ - if response: + if self.respect_retry_after_header and response: slept = self.sleep_for_retry(response) if slept: return @@ -314,8 +336,12 @@ class Retry(object): if self.status_forcelist and status_code in self.status_forcelist: return True - return (self.total and self.respect_retry_after_header and - has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) + return ( + self.total + and self.respect_retry_after_header + and has_retry_after + and (status_code in self.RETRY_AFTER_STATUS_CODES) + ) def is_exhausted(self): """ Are we out of retries? """ @@ -326,8 +352,15 @@ class Retry(object): return min(retry_counts) < 0 - def increment(self, method=None, url=None, response=None, error=None, - _pool=None, _stacktrace=None): + def increment( + self, + method=None, + url=None, + response=None, + error=None, + _pool=None, + _stacktrace=None, + ): """ Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not @@ -350,7 +383,7 @@ class Retry(object): read = self.read redirect = self.redirect status_count = self.status - cause = 'unknown' + cause = "unknown" status = None redirect_location = None @@ -372,7 +405,7 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 - cause = 'too many redirects' + cause = "too many redirects" redirect_location = response.get_redirect_location() status = response.status @@ -383,16 +416,21 @@ class Retry(object): if response and response.status: if status_count is not None: status_count -= 1 - cause = ResponseError.SPECIFIC_ERROR.format( - status_code=response.status) + cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status - history = self.history + (RequestHistory(method, url, error, status, redirect_location),) + history = self.history + ( + RequestHistory(method, url, error, status, redirect_location), + ) new_retry = self.new( total=total, - connect=connect, read=read, redirect=redirect, status=status_count, - history=history) + connect=connect, + read=read, + redirect=redirect, + status=status_count, + history=history, + ) if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) @@ -402,9 +440,10 @@ class Retry(object): return new_retry def __repr__(self): - return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' - 'read={self.read}, redirect={self.redirect}, status={self.status})').format( - cls=type(self), self=self) + return ( + "{cls.__name__}(total={self.total}, connect={self.connect}, " + "read={self.read}, redirect={self.redirect}, status={self.status})" + ).format(cls=type(self), self=self) # For backwards compatibility (equivalent to pre-v1.9): diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py index a0868c22..8f1abfe4 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/ssl_.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import errno import warnings import hmac -import socket +import sys from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 +from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning from ..packages import six @@ -17,11 +18,7 @@ IS_PYOPENSSL = False IS_SECURETRANSPORT = False # Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = { - 32: md5, - 40: sha1, - 64: sha256, -} +HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} def _const_compare_digest_backport(a, b): @@ -37,17 +34,27 @@ def _const_compare_digest_backport(a, b): return result == 0 -_const_compare_digest = getattr(hmac, 'compare_digest', - _const_compare_digest_backport) - +_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) try: # Test for SSL features import ssl - from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 + from ssl import wrap_socket, CERT_REQUIRED from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: # Platform-specific: Python 3.6 + from ssl import PROTOCOL_TLS + + PROTOCOL_SSLv23 = PROTOCOL_TLS +except ImportError: + try: + from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS + + PROTOCOL_SSLv23 = PROTOCOL_TLS + except ImportError: + PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 + try: from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION @@ -56,26 +63,6 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 and earlier didn't have inet_pton on non-Linux -# so we fallback on inet_aton in those cases. This means that -# we can only detect IPv4 addresses in this case. -if hasattr(socket, 'inet_pton'): - inet_pton = socket.inet_pton -else: - # Maybe we can use ipaddress if the user has urllib3[secure]? - try: - from pipenv.patched.notpip._vendor import ipaddress - - def inet_pton(_, host): - if isinstance(host, six.binary_type): - host = host.decode('ascii') - return ipaddress.ip_address(host) - - except ImportError: # Platform-specific: Non-Linux - def inet_pton(_, host): - return socket.inet_aton(host) - - # A secure default. # Sources for more information on TLS ciphers: # @@ -84,41 +71,39 @@ else: # - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # # The general intent is: -# - Prefer TLS 1.3 cipher suites # - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), # - prefer ECDHE over DHE for better performance, # - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and # security, # - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_CIPHERS = ':'.join([ - 'TLS13-AES-256-GCM-SHA384', - 'TLS13-CHACHA20-POLY1305-SHA256', - 'TLS13-AES-128-GCM-SHA256', - 'ECDH+AESGCM', - 'ECDH+CHACHA20', - 'DH+AESGCM', - 'DH+CHACHA20', - 'ECDH+AES256', - 'DH+AES256', - 'ECDH+AES128', - 'DH+AES', - 'RSA+AESGCM', - 'RSA+AES', - '!aNULL', - '!eNULL', - '!MD5', -]) +# - disable NULL authentication, MD5 MACs, DSS, and other +# insecure ciphers for security reasons. +# - NOTE: TLS 1.3 cipher suites are managed through a different interface +# not exposed by CPython (yet!) and are enabled by default if they're available. +DEFAULT_CIPHERS = ":".join( + [ + "ECDHE+AESGCM", + "ECDHE+CHACHA20", + "DHE+AESGCM", + "DHE+CHACHA20", + "ECDH+AESGCM", + "DH+AESGCM", + "ECDH+AES", + "DH+AES", + "RSA+AESGCM", + "RSA+AES", + "!aNULL", + "!eNULL", + "!MD5", + "!DSS", + ] +) try: from ssl import SSLContext # Modern SSL? except ImportError: - import sys - - class SSLContext(object): # Platform-specific: Python 2 & 3.1 - supports_set_ciphers = ((2, 7) <= sys.version_info < (3,) or - (3, 2) <= sys.version_info) + class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): self.protocol = protocol_version # Use default values from a real SSLContext @@ -141,36 +126,27 @@ except ImportError: raise SSLError("CA directories not supported in older Pythons") def set_ciphers(self, cipher_suite): - if not self.supports_set_ciphers: - raise TypeError( - 'Your version of Python does not support setting ' - 'a custom cipher suite. Please upgrade to Python ' - '2.7, 3.2, or later if you need this functionality.' - ) self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( - 'A true SSLContext object is not available. This prevents ' - 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. You can upgrade to a newer ' - 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - InsecurePlatformWarning + "A true SSLContext object is not available. This prevents " + "urllib3 from configuring SSL appropriately and may cause " + "certain SSL connections to fail. You can upgrade to a newer " + "version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + InsecurePlatformWarning, ) kwargs = { - 'keyfile': self.keyfile, - 'certfile': self.certfile, - 'ca_certs': self.ca_certs, - 'cert_reqs': self.verify_mode, - 'ssl_version': self.protocol, - 'server_side': server_side, + "keyfile": self.keyfile, + "certfile": self.certfile, + "ca_certs": self.ca_certs, + "cert_reqs": self.verify_mode, + "ssl_version": self.protocol, + "server_side": server_side, } - if self.supports_set_ciphers: # Platform-specific: Python 2.7+ - return wrap_socket(socket, ciphers=self.ciphers, **kwargs) - else: # Platform-specific: Python 2.6 - return wrap_socket(socket, **kwargs) + return wrap_socket(socket, ciphers=self.ciphers, **kwargs) def assert_fingerprint(cert, fingerprint): @@ -183,12 +159,11 @@ def assert_fingerprint(cert, fingerprint): Fingerprint as string of hexdigits, can be interspersed by colons. """ - fingerprint = fingerprint.replace(':', '').lower() + fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) hashfunc = HASHFUNC_MAP.get(digest_length) if not hashfunc: - raise SSLError( - 'Fingerprint of invalid length: {0}'.format(fingerprint)) + raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) @@ -196,8 +171,11 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() if not _const_compare_digest(cert_digest, fingerprint_bytes): - raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' - .format(fingerprint, hexlify(cert_digest))) + raise SSLError( + 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( + fingerprint, hexlify(cert_digest) + ) + ) def resolve_cert_reqs(candidate): @@ -212,12 +190,12 @@ def resolve_cert_reqs(candidate): constant which can directly be passed to wrap_socket. """ if candidate is None: - return CERT_NONE + return CERT_REQUIRED if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'CERT_' + candidate) + res = getattr(ssl, "CERT_" + candidate) return res return candidate @@ -228,19 +206,20 @@ def resolve_ssl_version(candidate): like resolve_cert_reqs """ if candidate is None: - return PROTOCOL_SSLv23 + return PROTOCOL_TLS if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'PROTOCOL_' + candidate) + res = getattr(ssl, "PROTOCOL_" + candidate) return res return candidate -def create_urllib3_context(ssl_version=None, cert_reqs=None, - options=None, ciphers=None): +def create_urllib3_context( + ssl_version=None, cert_reqs=None, options=None, ciphers=None +): """All arguments have the same meaning as ``ssl_wrap_socket``. By default, this function does a lot of the same work that @@ -274,7 +253,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context = SSLContext(ssl_version or PROTOCOL_TLS) + + context.set_ciphers(ciphers or DEFAULT_CIPHERS) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -291,21 +272,40 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options - if getattr(context, 'supports_set_ciphers', True): # Platform-specific: Python 2.6 - context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is + # necessary for conditional client cert authentication with TLS 1.3. + # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older + # versions of Python. We only enable on Python 3.7.4+ or if certificate + # verification is enabled to work around Python issue #37428 + # See: https://bugs.python.org/issue37428 + if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( + context, "post_handshake_auth", None + ) is not None: + context.post_handshake_auth = True context.verify_mode = cert_reqs - if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative # hostnames. So disable it here context.check_hostname = False return context -def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None, ciphers=None, ssl_context=None, - ca_cert_dir=None): +def ssl_wrap_socket( + sock, + keyfile=None, + certfile=None, + cert_reqs=None, + ca_certs=None, + server_hostname=None, + ssl_version=None, + ciphers=None, + ssl_context=None, + ca_cert_dir=None, + key_password=None, +): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have the same meaning as they do when using :func:`ssl.wrap_socket`. @@ -316,25 +316,25 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A pre-made :class:`SSLContext` object. If none is provided, one will be created using :func:`create_urllib3_context`. :param ciphers: - A string of ciphers we wish the client to support. This is not - supported on Python 2.6 as the ssl module does not support it. + A string of ciphers we wish the client to support. :param ca_cert_dir: A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to SSLContext.load_verify_locations(). + :param key_password: + Optional password if the keyfile is encrypted. """ context = ssl_context if context is None: # Note: This branch of code and all the variables in it are no longer # used by urllib3 itself. We should consider deprecating and removing # this code. - context = create_urllib3_context(ssl_version, cert_reqs, - ciphers=ciphers) + context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) if ca_certs or ca_cert_dir: try: context.load_verify_locations(ca_certs, ca_cert_dir) - except IOError as e: # Platform-specific: Python 2.6, 2.7, 3.2 + except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError # These are not equivalent unless we check the errno attribute @@ -342,55 +342,66 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise - elif getattr(context, 'load_default_certs', None) is not None: + + elif ssl_context is None and hasattr(context, "load_default_certs"): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() + # Attempt to detect if we get the goofy behavior of the + # keyfile being encrypted and OpenSSL asking for the + # passphrase via the terminal and instead error out. + if keyfile and key_password is None and _is_key_file_encrypted(keyfile): + raise SSLError("Client private key is encrypted, password is required") + if certfile: - context.load_cert_chain(certfile, keyfile) + if key_password is None: + context.load_cert_chain(certfile, keyfile) + else: + context.load_cert_chain(certfile, keyfile, key_password) # If we detect server_hostname is an IP address then the SNI # extension should not be used according to RFC3546 Section 3.1 # We shouldn't warn the user if SNI isn't available but we would # not be using SNI anyways due to IP address for server_hostname. - if ((server_hostname is not None and not is_ipaddress(server_hostname)) - or IS_SECURETRANSPORT): + if ( + server_hostname is not None and not is_ipaddress(server_hostname) + ) or IS_SECURETRANSPORT: if HAS_SNI and server_hostname is not None: return context.wrap_socket(sock, server_hostname=server_hostname) warnings.warn( - 'An HTTPS request has been made, but the SNI (Server Name ' - 'Indication) extension to TLS is not available on this platform. ' - 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. You can upgrade to ' - 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - SNIMissingWarning + "An HTTPS request has been made, but the SNI (Server Name " + "Indication) extension to TLS is not available on this platform. " + "This may cause the server to present an incorrect TLS " + "certificate, which can cause validation failures. You can upgrade to " + "a newer version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + SNIMissingWarning, ) return context.wrap_socket(sock) def is_ipaddress(hostname): - """Detects whether the hostname given is an IP address. + """Detects whether the hostname given is an IPv4 or IPv6 address. + Also detects IPv6 addresses with Zone IDs. :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, six.binary_type): + if not six.PY2 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. - hostname = hostname.decode('ascii') + hostname = hostname.decode("ascii") + return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname)) - families = [socket.AF_INET] - if hasattr(socket, 'AF_INET6'): - families.append(socket.AF_INET6) - for af in families: - try: - inet_pton(af, hostname) - except (socket.error, ValueError, OSError): - pass - else: - return True +def _is_key_file_encrypted(key_file): + """Detects if a key file is encrypted or not.""" + with open(key_file, "r") as f: + for line in f: + # Look for Proc-Type: 4,ENCRYPTED + if "ENCRYPTED" in line: + return True + return False diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/timeout.py b/pipenv/patched/notpip/_vendor/urllib3/util/timeout.py index cec817e6..c1dc1e97 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/timeout.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/timeout.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -45,19 +46,20 @@ class Timeout(object): :type total: integer, float, or None :param connect: - The maximum amount of time to wait for a connection attempt to a server - to succeed. Omitting the parameter will default the connect timeout to - the system default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait for a connection + attempt to a server to succeed. Omitting the parameter will default the + connect timeout to the system default, probably `the global default + timeout in socket.py <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. None will set an infinite timeout for connection attempts. :type connect: integer, float, or None :param read: - The maximum amount of time to wait between consecutive - read operations for a response from the server. Omitting - the parameter will default the read timeout to the system - default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait between consecutive + read operations for a response from the server. Omitting the parameter + will default the read timeout to the system default, probably `the + global default timeout in socket.py <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. None will set an infinite timeout. @@ -91,14 +93,18 @@ class Timeout(object): DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT def __init__(self, total=None, connect=_Default, read=_Default): - self._connect = self._validate_timeout(connect, 'connect') - self._read = self._validate_timeout(read, 'read') - self.total = self._validate_timeout(total, 'total') + self._connect = self._validate_timeout(connect, "connect") + self._read = self._validate_timeout(read, "read") + self.total = self._validate_timeout(total, "total") self._start_connect = None def __str__(self): - return '%s(connect=%r, read=%r, total=%r)' % ( - type(self).__name__, self._connect, self._read, self.total) + return "%s(connect=%r, read=%r, total=%r)" % ( + type(self).__name__, + self._connect, + self._read, + self.total, + ) @classmethod def _validate_timeout(cls, value, name): @@ -118,22 +124,31 @@ class Timeout(object): return value if isinstance(value, bool): - raise ValueError("Timeout cannot be a boolean value. It must " - "be an int, float or None.") + raise ValueError( + "Timeout cannot be a boolean value. It must " + "be an int, float or None." + ) try: float(value) except (TypeError, ValueError): - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) try: if value <= 0: - raise ValueError("Attempted to set %s timeout to %s, but the " - "timeout cannot be set to a value less " - "than or equal to 0." % (name, value)) - except TypeError: # Python 3 - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Attempted to set %s timeout to %s, but the " + "timeout cannot be set to a value less " + "than or equal to 0." % (name, value) + ) + except TypeError: + # Python 3 + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) return value @@ -165,8 +180,7 @@ class Timeout(object): # We can't use copy.deepcopy because that will also create a new object # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to # detect the user default. - return Timeout(connect=self._connect, read=self._read, - total=self.total) + return Timeout(connect=self._connect, read=self._read, total=self.total) def start_connect(self): """ Start the timeout clock, used during a connect() attempt @@ -182,14 +196,15 @@ class Timeout(object): def get_connect_duration(self): """ Gets the time elapsed since the call to :meth:`start_connect`. - :return: Elapsed time. + :return: Elapsed time in seconds. :rtype: float :raises urllib3.exceptions.TimeoutStateError: if you attempt to get duration for a timer that hasn't been started. """ if self._start_connect is None: - raise TimeoutStateError("Can't get connect duration for timer " - "that has not started.") + raise TimeoutStateError( + "Can't get connect duration for timer " "that has not started." + ) return current_time() - self._start_connect @property @@ -227,15 +242,16 @@ class Timeout(object): :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. """ - if (self.total is not None and - self.total is not self.DEFAULT_TIMEOUT and - self._read is not None and - self._read is not self.DEFAULT_TIMEOUT): + if ( + self.total is not None + and self.total is not self.DEFAULT_TIMEOUT + and self._read is not None + and self._read is not self.DEFAULT_TIMEOUT + ): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read - return max(0, min(self.total - self.get_connect_duration(), - self._read)) + return max(0, min(self.total - self.get_connect_duration(), self._read)) elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: return max(0, self.total - self.get_connect_duration()) else: diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/url.py b/pipenv/patched/notpip/_vendor/urllib3/util/url.py index 6b6f9968..007157ae 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/url.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/url.py @@ -1,34 +1,110 @@ from __future__ import absolute_import +import re from collections import namedtuple from ..exceptions import LocationParseError +from ..packages import six -url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] +url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] # We only want to normalize urls with an HTTP(S) scheme. # urllib3 infers URLs without a scheme (None) to be http. -NORMALIZABLE_SCHEMES = ('http', 'https', None) +NORMALIZABLE_SCHEMES = ("http", "https", None) + +# Almost all of these patterns were derived from the +# 'rfc3986' module: https://github.com/python-hyper/rfc3986 +PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") +SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") +URI_RE = re.compile( + r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" + r"(?://([^/?#]*))?" + r"([^?#]*)" + r"(?:\?([^#]*))?" + r"(?:#(.*))?$", + re.UNICODE | re.DOTALL, +) + +IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" +HEX_PAT = "[0-9A-Fa-f]{1,4}" +LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT) +_subs = {"hex": HEX_PAT, "ls32": LS32_PAT} +_variations = [ + # 6( h16 ":" ) ls32 + "(?:%(hex)s:){6}%(ls32)s", + # "::" 5( h16 ":" ) ls32 + "::(?:%(hex)s:){5}%(ls32)s", + # [ h16 ] "::" 4( h16 ":" ) ls32 + "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s", + # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s", + # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s", + # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s", + # [ *4( h16 ":" ) h16 ] "::" ls32 + "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s", + # [ *5( h16 ":" ) h16 ] "::" h16 + "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s", + # [ *6( h16 ":" ) h16 ] "::" + "(?:(?:%(hex)s:){0,6}%(hex)s)?::", +] + +UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._!\-~" +IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" +ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" +IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]" +REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" +TARGET_RE = re.compile(r"^(/[^?]*)(?:\?([^#]+))?(?:#(.*))?$") + +IPV4_RE = re.compile("^" + IPV4_PAT + "$") +IPV6_RE = re.compile("^" + IPV6_PAT + "$") +IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") +BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") +ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") + +SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( + REG_NAME_PAT, + IPV4_PAT, + IPV6_ADDRZ_PAT, +) +SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL) + +UNRESERVED_CHARS = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" +) +SUB_DELIM_CHARS = set("!$&'()*+,;=") +USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} +PATH_CHARS = USERINFO_CHARS | {"@", "/"} +QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} -class Url(namedtuple('Url', url_attrs)): +class Url(namedtuple("Url", url_attrs)): """ - Datastructure for representing an HTTP URL. Used as a return value for + Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ + __slots__ = () - def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, - query=None, fragment=None): - if path and not path.startswith('/'): - path = '/' + path - if scheme: + def __new__( + cls, + scheme=None, + auth=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + ): + if path and not path.startswith("/"): + path = "/" + path + if scheme is not None: scheme = scheme.lower() - if host and scheme in NORMALIZABLE_SCHEMES: - host = host.lower() - return super(Url, cls).__new__(cls, scheme, auth, host, port, path, - query, fragment) + return super(Url, cls).__new__( + cls, scheme, auth, host, port, path, query, fragment + ) @property def hostname(self): @@ -38,10 +114,10 @@ class Url(namedtuple('Url', url_attrs)): @property def request_uri(self): """Absolute path including the query string.""" - uri = self.path or '/' + uri = self.path or "/" if self.query is not None: - uri += '?' + self.query + uri += "?" + self.query return uri @@ -49,7 +125,7 @@ class Url(namedtuple('Url', url_attrs)): def netloc(self): """Network location including host and port""" if self.port: - return '%s:%d' % (self.host, self.port) + return "%s:%d" % (self.host, self.port) return self.host @property @@ -72,23 +148,23 @@ class Url(namedtuple('Url', url_attrs)): 'http://username:password@host.com:80/path?query#fragment' """ scheme, auth, host, port, path, query, fragment = self - url = '' + url = u"" # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + '://' + url += scheme + u"://" if auth is not None: - url += auth + '@' + url += auth + u"@" if host is not None: url += host if port is not None: - url += ':' + str(port) + url += u":" + str(port) if path is not None: url += path if query is not None: - url += '?' + query + url += u"?" + query if fragment is not None: - url += '#' + fragment + url += u"#" + fragment return url @@ -98,6 +174,8 @@ class Url(namedtuple('Url', url_attrs)): def split_first(s, delims): """ + .. deprecated:: 1.25 + Given a string and an iterable of delimiters, split on the first found delimiter. Return two split parts and the matched delimiter. @@ -124,15 +202,150 @@ def split_first(s, delims): min_delim = d if min_idx is None or min_idx < 0: - return s, '', None + return s, "", None - return s[:min_idx], s[min_idx + 1:], min_delim + return s[:min_idx], s[min_idx + 1 :], min_delim + + +def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): + """Percent-encodes a URI component without reapplying + onto an already percent-encoded component. + """ + if component is None: + return component + + component = six.ensure_text(component) + + # Try to see if the component we're encoding is already percent-encoded + # so we can skip all '%' characters but still encode all others. + percent_encodings = PERCENT_RE.findall(component) + + # Normalize existing percent-encoded bytes. + for enc in percent_encodings: + if not enc.isupper(): + component = component.replace(enc, enc.upper()) + + uri_bytes = component.encode("utf-8", "surrogatepass") + is_percent_encoded = len(percent_encodings) == uri_bytes.count(b"%") + + encoded_component = bytearray() + + for i in range(0, len(uri_bytes)): + # Will return a single character bytestring on both Python 2 & 3 + byte = uri_bytes[i : i + 1] + byte_ord = ord(byte) + if (is_percent_encoded and byte == b"%") or ( + byte_ord < 128 and byte.decode() in allowed_chars + ): + encoded_component.extend(byte) + continue + encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) + + return encoded_component.decode(encoding) + + +def _remove_path_dot_segments(path): + # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code + segments = path.split("/") # Turn the path into a list of segments + output = [] # Initialize the variable to use to store output + + for segment in segments: + # '.' is the current directory, so ignore it, it is superfluous + if segment == ".": + continue + # Anything other than '..', should be appended to the output + elif segment != "..": + output.append(segment) + # In this case segment == '..', if we can, we should pop the last + # element + elif output: + output.pop() + + # If the path starts with '/' and the output is empty or the first string + # is non-empty + if path.startswith("/") and (not output or output[0]): + output.insert(0, "") + + # If the path starts with '/.' or '/..' ensure we add one more empty + # string to add a trailing '/' + if path.endswith(("/.", "/..")): + output.append("") + + return "/".join(output) + + +def _normalize_host(host, scheme): + if host: + if isinstance(host, six.binary_type): + host = six.ensure_str(host) + + if scheme in NORMALIZABLE_SCHEMES: + is_ipv6 = IPV6_ADDRZ_RE.match(host) + if is_ipv6: + match = ZONE_ID_RE.search(host) + if match: + start, end = match.span(1) + zone_id = host[start:end] + + if zone_id.startswith("%25") and zone_id != "%25": + zone_id = zone_id[3:] + else: + zone_id = zone_id[1:] + zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS) + return host[:start].lower() + zone_id + host[end:] + else: + return host.lower() + elif not IPV4_RE.match(host): + return six.ensure_str( + b".".join([_idna_encode(label) for label in host.split(".")]) + ) + return host + + +def _idna_encode(name): + if name and any([ord(x) > 128 for x in name]): + try: + from pipenv.patched.notpip._vendor import idna + except ImportError: + six.raise_from( + LocationParseError("Unable to parse URL without the 'idna' module"), + None, + ) + try: + return idna.encode(name.lower(), strict=True, std3_rules=True) + except idna.IDNAError: + six.raise_from( + LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None + ) + return name.lower().encode("ascii") + + +def _encode_target(target): + """Percent-encodes a request target so that there are no invalid characters""" + if not target.startswith("/"): + return target + + path, query, fragment = TARGET_RE.match(target).groups() + target = _encode_invalid_chars(path, PATH_CHARS) + query = _encode_invalid_chars(query, QUERY_CHARS) + fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) + if query is not None: + target += "?" + query + if fragment is not None: + target += "#" + target + return target def parse_url(url): """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. + This parser is RFC 3986 compliant. + + The parser logic and helper functions are based heavily on + work done in the ``rfc3986`` module. + + :param str url: URL to parse into a :class:`.Url` namedtuple. Partly backwards-compatible with :mod:`urlparse`. @@ -145,81 +358,77 @@ def parse_url(url): >>> parse_url('/foo?bar') Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) """ - - # While this code has overlap with stdlib's urlparse, it is much - # simplified for our needs and less annoying. - # Additionally, this implementations does silly things to be optimal - # on CPython. - if not url: # Empty return Url() - scheme = None - auth = None - host = None - port = None - path = None - fragment = None - query = None + source_url = url + if not SCHEME_RE.search(url): + url = "//" + url - # Scheme - if '://' in url: - scheme, url = url.split('://', 1) + try: + scheme, authority, path, query, fragment = URI_RE.match(url).groups() + normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES - # Find the earliest Authority Terminator - # (http://tools.ietf.org/html/rfc3986#section-3.2) - url, path_, delim = split_first(url, ['/', '?', '#']) + if scheme: + scheme = scheme.lower() - if delim: - # Reassemble the path - path = delim + path_ - - # Auth - if '@' in url: - # Last '@' denotes end of auth part - auth, url = url.rsplit('@', 1) - - # IPv6 - if url and url[0] == '[': - host, url = url.split(']', 1) - host += ']' - - # Port - if ':' in url: - _host, port = url.split(':', 1) - - if not host: - host = _host - - if port: - # If given, ports must be integers. No whitespace, no plus or - # minus prefixes, no non-integer digits such as ^2 (superscript). - if not port.isdigit(): - raise LocationParseError(url) - try: - port = int(port) - except ValueError: - raise LocationParseError(url) + if authority: + auth, host, port = SUBAUTHORITY_RE.match(authority).groups() + if auth and normalize_uri: + auth = _encode_invalid_chars(auth, USERINFO_CHARS) + if port == "": + port = None else: - # Blank ports are cool, too. (rfc3986#section-3.2.3) - port = None + auth, host, port = None, None, None - elif not host and url: - host = url + if port is not None: + port = int(port) + if not (0 <= port <= 65535): + raise LocationParseError(url) + host = _normalize_host(host, scheme) + + if normalize_uri and path: + path = _remove_path_dot_segments(path) + path = _encode_invalid_chars(path, PATH_CHARS) + if normalize_uri and query: + query = _encode_invalid_chars(query, QUERY_CHARS) + if normalize_uri and fragment: + fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) + + except (ValueError, AttributeError): + return six.raise_from(LocationParseError(source_url), None) + + # For the sake of backwards compatibility we put empty + # string values for path if there are any defined values + # beyond the path in the URL. + # TODO: Remove this when we break backwards compatibility. if not path: - return Url(scheme, auth, host, port, path, query, fragment) + if query is not None or fragment is not None: + path = "" + else: + path = None - # Fragment - if '#' in path: - path, fragment = path.split('#', 1) + # Ensure that each part of the URL is a `str` for + # backwards compatibility. + if isinstance(url, six.text_type): + ensure_func = six.ensure_text + else: + ensure_func = six.ensure_str - # Query - if '?' in path: - path, query = path.split('?', 1) + def ensure_type(x): + return x if x is None else ensure_func(x) - return Url(scheme, auth, host, port, path, query, fragment) + return Url( + scheme=ensure_type(scheme), + auth=ensure_type(auth), + host=ensure_type(host), + port=port, + path=ensure_type(path), + query=ensure_type(query), + fragment=ensure_type(fragment), + ) def get_host(url): @@ -227,4 +436,4 @@ def get_host(url): Deprecated. Use :func:`parse_url` instead. """ p = parse_url(url) - return p.scheme or 'http', p.hostname, p.port + return p.scheme or "http", p.hostname, p.port diff --git a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py index fa686eff..d71d2fd7 100644 --- a/pipenv/patched/notpip/_vendor/urllib3/util/wait.py +++ b/pipenv/patched/notpip/_vendor/urllib3/util/wait.py @@ -2,6 +2,7 @@ import errno from functools import partial import select import sys + try: from time import monotonic except ImportError: @@ -40,12 +41,11 @@ if sys.version_info >= (3, 5): # Modern Python, that retries syscalls by default def _retry_on_intr(fn, timeout): return fn(timeout) + + else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): - if timeout is not None and timeout <= 0: - return fn(timeout) - if timeout is None: deadline = float("inf") else: @@ -117,7 +117,7 @@ def _have_working_poll(): # from libraries like eventlet/greenlet. try: poll_obj = select.poll() - poll_obj.poll(0) + _retry_on_intr(poll_obj.poll, 0) except (AttributeError, OSError): return False else: diff --git a/pipenv/patched/notpip/_vendor/vendor.txt b/pipenv/patched/notpip/_vendor/vendor.txt index 9389dd94..aadd3526 100644 --- a/pipenv/patched/notpip/_vendor/vendor.txt +++ b/pipenv/patched/notpip/_vendor/vendor.txt @@ -1,23 +1,23 @@ appdirs==1.4.3 -distlib==0.2.7 -distro==1.3.0 -html5lib==1.0.1 -six==1.11.0 -colorama==0.3.9 CacheControl==0.12.5 -msgpack-python==0.5.6 -lockfile==0.12.2 -progress==1.4 +colorama==0.4.1 +contextlib2==0.6.0 +distlib==0.2.9.post0 +distro==1.4.0 +html5lib==1.0.1 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==18.0 -pep517==0.2 -pyparsing==2.2.1 -pytoml==0.1.19 -retrying==1.3.3 -requests==2.19.1 +msgpack==0.6.2 +packaging==19.2 +pep517==0.7.0 +progress==1.5 +pyparsing==2.4.2 +pytoml==0.1.21 +requests==2.22.0 + certifi==2019.9.11 chardet==3.0.4 - idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -setuptools==40.4.3 + idna==2.8 + urllib3==1.25.6 +retrying==1.3.3 +setuptools==41.4.0 +six==1.12.0 webencodings==0.5.1 diff --git a/pipenv/patched/notpip/appdirs.LICENSE.txt b/pipenv/patched/notpip/appdirs.LICENSE.txt new file mode 100644 index 00000000..107c6140 --- /dev/null +++ b/pipenv/patched/notpip/appdirs.LICENSE.txt @@ -0,0 +1,23 @@ +# This is the MIT license + +Copyright (c) 2010 ActiveState Software Inc. + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/pipenv/patched/notpip/contextlib2.LICENSE.txt b/pipenv/patched/notpip/contextlib2.LICENSE.txt new file mode 100644 index 00000000..5de20277 --- /dev/null +++ b/pipenv/patched/notpip/contextlib2.LICENSE.txt @@ -0,0 +1,122 @@ + + +A. HISTORY OF THE SOFTWARE +========================== + +contextlib2 is a derivative of the contextlib module distributed by the PSF +as part of the Python standard library. According, it is itself redistributed +under the PSF license (reproduced in full below). As the contextlib module +was added only in Python 2.5, the licenses for earlier Python versions are +not applicable and have not been included. + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases that included the contextlib module. + + Release Derived Year Owner GPL- + from compatible? (1) + + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 3.0 2.6 2008 PSF yes + 3.0.1 3.0 2009 PSF yes + 3.1 3.0.1 2009 PSF yes + 3.1.1 3.1 2009 PSF yes + 3.1.2 3.1.1 2010 PSF yes + 3.1.3 3.1.2 2010 PSF yes + 3.1.4 3.1.3 2011 PSF yes + 3.2 3.1 2011 PSF yes + 3.2.1 3.2 2011 PSF yes + 3.2.2 3.2.1 2011 PSF yes + 3.3 3.2 2012 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011 Python Software Foundation; All Rights Reserved" are retained in Python +alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/patched/notpip/distro.LICENSE b/pipenv/patched/notpip/distro.LICENSE new file mode 100644 index 00000000..e06d2081 --- /dev/null +++ b/pipenv/patched/notpip/distro.LICENSE @@ -0,0 +1,202 @@ +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/idna.LICENSE.rst b/pipenv/patched/notpip/idna.LICENSE.rst new file mode 100644 index 00000000..3ee64fba --- /dev/null +++ b/pipenv/patched/notpip/idna.LICENSE.rst @@ -0,0 +1,80 @@ +License +------- + +Copyright (c) 2013-2018, Kim Davies. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +#. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +#. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided with + the distribution. + +#. Neither the name of the copyright holder nor the names of the + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +#. THIS SOFTWARE IS PROVIDED BY THE CONTRIBUTORS "AS IS" AND ANY + EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE + USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH + DAMAGE. + +Portions of the codec implementation and unit tests are derived from the +Python standard library, which carries the `Python Software Foundation +License <https://docs.python.org/2/license.html>`_: + + Copyright (c) 2001-2014 Python Software Foundation; All Rights Reserved + +Portions of the unit tests are derived from the Unicode standard, which +is subject to the Unicode, Inc. License Agreement: + + Copyright (c) 1991-2014 Unicode, Inc. All rights reserved. + Distributed under the Terms of Use in + <http://www.unicode.org/copyright.html>. + + Permission is hereby granted, free of charge, to any person obtaining + a copy of the Unicode data files and any associated documentation + (the "Data Files") or Unicode software and any associated documentation + (the "Software") to deal in the Data Files or Software + without restriction, including without limitation the rights to use, + copy, modify, merge, publish, distribute, and/or sell copies of + the Data Files or Software, and to permit persons to whom the Data Files + or Software are furnished to do so, provided that + + (a) this copyright and permission notice appear with all copies + of the Data Files or Software, + + (b) this copyright and permission notice appear in associated + documentation, and + + (c) there is clear notice in each modified Data File or in the Software + as well as in the documentation associated with the Data File(s) or + Software that the data or software has been modified. + + THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF + ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE + WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + NONINFRINGEMENT OF THIRD PARTY RIGHTS. + IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS + NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL + DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, + DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER + TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR + PERFORMANCE OF THE DATA FILES OR SOFTWARE. + + Except as contained in this notice, the name of a copyright holder + shall not be used in advertising or otherwise to promote the sale, + use or other dealings in these Data Files or Software without prior + written authorization of the copyright holder. diff --git a/pipenv/patched/notpip/ipaddress.LICENSE b/pipenv/patched/notpip/ipaddress.LICENSE new file mode 100644 index 00000000..41bd16ba --- /dev/null +++ b/pipenv/patched/notpip/ipaddress.LICENSE @@ -0,0 +1,50 @@ +This package is a modified version of cpython's ipaddress module. +It is therefore distributed under the PSF license, as follows: + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011, 2012, 2013, 2014 Python Software Foundation; All Rights Reserved" are +retained in Python alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/patched/notpip/msgpack.COPYING b/pipenv/patched/notpip/msgpack.COPYING new file mode 100644 index 00000000..f067af3a --- /dev/null +++ b/pipenv/patched/notpip/msgpack.COPYING @@ -0,0 +1,14 @@ +Copyright (C) 2008-2011 INADA Naoki <songofacandy@gmail.com> + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/pipenv/patched/notpip/packaging.LICENSE.APACHE b/pipenv/patched/notpip/packaging.LICENSE.APACHE new file mode 100644 index 00000000..4947287f --- /dev/null +++ b/pipenv/patched/notpip/packaging.LICENSE.APACHE @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/pipenv/patched/notpip/packaging.LICENSE.BSD b/pipenv/patched/notpip/packaging.LICENSE.BSD new file mode 100644 index 00000000..42ce7b75 --- /dev/null +++ b/pipenv/patched/notpip/packaging.LICENSE.BSD @@ -0,0 +1,23 @@ +Copyright (c) Donald Stufft and individual contributors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + 1. Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/patched/notpip/pyparsing.LICENSE b/pipenv/patched/notpip/pyparsing.LICENSE new file mode 100644 index 00000000..1bf98523 --- /dev/null +++ b/pipenv/patched/notpip/pyparsing.LICENSE @@ -0,0 +1,18 @@ +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/patched/notpip/retrying.LICENSE b/pipenv/patched/notpip/retrying.LICENSE new file mode 100644 index 00000000..7a4a3ea2 --- /dev/null +++ b/pipenv/patched/notpip/retrying.LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. \ No newline at end of file diff --git a/pipenv/patched/notpip/six.LICENSE b/pipenv/patched/notpip/six.LICENSE new file mode 100644 index 00000000..365d1074 --- /dev/null +++ b/pipenv/patched/notpip/six.LICENSE @@ -0,0 +1,18 @@ +Copyright (c) 2010-2018 Benjamin Peterson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/jinja2/LICENSE b/pipenv/patched/notpip/webencodings.LICENSE similarity index 95% rename from pipenv/vendor/jinja2/LICENSE rename to pipenv/patched/notpip/webencodings.LICENSE index 31bf900e..3d0d3e70 100644 --- a/pipenv/vendor/jinja2/LICENSE +++ b/pipenv/patched/notpip/webencodings.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2009 by the Jinja Team, see AUTHORS for more details. +Copyright (c) 2012 by Simon Sapin. Some rights reserved. diff --git a/pipenv/patched/patched.txt b/pipenv/patched/patched.txt index e7dadd8e..b8a2816a 100644 --- a/pipenv/patched/patched.txt +++ b/pipenv/patched/patched.txt @@ -1,5 +1,5 @@ -safety +safety==1.8.7 crayons==0.1.2 pipfile==0.0.2 -pip-tools==3.1.0 -pip==18.1 +pip-tools==4.3.0 +pip==19.3.1 diff --git a/pipenv/patched/piptools/__init__.py b/pipenv/patched/piptools/__init__.py index e69de29b..9f0c95aa 100644 --- a/pipenv/patched/piptools/__init__.py +++ b/pipenv/patched/piptools/__init__.py @@ -0,0 +1,11 @@ +import locale + +from piptools.click import secho + +# Needed for locale.getpreferredencoding(False) to work +# in pip._internal.utils.encoding.auto_decode +try: + locale.setlocale(locale.LC_ALL, "") +except locale.Error as e: # pragma: no cover + # setlocale can apparently crash if locale are uninitialized + secho("Ignoring error when setting locale: {}".format(e), fg="red") diff --git a/pipenv/patched/piptools/__main__.py b/pipenv/patched/piptools/__main__.py index 22bd7879..2d8b75e8 100644 --- a/pipenv/patched/piptools/__main__.py +++ b/pipenv/patched/piptools/__main__.py @@ -1,4 +1,5 @@ import click + from piptools.scripts import compile, sync @@ -7,10 +8,10 @@ def cli(): pass -cli.add_command(compile.cli, 'compile') -cli.add_command(sync.cli, 'sync') +cli.add_command(compile.cli, "compile") +cli.add_command(sync.cli, "sync") # Enable ``python -m piptools ...``. -if __name__ == '__main__': # pragma: no cover +if __name__ == "__main__": # pragma: no branch cli() diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py index c0ecec8a..fd8ecddd 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py @@ -1,34 +1,44 @@ # coding: utf-8 # flake8: noqa -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals import six -if six.PY2: - from .tempfile import TemporaryDirectory - from .contextlib import ExitStack -else: - from tempfile import TemporaryDirectory - from contextlib import ExitStack - from .pip_compat import ( - InstallRequirement, - parse_requirements, - RequirementSet, - user_cache_dir, + DEV_PKGS, FAVORITE_HASH, - is_file_url, - url_to_path, - PackageFinder, + PIP_VERSION, FormatControl, + InstallationCandidate, + InstallCommand, + InstallationError, + InstallRequirement, + Link, + PackageFinder, + PyPI, + RequirementSet, + RequirementTracker, + Resolver, + SafeFileCache, + VcsSupport, Wheel, - Command, + WheelCache, cmdoptions, get_installed_distributions, - PyPI, - SafeFileCache, - InstallationError, - install_req_from_line, install_req_from_editable, + install_req_from_line, + is_dir_url, + is_file_url, + is_vcs_url, + parse_requirements, + path_to_url, + pip_version, + stdlib_pkgs, + url_to_path, + user_cache_dir, ) + +if six.PY2: + from .tempfile import TemporaryDirectory +else: + from tempfile import TemporaryDirectory diff --git a/pipenv/patched/piptools/_compat/contextlib.py b/pipenv/patched/piptools/_compat/contextlib.py index b0e161bb..04039ccb 100644 --- a/pipenv/patched/piptools/_compat/contextlib.py +++ b/pipenv/patched/piptools/_compat/contextlib.py @@ -1,123 +1,18 @@ -# coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -import sys -from collections import deque - - -# Inspired by discussions on http://bugs.python.org/issue13585 -class ExitStack(object): - """Context manager for dynamic management of a stack of exit callbacks - - For example: - - with ExitStack() as stack: - files = [stack.enter_context(open(fname)) for fname in filenames] - # All opened files will automatically be closed at the end of - # the with statement, even if attempts to open files later - # in the list throw an exception - +# Ported from python 3.7 contextlib.py +class nullcontext(object): + """Context manager that does no additional processing. + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True """ - def __init__(self): - self._exit_callbacks = deque() - def pop_all(self): - """Preserve the context stack by transferring it to a new instance""" - new_stack = type(self)() - new_stack._exit_callbacks = self._exit_callbacks - self._exit_callbacks = deque() - return new_stack - - def _push_cm_exit(self, cm, cm_exit): - """Helper to correctly register callbacks to __exit__ methods""" - def _exit_wrapper(*exc_details): - return cm_exit(cm, *exc_details) - _exit_wrapper.__self__ = cm - self.push(_exit_wrapper) - - def push(self, exit): - """Registers a callback with the standard __exit__ method signature - - Can suppress exceptions the same way __exit__ methods can. - - Also accepts any object with an __exit__ method (registering the - method instead of the object itself) - """ - # We use an unbound method rather than a bound method to follow - # the standard lookup behaviour for special methods - _cb_type = type(exit) - try: - exit_method = _cb_type.__exit__ - except AttributeError: - # Not a context manager, so assume its a callable - self._exit_callbacks.append(exit) - else: - self._push_cm_exit(exit, exit_method) - return exit # Allow use as a decorator - - def callback(self, callback, *args, **kwds): - """Registers an arbitrary callback and arguments. - - Cannot suppress exceptions. - """ - def _exit_wrapper(exc_type, exc, tb): - callback(*args, **kwds) - # We changed the signature, so using @wraps is not appropriate, but - # setting __wrapped__ may still help with introspection - _exit_wrapper.__wrapped__ = callback - self.push(_exit_wrapper) - return callback # Allow use as a decorator - - def enter_context(self, cm): - """Enters the supplied context manager - - If successful, also pushes its __exit__ method as a callback and - returns the result of the __enter__ method. - """ - # We look up the special methods on the type to match the with - # statement - _cm_type = type(cm) - _exit = _cm_type.__exit__ - result = _cm_type.__enter__(cm) - self._push_cm_exit(cm, _exit) - return result - - def close(self): - """Immediately unwind the context stack""" - self.__exit__(None, None, None) + def __init__(self, enter_result=None): + self.enter_result = enter_result def __enter__(self): - return self + return self.enter_result - def __exit__(self, *exc_details): - if not self._exit_callbacks: - return - - # This looks complicated, but it is really just - # setting up a chain of try-expect statements to ensure - # that outer callbacks still get invoked even if an - # inner one throws an exception - def _invoke_next_callback(exc_details): - # Callbacks are removed from the list in FIFO order - # but the recursion means they're invoked in LIFO order - cb = self._exit_callbacks.popleft() - if not self._exit_callbacks: - # Innermost callback is invoked directly - return cb(*exc_details) - # More callbacks left, so descend another level in the stack - try: - suppress_exc = _invoke_next_callback(exc_details) - except: - suppress_exc = cb(*sys.exc_info()) - # Check if this cb suppressed the inner exception - if not suppress_exc: - raise - else: - # Check if inner cb suppressed the original exception - if suppress_exc: - exc_details = (None, None, None) - suppress_exc = cb(*exc_details) or suppress_exc - return suppress_exc - # Kick off the recursive chain - return _invoke_next_callback(exc_details) + def __exit__(self, *excinfo): + pass diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py index c466ef04..765bd49e 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py @@ -1,55 +1,94 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import +import importlib +import os +from appdirs import user_cache_dir +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") +import pip_shims.shims +from pip_shims.models import ShimmedPathCollection, ImportTypes -__all__ = [ - "InstallRequirement", - "parse_requirements", - "RequirementSet", - "user_cache_dir", - "FAVORITE_HASH", - "is_file_url", - "url_to_path", - "PackageFinder", - "FormatControl", - "Wheel", - "Command", - "cmdoptions", - "get_installed_distributions", - "PyPI", - "SafeFileCache", - "InstallationError", - "parse_version", - "pip_version", - "install_req_from_editable", - "install_req_from_line", - "user_cache_dir" -] +InstallationCandidate = ShimmedPathCollection("InstallationCandidate", ImportTypes.CLASS) +InstallationCandidate.create_path("models.candidate", "18.0", "9999") +InstallationCandidate.create_path("index", "7.0.3", "10.9.9") -from pipenv.vendor.appdirs import user_cache_dir -from pip_shims.shims import ( - InstallRequirement, - parse_requirements, - RequirementSet, - FAVORITE_HASH, - is_file_url, - url_to_path, - PackageFinder, - FormatControl, - Wheel, - Command, - cmdoptions, - get_installed_distributions, - PyPI, - SafeFileCache, - InstallationError, - parse_version, - pip_version, -) +PIP_VERSION = tuple(map(int, pip_shims.shims.parsed_pip_version.parsed_version.base_version.split("."))) + +RequirementTracker = pip_shims.shims.RequirementTracker + +def do_import(module_path, subimport=None, old_path=None): + old_path = old_path or module_path + pip_path = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") + prefixes = ["{}._internal".format(pip_path), pip_path] + paths = [module_path, old_path] + search_order = [ + "{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None + ] + package = subimport if subimport else None + for to_import in search_order: + if not subimport: + to_import, _, package = to_import.rpartition(".") + try: + imported = importlib.import_module(to_import) + except ImportError: + continue + else: + return getattr(imported, package) + + +InstallRequirement = pip_shims.shims.InstallRequirement +InstallationError = pip_shims.shims.InstallationError +parse_requirements = pip_shims.shims.parse_requirements +RequirementSet = pip_shims.shims.RequirementSet +SafeFileCache = pip_shims.shims.SafeFileCache +FAVORITE_HASH = pip_shims.shims.FAVORITE_HASH +path_to_url = pip_shims.shims.path_to_url +url_to_path = pip_shims.shims.url_to_path +PackageFinder = pip_shims.shims.PackageFinder +FormatControl = pip_shims.shims.FormatControl +InstallCommand = pip_shims.shims.InstallCommand +Wheel = pip_shims.shims.Wheel +cmdoptions = pip_shims.shims.cmdoptions +get_installed_distributions = pip_shims.shims.get_installed_distributions +PyPI = pip_shims.shims.PyPI +stdlib_pkgs = pip_shims.shims.stdlib_pkgs +DEV_PKGS = pip_shims.shims.DEV_PKGS +Link = pip_shims.shims.Link +Session = do_import("_vendor.requests.sessions", "Session") +Resolver = pip_shims.shims.Resolver +VcsSupport = pip_shims.shims.VcsSupport +WheelCache = pip_shims.shims.WheelCache +pip_version = pip_shims.shims.pip_version # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. -if parse_version(pip_version) < parse_version('18.1'): +if PIP_VERSION < (18, 1): install_req_from_line = InstallRequirement.from_line install_req_from_editable = InstallRequirement.from_editable else: - from pip_shims.shims import ( - install_req_from_editable, install_req_from_line + install_req_from_line = do_import("req.constructors", "install_req_from_line") + install_req_from_editable = do_import( + "req.constructors", "install_req_from_editable" ) + + +def is_vcs_url(link): + if PIP_VERSION < (19, 3): + _is_vcs_url = do_import("download", "is_vcs_url") + return _is_vcs_url(link) + + return link.is_vcs + + +def is_file_url(link): + if PIP_VERSION < (19, 3): + _is_file_url = do_import("download", "is_file_url") + return _is_file_url(link) + + return link.is_file + + +def is_dir_url(link): + if PIP_VERSION < (19, 3): + _is_dir_url = do_import("download", "is_dir_url") + return _is_dir_url(link) + + return link.is_existing_dir() diff --git a/pipenv/patched/piptools/_compat/tempfile.py b/pipenv/patched/piptools/_compat/tempfile.py index a003d080..dc7e9ef9 100644 --- a/pipenv/patched/piptools/_compat/tempfile.py +++ b/pipenv/patched/piptools/_compat/tempfile.py @@ -40,8 +40,10 @@ class TemporaryDirectory(object): # up due to missing globals if "None" not in str(ex): raise - print("ERROR: {!r} while cleaning up {!r}".format(ex, self,), - file=_sys.stderr) + print( + "ERROR: {!r} while cleaning up {!r}".format(ex, self), + file=_sys.stderr, + ) return self._closed = True diff --git a/pipenv/patched/piptools/cache.py b/pipenv/patched/piptools/cache.py index 610a4f37..14a276db 100644 --- a/pipenv/patched/piptools/cache.py +++ b/pipenv/patched/piptools/cache.py @@ -1,6 +1,5 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals import json import os @@ -19,23 +18,24 @@ class CorruptCacheError(PipToolsError): def __str__(self): lines = [ - 'The dependency cache seems to have been corrupted.', - 'Inspect, or delete, the following file:', - ' {}'.format(self.path), + "The dependency cache seems to have been corrupted.", + "Inspect, or delete, the following file:", + " {}".format(self.path), ] return os.linesep.join(lines) def read_cache_file(cache_file_path): - with open(cache_file_path, 'r') as cache_file: + with open(cache_file_path, "r") as cache_file: try: doc = json.load(cache_file) except ValueError: raise CorruptCacheError(cache_file_path) # Check version and load the contents - assert doc['__format__'] == 1, 'Unknown cache file format' - return doc['dependencies'] + if doc["__format__"] != 1: + raise AssertionError("Unknown cache file format") + return doc["dependencies"] class DependencyCache(object): @@ -48,13 +48,14 @@ class DependencyCache(object): Where X.Y indicates the Python version. """ + def __init__(self, cache_dir=None): if cache_dir is None: cache_dir = CACHE_DIR if not os.path.isdir(cache_dir): os.makedirs(cache_dir) - py_version = '.'.join(str(digit) for digit in sys.version_info[:2]) - cache_filename = 'depcache-py{}.json'.format(py_version) + py_version = ".".join(str(digit) for digit in sys.version_info[:2]) + cache_filename = "depcache-py{}.json".format(py_version) self._cache_file = os.path.join(cache_dir, cache_filename) self._cache = None @@ -71,13 +72,14 @@ class DependencyCache(object): def as_cache_key(self, ireq): """ - Given a requirement, return its cache key. This behavior is a little weird in order to allow backwards - compatibility with cache files. For a requirement without extras, this will return, for example: + Given a requirement, return its cache key. This behavior is a little weird + in order to allow backwards compatibility with cache files. For a requirement + without extras, this will return, for example: ("ipython", "2.1.0") - For a requirement with extras, the extras will be comma-separated and appended to the version, inside brackets, - like so: + For a requirement with extras, the extras will be comma-separated and appended + to the version, inside brackets, like so: ("ipython", "2.1.0[nbconvert,notebook]") """ @@ -97,11 +99,8 @@ class DependencyCache(object): def write_cache(self): """Writes the cache to disk as JSON.""" - doc = { - '__format__': 1, - 'dependencies': self._cache, - } - with open(self._cache_file, 'w') as f: + doc = {"__format__": 1, "dependencies": self._cache} + with open(self._cache_file, "w") as f: json.dump(doc, f, sort_keys=True) def clear(self): @@ -122,10 +121,6 @@ class DependencyCache(object): self.cache[pkgname][pkgversion_and_extras] = values self.write_cache() - def get(self, ireq, default=None): - pkgname, pkgversion_and_extras = self.as_cache_key(ireq) - return self.cache.get(pkgname, {}).get(pkgversion_and_extras, default) - def reverse_dependencies(self, ireqs): """ Returns a lookup table of reverse dependencies for all the given ireqs. @@ -157,8 +152,10 @@ class DependencyCache(object): 'pyflakes': ['flake8']} """ - # First, collect all the dependencies into a sequence of (parent, child) tuples, like [('flake8', 'pep8'), - # ('flake8', 'mccabe'), ...] - return lookup_table((key_from_req(Requirement(dep_name)), name) - for name, version_and_extras in cache_keys - for dep_name in self.cache[name][version_and_extras]) + # First, collect all the dependencies into a sequence of (parent, child) + # tuples, like [('flake8', 'pep8'), ('flake8', 'mccabe'), ...] + return lookup_table( + (key_from_req(Requirement(dep_name)), name) + for name, version_and_extras in cache_keys + for dep_name in self.cache[name][version_and_extras] + ) diff --git a/pipenv/patched/piptools/click.py b/pipenv/patched/piptools/click.py index 4bab11cb..86f1612c 100644 --- a/pipenv/patched/piptools/click.py +++ b/pipenv/patched/piptools/click.py @@ -1,6 +1,6 @@ from __future__ import absolute_import import click -click.disable_unicode_literals_warning = True - from click import * # noqa + +click.disable_unicode_literals_warning = True diff --git a/pipenv/patched/piptools/exceptions.py b/pipenv/patched/piptools/exceptions.py index 77c5bd40..5aac84bb 100644 --- a/pipenv/patched/piptools/exceptions.py +++ b/pipenv/patched/piptools/exceptions.py @@ -19,40 +19,35 @@ class NoCandidateFound(PipToolsError): else: versions.append(version) - lines = [ - 'Could not find a version that matches {}'.format(self.ireq), - ] + lines = ["Could not find a version that matches {}".format(self.ireq)] if versions: - lines.append('Tried: {}'.format(', '.join(versions))) + lines.append("Tried: {}".format(", ".join(versions))) if pre_versions: if self.finder.allow_all_prereleases: - line = 'Tried' + line = "Tried" else: - line = 'Skipped' + line = "Skipped" - line += ' pre-versions: {}'.format(', '.join(pre_versions)) + line += " pre-versions: {}".format(", ".join(pre_versions)) lines.append(line) if versions or pre_versions: - lines.append('There are incompatible versions in the resolved dependencies.') - else: - lines.append('No versions found') - lines.append('{} {} reachable?'.format( - 'Were' if len(self.finder.index_urls) > 1 else 'Was', ' or '.join(self.finder.index_urls)) + lines.append( + "There are incompatible versions in the resolved dependencies:" ) - return '\n'.join(lines) - - -class UnsupportedConstraint(PipToolsError): - def __init__(self, message, constraint): - super(UnsupportedConstraint, self).__init__(message) - self.constraint = constraint - - def __str__(self): - message = super(UnsupportedConstraint, self).__str__() - return '{} (constraint was: {})'.format(message, str(self.constraint)) + source_ireqs = getattr(self.ireq, "_source_ireqs", []) + lines.extend(" {}".format(ireq) for ireq in source_ireqs) + else: + lines.append("No versions found") + lines.append( + "{} {} reachable?".format( + "Were" if len(self.finder.index_urls) > 1 else "Was", + " or ".join(self.finder.index_urls), + ) + ) + return "\n".join(lines) class IncompatibleRequirements(PipToolsError): diff --git a/pipenv/patched/piptools/io.py b/pipenv/patched/piptools/io.py deleted file mode 100644 index b6bca675..00000000 --- a/pipenv/patched/piptools/io.py +++ /dev/null @@ -1,644 +0,0 @@ -# -*- coding: utf-8 -*- -# -# NOTE: -# The classes in this module are vendored from boltons: -# https://github.com/mahmoud/boltons/blob/master/boltons/fileutils.py -# -"""Virtually every Python programmer has used Python for wrangling -disk contents, and ``fileutils`` collects solutions to some of the -most commonly-found gaps in the standard library. -""" - -from __future__ import print_function - -import os -import re -import sys -import stat -import errno -import fnmatch -from shutil import copy2, copystat, Error - - -__all__ = ['mkdir_p', 'atomic_save', 'AtomicSaver', 'FilePerms', - 'iter_find_files', 'copytree'] - - -FULL_PERMS = 511 # 0777 that both Python 2 and 3 can digest -RW_PERMS = 438 -_SINGLE_FULL_PERM = 7 # or 07 in Python 2 -try: - basestring -except NameError: - unicode = str # Python 3 compat - basestring = (str, bytes) - - -def mkdir_p(path): - """Creates a directory and any parent directories that may need to - be created along the way, without raising errors for any existing - directories. This function mimics the behavior of the ``mkdir -p`` - command available in Linux/BSD environments, but also works on - Windows. - """ - try: - os.makedirs(path) - except OSError as exc: - if exc.errno == errno.EEXIST and os.path.isdir(path): - return - raise - return - - -class FilePerms(object): - """The :class:`FilePerms` type is used to represent standard POSIX - filesystem permissions: - - * Read - * Write - * Execute - - Across three classes of user: - - * Owning (u)ser - * Owner's (g)roup - * Any (o)ther user - - This class assists with computing new permissions, as well as - working with numeric octal ``777``-style and ``rwx``-style - permissions. Currently it only considers the bottom 9 permission - bits; it does not support sticky bits or more advanced permission - systems. - - Args: - user (str): A string in the 'rwx' format, omitting characters - for which owning user's permissions are not provided. - group (str): A string in the 'rwx' format, omitting characters - for which owning group permissions are not provided. - other (str): A string in the 'rwx' format, omitting characters - for which owning other/world permissions are not provided. - - There are many ways to use :class:`FilePerms`: - - >>> FilePerms(user='rwx', group='xrw', other='wxr') # note character order - FilePerms(user='rwx', group='rwx', other='rwx') - >>> int(FilePerms('r', 'r', '')) - 288 - >>> oct(288)[-3:] # XXX Py3k - '440' - - See also the :meth:`FilePerms.from_int` and - :meth:`FilePerms.from_path` classmethods for useful alternative - ways to construct :class:`FilePerms` objects. - """ - # TODO: consider more than the lower 9 bits - class _FilePermProperty(object): - _perm_chars = 'rwx' - _perm_set = frozenset('rwx') - _perm_val = {'r': 4, 'w': 2, 'x': 1} # for sorting - - def __init__(self, attribute, offset): - self.attribute = attribute - self.offset = offset - - def __get__(self, fp_obj, type_=None): - if fp_obj is None: - return self - return getattr(fp_obj, self.attribute) - - def __set__(self, fp_obj, value): - cur = getattr(fp_obj, self.attribute) - if cur == value: - return - try: - invalid_chars = set(str(value)) - self._perm_set - except TypeError: - raise TypeError('expected string, not %r' % value) - if invalid_chars: - raise ValueError('got invalid chars %r in permission' - ' specification %r, expected empty string' - ' or one or more of %r' - % (invalid_chars, value, self._perm_chars)) - - sort_key = (lambda c: self._perm_val[c]) - new_value = ''.join(sorted(set(value), - key=sort_key, reverse=True)) - setattr(fp_obj, self.attribute, new_value) - self._update_integer(fp_obj, new_value) - - def _update_integer(self, fp_obj, value): - mode = 0 - key = 'xwr' - for symbol in value: - bit = 2 ** key.index(symbol) - mode |= (bit << (self.offset * 3)) - fp_obj._integer |= mode - - def __init__(self, user='', group='', other=''): - self._user, self._group, self._other = '', '', '' - self._integer = 0 - self.user = user - self.group = group - self.other = other - - @classmethod - def from_int(cls, i): - """Create a :class:`FilePerms` object from an integer. - - >>> FilePerms.from_int(0o644) # note the leading zero-oh for octal - FilePerms(user='rw', group='r', other='r') - """ - i &= FULL_PERMS - key = ('', 'x', 'w', 'xw', 'r', 'rx', 'rw', 'rwx') - parts = [] - while i: - parts.append(key[i & _SINGLE_FULL_PERM]) - i >>= 3 - parts.reverse() - return cls(*parts) - - @classmethod - def from_path(cls, path): - """Make a new :class:`FilePerms` object based on the permissions - assigned to the file or directory at *path*. - - Args: - path (str): Filesystem path of the target file. - - >>> from os.path import expanduser - >>> 'r' in FilePerms.from_path(expanduser('~')).user # probably - True - """ - stat_res = os.stat(path) - return cls.from_int(stat.S_IMODE(stat_res.st_mode)) - - def __int__(self): - return self._integer - - # Sphinx tip: attribute docstrings come after the attribute - user = _FilePermProperty('_user', 2) - "Stores the ``rwx``-formatted *user* permission." - group = _FilePermProperty('_group', 1) - "Stores the ``rwx``-formatted *group* permission." - other = _FilePermProperty('_other', 0) - "Stores the ``rwx``-formatted *other* permission." - - def __repr__(self): - cn = self.__class__.__name__ - return ('%s(user=%r, group=%r, other=%r)' - % (cn, self.user, self.group, self.other)) - -#### - - -_TEXT_OPENFLAGS = os.O_RDWR | os.O_CREAT | os.O_EXCL -if hasattr(os, 'O_NOINHERIT'): - _TEXT_OPENFLAGS |= os.O_NOINHERIT -if hasattr(os, 'O_NOFOLLOW'): - _TEXT_OPENFLAGS |= os.O_NOFOLLOW -_BIN_OPENFLAGS = _TEXT_OPENFLAGS -if hasattr(os, 'O_BINARY'): - _BIN_OPENFLAGS |= os.O_BINARY - - -try: - import fcntl as fcntl -except ImportError: - def set_cloexec(fd): - "Dummy set_cloexec for platforms without fcntl support" - pass -else: - def set_cloexec(fd): - """Does a best-effort :func:`fcntl.fcntl` call to set a fd to be - automatically closed by any future child processes. - - Implementation from the :mod:`tempfile` module. - """ - try: - flags = fcntl.fcntl(fd, fcntl.F_GETFD, 0) - except IOError: - pass - else: - # flags read successfully, modify - flags |= fcntl.FD_CLOEXEC - fcntl.fcntl(fd, fcntl.F_SETFD, flags) - return - - -def atomic_save(dest_path, **kwargs): - """A convenient interface to the :class:`AtomicSaver` type. See the - :class:`AtomicSaver` documentation for details. - """ - return AtomicSaver(dest_path, **kwargs) - - -def path_to_unicode(path): - if isinstance(path, unicode): - return path - encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() - return path.decode(encoding) - - -if os.name == 'nt': - import ctypes - from ctypes import c_wchar_p - from ctypes.wintypes import DWORD, LPVOID - - _ReplaceFile = ctypes.windll.kernel32.ReplaceFile - _ReplaceFile.argtypes = [c_wchar_p, c_wchar_p, c_wchar_p, - DWORD, LPVOID, LPVOID] - - def replace(src, dst): - # argument names match stdlib docs, docstring below - try: - # ReplaceFile fails if the dest file does not exist, so - # first try to rename it into position - os.rename(src, dst) - return - except WindowsError as we: - if we.errno == errno.EEXIST: - pass # continue with the ReplaceFile logic below - else: - raise - - src = path_to_unicode(src) - dst = path_to_unicode(dst) - res = _ReplaceFile(c_wchar_p(dst), c_wchar_p(src), - None, 0, None, None) - if not res: - raise OSError('failed to replace %r with %r' % (dst, src)) - return - - def atomic_rename(src, dst, overwrite=False): - "Rename *src* to *dst*, replacing *dst* if *overwrite is True" - if overwrite: - replace(src, dst) - else: - os.rename(src, dst) - return -else: - # wrapper func for cross compat + docs - def replace(src, dst): - # os.replace does the same thing on unix - return os.rename(src, dst) - - def atomic_rename(src, dst, overwrite=False): - "Rename *src* to *dst*, replacing *dst* if *overwrite is True" - if overwrite: - os.rename(src, dst) - else: - os.link(src, dst) - os.unlink(dst) - return - - -_atomic_rename = atomic_rename # backwards compat - -replace.__doc__ = """Similar to :func:`os.replace` in Python 3.3+, -this function will atomically create or replace the file at path -*dst* with the file at path *src*. - -On Windows, this function uses the ReplaceFile API for maximum -possible atomicity on a range of filesystems. -""" - - -class AtomicSaver(object): - """``AtomicSaver`` is a configurable `context manager`_ that provides - a writable :class:`file` which will be moved into place as long as - no exceptions are raised within the context manager's block. These - "part files" are created in the same directory as the destination - path to ensure atomic move operations (i.e., no cross-filesystem - moves occur). - - Args: - dest_path (str): The path where the completed file will be - written. - overwrite (bool): Whether to overwrite the destination file if - it exists at completion time. Defaults to ``True``. - file_perms (int): Integer representation of file permissions - for the newly-created file. Defaults are, when the - destination path already exists, to copy the permissions - from the previous file, or if the file did not exist, to - respect the user's configured `umask`_, usually resulting - in octal 0644 or 0664. - part_file (str): Name of the temporary *part_file*. Defaults - to *dest_path* + ``.part``. Note that this argument is - just the filename, and not the full path of the part - file. To guarantee atomic saves, part files are always - created in the same directory as the destination path. - overwrite_part (bool): Whether to overwrite the *part_file*, - should it exist at setup time. Defaults to ``False``, - which results in an :exc:`OSError` being raised on - pre-existing part files. Be careful of setting this to - ``True`` in situations when multiple threads or processes - could be writing to the same part file. - rm_part_on_exc (bool): Remove *part_file* on exception cases. - Defaults to ``True``, but ``False`` can be useful for - recovery in some cases. Note that resumption is not - automatic and by default an :exc:`OSError` is raised if - the *part_file* exists. - - Practically, the AtomicSaver serves a few purposes: - - * Avoiding overwriting an existing, valid file with a partially - written one. - * Providing a reasonable guarantee that a part file only has one - writer at a time. - * Optional recovery of partial data in failure cases. - - .. _context manager: https://docs.python.org/3/reference/compound_stmts.html#with - .. _umask: https://en.wikipedia.org/wiki/Umask - - """ - _default_file_perms = RW_PERMS - - # TODO: option to abort if target file modify date has changed since start? - def __init__(self, dest_path, **kwargs): - self.dest_path = dest_path - self.overwrite = kwargs.pop('overwrite', True) - self.file_perms = kwargs.pop('file_perms', None) - self.overwrite_part = kwargs.pop('overwrite_part', False) - self.part_filename = kwargs.pop('part_file', None) - self.rm_part_on_exc = kwargs.pop('rm_part_on_exc', True) - self.text_mode = kwargs.pop('text_mode', False) # for windows - self.buffering = kwargs.pop('buffering', -1) - if kwargs: - raise TypeError('unexpected kwargs: %r' % (kwargs.keys(),)) - - self.dest_path = os.path.abspath(self.dest_path) - self.dest_dir = os.path.dirname(self.dest_path) - if not self.part_filename: - self.part_path = dest_path + '.part' - else: - self.part_path = os.path.join(self.dest_dir, self.part_filename) - self.mode = 'w+' if self.text_mode else 'w+b' - self.open_flags = _TEXT_OPENFLAGS if self.text_mode else _BIN_OPENFLAGS - - self.part_file = None - - def _open_part_file(self): - do_chmod = True - file_perms = self.file_perms - if file_perms is None: - try: - # try to copy from file being replaced - stat_res = os.stat(self.dest_path) - file_perms = stat.S_IMODE(stat_res.st_mode) - except (OSError, IOError): - # default if no destination file exists - file_perms = self._default_file_perms - do_chmod = False # respect the umask - - fd = os.open(self.part_path, self.open_flags, file_perms) - set_cloexec(fd) - self.part_file = os.fdopen(fd, self.mode, self.buffering) - - # if default perms are overridden by the user or previous dest_path - # chmod away the effects of the umask - if do_chmod: - try: - os.chmod(self.part_path, file_perms) - except (OSError, IOError): - self.part_file.close() - raise - return - - def setup(self): - """Called on context manager entry (the :keyword:`with` statement), - the ``setup()`` method creates the temporary file in the same - directory as the destination file. - - ``setup()`` tests for a writable directory with rename permissions - early, as the part file may not be written to immediately (not - using :func:`os.access` because of the potential issues of - effective vs. real privileges). - - If the caller is not using the :class:`AtomicSaver` as a - context manager, this method should be called explicitly - before writing. - """ - if os.path.lexists(self.dest_path): - if not self.overwrite: - raise OSError(errno.EEXIST, - 'Overwrite disabled and file already exists', - self.dest_path) - if self.overwrite_part and os.path.lexists(self.part_path): - os.unlink(self.part_path) - self._open_part_file() - return - - def __enter__(self): - self.setup() - return self.part_file - - def __exit__(self, exc_type, exc_val, exc_tb): - self.part_file.close() - if exc_type: - if self.rm_part_on_exc: - try: - os.unlink(self.part_path) - except Exception: - pass # avoid masking original error - return - try: - atomic_rename(self.part_path, self.dest_path, - overwrite=self.overwrite) - except OSError: - if self.rm_part_on_exc: - try: - os.unlink(self.part_path) - except Exception: - pass # avoid masking original error - raise # could not save destination file - return - - -def iter_find_files(directory, patterns, ignored=None): - """Returns a generator that yields file paths under a *directory*, - matching *patterns* using `glob`_ syntax (e.g., ``*.txt``). Also - supports *ignored* patterns. - - Args: - directory (str): Path that serves as the root of the - search. Yielded paths will include this as a prefix. - patterns (str or list): A single pattern or list of - glob-formatted patterns to find under *directory*. - ignored (str or list): A single pattern or list of - glob-formatted patterns to ignore. - - For example, finding Python files in the directory of this module: - - >>> files = set(iter_find_files(os.path.dirname(__file__), '*.py')) - - Or, Python files while ignoring emacs lockfiles: - - >>> filenames = iter_find_files('.', '*.py', ignored='.#*') - - .. _glob: https://en.wikipedia.org/wiki/Glob_%28programming%29 - - """ - if isinstance(patterns, basestring): - patterns = [patterns] - pats_re = re.compile('|'.join([fnmatch.translate(p) for p in patterns])) - - if not ignored: - ignored = [] - elif isinstance(ignored, basestring): - ignored = [ignored] - ign_re = re.compile('|'.join([fnmatch.translate(p) for p in ignored])) - for root, dirs, files in os.walk(directory): - for basename in files: - if pats_re.match(basename): - if ignored and ign_re.match(basename): - continue - filename = os.path.join(root, basename) - yield filename - return - - -def copy_tree(src, dst, symlinks=False, ignore=None): - """The ``copy_tree`` function is an exact copy of the built-in - :func:`shutil.copytree`, with one key difference: it will not - raise an exception if part of the tree already exists. It achieves - this by using :func:`mkdir_p`. - - Args: - src (str): Path of the source directory to copy. - dst (str): Destination path. Existing directories accepted. - symlinks (bool): If ``True``, copy symlinks rather than their - contents. - ignore (callable): A callable that takes a path and directory - listing, returning the files within the listing to be ignored. - - For more details, check out :func:`shutil.copytree` and - :func:`shutil.copy2`. - - """ - names = os.listdir(src) - if ignore is not None: - ignored_names = ignore(src, names) - else: - ignored_names = set() - - mkdir_p(dst) - errors = [] - for name in names: - if name in ignored_names: - continue - srcname = os.path.join(src, name) - dstname = os.path.join(dst, name) - try: - if symlinks and os.path.islink(srcname): - linkto = os.readlink(srcname) - os.symlink(linkto, dstname) - elif os.path.isdir(srcname): - copytree(srcname, dstname, symlinks, ignore) - else: - # Will raise a SpecialFileError for unsupported file types - copy2(srcname, dstname) - # catch the Error from the recursive copytree so that we can - # continue with other files - except Error as e: - errors.extend(e.args[0]) - except EnvironmentError as why: - errors.append((srcname, dstname, str(why))) - try: - copystat(src, dst) - except OSError as why: - if WindowsError is not None and isinstance(why, WindowsError): - # Copying file access times may fail on Windows - pass - else: - errors.append((src, dst, str(why))) - if errors: - raise Error(errors) - - -copytree = copy_tree # alias for drop-in replacement of shutil - - -try: - file -except NameError: - file = object - - -# like open(os.devnull) but with even fewer side effects -class DummyFile(file): - # TODO: raise ValueErrors on closed for all methods? - # TODO: enforce read/write - def __init__(self, path, mode='r', buffering=None): - self.name = path - self.mode = mode - self.closed = False - self.errors = None - self.isatty = False - self.encoding = None - self.newlines = None - self.softspace = 0 - - def close(self): - self.closed = True - - def fileno(self): - return -1 - - def flush(self): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def next(self): - raise StopIteration() - - def read(self, size=0): - if self.closed: - raise ValueError('I/O operation on a closed file') - return '' - - def readline(self, size=0): - if self.closed: - raise ValueError('I/O operation on a closed file') - return '' - - def readlines(self, size=0): - if self.closed: - raise ValueError('I/O operation on a closed file') - return [] - - def seek(self): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def tell(self): - if self.closed: - raise ValueError('I/O operation on a closed file') - return 0 - - def truncate(self): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def write(self, string): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def writelines(self, list_of_strings): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def __next__(self): - raise StopIteration() - - def __enter__(self): - if self.closed: - raise ValueError('I/O operation on a closed file') - return - - def __exit__(self, exc_type, exc_val, exc_tb): - return diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py index 0d460f64..7abf5c76 100644 --- a/pipenv/patched/piptools/locations.py +++ b/pipenv/patched/piptools/locations.py @@ -1,19 +1,27 @@ import os from shutil import rmtree -from .click import secho from ._compat import user_cache_dir +from .click import secho # The user_cache_dir helper comes straight from pipenv.patched.notpip itself -CACHE_DIR = user_cache_dir('pip-tools') +try: + from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR +except ImportError: + CACHE_DIR = user_cache_dir("pipenv") # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the # preferred place to store caches for any platform. This has been addressed # in pip-tools==1.0.5, but to be good citizens, we point this out explicitly # to the user when this directory is still found. -LEGACY_CACHE_DIR = os.path.expanduser('~/.pip-tools') +LEGACY_CACHE_DIR = os.path.expanduser("~/.pip-tools") if os.path.exists(LEGACY_CACHE_DIR): - secho('Removing old cache dir {} (new cache dir is {})'.format(LEGACY_CACHE_DIR, CACHE_DIR), fg='yellow') + secho( + "Removing old cache dir {} (new cache dir is {})".format( + LEGACY_CACHE_DIR, CACHE_DIR + ), + fg="yellow", + ) rmtree(LEGACY_CACHE_DIR) diff --git a/pipenv/patched/piptools/logging.py b/pipenv/patched/piptools/logging.py index 98f05287..488a8a2e 100644 --- a/pipenv/patched/piptools/logging.py +++ b/pipenv/patched/piptools/logging.py @@ -1,34 +1,31 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) - -import sys +from __future__ import absolute_import, division, print_function, unicode_literals from . import click class LogContext(object): - def __init__(self, verbose=False): - self.verbose = verbose + def __init__(self, verbosity=0): + self.verbosity = verbosity def log(self, *args, **kwargs): + kwargs.setdefault("err", True) click.secho(*args, **kwargs) def debug(self, *args, **kwargs): - if self.verbose: + if self.verbosity >= 1: self.log(*args, **kwargs) def info(self, *args, **kwargs): - self.log(*args, **kwargs) + if self.verbosity >= 0: + self.log(*args, **kwargs) def warning(self, *args, **kwargs): - kwargs.setdefault('fg', 'yellow') - kwargs.setdefault('file', sys.stderr) + kwargs.setdefault("fg", "yellow") self.log(*args, **kwargs) def error(self, *args, **kwargs): - kwargs.setdefault('fg', 'red') - kwargs.setdefault('file', sys.stderr) + kwargs.setdefault("fg", "red") self.log(*args, **kwargs) diff --git a/pipenv/patched/piptools/repositories/base.py b/pipenv/patched/piptools/repositories/base.py index 57e85fda..dd73ff32 100644 --- a/pipenv/patched/piptools/repositories/base.py +++ b/pipenv/patched/piptools/repositories/base.py @@ -1,6 +1,5 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals from abc import ABCMeta, abstractmethod from contextlib import contextmanager @@ -10,7 +9,6 @@ from six import add_metaclass @add_metaclass(ABCMeta) class BaseRepository(object): - def clear_caches(self): """Should clear any caches used by the implementation.""" @@ -27,7 +25,7 @@ class BaseRepository(object): @abstractmethod def get_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py index 480ad1ed..c1bcf9d1 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py @@ -1,12 +1,12 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals from contextlib import contextmanager -from piptools.utils import as_tuple, key_from_req, make_install_requirement -from .base import BaseRepository from .._compat import FAVORITE_HASH +from .base import BaseRepository + +from piptools.utils import as_tuple, key_from_req, make_install_requirement def ireq_satisfied_by_existing_pin(ireq, existing_pin): @@ -28,10 +28,15 @@ class LocalRequirementsRepository(BaseRepository): requirements file, we prefer that version over the best match found in PyPI. This keeps updates to the requirements.txt down to a minimum. """ + def __init__(self, existing_pins, proxied_repository): self.repository = proxied_repository self.existing_pins = existing_pins + @property + def options(self): + return self.repository.options + @property def finder(self): return self.repository.finder @@ -56,7 +61,8 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers + project, version, ireq.extras, ireq.markers, + constraint=ireq.constraint ) else: return self.repository.find_best_match(ireq, prereleases) @@ -68,12 +74,11 @@ class LocalRequirementsRepository(BaseRepository): key = key_from_req(ireq.req) existing_pin = self.existing_pins.get(key) if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): - hashes = existing_pin.options.get('hashes', {}) + hashes = existing_pin.options.get("hashes", {}) hexdigests = hashes.get(FAVORITE_HASH) if hexdigests: return { - ':'.join([FAVORITE_HASH, hexdigest]) - for hexdigest in hexdigests + ":".join([FAVORITE_HASH, hexdigest]) for hexdigest in hexdigests } return self.repository.get_hashes(ireq) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2a0743a3..ff02d36c 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,44 +1,59 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals + +import collections import copy import hashlib import os from contextlib import contextmanager +from functools import partial from shutil import rmtree -from .._compat import ( - is_file_url, - url_to_path, - PackageFinder, - RequirementSet, - Wheel, - FAVORITE_HASH, - TemporaryDirectory, - PyPI, - InstallRequirement, - SafeFileCache -) -os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") -from pip_shims.shims import do_import, VcsSupport, WheelCache from packaging.requirements import Requirement -from packaging.specifiers import SpecifierSet, Specifier -InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -from pipenv.patched.notpip._internal.resolve import Resolver as PipResolver +from packaging.specifiers import Specifier, SpecifierSet - -from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR +from .._compat import ( + FAVORITE_HASH, + PIP_VERSION, + InstallationError, + InstallRequirement, + Link, + PyPI, + RequirementSet, + RequirementTracker, + Resolver as PipResolver, + SafeFileCache, + TemporaryDirectory, + VcsSupport, + Wheel, + WheelCache, + contextlib, + is_dir_url, + is_file_url, + is_vcs_url, + path_to_url, + pip_version, + url_to_path, +) +from ..cache import CACHE_DIR +from ..click import progressbar from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, - make_install_requirement, clean_requires_python) +from ..logging import log +from ..utils import ( + dedup, + clean_requires_python, + create_install_command, + fs_str, + is_pinned_requirement, + is_url_requirement, + lookup_table, + make_install_requirement, +) from .base import BaseRepository -try: - from pipenv.patched.notpip._internal.req.req_tracker import RequirementTracker -except ImportError: - @contextmanager - def RequirementTracker(): - yield +os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") +FILE_CHUNK_SIZE = 4096 +FileStream = collections.namedtuple("FileStream", "stream size") class HashCache(SafeFileCache): @@ -74,7 +89,7 @@ class HashCache(SafeFileCache): def _get_file_hash(self, location): h = hashlib.new(FAVORITE_HASH) - with open_local_or_remote_file(location, self.session) as fp: + with open_local_or_remote_file(location, self.session) as (fp, size): for chunk in iter(lambda: fp.read(8096), b""): h.update(chunk) return ":".join([FAVORITE_HASH, h.hexdigest()]) @@ -89,22 +104,22 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ - def __init__(self, pip_options, session, use_json=False): - self.session = session + + def __init__(self, pip_args, session=None, build_isolation=False, use_json=False): + self.build_isolation = build_isolation self.use_json = use_json - self.pip_options = pip_options - index_urls = [pip_options.index_url] + pip_options.extra_index_urls - if pip_options.no_index: - index_urls = [] + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, + # and pre) are deferred to pip. + command = create_install_command() + self.options, _ = command.parse_args(pip_args) - self.finder = PackageFinder( - find_links=pip_options.find_links, - index_urls=index_urls, - trusted_hosts=pip_options.trusted_hosts, - allow_all_prereleases=pip_options.pre, - process_dependency_links=pip_options.process_dependency_links, - session=self.session, + if session is None: + session = command._build_session(self.options) + self.session = session + self.finder = command._build_package_finder( + options=self.options, session=self.session ) # Caches @@ -124,16 +139,16 @@ class PyPIRepository(BaseRepository): # Setup file paths self.freshen_build_caches() - self._download_dir = fs_str(os.path.join(CACHE_DIR, 'pkgs')) - self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, 'wheels')) + self._download_dir = fs_str(os.path.join(CACHE_DIR, "pkgs")) + self._wheel_download_dir = fs_str(os.path.join(CACHE_DIR, "wheels")) def freshen_build_caches(self): """ Start with fresh build/source caches. Will remove any old build caches from disk automatically. """ - self._build_dir = TemporaryDirectory(fs_str('build')) - self._source_dir = TemporaryDirectory(fs_str('source')) + self._build_dir = TemporaryDirectory(fs_str("build")) + self._source_dir = TemporaryDirectory(fs_str("source")) @property def build_dir(self): @@ -158,14 +173,16 @@ class PyPIRepository(BaseRepository): Returns a Version object that indicates the best match for the given InstallRequirement according to the external repository. """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq): return ireq # return itself as the best match all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) + candidates_by_version = lookup_table( + all_candidates, key=lambda c: c.version, unique=True + ) try: matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) + prereleases=prereleases) except TypeError: matching_versions = [candidate.version for candidate in all_candidates] @@ -173,12 +190,46 @@ class PyPIRepository(BaseRepository): matching_candidates = [candidates_by_version[ver] for ver in matching_versions] if not matching_candidates: raise NoCandidateFound(ireq, all_candidates, self.finder) - best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) + + if PIP_VERSION < (19, 1): + best_candidate = max( + matching_candidates, key=self.finder._candidate_sort_key + ) + elif PIP_VERSION < (19, 2): + evaluator = self.finder.candidate_evaluator + best_candidate = evaluator.get_best_candidate(matching_candidates) + elif PIP_VERSION < (19, 3): + evaluator = self.finder.make_candidate_evaluator(ireq.name) + best_candidate = evaluator.get_best_candidate(matching_candidates) + else: + evaluator = self.finder.make_candidate_evaluator(ireq.name) + best_candidate_result = evaluator.compute_best_candidate( + matching_candidates + ) + best_candidate = best_candidate_result.best_candidate # Turn the candidate into a pinned InstallRequirement return make_install_requirement( - best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint - ) + best_candidate.project, + best_candidate.version, + ireq.extras, + ireq.markers, + constraint=ireq.constraint, + ) + + def get_dependencies(self, ireq): + json_results = set() + + if self.use_json: + try: + json_results = self.get_json_dependencies(ireq) + except TypeError: + json_results = set() + + legacy_results = self.get_legacy_dependencies(ireq) + json_results.update(legacy_results) + + return json_results def get_json_dependencies(self, ireq): @@ -222,31 +273,16 @@ class PyPIRepository(BaseRepository): except Exception: return set() - def get_dependencies(self, ireq): - json_results = set() - - if self.use_json: - try: - json_results = self.get_json_dependencies(ireq) - except TypeError: - json_results = set() - - legacy_results = self.get_legacy_dependencies(ireq) - json_results.update(legacy_results) - - return json_results - - def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None - setup_requires = {} - dist = None - ireq.isolated = False + ireq.isolated = self.build_isolation ireq._wheel_cache = wheel_cache + if ireq and not ireq.link: + ireq.populate_link(self.finder, False, False) + if ireq.link and not ireq.link.is_wheel: + ireq.ensure_has_source_dir(self.source_dir) - try: - from pipenv.patched.notpip._internal.operations.prepare import RequirementPreparer - except ImportError: - # Pip 9 and below + if PIP_VERSION < (10,): reqset = RequirementSet( self.build_dir, self.source_dir, @@ -255,152 +291,91 @@ class PyPIRepository(BaseRepository): session=self.session, ignore_installed=True, ignore_compatibility=False, - wheel_cache=wheel_cache + wheel_cache=wheel_cache, ) results = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) else: - # pip >= 10 + from pip_shims.shims import RequirementPreparer + preparer_kwargs = { - 'build_dir': self.build_dir, - 'src_dir': self.source_dir, - 'download_dir': download_dir, - 'wheel_download_dir': self._wheel_download_dir, - 'progress_bar': 'off', - 'build_isolation': False + "build_dir": self.build_dir, + "src_dir": self.source_dir, + "download_dir": download_dir, + "wheel_download_dir": self._wheel_download_dir, + "progress_bar": "off", + "build_isolation": self.build_isolation, } resolver_kwargs = { - 'finder': self.finder, - 'session': self.session, - 'upgrade_strategy': "to-satisfy-only", - 'force_reinstall': True, - 'ignore_dependencies': False, - 'ignore_requires_python': True, - 'ignore_installed': True, - 'isolated': False, - 'wheel_cache': wheel_cache, - 'use_user_site': False, - 'ignore_compatibility': False + "finder": self.finder, + "session": self.session, + "upgrade_strategy": "to-satisfy-only", + "force_reinstall": False, + "ignore_dependencies": False, + "ignore_requires_python": True, + "ignore_installed": True, + "use_user_site": False, + "ignore_compatibility": False, + "use_pep517": True, } + make_install_req_kwargs = {"isolated": False, "wheel_cache": wheel_cache} + + if PIP_VERSION < (19, 3): + resolver_kwargs.update(**make_install_req_kwargs) + else: + from pipenv.vendor.pip_shims.shims import install_req_from_req_string + + make_install_req = partial( + install_req_from_req_string, **make_install_req_kwargs + ) + resolver_kwargs["make_install_req"] = make_install_req + del resolver_kwargs["use_pep517"] + + if PIP_VERSION >= (20,): + preparer_kwargs["session"] = self.session + del resolver_kwargs["session"] + resolver = None preparer = None + reqset = None with RequirementTracker() as req_tracker: # Pip 18 uses a requirement tracker to prevent fork bombs if req_tracker: - preparer_kwargs['req_tracker'] = req_tracker + preparer_kwargs["req_tracker"] = req_tracker preparer = RequirementPreparer(**preparer_kwargs) - resolver_kwargs['preparer'] = preparer + resolver_kwargs["preparer"] = preparer reqset = RequirementSet() ireq.is_direct = True - # reqset.add_requirement(ireq) + resolver = PipResolver(**resolver_kwargs) - resolver.require_hashes = False - results = resolver._resolve_one(reqset, ireq) - - cleanup_fn = getattr(reqset, "cleanup_files", None) - if cleanup_fn is not None: - try: - cleanup_fn() - except OSError: - pass - - if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): - if ireq.editable: - self._source_dir = TemporaryDirectory(fs_str("source")) - ireq.ensure_has_source_dir(self.source_dir) - - if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): - # Collect setup_requires info from local eggs. - # Do this after we call the preparer on these reqs to make sure their - # egg info has been created - from pipenv.utils import chdir - with chdir(ireq.setup_py_dir): - try: - from setuptools.dist import distutils - dist = distutils.core.run_setup(ireq.setup_py) - except InstallationError: - ireq.run_egg_info() - except (TypeError, ValueError, AttributeError): - pass - if not dist: - try: - dist = ireq.get_dist() - except (ImportError, ValueError, TypeError, AttributeError): - pass - if ireq.editable and dist: - setup_requires = getattr(dist, "extras_require", None) - if not setup_requires: - setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} - if not getattr(ireq, 'req', None): - try: - ireq.req = dist.as_requirement() if dist else None - except (ValueError, TypeError) as e: - pass - - # Convert setup_requires dict into a somewhat usable form. - if setup_requires: - for section in setup_requires: - python_version = section - not_python = not (section.startswith('[') and ':' in section) - - # This is for cleaning up :extras: formatted markers - # by adding them to the results of the resolver - # since any such extra would have been returned as a result anyway - for value in setup_requires[section]: - # This is a marker. - if value.startswith('[') and ':' in value: - python_version = value[1:-1] - not_python = False - # Strip out other extras. - if value.startswith('[') and ':' not in value: - not_python = True - - if ':' not in value: - try: - if not not_python: - results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) - # Anything could go wrong here -- can't be too careful. - except Exception: - pass - - # this section properly creates 'python_version' markers for cross-python - # virtualenv creation and for multi-python compatibility. - requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python - if requires_python: - marker_str = '' - # This corrects a logic error from the previous code which said that if - # we Encountered any 'requires_python' attributes, basically only create a - # single result no matter how many we resolved. This should fix - # a majority of the remaining non-deterministic resolution issues. - if any(requires_python.startswith(op) for op in Specifier._operators.keys()): - # We are checking first if we have leading specifier operator - # if not, we can assume we should be doing a == comparison - specifierset = SpecifierSet(requires_python) - # for multiple specifiers, the correct way to represent that in - # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` - from passa.internals.specifiers import cleanup_pyspecs - marker_str = str(Marker(" and ".join(dedup([ - "python_version {0[0]} '{0[1]}'".format(spec) - for spec in cleanup_pyspecs(specifierset) - ])))) - # The best way to add markers to a requirement is to make a separate requirement - # with only markers on it, and then to transfer the object istelf - marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker - if ireq in results: - results.remove(ireq) - print(marker_to_add) - ireq.req.marker = marker_to_add + require_hashes = False + if PIP_VERSION < (20,): + resolver.require_hashes = require_hashes + results = resolver._resolve_one(reqset, ireq) + else: + results = resolver._resolve_one(reqset, ireq, require_hashes) + try: + reqset.cleanup_files() + except (AttributeError, OSError): + pass results = set(results) if results else set() + return results, ireq def get_legacy_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). They indicate the secondary dependencies for the given requirement. """ - if not (ireq.editable or is_pinned_requirement(ireq)): - raise TypeError('Expected pinned or editable InstallRequirement, got {}'.format(ireq)) + if not ( + ireq.editable or is_url_requirement(ireq) or is_pinned_requirement(ireq) + ): + raise TypeError( + "Expected url, pinned or editable InstallRequirement, got {}".format( + ireq + ) + ) if ireq not in self._dependencies_cache: if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): @@ -408,8 +383,7 @@ class PyPIRepository(BaseRepository): # If a download_dir is passed, pip will unnecessarely # archive the entire source directory download_dir = None - - elif ireq.link and not ireq.link.is_artifact: + elif ireq.link and is_vcs_url(ireq.link): # No download_dir for VCS sources. This also works around pip # using git-checkout-index, which gets rid of the .git dir. download_dir = None @@ -420,57 +394,72 @@ class PyPIRepository(BaseRepository): if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) - wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) - prev_tracker = os.environ.get('PIP_REQ_TRACKER') + wheel_cache = WheelCache(CACHE_DIR, self.options.format_control) + prev_tracker = os.environ.get("PIP_REQ_TRACKER") try: results, ireq = self.resolve_reqs(download_dir, ireq, wheel_cache) self._dependencies_cache[ireq] = results finally: - if 'PIP_REQ_TRACKER' in os.environ: + if "PIP_REQ_TRACKER" in os.environ: if prev_tracker: - os.environ['PIP_REQ_TRACKER'] = prev_tracker + os.environ["PIP_REQ_TRACKER"] = prev_tracker else: - del os.environ['PIP_REQ_TRACKER'] - try: - self.wheel_cache.cleanup() - except AttributeError: - pass + del os.environ["PIP_REQ_TRACKER"] + + # WheelCache.cleanup() introduced in pip==10.0.0 + if PIP_VERSION >= (10,): + wheel_cache.cleanup() return self._dependencies_cache[ireq] def get_hashes(self, ireq): """ Given an InstallRequirement, return a set of hashes that represent all - of the files for a given requirement. Editable requirements return an + of the files for a given requirement. Unhashable requirements return an empty set. Unpinned requirements raise a TypeError. """ - if ireq.editable: - return set() - vcs = VcsSupport() - if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: - return set() + if ireq.link: + link = ireq.link + + if is_vcs_url(link) or (is_file_url(link) and is_dir_url(link)): + # Return empty set for unhashable requirements. + # Unhashable logic modeled on pip's + # RequirementPreparer.prepare_linked_requirement + return set() + + if is_url_requirement(ireq): + # Directly hash URL requirements. + # URL requirements may have been previously downloaded and cached + # locally by self.resolve_reqs() + cached_path = os.path.join(self._download_dir, link.filename) + if os.path.exists(cached_path): + cached_link = Link(path_to_url(cached_path)) + else: + cached_link = link + return {self._hash_cache._get_file_hash(cached_link)} if not is_pinned_requirement(ireq): - raise TypeError( - "Expected pinned requirement, got {}".format(ireq)) + raise TypeError("Expected pinned requirement, got {}".format(ireq)) # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - ### Modification -- this is much more efficient.... - ### modification again -- still more efficient matching_candidates = ( c for c in clean_requires_python(self.find_all_candidates(ireq.name)) if c.version in ireq.specifier ) - # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - # matching_versions = list( - # ireq.specifier.filter((candidate.version for candidate in all_candidates))) - # matching_candidates = candidates_by_version[matching_versions[0]] + + log.debug(" {}".format(ireq.name)) + + def get_candidate_link(candidate): + if PIP_VERSION < (19, 2): + return candidate.location + return candidate.link return { - h for h in map(lambda c: self._hash_cache.get_hash(c.location), - matching_candidates) if h is not None + h for h in + map(lambda c: self._hash_cache.get_hash(get_candidate_link(c)), matching_candidates) + if h is not None } @contextmanager @@ -478,9 +467,10 @@ class PyPIRepository(BaseRepository): """ Monkey patches pip.Wheel to allow wheels from all platforms and Python versions. - This also saves the candidate cache and set a new one, or else the results from the - previous non-patched calls will interfere. + This also saves the candidate cache and set a new one, or else the results from + the previous non-patched calls will interfere. """ + def _wheel_supported(self, tags=None): # Ignore current platform. Support everything. return True @@ -513,7 +503,7 @@ def open_local_or_remote_file(link, session): :type link: pip.index.Link :type session: requests.Session :raises ValueError: If link points to a local directory. - :return: a context manager to the opened file-like object + :return: a context manager to a FileStream with the opened file-like object """ url = link.url_without_fragment @@ -523,13 +513,21 @@ def open_local_or_remote_file(link, session): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(url)) else: - with open(local_path, 'rb') as local_file: - yield local_file + st = os.stat(local_path) + with open(local_path, "rb") as local_file: + yield FileStream(stream=local_file, size=st.st_size) else: # Remote URL headers = {"Accept-Encoding": "identity"} response = session.get(url, headers=headers, stream=True) + + # Content length must be int or None try: - yield response.raw + content_length = int(response.headers["content-length"]) + except (ValueError, KeyError, TypeError): + content_length = None + + try: + yield FileStream(stream=response.raw, size=content_length) finally: response.close() diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py index 1f3e18c1..7e856fe6 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py @@ -1,34 +1,43 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals import copy +import os from functools import partial from itertools import chain, count -import os -from ._compat import install_req_from_line +from pipenv.vendor.requirementslib.models.markers import normalize_marker_str +from packaging.markers import Marker from . import click +from ._compat import install_req_from_line from .cache import DependencyCache -from .exceptions import UnsupportedConstraint from .logging import log -from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, - is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) +from .utils import ( + UNSAFE_PACKAGES, + format_requirement, + format_specifier, + full_groupby, + is_pinned_requirement, + is_url_requirement, + key_from_ireq, + key_from_req, +) -green = partial(click.style, fg='green') -magenta = partial(click.style, fg='magenta') +green = partial(click.style, fg="green") +magenta = partial(click.style, fg="magenta") class RequirementSummary(object): """ Summary of a requirement's properties for comparison purposes. """ + def __init__(self, ireq): self.req = ireq.req self.key = key_from_req(ireq.req) - self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) + self.markers = ireq.markers self.specifier = str(ireq.specifier) def __eq__(self, other): @@ -41,8 +50,71 @@ class RequirementSummary(object): return repr([self.key, self.specifier, self.extras]) +def combine_install_requirements(ireqs): + """ + Return a single install requirement that reflects a combination of + all the inputs. + """ + # We will store the source ireqs in a _source_ireqs attribute; + # if any of the inputs have this, then use those sources directly. + source_ireqs = [] + for ireq in ireqs: + source_ireqs.extend(getattr(ireq, "_source_ireqs", [ireq])) + + # deepcopy the accumulator so as to not modify the inputs + combined_ireq = copy.deepcopy(source_ireqs[0]) + for ireq in source_ireqs[1:]: + # NOTE we may be losing some info on dropped reqs here + combined_ireq.req.specifier &= ireq.req.specifier + combined_ireq.constraint &= ireq.constraint + if ireq.markers and not combined_ireq.markers: + combined_ireq.markers = copy.deepcopy(ireq.markers) + elif ireq.markers and combined_ireq.markers: + _markers = [] # type: List[Marker] + for marker in [ireq.markers, combined_ireq.markers]: + if isinstance(marker, str): + _markers.append(Marker(marker)) + else: + _markers.append(marker) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + combined_ireq.markers = Marker(marker_str) + # Return a sorted, de-duped tuple of extras + combined_ireq.extras = tuple( + sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras))) + ) + + # InstallRequirements objects are assumed to come from only one source, and + # so they support only a single comes_from entry. This function breaks this + # model. As a workaround, we deterministically choose a single source for + # the comes_from entry, and add an extra _source_ireqs attribute to keep + # track of multiple sources for use within pip-tools. + if len(source_ireqs) > 1: + if any(ireq.comes_from is None for ireq in source_ireqs): + # None indicates package was directly specified. + combined_ireq.comes_from = None + else: + # Populate the comes_from field from one of the sources. + # Requirement input order is not stable, so we need to sort: + # We choose the shortest entry in order to keep the printed + # representation as concise as possible. + combined_ireq.comes_from = min( + (ireq.comes_from for ireq in source_ireqs), + key=lambda x: (len(str(x)), str(x)), + ) + combined_ireq._source_ireqs = source_ireqs + return combined_ireq + + class Resolver(object): - def __init__(self, constraints, repository, cache=None, prereleases=False, clear_caches=False, allow_unsafe=False): + def __init__( + self, + constraints, + repository, + cache=None, + prereleases=False, + clear_caches=False, + allow_unsafe=False, + ): """ This class resolves a given set of constraints (a collection of InstallRequirement objects) by consulting the given Repository and the @@ -61,13 +133,21 @@ class Resolver(object): @property def constraints(self): - return set(self._group_constraints(chain(self.our_constraints, - self.their_constraints))) + return set( + self._group_constraints( + chain( + sorted(self.our_constraints, key=str), + sorted(self.their_constraints, key=str), + ) + ) + ) def resolve_hashes(self, ireqs): """ Finds acceptable hashes for all of the given InstallRequirements. """ + log.debug("") + log.debug("Generating hashes:") with self.repository.allow_all_wheels(): return {ireq: self.repository.get_hashes(ireq) for ireq in ireqs} @@ -85,24 +165,28 @@ class Resolver(object): self.dependency_cache.clear() self.repository.clear_caches() - self.check_constraints(chain(self.our_constraints, - self.their_constraints)) - # Ignore existing packages - os.environ[str('PIP_EXISTS_ACTION')] = str('i') # NOTE: str() wrapping necessary for Python 2/3 compat - for current_round in count(start=1): + os.environ[str("PIP_EXISTS_ACTION")] = str( + "i" + ) # NOTE: str() wrapping necessary for Python 2/3 compat + for current_round in count(start=1): # pragma: no branch if current_round > max_rounds: - raise RuntimeError('No stable configuration of concrete packages ' - 'could be found for the given constraints after ' - '%d rounds of resolving.\n' - 'This is likely a bug.' % max_rounds) + raise RuntimeError( + "No stable configuration of concrete packages " + "could be found for the given constraints after " + "%d rounds of resolving.\n" + "This is likely a bug." % max_rounds + ) - log.debug('') - log.debug(magenta('{:^60}'.format('ROUND {}'.format(current_round)))) + log.debug("") + log.debug(magenta("{:^60}".format("ROUND {}".format(current_round)))) has_changed, best_matches = self._resolve_one_round() - log.debug('-' * 60) - log.debug('Result of round {}: {}'.format(current_round, - 'not stable' if has_changed else 'stable, done')) + log.debug("-" * 60) + log.debug( + "Result of round {}: {}".format( + current_round, "not stable" if has_changed else "stable, done" + ) + ) if not has_changed: break @@ -113,17 +197,29 @@ class Resolver(object): # build cache dir for every round, so this can never happen. self.repository.freshen_build_caches() - del os.environ['PIP_EXISTS_ACTION'] - # Only include hard requirements and not pip constraints - return {req for req in best_matches if not req.constraint} + del os.environ["PIP_EXISTS_ACTION"] - @staticmethod - def check_constraints(constraints): - for constraint in constraints: - if constraint.link is not None and not constraint.editable and not constraint.is_wheel: - msg = ('pip-compile does not support URLs as packages, unless they are editable. ' - 'Perhaps add -e option?') - raise UnsupportedConstraint(msg, constraint) + # Only include hard requirements and not pip constraints + results = {req for req in best_matches if not req.constraint} + + # Filter out unsafe requirements. + self.unsafe_constraints = set() + if not self.allow_unsafe: + # reverse_dependencies is used to filter out packages that are only + # required by unsafe packages. This logic is incomplete, as it would + # fail to filter sub-sub-dependencies of unsafe packages. None of the + # UNSAFE_PACKAGES currently have any dependencies at all (which makes + # sense for installation tools) so this seems sufficient. + reverse_dependencies = self.reverse_dependencies(results) + for req in results.copy(): + required_by = reverse_dependencies.get(req.name.lower(), []) + if req.name in UNSAFE_PACKAGES or ( + required_by and all(name in UNSAFE_PACKAGES for name in required_by) + ): + self.unsafe_constraints.add(req) + results.remove(req) + + return results def _group_constraints(self, constraints): """ @@ -145,27 +241,11 @@ class Resolver(object): ireqs = list(ireqs) editable_ireq = next((ireq for ireq in ireqs if ireq.editable), None) if editable_ireq: - yield editable_ireq # ignore all the other specs: the editable one is the one that counts + # ignore all the other specs: the editable one is the one that counts + yield editable_ireq continue - ireqs = iter(ireqs) - # deepcopy the accumulator so as to not modify the self.our_constraints invariant - combined_ireq = copy.deepcopy(next(ireqs)) - combined_ireq.comes_from = None - for ireq in ireqs: - # NOTE we may be losing some info on dropped reqs here - combined_ireq.req.specifier &= ireq.req.specifier - combined_ireq.constraint &= ireq.constraint - if not combined_ireq.markers: - combined_ireq.markers = ireq.markers - else: - _markers = combined_ireq.markers._markers - if not isinstance(_markers[0], (tuple, list)): - combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] - - # Return a sorted, de-duped tuple of extras - combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) - yield combined_ireq + yield combine_install_requirements(ireqs) def _resolve_one_round(self): """ @@ -180,60 +260,48 @@ class Resolver(object): """ # Sort this list for readability of terminal output constraints = sorted(self.constraints, key=key_from_ireq) - unsafe_constraints = [] - original_constraints = copy.copy(constraints) - if not self.allow_unsafe: - for constraint in original_constraints: - if constraint.name in UNSAFE_PACKAGES: - constraints.remove(constraint) - constraint.req.specifier = None - unsafe_constraints.append(constraint) - log.debug('Current constraints:') + log.debug("Current constraints:") for constraint in constraints: - log.debug(' {}'.format(constraint)) + log.debug(" {}".format(constraint)) - log.debug('') - log.debug('Finding the best candidates:') + log.debug("") + log.debug("Finding the best candidates:") best_matches = {self.get_best_match(ireq) for ireq in constraints} # Find the new set of secondary dependencies - log.debug('') - log.debug('Finding secondary dependencies:') + log.debug("") + log.debug("Finding secondary dependencies:") - safe_constraints = [] + their_constraints = [] for best_match in best_matches: - for dep in self._iter_dependencies(best_match): - if self.allow_unsafe or dep.name not in UNSAFE_PACKAGES: - safe_constraints.append(dep) + their_constraints.extend(self._iter_dependencies(best_match)) # Grouping constraints to make clean diff between rounds - theirs = set(self._group_constraints(safe_constraints)) + theirs = set(self._group_constraints(sorted(their_constraints, key=str))) # NOTE: We need to compare RequirementSummary objects, since # InstallRequirement does not define equality - diff = {RequirementSummary(t) for t in theirs} - {RequirementSummary(t) for t in self.their_constraints} - removed = ({RequirementSummary(t) for t in self.their_constraints} - - {RequirementSummary(t) for t in theirs}) - unsafe = ({RequirementSummary(t) for t in unsafe_constraints} - - {RequirementSummary(t) for t in self.unsafe_constraints}) + diff = {RequirementSummary(t) for t in theirs} - { + RequirementSummary(t) for t in self.their_constraints + } + removed = {RequirementSummary(t) for t in self.their_constraints} - { + RequirementSummary(t) for t in theirs + } - has_changed = len(diff) > 0 or len(removed) > 0 or len(unsafe) > 0 + has_changed = len(diff) > 0 or len(removed) > 0 if has_changed: - log.debug('') - log.debug('New dependencies found in this round:') + log.debug("") + log.debug("New dependencies found in this round:") for new_dependency in sorted(diff, key=lambda req: key_from_req(req.req)): - log.debug(' adding {}'.format(new_dependency)) - log.debug('Removed dependencies in this round:') - for removed_dependency in sorted(removed, key=lambda req: key_from_req(req.req)): - log.debug(' removing {}'.format(removed_dependency)) - log.debug('Unsafe dependencies in this round:') - for unsafe_dependency in sorted(unsafe, key=lambda req: key_from_req(req.req)): - log.debug(' remembering unsafe {}'.format(unsafe_dependency)) + log.debug(" adding {}".format(new_dependency)) + log.debug("Removed dependencies in this round:") + for removed_dependency in sorted( + removed, key=lambda req: key_from_req(req.req) + ): + log.debug(" removing {}".format(removed_dependency)) # Store the last round's results in the their_constraints self.their_constraints = theirs - # Store the last round's unsafe constraints - self.unsafe_constraints = unsafe_constraints return has_changed, best_matches def get_best_match(self, ireq): @@ -251,7 +319,7 @@ class Resolver(object): Flask==0.10.1 => Flask==0.10.1 """ - if ireq.editable: + if ireq.editable or is_url_requirement(ireq): # NOTE: it's much quicker to immediately return instead of # hitting the index server best_match = ireq @@ -260,23 +328,29 @@ class Resolver(object): # hitting the index server best_match = ireq else: - best_match = self.repository.find_best_match(ireq, prereleases=self.prereleases) + best_match = self.repository.find_best_match( + ireq, prereleases=self.prereleases + ) # Format the best match - log.debug(' found candidate {} (constraint was {})'.format(format_requirement(best_match), - format_specifier(ireq))) + log.debug( + " found candidate {} (constraint was {})".format( + format_requirement(best_match), format_specifier(ireq) + ) + ) + best_match.comes_from = ireq.comes_from return best_match def _iter_dependencies(self, ireq): """ - Given a pinned or editable InstallRequirement, collects all the + Given a pinned, url, or editable InstallRequirement, collects all the secondary dependencies for them, either by looking them up in a local cache, or by reaching out to the repository. Editable requirements will never be looked up, as they may have changed at any time. """ - if ireq.editable: + if ireq.editable or (is_url_requirement(ireq) and not ireq.link.is_wheel): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -290,24 +364,40 @@ class Resolver(object): ireq.extras = ireq.extra elif not is_pinned_requirement(ireq): - raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) + raise TypeError( + "Expected pinned or editable requirement, got {}".format(ireq) + ) # Now, either get the dependencies from the dependency cache (for # speed), or reach out to the external repository to # download and inspect the package version and get dependencies # from there if ireq not in self.dependency_cache: - log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') + log.debug( + " {} not in cache, need to check index".format( + format_requirement(ireq) + ), + fg="yellow", + ) dependencies = self.repository.get_dependencies(ireq) self.dependency_cache[ireq] = sorted(set(format_requirement(ireq) for ireq in dependencies)) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] - log.debug(' {:25} requires {}'.format(format_requirement(ireq), - ', '.join(sorted(dependency_strings, key=lambda s: s.lower())) or '-')) + log.debug( + " {:25} requires {}".format( + format_requirement(ireq), + ", ".join(sorted(dependency_strings, key=lambda s: s.lower())) or "-", + ) + ) for dependency_string in dependency_strings: - yield install_req_from_line(dependency_string, constraint=ireq.constraint) + yield install_req_from_line( + dependency_string, constraint=ireq.constraint, comes_from=ireq + ) def reverse_dependencies(self, ireqs): - non_editable = [ireq for ireq in ireqs if not ireq.editable] + is_non_wheel_url = lambda r: is_url_requirement(r) and not r.link.is_wheel + non_editable = [ + ireq for ireq in ireqs if not (ireq.editable or is_non_wheel_url(ireq)) + ] return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/pipenv/patched/piptools/scripts/compile.py b/pipenv/patched/piptools/scripts/compile.py index 4625618a..5ac16e35 100644 --- a/pipenv/patched/piptools/scripts/compile.py +++ b/pipenv/patched/piptools/scripts/compile.py @@ -1,193 +1,361 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals -import optparse import os import sys import tempfile -from .._compat import ( - install_req_from_line, - parse_requirements, - cmdoptions, - Command, -) +from click.utils import safecall from .. import click +from .._compat import install_req_from_line, parse_requirements from ..exceptions import PipToolsError from ..logging import log from ..repositories import LocalRequirementsRepository, PyPIRepository from ..resolver import Resolver -from ..utils import (dedup, is_pinned_requirement, key_from_req, UNSAFE_PACKAGES) +from ..utils import ( + UNSAFE_PACKAGES, + create_install_command, + dedup, + get_trusted_hosts, + is_pinned_requirement, + key_from_ireq, + key_from_req, +) from ..writer import OutputWriter -DEFAULT_REQUIREMENTS_FILE = 'requirements.in' +DEFAULT_REQUIREMENTS_FILE = "requirements.in" +DEFAULT_REQUIREMENTS_OUTPUT_FILE = "requirements.txt" - -class PipCommand(Command): - name = 'PipCommand' +# Get default values of the pip's options (including options from pipenv.patched.notpip.conf). +install_command = create_install_command() +pip_defaults = install_command.parser.get_default_values() @click.command() @click.version_option() -@click.option('-v', '--verbose', is_flag=True, help="Show more output") -@click.option('-n', '--dry-run', is_flag=True, help="Only show what would happen, don't change anything") -@click.option('-p', '--pre', is_flag=True, default=None, help="Allow resolving to prereleases (default is not)") -@click.option('-r', '--rebuild', is_flag=True, help="Clear any caches upfront, rebuild from scratch") -@click.option('-f', '--find-links', multiple=True, help="Look for archives in this directory or on this HTML page", envvar='PIP_FIND_LINKS') # noqa -@click.option('-i', '--index-url', help="Change index URL (defaults to PyPI)", envvar='PIP_INDEX_URL') -@click.option('--extra-index-url', multiple=True, help="Add additional index URL to search", envvar='PIP_EXTRA_INDEX_URL') # noqa -@click.option('--cert', help="Path to alternate CA bundle.") -@click.option('--client-cert', help="Path to SSL client certificate, a single file containing the private key and the certificate in PEM format.") # noqa -@click.option('--trusted-host', multiple=True, envvar='PIP_TRUSTED_HOST', - help="Mark this host as trusted, even though it does not have " - "valid or any HTTPS.") -@click.option('--header/--no-header', is_flag=True, default=True, - help="Add header to generated file") -@click.option('--index/--no-index', is_flag=True, default=True, - help="Add index URL to generated file") -@click.option('--emit-trusted-host/--no-emit-trusted-host', is_flag=True, - default=True, help="Add trusted host option to generated file") -@click.option('--annotate/--no-annotate', is_flag=True, default=True, - help="Annotate results, indicating where dependencies come from") -@click.option('-U', '--upgrade', is_flag=True, default=False, - help='Try to upgrade all dependencies to their latest versions') -@click.option('-P', '--upgrade-package', 'upgrade_packages', nargs=1, multiple=True, - help="Specify particular packages to upgrade.") -@click.option('-o', '--output-file', nargs=1, type=str, default=None, - help=('Output file name. Required if more than one input file is given. ' - 'Will be derived from input file otherwise.')) -@click.option('--allow-unsafe', is_flag=True, default=False, - help="Pin packages considered unsafe: {}".format(', '.join(sorted(UNSAFE_PACKAGES)))) -@click.option('--generate-hashes', is_flag=True, default=False, - help="Generate pip 8 style hashes in the resulting requirements file.") -@click.option('--max-rounds', default=10, - help="Maximum number of rounds before resolving the requirements aborts.") -@click.argument('src_files', nargs=-1, type=click.Path(exists=True, allow_dash=True)) -def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, - cert, client_cert, trusted_host, header, index, emit_trusted_host, annotate, - upgrade, upgrade_packages, output_file, allow_unsafe, generate_hashes, - src_files, max_rounds): +@click.pass_context +@click.option("-v", "--verbose", count=True, help="Show more output") +@click.option("-q", "--quiet", count=True, help="Give less output") +@click.option( + "-n", + "--dry-run", + is_flag=True, + help="Only show what would happen, don't change anything", +) +@click.option( + "-p", + "--pre", + is_flag=True, + default=None, + help="Allow resolving to prereleases (default is not)", +) +@click.option( + "-r", + "--rebuild", + is_flag=True, + help="Clear any caches upfront, rebuild from scratch", +) +@click.option( + "-f", + "--find-links", + multiple=True, + help="Look for archives in this directory or on this HTML page", + envvar="PIP_FIND_LINKS", +) +@click.option( + "-i", + "--index-url", + help="Change index URL (defaults to {})".format(pip_defaults.index_url), + envvar="PIP_INDEX_URL", +) +@click.option( + "--extra-index-url", + multiple=True, + help="Add additional index URL to search", + envvar="PIP_EXTRA_INDEX_URL", +) +@click.option("--cert", help="Path to alternate CA bundle.") +@click.option( + "--client-cert", + help="Path to SSL client certificate, a single file containing " + "the private key and the certificate in PEM format.", +) +@click.option( + "--trusted-host", + multiple=True, + envvar="PIP_TRUSTED_HOST", + help="Mark this host as trusted, even though it does not have " + "valid or any HTTPS.", +) +@click.option( + "--header/--no-header", + is_flag=True, + default=True, + help="Add header to generated file", +) +@click.option( + "--index/--no-index", + is_flag=True, + default=True, + help="Add index URL to generated file", +) +@click.option( + "--emit-trusted-host/--no-emit-trusted-host", + is_flag=True, + default=True, + help="Add trusted host option to generated file", +) +@click.option( + "--annotate/--no-annotate", + is_flag=True, + default=True, + help="Annotate results, indicating where dependencies come from", +) +@click.option( + "-U", + "--upgrade", + is_flag=True, + default=False, + help="Try to upgrade all dependencies to their latest versions", +) +@click.option( + "-P", + "--upgrade-package", + "upgrade_packages", + nargs=1, + multiple=True, + help="Specify particular packages to upgrade.", +) +@click.option( + "-o", + "--output-file", + nargs=1, + default=None, + type=click.File("w+b", atomic=True, lazy=True), + help=( + "Output file name. Required if more than one input file is given. " + "Will be derived from input file otherwise." + ), +) +@click.option( + "--allow-unsafe", + is_flag=True, + default=False, + help="Pin packages considered unsafe: {}".format( + ", ".join(sorted(UNSAFE_PACKAGES)) + ), +) +@click.option( + "--generate-hashes", + is_flag=True, + default=False, + help="Generate pip 8 style hashes in the resulting requirements file.", +) +@click.option( + "--max-rounds", + default=10, + help="Maximum number of rounds before resolving the requirements aborts.", +) +@click.argument("src_files", nargs=-1, type=click.Path(exists=True, allow_dash=True)) +@click.option( + "--build-isolation/--no-build-isolation", + is_flag=True, + default=False, + help="Enable isolation when building a modern source distribution. " + "Build dependencies specified by PEP 518 must be already installed " + "if build isolation is disabled.", +) +@click.option( + "--emit-find-links/--no-emit-find-links", + is_flag=True, + default=True, + help="Add the find-links option to generated file", +) +def cli( + ctx, + verbose, + quiet, + dry_run, + pre, + rebuild, + find_links, + index_url, + extra_index_url, + cert, + client_cert, + trusted_host, + header, + index, + emit_trusted_host, + annotate, + upgrade, + upgrade_packages, + output_file, + allow_unsafe, + generate_hashes, + src_files, + max_rounds, + build_isolation, + emit_find_links, +): """Compiles requirements.txt from requirements.in specs.""" - log.verbose = verbose + log.verbosity = verbose - quiet if len(src_files) == 0: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE,) - elif os.path.exists('setup.py'): - src_files = ('setup.py',) - if not output_file: - output_file = 'requirements.txt' + elif os.path.exists("setup.py"): + src_files = ("setup.py",) else: - raise click.BadParameter(("If you do not specify an input file, " - "the default is {} or setup.py").format(DEFAULT_REQUIREMENTS_FILE)) + raise click.BadParameter( + ( + "If you do not specify an input file, " + "the default is {} or setup.py" + ).format(DEFAULT_REQUIREMENTS_FILE) + ) - if len(src_files) == 1 and src_files[0] == '-': - if not output_file: - raise click.BadParameter('--output-file is required if input is from stdin') + if not output_file: + # An output file must be provided for stdin + if src_files == ("-",): + raise click.BadParameter("--output-file is required if input is from stdin") + # Use default requirements output file if there is a setup.py the source file + elif src_files == ("setup.py",): + file_name = DEFAULT_REQUIREMENTS_OUTPUT_FILE + # An output file must be provided if there are multiple source files + elif len(src_files) > 1: + raise click.BadParameter( + "--output-file is required if two or more input files are given." + ) + # Otherwise derive the output file from the source file + else: + base_name = src_files[0].rsplit(".", 1)[0] + file_name = base_name + ".txt" - if len(src_files) > 1 and not output_file: - raise click.BadParameter('--output-file is required if two or more input files are given.') + output_file = click.open_file(file_name, "w+b", atomic=True, lazy=True) - if output_file: - dst_file = output_file - else: - base_name = src_files[0].rsplit('.', 1)[0] - dst_file = base_name + '.txt' - - if upgrade and upgrade_packages: - raise click.BadParameter('Only one of --upgrade or --upgrade-package can be provided as an argument.') + # Close the file at the end of the context execution + ctx.call_on_close(safecall(output_file.close_intelligently)) ### # Setup ### - pip_command = get_pip_command() - pip_args = [] if find_links: for link in find_links: - pip_args.extend(['-f', link]) + pip_args.extend(["-f", link]) if index_url: - pip_args.extend(['-i', index_url]) + pip_args.extend(["-i", index_url]) if extra_index_url: for extra_index in extra_index_url: - pip_args.extend(['--extra-index-url', extra_index]) + pip_args.extend(["--extra-index-url", extra_index]) if cert: - pip_args.extend(['--cert', cert]) + pip_args.extend(["--cert", cert]) if client_cert: - pip_args.extend(['--client-cert', client_cert]) + pip_args.extend(["--client-cert", client_cert]) if pre: - pip_args.extend(['--pre']) + pip_args.extend(["--pre"]) if trusted_host: for host in trusted_host: - pip_args.extend(['--trusted-host', host]) + pip_args.extend(["--trusted-host", host]) - pip_options, _ = pip_command.parse_args(pip_args) + repository = PyPIRepository(pip_args, build_isolation=build_isolation) - session = pip_command._build_session(pip_options) - repository = PyPIRepository(pip_options, session) + # Parse all constraints coming from --upgrade-package/-P + upgrade_reqs_gen = (install_req_from_line(pkg) for pkg in upgrade_packages) + upgrade_install_reqs = { + key_from_req(install_req.req): install_req for install_req in upgrade_reqs_gen + } # Proxy with a LocalRequirementsRepository if --upgrade is not specified # (= default invocation) - if not upgrade and os.path.exists(dst_file): - ireqs = parse_requirements(dst_file, finder=repository.finder, session=repository.session, options=pip_options) - # Exclude packages from --upgrade-package/-P from the existing pins: We want to upgrade. - upgrade_pkgs_key = {key_from_req(install_req_from_line(pkg).req) for pkg in upgrade_packages} - existing_pins = {key_from_req(ireq.req): ireq - for ireq in ireqs - if is_pinned_requirement(ireq) and key_from_req(ireq.req) not in upgrade_pkgs_key} + if not upgrade and os.path.exists(output_file.name): + ireqs = parse_requirements( + output_file.name, + finder=repository.finder, + session=repository.session, + options=repository.options, + ) + + # Exclude packages from --upgrade-package/-P from the existing + # constraints + existing_pins = { + key_from_req(ireq.req): ireq + for ireq in ireqs + if is_pinned_requirement(ireq) + and key_from_req(ireq.req) not in upgrade_install_reqs + } repository = LocalRequirementsRepository(existing_pins, repository) - log.debug('Using indexes:') - # remove duplicate index urls before processing - repository.finder.index_urls = list(dedup(repository.finder.index_urls)) - for index_url in repository.finder.index_urls: - log.debug(' {}'.format(index_url)) - - if repository.finder.find_links: - log.debug('') - log.debug('Configuration:') - for find_link in repository.finder.find_links: - log.debug(' -f {}'.format(find_link)) - ### # Parsing/collecting initial requirements ### constraints = [] for src_file in src_files: - is_setup_file = os.path.basename(src_file) == 'setup.py' - if is_setup_file or src_file == '-': + is_setup_file = os.path.basename(src_file) == "setup.py" + if is_setup_file or src_file == "-": # pip requires filenames and not files. Since we want to support # piping from stdin, we need to briefly save the input from stdin # to a temporary file and have pip read that. also used for # reading requirements from install_requires in setup.py. - tmpfile = tempfile.NamedTemporaryFile(mode='wt', delete=False) + tmpfile = tempfile.NamedTemporaryFile(mode="wt", delete=False) if is_setup_file: from distutils.core import run_setup + dist = run_setup(src_file) - tmpfile.write('\n'.join(dist.install_requires)) + tmpfile.write("\n".join(dist.install_requires)) else: tmpfile.write(sys.stdin.read()) tmpfile.flush() - constraints.extend(parse_requirements( - tmpfile.name, finder=repository.finder, session=repository.session, options=pip_options)) + constraints.extend( + parse_requirements( + tmpfile.name, + finder=repository.finder, + session=repository.session, + options=repository.options, + ) + ) else: - constraints.extend(parse_requirements( - src_file, finder=repository.finder, session=repository.session, options=pip_options)) + constraints.extend( + parse_requirements( + src_file, + finder=repository.finder, + session=repository.session, + options=repository.options, + ) + ) + + primary_packages = { + key_from_ireq(ireq) for ireq in constraints if not ireq.constraint + } + + constraints.extend(upgrade_install_reqs.values()) # Filter out pip environment markers which do not match (PEP496) - constraints = [req for req in constraints - if req.markers is None or req.markers.evaluate()] + constraints = [ + req for req in constraints if req.markers is None or req.markers.evaluate() + ] - # Check the given base set of constraints first - Resolver.check_constraints(constraints) + log.debug("Using indexes:") + for index_url in dedup(repository.finder.index_urls): + log.debug(" {}".format(index_url)) + + if repository.finder.find_links: + log.debug("") + log.debug("Configuration:") + for find_link in dedup(repository.finder.find_links): + log.debug(" -f {}".format(find_link)) try: - resolver = Resolver(constraints, repository, prereleases=pre, - clear_caches=rebuild, allow_unsafe=allow_unsafe) + resolver = Resolver( + constraints, + repository, + prereleases=repository.finder.allow_all_prereleases or pre, + clear_caches=rebuild, + allow_unsafe=allow_unsafe, + ) results = resolver.resolve(max_rounds=max_rounds) if generate_hashes: hashes = resolver.resolve_hashes(results) @@ -197,7 +365,7 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, log.error(str(e)) sys.exit(2) - log.debug('') + log.debug("") ## # Output @@ -213,12 +381,14 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, # TODO (1b): perhaps it's easiest if the dependency cache has an API # that could take InstallRequirements directly, like: # - # cache.set(ireq, ...) + # cache.set(ireq, ...) # # then, when ireq is editable, it would store in # # editables[egg_name][link_without_fragment] = deps - # editables['pip-tools']['git+...ols.git@future'] = {'click>=3.0', 'six'} + # editables['pip-tools']['git+...ols.git@future'] = { + # 'click>=3.0', 'six' + # } # # otherwise: # @@ -228,40 +398,34 @@ def cli(verbose, dry_run, pre, rebuild, find_links, index_url, extra_index_url, if annotate: reverse_dependencies = resolver.reverse_dependencies(results) - writer = OutputWriter(src_files, dst_file, dry_run=dry_run, - emit_header=header, emit_index=index, - emit_trusted_host=emit_trusted_host, - annotate=annotate, - generate_hashes=generate_hashes, - default_index_url=repository.DEFAULT_INDEX_URL, - index_urls=repository.finder.index_urls, - trusted_hosts=pip_options.trusted_hosts, - format_control=repository.finder.format_control) - writer.write(results=results, - unsafe_requirements=resolver.unsafe_constraints, - reverse_dependencies=reverse_dependencies, - primary_packages={key_from_req(ireq.req) for ireq in constraints if not ireq.constraint}, - markers={key_from_req(ireq.req): ireq.markers - for ireq in constraints if ireq.markers}, - hashes=hashes, - allow_unsafe=allow_unsafe) + writer = OutputWriter( + src_files, + output_file, + click_ctx=ctx, + dry_run=dry_run, + emit_header=header, + emit_index=index, + emit_trusted_host=emit_trusted_host, + annotate=annotate, + generate_hashes=generate_hashes, + default_index_url=repository.DEFAULT_INDEX_URL, + index_urls=repository.finder.index_urls, + trusted_hosts=get_trusted_hosts(repository.finder), + format_control=repository.finder.format_control, + allow_unsafe=allow_unsafe, + find_links=repository.finder.find_links, + emit_find_links=emit_find_links, + ) + writer.write( + results=results, + unsafe_requirements=resolver.unsafe_constraints, + reverse_dependencies=reverse_dependencies, + primary_packages=primary_packages, + markers={ + key_from_ireq(ireq): ireq.markers for ireq in constraints if ireq.markers + }, + hashes=hashes, + ) if dry_run: - log.warning('Dry-run, so nothing updated.') - - -def get_pip_command(): - # Use pip's parser for pip.conf management and defaults. - # General options (find_links, index_url, extra_index_url, trusted_host, - # and pre) are defered to pip. - pip_command = PipCommand() - pip_command.parser.add_option(cmdoptions.no_binary()) - pip_command.parser.add_option(cmdoptions.only_binary()) - index_opts = cmdoptions.make_option_group( - cmdoptions.index_group, - pip_command.parser, - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option(optparse.Option('--pre', action='store_true', default=False)) - - return pip_command + log.info("Dry-run, so nothing updated.") diff --git a/pipenv/patched/piptools/scripts/sync.py b/pipenv/patched/piptools/scripts/sync.py index 610c1d5e..40c086a4 100644 --- a/pipenv/patched/piptools/scripts/sync.py +++ b/pipenv/patched/piptools/scripts/sync.py @@ -1,52 +1,112 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals import os import sys - from .. import click, sync -from .._compat import parse_requirements, get_installed_distributions +from .._compat import get_installed_distributions, parse_requirements from ..exceptions import PipToolsError from ..logging import log from ..utils import flat_map -DEFAULT_REQUIREMENTS_FILE = 'requirements.txt' +DEFAULT_REQUIREMENTS_FILE = "requirements.txt" @click.command() @click.version_option() -@click.option('-n', '--dry-run', is_flag=True, help="Only show what would happen, don't change anything") -@click.option('--force', is_flag=True, help="Proceed even if conflicts are found") -@click.option('-f', '--find-links', multiple=True, help="Look for archives in this directory or on this HTML page", envvar='PIP_FIND_LINKS') # noqa -@click.option('-i', '--index-url', help="Change index URL (defaults to PyPI)", envvar='PIP_INDEX_URL') -@click.option('--extra-index-url', multiple=True, help="Add additional index URL to search", envvar='PIP_EXTRA_INDEX_URL') # noqa -@click.option('--no-index', is_flag=True, help="Ignore package index (only looking at --find-links URLs instead)") -@click.option('-q', '--quiet', default=False, is_flag=True, help="Give less output") -@click.option('--user', 'user_only', is_flag=True, help="Restrict attention to user directory") -@click.argument('src_files', required=False, type=click.Path(exists=True), nargs=-1) -def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, user_only, src_files): +@click.option( + "-a", + "--ask", + is_flag=True, + help="Show what would happen, then ask whether to continue", +) +@click.option( + "-n", + "--dry-run", + is_flag=True, + help="Only show what would happen, don't change anything", +) +@click.option("--force", is_flag=True, help="Proceed even if conflicts are found") +@click.option( + "-f", + "--find-links", + multiple=True, + help="Look for archives in this directory or on this HTML page", + envvar="PIP_FIND_LINKS", +) +@click.option( + "-i", + "--index-url", + help="Change index URL (defaults to PyPI)", + envvar="PIP_INDEX_URL", +) +@click.option( + "--extra-index-url", + multiple=True, + help="Add additional index URL to search", + envvar="PIP_EXTRA_INDEX_URL", +) +@click.option( + "--trusted-host", + multiple=True, + help="Mark this host as trusted, even though it does not have valid or any HTTPS.", +) +@click.option( + "--no-index", + is_flag=True, + help="Ignore package index (only looking at --find-links URLs instead)", +) +@click.option("-q", "--quiet", default=False, is_flag=True, help="Give less output") +@click.option( + "--user", "user_only", is_flag=True, help="Restrict attention to user directory" +) +@click.option("--cert", help="Path to alternate CA bundle.") +@click.option( + "--client-cert", + help="Path to SSL client certificate, a single file containing " + "the private key and the certificate in PEM format.", +) +@click.argument("src_files", required=False, type=click.Path(exists=True), nargs=-1) +def cli( + ask, + dry_run, + force, + find_links, + index_url, + extra_index_url, + trusted_host, + no_index, + quiet, + user_only, + cert, + client_cert, + src_files, +): """Synchronize virtual environment with requirements.txt.""" if not src_files: if os.path.exists(DEFAULT_REQUIREMENTS_FILE): src_files = (DEFAULT_REQUIREMENTS_FILE,) else: - msg = 'No requirement files given and no {} found in the current directory' + msg = "No requirement files given and no {} found in the current directory" log.error(msg.format(DEFAULT_REQUIREMENTS_FILE)) sys.exit(2) - if any(src_file.endswith('.in') for src_file in src_files): - msg = ('Some input files have the .in extension, which is most likely an error and can ' - 'cause weird behaviour. You probably meant to use the corresponding *.txt file?') + if any(src_file.endswith(".in") for src_file in src_files): + msg = ( + "Some input files have the .in extension, which is most likely an error " + "and can cause weird behaviour. You probably meant to use " + "the corresponding *.txt file?" + ) if force: - log.warning('WARNING: ' + msg) + log.warning("WARNING: " + msg) else: - log.error('ERROR: ' + msg) + log.error("ERROR: " + msg) sys.exit(2) - requirements = flat_map(lambda src: parse_requirements(src, session=True), - src_files) + requirements = flat_map( + lambda src: parse_requirements(src, session=True), src_files + ) try: requirements = sync.merge(requirements, ignore_conflicts=force) @@ -59,16 +119,31 @@ def cli(dry_run, force, find_links, index_url, extra_index_url, no_index, quiet, install_flags = [] for link in find_links or []: - install_flags.extend(['-f', link]) + install_flags.extend(["-f", link]) if no_index: - install_flags.append('--no-index') + install_flags.append("--no-index") if index_url: - install_flags.extend(['-i', index_url]) + install_flags.extend(["-i", index_url]) if extra_index_url: for extra_index in extra_index_url: - install_flags.extend(['--extra-index-url', extra_index]) + install_flags.extend(["--extra-index-url", extra_index]) + if trusted_host: + for host in trusted_host: + install_flags.extend(["--trusted-host", host]) if user_only: - install_flags.append('--user') + install_flags.append("--user") + if cert: + install_flags.extend(["--cert", cert]) + if client_cert: + install_flags.extend(["--client-cert", client_cert]) - sys.exit(sync.sync(to_install, to_uninstall, verbose=(not quiet), dry_run=dry_run, - install_flags=install_flags)) + sys.exit( + sync.sync( + to_install, + to_uninstall, + verbose=(not quiet), + dry_run=dry_run, + install_flags=install_flags, + ask=ask, + ) + ) diff --git a/pipenv/patched/piptools/sync.py b/pipenv/patched/piptools/sync.py index 5c473916..00b1ae8e 100644 --- a/pipenv/patched/piptools/sync.py +++ b/pipenv/patched/piptools/sync.py @@ -1,21 +1,26 @@ import collections import os import sys -from subprocess import check_call +import tempfile +from subprocess import check_call # nosec from . import click -from .exceptions import IncompatibleRequirements, UnsupportedConstraint -from .utils import flat_map, format_requirement, key_from_ireq, key_from_req +from ._compat import DEV_PKGS, stdlib_pkgs +from .exceptions import IncompatibleRequirements +from .utils import ( + flat_map, + format_requirement, + get_hashes_from_ireq, + is_url_requirement, + key_from_ireq, + key_from_req, +) -PACKAGES_TO_IGNORE = [ - '-markerlib', - 'pip', - 'pip-tools', - 'pip-review', - 'pkg-resources', - 'setuptools', - 'wheel', -] +PACKAGES_TO_IGNORE = ( + ["-markerlib", "pip", "pip-tools", "pip-review", "pkg-resources"] + + list(stdlib_pkgs) + + list(DEV_PKGS) +) def dependency_tree(installed_keys, root_key): @@ -63,19 +68,19 @@ def get_dists_to_ignore(installed): requirements. """ installed_keys = {key_from_req(r): r for r in installed} - return list(flat_map(lambda req: dependency_tree(installed_keys, req), PACKAGES_TO_IGNORE)) + return list( + flat_map(lambda req: dependency_tree(installed_keys, req), PACKAGES_TO_IGNORE) + ) def merge(requirements, ignore_conflicts): by_key = {} for ireq in requirements: - if ireq.link is not None and not ireq.editable: - msg = ('pip-compile does not support URLs as packages, unless they are editable. ' - 'Perhaps add -e option?') - raise UnsupportedConstraint(msg, ireq) - - key = ireq.link or key_from_req(ireq.req) + # Limitation: URL requirements are merged by precise string match, so + # "file:///example.zip#egg=example", "file:///example.zip", and + # "example==1.0" will not merge with each other + key = key_from_ireq(ireq) if not ignore_conflicts: existing_ireq = by_key.get(key) @@ -87,16 +92,35 @@ def merge(requirements, ignore_conflicts): # TODO: Always pick the largest specifier in case of a conflict by_key[key] = ireq - return by_key.values() +def diff_key_from_ireq(ireq): + """ + Calculate a key for comparing a compiled requirement with installed modules. + For URL requirements, only provide a useful key if the url includes + #egg=name==version, which will set ireq.req.name and ireq.specifier. + Otherwise return ireq.link so the key will not match and the package will + reinstall. Reinstall is necessary to ensure that packages will reinstall + if the URL is changed but the version is not. + """ + if is_url_requirement(ireq): + if ( + ireq.req + and (getattr(ireq.req, "key", None) or getattr(ireq.req, "name", None)) + and ireq.specifier + ): + return key_from_ireq(ireq) + return str(ireq.link) + return key_from_ireq(ireq) + + def diff(compiled_requirements, installed_dists): """ Calculate which packages should be installed or uninstalled, given a set of compiled requirements and a list of currently installed modules. """ - requirements_lut = {r.link or key_from_req(r.req): r for r in compiled_requirements} + requirements_lut = {diff_key_from_ireq(r): r for r in compiled_requirements} satisfied = set() # holds keys to_install = set() # holds InstallRequirement objects @@ -120,47 +144,72 @@ def diff(compiled_requirements, installed_dists): return (to_install, to_uninstall) -def sync(to_install, to_uninstall, verbose=False, dry_run=False, pip_flags=None, install_flags=None): +def sync( + to_install, + to_uninstall, + verbose=False, + dry_run=False, + install_flags=None, + ask=False, +): """ Install and uninstalls the given sets of modules. """ if not to_uninstall and not to_install: - click.echo("Everything up-to-date") - - if pip_flags is None: - pip_flags = [] + if verbose: + click.echo("Everything up-to-date") + return 0 + pip_flags = [] if not verbose: - pip_flags += ['-q'] + pip_flags += ["-q"] - if os.environ.get('VIRTUAL_ENV'): - # find pip via PATH - pip = 'pip' - else: - # find pip in same directory as pip-sync entry-point script - pip = os.path.join(os.path.dirname(os.path.abspath(sys.argv[0])), 'pip') + if ask: + dry_run = True - if to_uninstall: - if dry_run: + if dry_run: + if to_uninstall: click.echo("Would uninstall:") for pkg in to_uninstall: click.echo(" {}".format(pkg)) - else: - check_call([pip, 'uninstall', '-y'] + pip_flags + sorted(to_uninstall)) - if to_install: - if install_flags is None: - install_flags = [] - if dry_run: + if to_install: click.echo("Would install:") for ireq in to_install: click.echo(" {}".format(format_requirement(ireq))) - else: - package_args = [] + + if ask and click.confirm("Would you like to proceed with these changes?"): + dry_run = False + + if not dry_run: + if to_uninstall: + check_call( # nosec + [sys.executable, "-m", "pip", "uninstall", "-y"] + + pip_flags + + sorted(to_uninstall) + ) + + if to_install: + if install_flags is None: + install_flags = [] + # prepare requirement lines + req_lines = [] for ireq in sorted(to_install, key=key_from_ireq): - if ireq.editable: - package_args.extend(['-e', str(ireq.link or ireq.req)]) - else: - package_args.append(str(ireq.req)) - check_call([pip, 'install'] + pip_flags + install_flags + package_args) + ireq_hashes = get_hashes_from_ireq(ireq) + req_lines.append(format_requirement(ireq, hashes=ireq_hashes)) + + # save requirement lines to a temporary file + tmp_req_file = tempfile.NamedTemporaryFile(mode="wt", delete=False) + tmp_req_file.write("\n".join(req_lines)) + tmp_req_file.close() + + try: + check_call( # nosec + [sys.executable, "-m", "pip", "install", "-r", tmp_req_file.name] + + pip_flags + + install_flags + ) + finally: + os.unlink(tmp_req_file.name) + return 0 diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py index 6f62eb9d..6bd01c0b 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py @@ -1,23 +1,31 @@ # coding: utf-8 -from __future__ import (absolute_import, division, print_function, - unicode_literals) +from __future__ import absolute_import, division, print_function, unicode_literals import os import sys -import six -from itertools import chain, groupby from collections import OrderedDict -from contextlib import contextmanager +from itertools import chain, groupby -from ._compat import install_req_from_line +import six +from click.utils import LazyFile +from six.moves import shlex_quote +from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier +from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable + +from ._compat import PIP_VERSION, InstallCommand, install_req_from_line from .click import style -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable - -UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} +UNSAFE_PACKAGES = {"setuptools", "distribute", "pip"} +COMPILE_EXCLUDE_OPTIONS = { + "--dry-run", + "--quiet", + "--rebuild", + "--upgrade", + "--upgrade-package", + "--verbose", +} def simplify_markers(ireq): @@ -68,11 +76,11 @@ def clean_requires_python(candidates): all_candidates = [] py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) for c in candidates: - if c.requires_python: + if getattr(c, "requires_python", None): # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,<digit+1' - if c.requires_python.isdigit(): - c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1) + if len(c.requires_python) == 1 and c.requires_python in ("2", "3"): + c.requires_python = '>={0},<{1!s}'.format(c.requires_python, int(c.requires_python) + 1) try: specifierset = SpecifierSet(c.requires_python) except InvalidSpecifier: @@ -94,19 +102,19 @@ def key_from_ireq(ireq): def key_from_req(req): """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, 'key'): + if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync key = req.key else: # from packaging, such as install requirements from requirements.txt key = req.name - key = key.replace('_', '-').lower() + key = key.replace("_", "-").lower() return key def comment(text): - return style(text, fg='green') + return style(text, fg="green") def make_install_requirement(name, version, extras, markers, constraint=False): @@ -156,14 +164,32 @@ def _requirement_to_str_lowercase_name(requirement): return "".join(parts) -def format_requirement(ireq, marker=None): +def is_url_requirement(ireq): + """ + Return True if requirement was specified as a path or URL. + ireq.original_link will have been set by InstallRequirement.__init__ + """ + return bool(ireq.original_link) + + +def format_requirement(ireq, marker=None, hashes=None): + """ + Generic formatter for pretty printing InstallRequirements to the terminal + in a less verbose way than using its `__str__` method. + """ if ireq.editable: - line = '-e {}'.format(ireq.link) + line = "-e {}".format(ireq.link.url) + elif is_url_requirement(ireq): + line = ireq.link.url else: line = _requirement_to_str_lowercase_name(ireq.req) if marker and ';' not in line: - line = '{}; {}'.format(line, marker) + line = "{}; {}".format(line, marker) + + if hashes: + for hash_ in sorted(hashes): + line += " \\\n --hash={}".format(hash_) return line @@ -176,7 +202,7 @@ def format_specifier(ireq): # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier._specs if ireq.req is not None else [] specs = sorted(specs, key=lambda x: x._spec[1]) - return ','.join(str(s) for s in specs) or '<any>' + return ",".join(str(s) for s in specs) or "<any>" def is_pinned_requirement(ireq): @@ -199,19 +225,20 @@ def is_pinned_requirement(ireq): if ireq.editable: return False - if len(ireq.specifier._specs) != 1: + if ireq.req is None or len(ireq.specifier._specs) != 1: return False op, version = next(iter(ireq.specifier._specs))._spec - return (op == '==' or op == '===') and not version.endswith('.*') + return (op == "==" or op == "===") and not version.endswith(".*") def as_tuple(ireq): """ - Pulls out the (name: str, version:str, extras:(str)) tuple from the pinned InstallRequirement. + Pulls out the (name: str, version:str, extras:(str)) tuple from + the pinned InstallRequirement. """ if not is_pinned_requirement(ireq): - raise TypeError('Expected a pinned InstallRequirement, got {}'.format(ireq)) + raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) name = key_from_req(ireq.req) version = next(iter(ireq.specifier._specs))._spec[1] @@ -252,6 +279,16 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): ... 'q': 'quux' ... } + For the values represented as lists, set use_lists=True: + + >>> assert lookup_table( + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], + ... use_lists=True) == { + ... 'b': ['bar', 'baz'], + ... 'f': ['foo'], + ... 'q': ['qux', 'quux'] + ... } + The values of the resulting lookup table will be values, not sets. For extra power, you can even change the values while building up the LUT. @@ -268,9 +305,14 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): """ if keyval is None: if key is None: - keyval = (lambda v: v) + + def keyval(v): + return v + else: - keyval = (lambda v: (key(v), v)) + + def keyval(v): + return (key(v), v) if unique: return dict(keyval(v) for v in values) @@ -301,7 +343,7 @@ def dedup(iterable): def name_from_req(req): """Get the name of the requirement""" - if hasattr(req, 'project_name'): + if hasattr(req, "project_name"): # from pkg_resources, such as installed dists for pip-sync return req.project_name else: @@ -319,32 +361,136 @@ def fs_str(string): On Python 3 returns the string as is, since Python 3 uses unicode paths and the input string shouldn't be bytes. - >>> fs_str(u'some path component/Something') - 'some path component/Something' - >>> assert isinstance(fs_str('whatever'), str) - >>> assert isinstance(fs_str(u'whatever'), str) - :type string: str|unicode :rtype: str """ if isinstance(string, str): return string - assert not isinstance(string, bytes) + if isinstance(string, bytes): + raise AssertionError return string.encode(_fs_encoding) _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() -# Borrowed from pew to avoid importing pew which imports psutil -# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 -@contextmanager -def temp_environ(): - """Allow the ability to set os.environ temporarily""" - environ = dict(os.environ) - try: - yield +def get_hashes_from_ireq(ireq): + """ + Given an InstallRequirement, return a list of string hashes in + the format "{algorithm}:{hash}". Return an empty list if there are no hashes + in the requirement options. + """ + result = [] + ireq_hashes = ireq.options.get("hashes", {}) + for algorithm, hexdigests in ireq_hashes.items(): + for hash_ in hexdigests: + result.append("{}:{}".format(algorithm, hash_)) + return result - finally: - os.environ.clear() - os.environ.update(environ) + +def force_text(s): + """ + Return a string representing `s`. + """ + if s is None: + return "" + if not isinstance(s, six.string_types): + return six.text_type(s) + return s + + +def get_compile_command(click_ctx): + """ + Returns a normalized compile command depending on cli context. + + The command will be normalized by: + - expanding options short to long + - removing values that are already default + - sorting the arguments + - removing one-off arguments like '--upgrade' + - removing arguments that don't change build behaviour like '--verbose' + """ + from piptools.scripts.compile import cli + + # Map of the compile cli options (option name -> click.Option) + compile_options = {option.name: option for option in cli.params} + + left_args = [] + right_args = [] + + for option_name, value in click_ctx.params.items(): + option = compile_options[option_name] + + # Get the latest option name (usually it'll be a long name) + option_long_name = option.opts[-1] + + # Collect variadic args separately, they will be added + # at the end of the command later + if option.nargs < 0: + right_args.extend([shlex_quote(force_text(val)) for val in value]) + continue + + # Exclude one-off options (--upgrade/--upgrade-package/--rebuild/...) + # or options that don't change compile behaviour (--verbose/--dry-run/...) + if option_long_name in COMPILE_EXCLUDE_OPTIONS: + continue + + # Skip options without a value + if option.default is None and not value: + continue + + # Skip options with a default value + if option.default == value: + continue + + # Use a file name for file-like objects + if isinstance(value, LazyFile): + value = value.name + + # Convert value to the list + if not isinstance(value, (tuple, list)): + value = [value] + + for val in value: + # Flags don't have a value, thus add to args true or false option long name + if option.is_flag: + # If there are false-options, choose an option name depending on a value + if option.secondary_opts: + # Get the latest false-option + secondary_option_long_name = option.secondary_opts[-1] + arg = option_long_name if val else secondary_option_long_name + # There are no false-options, use true-option + else: + arg = option_long_name + left_args.append(shlex_quote(arg)) + # Append to args the option with a value + else: + left_args.append( + "{option}={value}".format( + option=option_long_name, value=shlex_quote(force_text(val)) + ) + ) + + return " ".join(["pip-compile"] + sorted(left_args) + sorted(right_args)) + + +def create_install_command(): + """ + Return an instance of InstallCommand. + """ + if PIP_VERSION < (19, 3): + return InstallCommand() + + from pipenv.patched.notpip._internal.commands import create_command + + return create_command("install") + + +def get_trusted_hosts(finder): + """ + Returns an iterable of trusted hosts from a given finder. + """ + if PIP_VERSION < (19, 2): + return (host for _, host, _ in finder.secure_origins) + + return finder.trusted_hosts diff --git a/pipenv/patched/piptools/writer.py b/pipenv/patched/piptools/writer.py index 97b6df94..47cfbbc4 100644 --- a/pipenv/patched/piptools/writer.py +++ b/pipenv/patched/piptools/writer.py @@ -1,19 +1,64 @@ +from __future__ import unicode_literals + import os from itertools import chain -from ._compat import ExitStack from .click import unstyle -from .io import AtomicSaver from .logging import log -from .utils import comment, dedup, format_requirement, key_from_req, UNSAFE_PACKAGES +from .utils import ( + UNSAFE_PACKAGES, + comment, + dedup, + format_requirement, + get_compile_command, + key_from_ireq, +) + +MESSAGE_UNHASHED_PACKAGE = comment( + "# WARNING: pip install will require the following package to be hashed." + "\n# Consider using a hashable URL like " + "https://github.com/jazzband/pip-tools/archive/SOMECOMMIT.zip" +) + +MESSAGE_UNSAFE_PACKAGES_UNPINNED = comment( + "# WARNING: The following packages were not pinned, but pip requires them to be" + "\n# pinned when the requirements file includes hashes. " + "Consider using the --allow-unsafe flag." +) + +MESSAGE_UNSAFE_PACKAGES = comment( + "# The following packages are considered to be unsafe in a requirements file:" +) + +MESSAGE_UNINSTALLABLE = ( + "The generated requirements file may be rejected by pip install. " + "See # WARNING lines for details." +) class OutputWriter(object): - def __init__(self, src_files, dst_file, dry_run, emit_header, emit_index, - emit_trusted_host, annotate, generate_hashes, - default_index_url, index_urls, trusted_hosts, format_control): + def __init__( + self, + src_files, + dst_file, + click_ctx, + dry_run, + emit_header, + emit_index, + emit_trusted_host, + annotate, + generate_hashes, + default_index_url, + index_urls, + trusted_hosts, + format_control, + allow_unsafe, + find_links, + emit_find_links, + ): self.src_files = src_files self.dst_file = dst_file + self.click_ctx = click_ctx self.dry_run = dry_run self.emit_header = emit_header self.emit_index = emit_index @@ -24,120 +69,177 @@ class OutputWriter(object): self.index_urls = index_urls self.trusted_hosts = trusted_hosts self.format_control = format_control + self.allow_unsafe = allow_unsafe + self.find_links = find_links + self.emit_find_links = emit_find_links def _sort_key(self, ireq): return (not ireq.editable, str(ireq.req).lower()) def write_header(self): if self.emit_header: - yield comment('#') - yield comment('# This file is autogenerated by pip-compile') - yield comment('# To update, run:') - yield comment('#') - custom_cmd = os.environ.get('CUSTOM_COMPILE_COMMAND') - if custom_cmd: - yield comment('# {}'.format(custom_cmd)) - else: - params = [] - if not self.emit_index: - params += ['--no-index'] - if not self.emit_trusted_host: - params += ['--no-emit-trusted-host'] - if not self.annotate: - params += ['--no-annotate'] - if self.generate_hashes: - params += ["--generate-hashes"] - params += ['--output-file', self.dst_file] - params += self.src_files - yield comment('# pip-compile {}'.format(' '.join(params))) - yield comment('#') + yield comment("#") + yield comment("# This file is autogenerated by pip-compile") + yield comment("# To update, run:") + yield comment("#") + compile_command = os.environ.get( + "CUSTOM_COMPILE_COMMAND" + ) or get_compile_command(self.click_ctx) + yield comment("# {}".format(compile_command)) + yield comment("#") def write_index_options(self): if self.emit_index: for index, index_url in enumerate(dedup(self.index_urls)): - if index_url.rstrip('/') == self.default_index_url: + if index_url.rstrip("/") == self.default_index_url: continue - flag = '--index-url' if index == 0 else '--extra-index-url' - yield '{} {}'.format(flag, index_url) + flag = "--index-url" if index == 0 else "--extra-index-url" + yield "{} {}".format(flag, index_url) def write_trusted_hosts(self): if self.emit_trusted_host: for trusted_host in dedup(self.trusted_hosts): - yield '--trusted-host {}'.format(trusted_host) + yield "--trusted-host {}".format(trusted_host) def write_format_controls(self): for nb in dedup(self.format_control.no_binary): - yield '--no-binary {}'.format(nb) + yield "--no-binary {}".format(nb) for ob in dedup(self.format_control.only_binary): - yield '--only-binary {}'.format(ob) + yield "--only-binary {}".format(ob) + + def write_find_links(self): + if self.emit_find_links: + for find_link in dedup(self.find_links): + yield "--find-links {}".format(find_link) def write_flags(self): emitted = False - for line in chain(self.write_index_options(), - self.write_trusted_hosts(), - self.write_format_controls()): + for line in chain( + self.write_index_options(), + self.write_find_links(), + self.write_trusted_hosts(), + self.write_format_controls(), + ): emitted = True yield line if emitted: - yield '' + yield "" + + def _iter_lines( + self, + results, + unsafe_requirements=None, + reverse_dependencies=None, + primary_packages=None, + markers=None, + hashes=None, + ): + # default values + unsafe_requirements = unsafe_requirements or [] + reverse_dependencies = reverse_dependencies or {} + primary_packages = primary_packages or [] + markers = markers or {} + hashes = hashes or {} + + # Check for unhashed or unpinned packages if at least one package does have + # hashes, which will trigger pip install's --require-hashes mode. + warn_uninstallable = False + has_hashes = hashes and any(hash for hash in hashes.values()) + + yielded = False - def _iter_lines(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): for line in self.write_header(): yield line + yielded = True for line in self.write_flags(): yield line + yielded = True - unsafe_requirements = {r for r in results if r.name in UNSAFE_PACKAGES} if not unsafe_requirements else unsafe_requirements # noqa + unsafe_requirements = ( + {r for r in results if r.name in UNSAFE_PACKAGES} + if not unsafe_requirements + else unsafe_requirements + ) packages = {r for r in results if r.name not in UNSAFE_PACKAGES} - packages = sorted(packages, key=self._sort_key) - - for ireq in packages: - line = self._format_requirement( - ireq, reverse_dependencies, primary_packages, - markers.get(key_from_req(ireq.req)), hashes=hashes) - yield line + if packages: + packages = sorted(packages, key=self._sort_key) + for ireq in packages: + if has_hashes and not hashes.get(ireq): + yield MESSAGE_UNHASHED_PACKAGE + warn_uninstallable = True + line = self._format_requirement( + ireq, + reverse_dependencies, + primary_packages, + markers.get(key_from_ireq(ireq)), + hashes=hashes, + ) + yield line + yielded = True if unsafe_requirements: unsafe_requirements = sorted(unsafe_requirements, key=self._sort_key) - yield '' - yield comment('# The following packages are considered to be unsafe in a requirements file:') + yield "" + yielded = True + if has_hashes and not self.allow_unsafe: + yield MESSAGE_UNSAFE_PACKAGES_UNPINNED + warn_uninstallable = True + else: + yield MESSAGE_UNSAFE_PACKAGES for ireq in unsafe_requirements: - req = self._format_requirement(ireq, - reverse_dependencies, - primary_packages, - marker=markers.get(key_from_req(ireq.req)), - hashes=hashes) - if not allow_unsafe: - yield comment('# {}'.format(req)) + ireq_key = key_from_ireq(ireq) + if not self.allow_unsafe: + yield comment("# {}".format(ireq_key)) else: - yield req + line = self._format_requirement( + ireq, + reverse_dependencies, + primary_packages, + marker=markers.get(ireq_key), + hashes=hashes, + ) + yield line - def write(self, results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=False): - with ExitStack() as stack: - f = None + # Yield even when there's no real content, so that blank files are written + if not yielded: + yield "" + + if warn_uninstallable: + log.warning(MESSAGE_UNINSTALLABLE) + + def write( + self, + results, + unsafe_requirements, + reverse_dependencies, + primary_packages, + markers, + hashes, + ): + + for line in self._iter_lines( + results, + unsafe_requirements, + reverse_dependencies, + primary_packages, + markers, + hashes, + ): + log.info(line) if not self.dry_run: - f = stack.enter_context(AtomicSaver(self.dst_file)) - - for line in self._iter_lines(results, unsafe_requirements, reverse_dependencies, - primary_packages, markers, hashes, allow_unsafe=allow_unsafe): - log.info(line) - if f: - f.write(unstyle(line).encode('utf-8')) - f.write(os.linesep.encode('utf-8')) - - def _format_requirement(self, ireq, reverse_dependencies, primary_packages, marker=None, hashes=None): - line = format_requirement(ireq, marker=marker) + self.dst_file.write(unstyle(line).encode("utf-8")) + self.dst_file.write(os.linesep.encode("utf-8")) + def _format_requirement( + self, ireq, reverse_dependencies, primary_packages, marker=None, hashes=None + ): ireq_hashes = (hashes if hashes is not None else {}).get(ireq) - if ireq_hashes: - for hash_ in sorted(ireq_hashes): - line += " \\\n --hash={}".format(hash_) - if not self.annotate or key_from_req(ireq.req) in primary_packages: + line = format_requirement(ireq, marker=marker, hashes=ireq_hashes) + + if not self.annotate or key_from_ireq(ireq) in primary_packages: return line # Annotate what packages this package is required by @@ -147,5 +249,6 @@ class OutputWriter(object): line = "{:24}{}{}".format( line, " \\\n " if ireq_hashes else " ", - comment("# via " + annotation)) + comment("# via " + annotation), + ) return line diff --git a/pipenv/patched/safety/LICENSE b/pipenv/patched/safety.LICENSE similarity index 99% rename from pipenv/patched/safety/LICENSE rename to pipenv/patched/safety.LICENSE index 55a1eb03..c5fda558 100644 --- a/pipenv/patched/safety/LICENSE +++ b/pipenv/patched/safety.LICENSE @@ -1,4 +1,3 @@ - MIT License Copyright (c) 2016, pyup.io diff --git a/pipenv/patched/safety.zip b/pipenv/patched/safety.zip deleted file mode 100644 index b839971a..00000000 Binary files a/pipenv/patched/safety.zip and /dev/null differ diff --git a/pipenv/patched/safety/__init__.py b/pipenv/patched/safety/__init__.py index 56b497d2..e9a6e965 100644 --- a/pipenv/patched/safety/__init__.py +++ b/pipenv/patched/safety/__init__.py @@ -2,4 +2,4 @@ __author__ = """pyup.io""" __email__ = 'support@pyup.io' -__version__ = '1.8.4' +__version__ = '1.8.7' diff --git a/pipenv/patched/safety/__main__.py b/pipenv/patched/safety/__main__.py index d9a0bdab..f905408a 100644 --- a/pipenv/patched/safety/__main__.py +++ b/pipenv/patched/safety/__main__.py @@ -1,8 +1,51 @@ """Allow safety to be executable through `python -m safety`.""" from __future__ import absolute_import -from .cli import cli +import os +import sys +import sysconfig + + +PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PIPENV_DIR = os.path.dirname(PATCHED_DIR) +VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor") + + +def get_site_packages(): + prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} + try: + prefixes.add(sys.real_prefix) + except AttributeError: + pass + form = sysconfig.get_path('purelib', expand=False) + py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) + return { + form.format(base=prefix, py_version_short=py_version_short) + for prefix in prefixes + } + + +def insert_before_site_packages(*paths): + site_packages = get_site_packages() + index = None + for i, path in enumerate(sys.path): + if path in site_packages: + index = i + break + if index is None: + sys.path += list(paths) + else: + sys.path = sys.path[:index] + list(paths) + sys.path[index:] + + +def insert_pipenv_dirs(): + insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR) if __name__ == "__main__": # pragma: no cover + insert_pipenv_dirs() + yaml_lib = "pipenv.patched.yaml{0}".format(sys.version_info[0]) + locals()[yaml_lib] = __import__(yaml_lib) + sys.modules["yaml"] = sys.modules[yaml_lib] + from safety.cli import cli cli(prog_name="safety") diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index 9ebd9e8a..2e8f88df 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -6,17 +6,13 @@ from safety import __version__ from safety import safety from safety.formatter import report import itertools -from safety.util import read_requirements +from safety.util import read_requirements, read_vulnerabilities from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError - try: - # pip 9 - from pipenv.patched.notpip import get_installed_distributions + from json.decoder import JSONDecodeError except ImportError: - # pip 10 - from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - + JSONDecodeError = ValueError @click.group() @click.version_option(version=__version__) @@ -46,10 +42,17 @@ def cli(): help="Read input from one (or multiple) requirement files. Default: empty") @click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[], help="Ignore one (or multiple) vulnerabilities by ID. Default: empty") -def check(key, db, json, full_report, bare, stdin, files, cache, ignore): - +@click.option("--output", "-o", default="", + help="Path to where output file will be placed. Default: empty") +@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, + help="Proxy host IP or DNS --proxy-host") +@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80, + help="Proxy port number --proxy-port") +@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', + help="Proxy protocol (https or http) --proxy-protocol") +def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport): if files and stdin: - click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red") + click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr) sys.exit(-1) if files: @@ -57,33 +60,72 @@ def check(key, db, json, full_report, bare, stdin, files, cache, ignore): elif stdin: packages = list(read_requirements(sys.stdin)) else: - packages = get_installed_distributions() - + import pkg_resources + packages = [ + d for d in pkg_resources.working_set + if d.key not in {"python", "wsgiref", "argparse"} + ] + proxy_dictionary = {} + if proxyhost is not None: + if proxyprotocol in ["http", "https"]: + proxy_dictionary = {proxyprotocol: "{0}://{1}:{2}".format(proxyprotocol, proxyhost, str(proxyport))} + else: + click.secho("Proxy Protocol should be http or https only.", fg="red") + sys.exit(-1) try: - vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore) - click.secho(report( - vulns=vulns, - full=full_report, - json_report=json, - bare_report=bare, - checked_packages=len(packages), - db=db, - key=key - ) - ) + vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary) + output_report = report(vulns=vulns, + full=full_report, + json_report=json, + bare_report=bare, + checked_packages=len(packages), + db=db, + key=key) + + if output: + with open(output, 'w+') as output_file: + output_file.write(output_report) + else: + click.secho(output_report, nl=False if bare and not vulns else True) sys.exit(-1 if vulns else 0) except InvalidKeyError: click.secho("Your API Key '{key}' is invalid. See {link}".format( key=key, link='https://goo.gl/O7Y1rS'), - fg="red") + fg="red", + file=sys.stderr) sys.exit(-1) except DatabaseFileNotFoundError: - click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red") + click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red", file=sys.stderr) sys.exit(-1) except DatabaseFetchError: - click.secho("Unable to load vulnerability database", fg="red") + click.secho("Unable to load vulnerability database", fg="red", file=sys.stderr) sys.exit(-1) +@cli.command() +@click.option("--full-report/--short-report", default=False, + help='Full reports include a security advisory (if available). Default: ' + '--short-report') +@click.option("--bare/--not-bare", default=False, + help='Output vulnerable packages only. Useful in combination with other tools.' + 'Default: --not-bare') +@click.option("file", "--file", "-f", type=click.File(), required=True, + help="Read input from an insecure report file. Default: empty") +def review(full_report, bare, file): + if full_report and bare: + click.secho("Can't choose both --bare and --full-report/--short-report", fg="red") + sys.exit(-1) + + try: + input_vulns = read_vulnerabilities(file) + except JSONDecodeError: + click.secho("Not a valid JSON file", fg="red") + sys.exit(-1) + + vulns = safety.review(input_vulns) + output_report = report(vulns=vulns, full=full_report, bare_report=bare) + click.secho(output_report, nl=False if bare and not vulns else True) + + if __name__ == "__main__": cli() diff --git a/pipenv/patched/safety/formatter.py b/pipenv/patched/safety/formatter.py index 8bc57ec1..c19bff1b 100644 --- a/pipenv/patched/safety/formatter.py +++ b/pipenv/patched/safety/formatter.py @@ -3,6 +3,7 @@ import platform import sys import json import os +import textwrap # python 2.7 compat try: @@ -110,9 +111,10 @@ class SheetReport(object): descr = get_advisory(vuln) - for chunk in [descr[i:i + 76] for i in range(0, len(descr), 76)]: - - for line in chunk.splitlines(): + for pn, paragraph in enumerate(descr.replace('\r', '').split('\n\n')): + if pn: + table.append("│ {:76} │".format('')) + for line in textwrap.wrap(paragraph, width=76): try: table.append("│ {:76} │".format(line.encode('utf-8'))) except TypeError: diff --git a/pipenv/patched/safety/safety.py b/pipenv/patched/safety/safety.py index 99b99125..2fca3eb2 100644 --- a/pipenv/patched/safety/safety.py +++ b/pipenv/patched/safety/safety.py @@ -9,6 +9,7 @@ import json import time import errno + class Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id"])): pass @@ -64,7 +65,7 @@ def write_to_cache(db_name, data): f.write(json.dumps(cache)) -def fetch_database_url(mirror, db_name, key, cached): +def fetch_database_url(mirror, db_name, key, cached, proxy): headers = {} if key: @@ -74,9 +75,8 @@ def fetch_database_url(mirror, db_name, key, cached): cached_data = get_from_cache(db_name=db_name) if cached_data: return cached_data - url = mirror + db_name - r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers) + r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy) if r.status_code == 200: data = r.json() if cached: @@ -94,7 +94,7 @@ def fetch_database_file(path, db_name): return json.loads(f.read()) -def fetch_database(full=False, key=False, db=False, cached=False): +def fetch_database(full=False, key=False, db=False, cached=False, proxy={}): if db: mirrors = [db] @@ -105,7 +105,7 @@ def fetch_database(full=False, key=False, db=False, cached=False): for mirror in mirrors: # mirror can either be a local path or a URL if mirror.startswith("http://") or mirror.startswith("https://"): - data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached) + data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy) else: data = fetch_database_file(mirror, db_name=db_name) if data: @@ -120,10 +120,9 @@ def get_vulnerabilities(pkg, spec, db): yield entry -def check(packages, key, db_mirror, cached, ignore_ids): - +def check(packages, key, db_mirror, cached, ignore_ids, proxy): key = key if key else os.environ.get("SAFETY_API_KEY", False) - db = fetch_database(key=key, db=db_mirror, cached=cached) + db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy) db_full = None vulnerable_packages = frozenset(db.keys()) vulnerable = [] @@ -138,7 +137,7 @@ def check(packages, key, db_mirror, cached, ignore_ids): spec_set = SpecifierSet(specifiers=specifier) if spec_set.contains(pkg.version): if not db_full: - db_full = fetch_database(full=True, key=key, db=db_mirror) + db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy) for data in get_vulnerabilities(pkg=name, spec=specifier, db=db_full): vuln_id = data.get("id").replace("pyup.io-", "") if vuln_id and vuln_id not in ignore_ids: @@ -152,3 +151,19 @@ def check(packages, key, db_mirror, cached, ignore_ids): ) ) return vulnerable + + +def review(vulnerabilities): + vulnerable = [] + for vuln in vulnerabilities: + current_vuln = { + "name": vuln[0], + "spec": vuln[1], + "version": vuln[2], + "advisory": vuln[3], + "vuln_id": vuln[4], + } + vulnerable.append( + Vulnerability(**current_vuln) + ) + return vulnerable diff --git a/pipenv/patched/safety/util.py b/pipenv/patched/safety/util.py index 2e6efaec..16062f41 100644 --- a/pipenv/patched/safety/util.py +++ b/pipenv/patched/safety/util.py @@ -1,11 +1,17 @@ from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements from collections import namedtuple import click +import sys +import json import os Package = namedtuple("Package", ["key", "version"]) RequirementFile = namedtuple("RequirementFile", ["path"]) +def read_vulnerabilities(fh): + return json.load(fh) + + def iter_lines(fh, lineno=0): for line in fh.readlines()[lineno:]: yield line @@ -85,7 +91,8 @@ def read_requirements(fh, resolve=False): "Warning: unpinned requirement '{req}' found in {fname}, " "unable to check.".format(req=req.name, fname=fname), - fg="yellow" + fg="yellow", + file=sys.stderr ) except ValueError: continue diff --git a/pipenv/patched/yaml2/LICENSE b/pipenv/patched/yaml2/LICENSE new file mode 100644 index 00000000..3d82c281 --- /dev/null +++ b/pipenv/patched/yaml2/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/yaml2/__init__.py b/pipenv/patched/yaml2/__init__.py new file mode 100644 index 00000000..211fc866 --- /dev/null +++ b/pipenv/patched/yaml2/__init__.py @@ -0,0 +1,431 @@ + +from error import * + +from tokens import * +from events import * +from nodes import * + +from loader import * +from dumper import * + +__version__ = '5.3.1' + +try: + from cyaml import * + __with_libyaml__ = True +except ImportError: + __with_libyaml__ = False + + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = ( + "calling yaml.%s() without Loader=... is deprecated, as the " + "default Loader is unsafe. Please read " + "https://msg.pyyaml.org/load for full details." + ) % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding representation trees. + """ + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + +def load(stream, Loader=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + if Loader is None: + load_warning('load') + Loader = FullLoader + + loader = Loader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + +def load_all(stream, Loader=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + + loader = Loader(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load(stream, SafeLoader) + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load_all(stream, SafeLoader) + +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + from StringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + from StringIO import StringIO + else: + from cStringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding='utf-8', explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + from StringIO import StringIO + else: + from cStringIO import StringIO + stream = StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=None, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=None): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(object): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __metaclass__ = YAMLObjectMetaclass + __slots__ = () # no direct instantiation, so allow immutable subclasses + + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + from_yaml = classmethod(from_yaml) + + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + to_yaml = classmethod(to_yaml) + diff --git a/pipenv/patched/yaml2/composer.py b/pipenv/patched/yaml2/composer.py new file mode 100644 index 00000000..df85ef65 --- /dev/null +++ b/pipenv/patched/yaml2/composer.py @@ -0,0 +1,139 @@ + +__all__ = ['Composer', 'ComposerError'] + +from error import MarkedYAMLError +from events import * +from nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer(object): + + def __init__(self): + self.anchors = {} + + def check_node(self): + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def get_single_node(self): + # Drop the STREAM-START event. + self.get_event() + + # Compose a document if the stream is not empty. + document = None + if not self.check_event(StreamEndEvent): + document = self.compose_document() + + # Ensure that the stream contains no more documents. + if not self.check_event(StreamEndEvent): + event = self.get_event() + raise ComposerError("expected a single document in the stream", + document.start_mark, "but found another document", + event.start_mark) + + # Drop the STREAM-END event. + self.get_event() + + return document + + def compose_document(self): + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor.encode('utf-8'), event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" + % anchor.encode('utf-8'), self.anchors[anchor].start_mark, + "second occurrence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == u'!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == u'!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + #key_event = self.peek_event() + item_key = self.compose_node(node, None) + #if item_key in node.value: + # raise ComposerError("while composing a mapping", start_event.start_mark, + # "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + #node.value[item_key] = item_value + node.value.append((item_key, item_value)) + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/pipenv/patched/yaml2/constructor.py b/pipenv/patched/yaml2/constructor.py new file mode 100644 index 00000000..794681cb --- /dev/null +++ b/pipenv/patched/yaml2/constructor.py @@ -0,0 +1,760 @@ + +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] + +from error import * +from nodes import * + +import datetime + +import binascii, re, sys, types + +class ConstructorError(MarkedYAMLError): + pass + + +class timezone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + seconds = abs(offset).total_seconds() + self._name = 'UTC%s%02d:%02d' % ( + '-' if offset.days < 0 else '+', + seconds // 3600, + seconds % 3600 // 60 + ) + + def tzname(self, dt=None): + return self._name + + def utcoffset(self, dt=None): + return self._offset + + def dst(self, dt=None): + return datetime.timedelta(0) + + __repr__ = __str__ = tzname + + +class BaseConstructor(object): + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def get_single_data(self): + # Ensure that the stream contains a single document and construct it. + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if tag_prefix is not None and node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = generator.next() + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + try: + hash(key) + except TypeError, exc: + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unacceptable key (%s)" % exc, key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + add_constructor = classmethod(add_constructor) + + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + add_multi_constructor = classmethod(add_multi_constructor) + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == u'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return BaseConstructor.construct_scalar(self, node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == u'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == u'tag:yaml.org,2002:value': + key_node.tag = u'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return BaseConstructor.construct_mapping(self, node, deep=deep) + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + u'yes': True, + u'no': False, + u'true': True, + u'false': False, + u'on': True, + u'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + def construct_yaml_float(self, node): + value = str(self.construct_scalar(node)) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_yaml_binary(self, node): + value = self.construct_scalar(node) + try: + return str(value).decode('base64') + except (binascii.Error, UnicodeEncodeError), exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + ur'''^(?P<year>[0-9][0-9][0-9][0-9]) + -(?P<month>[0-9][0-9]?) + -(?P<day>[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P<hour>[0-9][0-9]?) + :(?P<minute>[0-9][0-9]) + :(?P<second>[0-9][0-9]) + (?:\.(?P<fraction>[0-9]*))? + (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?) + (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + year = int(values['year']) + month = int(values['month']) + day = int(values['day']) + if not values['hour']: + return datetime.date(year, month, day) + hour = int(values['hour']) + minute = int(values['minute']) + second = int(values['second']) + fraction = 0 + tzinfo = None + if values['fraction']: + fraction = values['fraction'][:6] + while len(fraction) < 6: + fraction += '0' + fraction = int(fraction) + if values['tz_sign']: + tz_hour = int(values['tz_hour']) + tz_minute = int(values['tz_minute'] or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if values['tz_sign'] == '-': + delta = -delta + tzinfo = timezone(delta) + elif values['tz']: + tzinfo = timezone(datetime.timedelta(0)) + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + omap.append((key, value)) + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + pairs.append((key, value)) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + value = self.construct_scalar(node) + try: + return value.encode('ascii') + except UnicodeEncodeError: + return value + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) + data.__setstate__(state) + else: + state = self.construct_mapping(node) + data.__dict__.update(state) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag.encode('utf-8'), + node.start_mark) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + u'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp + + def construct_python_str(self, node): + return self.construct_scalar(node).encode('utf-8') + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_long(self, node): + return long(self.construct_yaml_int(node)) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def find_python_module(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + if unsafe: + try: + __import__(name) + except ImportError, exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name.encode('utf-8'), exc), mark) + if name not in sys.modules: + raise ConstructorError("while constructing a Python module", mark, + "module %r is not imported" % name.encode('utf-8'), mark) + return sys.modules[name] + + def find_python_name(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if u'.' in name: + module_name, object_name = name.rsplit('.', 1) + else: + module_name = '__builtin__' + object_name = name + if unsafe: + try: + __import__(module_name) + except ImportError, exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name.encode('utf-8'), exc), mark) + if module_name not in sys.modules: + raise ConstructorError("while constructing a Python object", mark, + "module %r is not imported" % module_name.encode('utf-8'), mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" % (object_name.encode('utf-8'), + module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value.encode('utf-8'), + node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + class classobj: pass + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False, unsafe=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type) or isinstance(cls, type(self.classobj))): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) + if newobj and isinstance(cls, type(self.classobj)) \ + and not args and not kwds: + instance = self.classobj() + instance.__class__ = cls + return instance + elif newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state, unsafe=False): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) + setattr(instance, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) + self.set_python_instance_state(instance, state) + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node, deep=True) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node, deep=True) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + u'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/module:', + FullConstructor.construct_python_module) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object:', + FullConstructor.construct_python_object) + +FullConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/new:', + FullConstructor.construct_python_object_new) + +class UnsafeConstructor(FullConstructor): + + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) + + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) + + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) + + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + +UnsafeConstructor.add_multi_constructor( + u'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/pipenv/patched/yaml2/cyaml.py b/pipenv/patched/yaml2/cyaml.py new file mode 100644 index 00000000..ebb89593 --- /dev/null +++ b/pipenv/patched/yaml2/cyaml.py @@ -0,0 +1,101 @@ + +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] + +from _yaml import CParser, CEmitter + +from constructor import * + +from serializer import * +from representer import * + +from resolver import * + +class CBaseLoader(CParser, BaseConstructor, BaseResolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class CSafeLoader(CParser, SafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + +class CLoader(CParser, Constructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + Constructor.__init__(self) + Resolver.__init__(self) + +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CSafeDumper(CEmitter, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CDumper(CEmitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml2/dumper.py b/pipenv/patched/yaml2/dumper.py new file mode 100644 index 00000000..f9cd49fd --- /dev/null +++ b/pipenv/patched/yaml2/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from emitter import * +from serializer import * +from representer import * +from resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml2/emitter.py b/pipenv/patched/yaml2/emitter.py new file mode 100644 index 00000000..23c25ca8 --- /dev/null +++ b/pipenv/patched/yaml2/emitter.py @@ -0,0 +1,1144 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +import sys + +from error import YAMLError +from events import * + +has_ucs4 = sys.maxunicode > 0xffff + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis(object): + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter(object): + + DEFAULT_TAG_PREFIXES = { + u'!' : u'!', + u'tag:yaml.org,2002:' : u'!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overridden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Whether the document requires an explicit document indicator + self.open_ended = False + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = u'\n' + if line_break in [u'\r', u'\n', u'\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding and not getattr(self.stream, 'encoding', None): + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if (self.event.version or self.event.tags) and self.open_ended: + self.write_indicator(u'...', True) + self.write_indent() + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = self.event.tags.keys() + handles.sort() + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator(u'---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + if self.open_ended: + self.write_indicator(u'...', True) + self.write_indent() + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator(u'...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor(u'&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor(u'*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator(u'[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u']', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator(u'{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(u',', False) + self.write_indent() + self.write_indicator(u'}', False) + self.state = self.states.pop() + else: + self.write_indicator(u',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(u':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator(u'-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator(u'?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(u':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(u':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == u'') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = u'!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if (not self.flow_level and not self.simple_key_context + and self.analysis.allow_block): + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return u'%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != u'!' or handle[-1] != u'!': + raise EmitterError("tag handle must start and end with '!': %r" + % (handle.encode('utf-8'))) + for ch in handle[1:-1]: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch.encode('utf-8'), handle.encode('utf-8'))) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == u'!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return u''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == u'!': + return tag + handle = None + suffix = tag + prefixes = self.tag_prefixes.keys() + prefixes.sort() + for prefix in prefixes: + if tag.startswith(prefix) \ + and (prefix == u'!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?:@&=+$,_.~*\'()[]' \ + or (ch == u'!' and handle != u'!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append(u'%%%02X' % ord(ch)) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = u''.join(chunks) + if handle: + return u'%s%s' % (handle, suffix_text) + else: + return u'!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not (u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch.encode('utf-8'), anchor.encode('utf-8'))) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Important whitespace combinations. + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + + # Check document indicators. + if scalar.startswith(u'---') or scalar.startswith(u'...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceded_by_whitespace = True + + # Last character or followed by a whitespace. + followed_by_whitespace = (len(scalar) == 1 or + scalar[1] in u'\0 \t\r\n\x85\u2028\u2029') + + # The previous character is a space. + previous_space = False + + # The previous character is a break. + previous_break = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + if index == 0: + # Leading indicators are special characters. + if ch in u'#,[]{}&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in u'?:': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == u'-' and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in u',?[]{}': + flow_indicators = True + if ch == u':': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == u'#' and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in u'\n\x85\u2028\u2029': + line_breaks = True + if not (ch == u'\n' or u'\x20' <= ch <= u'\x7E'): + if (ch == u'\x85' or u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD' + or (u'\U00010000' <= ch < u'\U0010ffff')) and ch != u'\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Detect important whitespace combinations. + if ch == u' ': + if index == 0: + leading_space = True + if index == len(scalar)-1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in u'\n\x85\u2028\u2029': + if index == 0: + leading_break = True + if index == len(scalar)-1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + + # Prepare for the next character. + index += 1 + preceded_by_whitespace = (ch in u'\0 \t\r\n\x85\u2028\u2029') + followed_by_whitespace = (index+1 >= len(scalar) or + scalar[index+1] in u'\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespaces are bad for plain scalars. + if (leading_space or leading_break + or trailing_space or trailing_break): + allow_flow_plain = allow_block_plain = False + + # We do not permit trailing spaces for block scalars. + if trailing_space: + allow_block = False + + # Spaces at the beginning of a new line are only acceptable for block + # scalars. + if break_space: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Spaces followed by breaks, as well as special character are only + # allowed for double quoted scalars. + if space_break or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # Although the plain scalar writer supports breaks, we never emit + # multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write(u'\uFEFF'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = u' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = u' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = u'%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = u'%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator(u'\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != u' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029' or ch == u'\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == u'\'': + data = u'\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + self.write_indicator(u'\'', False) + + ESCAPE_REPLACEMENTS = { + u'\0': u'0', + u'\x07': u'a', + u'\x08': u'b', + u'\x09': u't', + u'\x0A': u'n', + u'\x0B': u'v', + u'\x0C': u'f', + u'\x0D': u'r', + u'\x1B': u'e', + u'\"': u'\"', + u'\\': u'\\', + u'\x85': u'N', + u'\xA0': u'_', + u'\u2028': u'L', + u'\u2029': u'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator(u'"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in u'"\\\x85\u2028\u2029\uFEFF' \ + or not (u'\x20' <= ch <= u'\x7E' + or (self.allow_unicode + and (u'\xA0' <= ch <= u'\uD7FF' + or u'\uE000' <= ch <= u'\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = u'\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= u'\xFF': + data = u'\\x%02X' % ord(ch) + elif ch <= u'\uFFFF': + data = u'\\u%04X' % ord(ch) + else: + data = u'\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == u' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+u'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == u' ': + data = u'\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator(u'"', False) + + def determine_block_hints(self, text): + hints = u'' + if text: + if text[0] in u' \n\x85\u2028\u2029': + hints += unicode(self.best_indent) + if text[-1] not in u'\n\x85\u2028\u2029': + hints += u'-' + elif len(text) == 1 or text[-2] in u'\n\x85\u2028\u2029': + hints += u'+' + return hints + + def write_folded(self, text): + hints = self.determine_block_hints(text) + self.write_indicator(u'>'+hints, True) + if hints[-1:] == u'+': + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != u' ' \ + and text[start] == u'\n': + self.write_line_break() + leading_space = (ch == u' ') + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + spaces = (ch == u' ') + end += 1 + + def write_literal(self, text): + hints = self.determine_block_hints(text) + self.write_indicator(u'|'+hints, True) + if hints[-1:] == u'+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in u'\n\x85\u2028\u2029': + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in u'\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + data = u' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != u' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in u'\n\x85\u2028\u2029': + if text[start] == u'\n': + self.write_line_break() + for br in text[start:end]: + if br == u'\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in u' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == u' ') + breaks = (ch in u'\n\x85\u2028\u2029') + end += 1 diff --git a/pipenv/patched/yaml2/error.py b/pipenv/patched/yaml2/error.py new file mode 100644 index 00000000..577686db --- /dev/null +++ b/pipenv/patched/yaml2/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark(object): + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in u'\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in u'\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end].encode('utf-8') + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/pipenv/patched/yaml2/events.py b/pipenv/patched/yaml2/events.py new file mode 100644 index 00000000..f79ad389 --- /dev/null +++ b/pipenv/patched/yaml2/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event(object): + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/pipenv/patched/yaml2/loader.py b/pipenv/patched/yaml2/loader.py new file mode 100644 index 00000000..4d773c3c --- /dev/null +++ b/pipenv/patched/yaml2/loader.py @@ -0,0 +1,63 @@ + +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] + +from reader import * +from scanner import * +from parser import * +from composer import * +from constructor import * +from resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatibility. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/pipenv/patched/yaml2/nodes.py b/pipenv/patched/yaml2/nodes.py new file mode 100644 index 00000000..c4f070c4 --- /dev/null +++ b/pipenv/patched/yaml2/nodes.py @@ -0,0 +1,49 @@ + +class Node(object): + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '<empty>' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/pipenv/patched/yaml2/parser.py b/pipenv/patched/yaml2/parser.py new file mode 100644 index 00000000..f9e3057f --- /dev/null +++ b/pipenv/patched/yaml2/parser.py @@ -0,0 +1,589 @@ + +# The following YAML grammar is LL(1) and is parsed by a recursive descent +# parser. +# +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# implicit_document ::= block_node DOCUMENT-END* +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +# block_node_or_indentless_sequence ::= +# ALIAS +# | properties (block_content | indentless_block_sequence)? +# | block_content +# | indentless_block_sequence +# block_node ::= ALIAS +# | properties block_content? +# | block_content +# flow_node ::= ALIAS +# | properties flow_content? +# | flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# flow_collection ::= flow_sequence | flow_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +# block_mapping ::= BLOCK-MAPPING_START +# ((KEY block_node_or_indentless_sequence?)? +# (VALUE block_node_or_indentless_sequence?)?)* +# BLOCK-END +# flow_sequence ::= FLOW-SEQUENCE-START +# (flow_sequence_entry FLOW-ENTRY)* +# flow_sequence_entry? +# FLOW-SEQUENCE-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping ::= FLOW-MAPPING-START +# (flow_mapping_entry FLOW-ENTRY)* +# flow_mapping_entry? +# FLOW-MAPPING-END +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# +# FIRST sets: +# +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from error import MarkedYAMLError +from tokens import * +from events import * +from scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser(object): + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + + DEFAULT_TAGS = { + u'!': u'!', + u'!!': u'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.states = [] + self.marks = [] + self.state = self.parse_stream_start + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + return self.current_event + + def get_event(self): + # Get the next event and proceed further. + if self.current_event is None: + if self.state: + self.current_event = self.state() + value = self.current_event + self.current_event = None + return value + + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END + # implicit_document ::= block_node DOCUMENT-END* + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + + def parse_stream_start(self): + + # Parse the stream start. + token = self.get_token() + event = StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Prepare the next state. + self.state = self.parse_implicit_document_start + + return event + + def parse_implicit_document_start(self): + + # Parse an implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=False) + + # Prepare the next state. + self.states.append(self.parse_document_end) + self.state = self.parse_block_node + + return event + + else: + return self.parse_document_start() + + def parse_document_start(self): + + # Parse any extra document end indicators. + while self.check_token(DocumentEndToken): + self.get_token() + + # Parse an explicit document. + if not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '<document start>', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + self.states.append(self.parse_document_end) + self.state = self.parse_document_content + else: + # Parse the end of the stream. + token = self.get_token() + event = StreamEndEvent(token.start_mark, token.end_mark) + assert not self.states + assert not self.marks + self.state = None + return event + + def parse_document_end(self): + + # Parse the document end. + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + if self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + event = DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Prepare the next state. + self.state = self.parse_document_start + + return event + + def parse_document_content(self): + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + event = self.process_empty_scalar(self.peek_token().start_mark) + self.state = self.states.pop() + return event + else: + return self.parse_block_node() + + def process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == u'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == u'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle.encode('utf-8'), + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + # block_node_or_indentless_sequence ::= ALIAS + # | properties (block_content | indentless_block_sequence)? + # | block_content + # | indentless_block_sequence + # block_node ::= ALIAS + # | properties block_content? + # | block_content + # flow_node ::= ALIAS + # | properties flow_content? + # | flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # flow_collection ::= flow_sequence | flow_mapping + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + if self.check_token(AliasToken): + token = self.get_token() + event = AliasEvent(token.value, token.start_mark, token.end_mark) + self.state = self.states.pop() + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle.encode('utf-8'), + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == u'!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + implicit = (tag is None or tag == u'!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + self.state = self.parse_indentless_sequence_entry + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == u'!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + self.state = self.states.pop() + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_sequence_first_entry + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_mapping_first_key + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_sequence_first_entry + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_mapping_first_key + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), u'', + start_mark, end_mark) + self.state = self.states.pop() + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while parsing a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + return event + + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + + def parse_block_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_sequence_entry() + + def parse_block_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + self.states.append(self.parse_block_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_block_sequence_entry + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block collection", self.marks[-1], + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + + def parse_indentless_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_indentless_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_indentless_sequence_entry + return self.process_empty_scalar(token.end_mark) + token = self.peek_token() + event = SequenceEndEvent(token.start_mark, token.start_mark) + self.state = self.states.pop() + return event + + # block_mapping ::= BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + + def parse_block_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_mapping_key() + + def parse_block_mapping_key(self): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_value) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_value + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block mapping", self.marks[-1], + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_block_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_key) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_block_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + + def parse_flow_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_sequence_entry(first=True) + + def parse_flow_sequence_entry(self, first=False): + if not self.check_token(FlowSequenceEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow sequence", self.marks[-1], + "expected ',' or ']', but got %r" % token.id, token.start_mark) + + if self.check_token(KeyToken): + token = self.peek_token() + event = MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + self.state = self.parse_flow_sequence_entry_mapping_key + return event + elif not self.check_token(FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry) + return self.parse_flow_node() + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_sequence_entry_mapping_key(self): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_value + return self.process_empty_scalar(token.end_mark) + + def parse_flow_sequence_entry_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_end) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_end + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_sequence_entry_mapping_end + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_sequence_entry_mapping_end(self): + self.state = self.parse_flow_sequence_entry + token = self.peek_token() + return MappingEndEvent(token.start_mark, token.start_mark) + + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + + def parse_flow_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_mapping_key(first=True) + + def parse_flow_mapping_key(self, first=False): + if not self.check_token(FlowMappingEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow mapping", self.marks[-1], + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_value + return self.process_empty_scalar(token.end_mark) + elif not self.check_token(FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_empty_value) + return self.parse_flow_node() + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_key) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_mapping_empty_value(self): + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(self.peek_token().start_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), u'', mark, mark) + diff --git a/pipenv/patched/yaml2/reader.py b/pipenv/patched/yaml2/reader.py new file mode 100644 index 00000000..4b377d61 --- /dev/null +++ b/pipenv/patched/yaml2/reader.py @@ -0,0 +1,188 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from error import YAMLError, Mark + +import codecs, re, sys + +has_ucs4 = sys.maxunicode > 0xffff + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, str): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (self.character, self.reason, + self.name, self.position) + +class Reader(object): + # Reader: + # - determines the data encoding and converts it to unicode, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `str` object, + # - a `unicode` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = u'' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, unicode): + self.name = "<unicode string>" + self.check_printable(stream) + self.buffer = stream+u'\0' + elif isinstance(stream, str): + self.name = "<string>" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "<file>") + self.eof = False + self.raw_buffer = '' + self.determine_encoding() + + def peek(self, index=0): + try: + return self.buffer[self.pointer+index] + except IndexError: + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + while length: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in u'\n\x85\u2028\u2029' \ + or (ch == u'\r' and self.buffer[self.pointer] != u'\n'): + self.line += 1 + self.column = 0 + elif ch != u'\uFEFF': + self.column += 1 + length -= 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and len(self.raw_buffer) < 2: + self.update_raw() + if not isinstance(self.raw_buffer, unicode): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = codecs.utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = codecs.utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = codecs.utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + if has_ucs4: + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + else: + NON_PRINTABLE = re.compile(u'[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uFFFD]|(?:^|[^\uD800-\uDBFF])[\uDC00-\uDFFF]|[\uD800-\uDBFF](?:[^\uDC00-\uDFFF]|$)') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, ord(character), + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError, exc: + character = exc.object[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += u'\0' + self.raw_buffer = None + break + + def update_raw(self, size=1024): + data = self.stream.read(size) + if data: + self.raw_buffer += data + self.stream_pointer += len(data) + else: + self.eof = True diff --git a/pipenv/patched/yaml2/representer.py b/pipenv/patched/yaml2/representer.py new file mode 100644 index 00000000..93e09b67 --- /dev/null +++ b/pipenv/patched/yaml2/representer.py @@ -0,0 +1,489 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from error import * + +from nodes import * + +import datetime + +import copy_reg, types + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter(object): + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=False, sort_keys=True): + self.default_style = default_style + self.default_flow_style = default_flow_style + self.sort_keys = sort_keys + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def get_classobj_bases(self, cls): + bases = [cls] + for base in cls.__bases__: + bases.extend(self.get_classobj_bases(base)) + return bases + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + node = self.represented_objects[self.alias_key] + #if node is None: + # raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + #self.represented_objects[alias_key] = None + self.object_keeper.append(data) + data_types = type(data).__mro__ + if type(data) is types.InstanceType: + data_types = self.get_classobj_bases(data.__class__)+list(data_types) + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, unicode(data)) + #if alias_key is not None: + # self.represented_objects[alias_key] = node + return node + + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + add_representer = classmethod(add_representer) + + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + add_multi_representer = classmethod(add_multi_representer) + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + mapping = mapping.items() + if self.sort_keys: + mapping.sort() + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, unicode, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:null', + u'null') + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:str', data) + + def represent_bool(self, data): + if data: + value = u'true' + else: + value = u'false' + return self.represent_scalar(u'tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + def represent_long(self, data): + return self.represent_scalar(u'tag:yaml.org,2002:int', unicode(data)) + + inf_value = 1e300 + while repr(inf_value) != repr(inf_value*inf_value): + inf_value *= inf_value + + def represent_float(self, data): + if data != data or (data == 0.0 and data == 1.0): + value = u'.nan' + elif data == self.inf_value: + value = u'.inf' + elif data == -self.inf_value: + value = u'-.inf' + else: + value = unicode(repr(data)).lower() + # Note that in some cases `repr(data)` represents a float number + # without the decimal parts. For instance: + # >>> repr(1e17) + # '1e17' + # Unfortunately, this is not a valid float representation according + # to the definition of the `!!float` tag. We fix this by adding + # '.0' before the 'e' symbol. + if u'.' not in value and u'e' in value: + value = value.replace(u'e', u'.0e', 1) + return self.represent_scalar(u'tag:yaml.org,2002:float', value) + + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence(u'tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping(u'tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping(u'tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = unicode(data.isoformat()) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = unicode(data.isoformat(' ')) + return self.represent_scalar(u'tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(unicode, + SafeRepresenter.represent_unicode) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(long, + SafeRepresenter.represent_long) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + +SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_str(self, data): + tag = None + style = None + try: + data = unicode(data, 'ascii') + tag = u'tag:yaml.org,2002:str' + except UnicodeDecodeError: + try: + data = unicode(data, 'utf-8') + tag = u'tag:yaml.org,2002:python/str' + except UnicodeDecodeError: + data = data.encode('base64') + tag = u'tag:yaml.org,2002:binary' + style = '|' + return self.represent_scalar(tag, data, style=style) + + def represent_unicode(self, data): + tag = None + try: + data.encode('ascii') + tag = u'tag:yaml.org,2002:python/unicode' + except UnicodeEncodeError: + tag = u'tag:yaml.org,2002:str' + return self.represent_scalar(tag, data) + + def represent_long(self, data): + tag = u'tag:yaml.org,2002:int' + if int(data) is not data: + tag = u'tag:yaml.org,2002:python/long' + return self.represent_scalar(tag, unicode(data)) + + def represent_complex(self, data): + if data.imag == 0.0: + data = u'%r' % data.real + elif data.real == 0.0: + data = u'%rj' % data.imag + elif data.imag > 0: + data = u'%r+%rj' % (data.real, data.imag) + else: + data = u'%r%rj' % (data.real, data.imag) + return self.represent_scalar(u'tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence(u'tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = u'%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar(u'tag:yaml.org,2002:python/name:'+name, u'') + + def represent_module(self, data): + return self.represent_scalar( + u'tag:yaml.org,2002:python/module:'+data.__name__, u'') + + def represent_instance(self, data): + # For instances of classic classes, we use __getinitargs__ and + # __getstate__ to serialize the data. + + # If data.__getinitargs__ exists, the object must be reconstructed by + # calling cls(**args), where args is a tuple returned by + # __getinitargs__. Otherwise, the cls.__init__ method should never be + # called and the class instance is created by instantiating a trivial + # class and assigning to the instance's __class__ variable. + + # If data.__getstate__ exists, it returns the state of the object. + # Otherwise, the state of the object is data.__dict__. + + # We produce either a !!python/object or !!python/object/new node. + # If data.__getinitargs__ does not exist and state is a dictionary, we + # produce a !!python/object node . Otherwise we produce a + # !!python/object/new node. + + cls = data.__class__ + class_name = u'%s.%s' % (cls.__module__, cls.__name__) + args = None + state = None + if hasattr(data, '__getinitargs__'): + args = list(data.__getinitargs__()) + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__ + if args is None and isinstance(state, dict): + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+class_name, state) + if isinstance(state, dict) and not state: + return self.represent_sequence( + u'tag:yaml.org,2002:python/object/new:'+class_name, args) + value = {} + if args: + value['args'] = args + value['state'] = state + return self.represent_mapping( + u'tag:yaml.org,2002:python/object/new:'+class_name, value) + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copy_reg.dispatch_table: + reduce = copy_reg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent an object", data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = u'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = u'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = u'%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return self.represent_mapping( + u'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + +Representer.add_representer(str, + Representer.represent_str) + +Representer.add_representer(unicode, + Representer.represent_unicode) + +Representer.add_representer(long, + Representer.represent_long) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(types.ClassType, + Representer.represent_name) + +Representer.add_representer(types.FunctionType, + Representer.represent_name) + +Representer.add_representer(types.BuiltinFunctionType, + Representer.represent_name) + +Representer.add_representer(types.ModuleType, + Representer.represent_module) + +Representer.add_multi_representer(types.InstanceType, + Representer.represent_instance) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/pipenv/patched/yaml2/resolver.py b/pipenv/patched/yaml2/resolver.py new file mode 100644 index 00000000..528fbc0e --- /dev/null +++ b/pipenv/patched/yaml2/resolver.py @@ -0,0 +1,227 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from error import * +from nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver(object): + + DEFAULT_SCALAR_TAG = u'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = u'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = u'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + implicit_resolvers = {} + for key in cls.yaml_implicit_resolvers: + implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:] + cls.yaml_implicit_resolvers = implicit_resolvers + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + add_implicit_resolver = classmethod(add_implicit_resolver) + + def add_path_resolver(cls, tag, path, kind=None): + # Note: `add_path_resolver` is experimental. The API could be changed. + # `new_path` is a pattern that is matched against the path from the + # root to the node that is being considered. `node_path` elements are + # tuples `(node_check, index_check)`. `node_check` is a node class: + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None` + # matches any kind of a node. `index_check` could be `None`, a boolean + # value, a string value, or a number. `None` and `False` match against + # any _value_ of sequence and mapping nodes. `True` matches against + # any _key_ of a mapping node. A string `index_check` matches against + # a mapping value that corresponds to a scalar key which content is + # equal to the `index_check` value. An integer `index_check` matches + # against a sequence value with the index equal to `index_check`. + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is dict: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, basestring) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (basestring, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is dict: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + add_path_resolver = classmethod(add_path_resolver) + + def descend_resolver(self, current_node, current_index): + if not self.yaml_path_resolvers: + return + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + if not self.yaml_path_resolvers: + return + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, basestring): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if (index_check is False or index_check is None) \ + and current_index is None: + return + if isinstance(index_check, basestring): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int) and not isinstance(index_check, bool): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == u'': + resolvers = self.yaml_implicit_resolvers.get(u'', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + resolvers += self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:bool', + re.compile(ur'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list(u'yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:float', + re.compile(ur'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list(u'-+0123456789.')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:int', + re.compile(ur'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list(u'-+0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:merge', + re.compile(ur'^(?:<<)$'), + [u'<']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:null', + re.compile(ur'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + [u'~', u'n', u'N', u'']) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:timestamp', + re.compile(ur'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list(u'0123456789')) + +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:value', + re.compile(ur'^(?:=)$'), + [u'=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + u'tag:yaml.org,2002:yaml', + re.compile(ur'^(?:!|&|\*)$'), + list(u'!&*')) + diff --git a/pipenv/patched/yaml2/scanner.py b/pipenv/patched/yaml2/scanner.py new file mode 100644 index 00000000..098ea7be --- /dev/null +++ b/pipenv/patched/yaml2/scanner.py @@ -0,0 +1,1444 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain, style) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from error import MarkedYAMLError +from tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey(object): + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner(object): + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + # Return None if no more tokens. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + else: + return None + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == u'\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == u'%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == u'-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == u'.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == u'\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == u'[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == u'{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == u']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == u'}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch == u',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch == u'-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == u'?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == u':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == u'*': + return self.fetch_alias() + + # Is it an anchor? + if ch == u'&': + return self.fetch_anchor() + + # Is it a tag? + if ch == u'!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == u'|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == u'>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == u'\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == u'\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" + % ch.encode('utf-8'), self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in self.possible_simple_keys.keys(): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + + del self.possible_simple_keys[self.flow_level] + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid indentation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not necessary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be caught by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # If this value starts a new block mapping, we need to add + # BLOCK-MAPPING-START. It will be detected as an error later by + # the parser. + if not self.flow_level: + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'---' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == u'...' \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in u'\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in u'\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in u'\0 \t\r\n\x85\u2028\u2029' + and (ch == u'-' or (not self.flow_level and ch in u'?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if <TAB>: + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == u'\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == u'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == u'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" + % self.peek().encode('utf-8'), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not (u'0' <= ch <= u'9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 0 + while u'0' <= self.peek(length) <= u'9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == u' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != u' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpreted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == u'*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in u'\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch.encode('utf-8'), self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == u'<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != u'>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek().encode('utf-8'), + self.get_mark()) + self.forward() + elif ch in u'\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = u'!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in u'\0 \r\n\x85\u2028\u2029': + if ch == u'!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = u'!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = u'!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch.encode('utf-8'), + self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = u'' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != u'\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in u' \t' + length = 0 + while self.peek(length) not in u'\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != u'\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == u'\n' \ + and leading_non_space and self.peek() not in u' \t': + if not breaks: + chunks.append(u' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == u'\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(u' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in u'0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in u'+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in u'\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch.encode('utf-8'), self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == u' ': + self.forward() + if self.peek() == u'#': + while self.peek() not in u'\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in u'\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" + % ch.encode('utf-8'), self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() != u' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + while self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == u' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(u''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + u'0': u'\0', + u'a': u'\x07', + u'b': u'\x08', + u't': u'\x09', + u'\t': u'\x09', + u'n': u'\x0A', + u'v': u'\x0B', + u'f': u'\x0C', + u'r': u'\x0D', + u'e': u'\x1B', + u' ': u'\x20', + u'\"': u'\"', + u'\\': u'\\', + u'/': u'/', + u'N': u'\x85', + u'_': u'\xA0', + u'L': u'\u2028', + u'P': u'\u2029', + } + + ESCAPE_CODES = { + u'x': 2, + u'u': 4, + u'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in u'\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == u'\'' and self.peek(1) == u'\'': + chunks.append(u'\'') + self.forward(2) + elif (double and ch == u'\'') or (not double and ch in u'\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == u'\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k).encode('utf-8')), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(unichr(code)) + self.forward(length) + elif ch in u'\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch.encode('utf-8'), self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in u' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == u'\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in u' \t': + self.forward() + if self.peek() in u'\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',' or '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == u'#': + break + while True: + ch = self.peek(length) + if ch in u'\0 \t\r\n\x85\u2028\u2029' \ + or (ch == u':' and + self.peek(length+1) in u'\0 \t\r\n\x85\u2028\u2029' + + (u',[]{}' if self.flow_level else u''))\ + or (self.flow_level and ch in u',?[]{}'): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == u'#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(u''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in u' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in u'\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in u' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == u'---' or prefix == u'...') \ + and self.peek(3) in u'\0 \t\r\n\x85\u2028\u2029': + return + if line_break != u'\n': + chunks.append(line_break) + elif not breaks: + chunks.append(u' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != u'!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != u' ': + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-_': + length += 1 + ch = self.peek(length) + if ch != u'!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch.encode('utf-8'), + self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while u'0' <= ch <= u'9' or u'A' <= ch <= u'Z' or u'a' <= ch <= u'z' \ + or ch in u'-;/?:@&=+$,_.!~*\'()[]%': + if ch == u'%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch.encode('utf-8'), + self.get_mark()) + return u''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + bytes = [] + mark = self.get_mark() + while self.peek() == u'%': + self.forward() + for k in range(2): + if self.peek(k) not in u'0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" % + (self.peek(k).encode('utf-8')), self.get_mark()) + bytes.append(chr(int(self.prefix(2), 16))) + self.forward(2) + try: + value = unicode(''.join(bytes), 'utf-8') + except UnicodeDecodeError, exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in u'\r\n\x85': + if self.prefix(2) == u'\r\n': + self.forward(2) + else: + self.forward() + return u'\n' + elif ch in u'\u2028\u2029': + self.forward() + return ch + return u'' diff --git a/pipenv/patched/yaml2/serializer.py b/pipenv/patched/yaml2/serializer.py new file mode 100644 index 00000000..0bf1e96d --- /dev/null +++ b/pipenv/patched/yaml2/serializer.py @@ -0,0 +1,111 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from error import YAMLError +from events import * +from nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer(object): + + ANCHOR_TEMPLATE = u'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/pipenv/patched/yaml2/tokens.py b/pipenv/patched/yaml2/tokens.py new file mode 100644 index 00000000..4d0b48a3 --- /dev/null +++ b/pipenv/patched/yaml2/tokens.py @@ -0,0 +1,104 @@ + +class Token(object): + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '<byte order mark>' + +class DirectiveToken(Token): + id = '<directive>' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '<document start>' + +class DocumentEndToken(Token): + id = '<document end>' + +class StreamStartToken(Token): + id = '<stream start>' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '<stream end>' + +class BlockSequenceStartToken(Token): + id = '<block sequence start>' + +class BlockMappingStartToken(Token): + id = '<block mapping start>' + +class BlockEndToken(Token): + id = '<block end>' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '<alias>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '<anchor>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '<tag>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '<scalar>' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/pipenv/patched/yaml3/LICENSE b/pipenv/patched/yaml3/LICENSE new file mode 100644 index 00000000..3d82c281 --- /dev/null +++ b/pipenv/patched/yaml3/LICENSE @@ -0,0 +1,20 @@ +Copyright (c) 2017-2020 Ingy döt Net +Copyright (c) 2006-2016 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/patched/yaml3/__init__.py b/pipenv/patched/yaml3/__init__.py new file mode 100644 index 00000000..13d687c5 --- /dev/null +++ b/pipenv/patched/yaml3/__init__.py @@ -0,0 +1,427 @@ + +from .error import * + +from .tokens import * +from .events import * +from .nodes import * + +from .loader import * +from .dumper import * + +__version__ = '5.3.1' +try: + from .cyaml import * + __with_libyaml__ = True +except ImportError: + __with_libyaml__ = False + +import io + +#------------------------------------------------------------------------------ +# Warnings control +#------------------------------------------------------------------------------ + +# 'Global' warnings state: +_warnings_enabled = { + 'YAMLLoadWarning': True, +} + +# Get or set global warnings' state +def warnings(settings=None): + if settings is None: + return _warnings_enabled + + if type(settings) is dict: + for key in settings: + if key in _warnings_enabled: + _warnings_enabled[key] = settings[key] + +# Warn when load() is called without Loader=... +class YAMLLoadWarning(RuntimeWarning): + pass + +def load_warning(method): + if _warnings_enabled['YAMLLoadWarning'] is False: + return + + import warnings + + message = ( + "calling yaml.%s() without Loader=... is deprecated, as the " + "default Loader is unsafe. Please read " + "https://msg.pyyaml.org/load for full details." + ) % method + + warnings.warn(message, YAMLLoadWarning, stacklevel=3) + +#------------------------------------------------------------------------------ +def scan(stream, Loader=Loader): + """ + Scan a YAML stream and produce scanning tokens. + """ + loader = Loader(stream) + try: + while loader.check_token(): + yield loader.get_token() + finally: + loader.dispose() + +def parse(stream, Loader=Loader): + """ + Parse a YAML stream and produce parsing events. + """ + loader = Loader(stream) + try: + while loader.check_event(): + yield loader.get_event() + finally: + loader.dispose() + +def compose(stream, Loader=Loader): + """ + Parse the first YAML document in a stream + and produce the corresponding representation tree. + """ + loader = Loader(stream) + try: + return loader.get_single_node() + finally: + loader.dispose() + +def compose_all(stream, Loader=Loader): + """ + Parse all YAML documents in a stream + and produce corresponding representation trees. + """ + loader = Loader(stream) + try: + while loader.check_node(): + yield loader.get_node() + finally: + loader.dispose() + +def load(stream, Loader=None): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + """ + if Loader is None: + load_warning('load') + Loader = FullLoader + + loader = Loader(stream) + try: + return loader.get_single_data() + finally: + loader.dispose() + +def load_all(stream, Loader=None): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + """ + if Loader is None: + load_warning('load_all') + Loader = FullLoader + + loader = Loader(stream) + try: + while loader.check_data(): + yield loader.get_data() + finally: + loader.dispose() + +def full_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load(stream, FullLoader) + +def full_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags except those known to be + unsafe on untrusted input. + """ + return load_all(stream, FullLoader) + +def safe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load(stream, SafeLoader) + +def safe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve only basic YAML tags. This is known + to be safe for untrusted input. + """ + return load_all(stream, SafeLoader) + +def unsafe_load(stream): + """ + Parse the first YAML document in a stream + and produce the corresponding Python object. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load(stream, UnsafeLoader) + +def unsafe_load_all(stream): + """ + Parse all YAML documents in a stream + and produce corresponding Python objects. + + Resolve all tags, even those known to be + unsafe on untrusted input. + """ + return load_all(stream, UnsafeLoader) + +def emit(events, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + """ + Emit YAML parsing events into a stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + stream = io.StringIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + try: + for event in events: + dumper.emit(event) + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize_all(nodes, stream=None, Dumper=Dumper, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None): + """ + Serialize a sequence of representation trees into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end) + try: + dumper.open() + for node in nodes: + dumper.serialize(node) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def serialize(node, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a representation tree into a YAML stream. + If stream is None, return the produced string instead. + """ + return serialize_all([node], stream, Dumper=Dumper, **kwds) + +def dump_all(documents, stream=None, Dumper=Dumper, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + """ + Serialize a sequence of Python objects into a YAML stream. + If stream is None, return the produced string instead. + """ + getvalue = None + if stream is None: + if encoding is None: + stream = io.StringIO() + else: + stream = io.BytesIO() + getvalue = stream.getvalue + dumper = Dumper(stream, default_style=default_style, + default_flow_style=default_flow_style, + canonical=canonical, indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break, + encoding=encoding, version=version, tags=tags, + explicit_start=explicit_start, explicit_end=explicit_end, sort_keys=sort_keys) + try: + dumper.open() + for data in documents: + dumper.represent(data) + dumper.close() + finally: + dumper.dispose() + if getvalue: + return getvalue() + +def dump(data, stream=None, Dumper=Dumper, **kwds): + """ + Serialize a Python object into a YAML stream. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=Dumper, **kwds) + +def safe_dump_all(documents, stream=None, **kwds): + """ + Serialize a sequence of Python objects into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all(documents, stream, Dumper=SafeDumper, **kwds) + +def safe_dump(data, stream=None, **kwds): + """ + Serialize a Python object into a YAML stream. + Produce only basic YAML tags. + If stream is None, return the produced string instead. + """ + return dump_all([data], stream, Dumper=SafeDumper, **kwds) + +def add_implicit_resolver(tag, regexp, first=None, + Loader=None, Dumper=Dumper): + """ + Add an implicit scalar detector. + If an implicit scalar value matches the given regexp, + the corresponding tag is assigned to the scalar. + first is a sequence of possible initial characters or None. + """ + if Loader is None: + loader.Loader.add_implicit_resolver(tag, regexp, first) + loader.FullLoader.add_implicit_resolver(tag, regexp, first) + loader.UnsafeLoader.add_implicit_resolver(tag, regexp, first) + else: + Loader.add_implicit_resolver(tag, regexp, first) + Dumper.add_implicit_resolver(tag, regexp, first) + +def add_path_resolver(tag, path, kind=None, Loader=None, Dumper=Dumper): + """ + Add a path based resolver for the given tag. + A path is a list of keys that forms a path + to a node in the representation tree. + Keys can be string values, integers, or None. + """ + if Loader is None: + loader.Loader.add_path_resolver(tag, path, kind) + loader.FullLoader.add_path_resolver(tag, path, kind) + loader.UnsafeLoader.add_path_resolver(tag, path, kind) + else: + Loader.add_path_resolver(tag, path, kind) + Dumper.add_path_resolver(tag, path, kind) + +def add_constructor(tag, constructor, Loader=None): + """ + Add a constructor for the given tag. + Constructor is a function that accepts a Loader instance + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_constructor(tag, constructor) + loader.FullLoader.add_constructor(tag, constructor) + loader.UnsafeLoader.add_constructor(tag, constructor) + else: + Loader.add_constructor(tag, constructor) + +def add_multi_constructor(tag_prefix, multi_constructor, Loader=None): + """ + Add a multi-constructor for the given tag prefix. + Multi-constructor is called for a node if its tag starts with tag_prefix. + Multi-constructor accepts a Loader instance, a tag suffix, + and a node object and produces the corresponding Python object. + """ + if Loader is None: + loader.Loader.add_multi_constructor(tag_prefix, multi_constructor) + loader.FullLoader.add_multi_constructor(tag_prefix, multi_constructor) + loader.UnsafeLoader.add_multi_constructor(tag_prefix, multi_constructor) + else: + Loader.add_multi_constructor(tag_prefix, multi_constructor) + +def add_representer(data_type, representer, Dumper=Dumper): + """ + Add a representer for the given type. + Representer is a function accepting a Dumper instance + and an instance of the given data type + and producing the corresponding representation node. + """ + Dumper.add_representer(data_type, representer) + +def add_multi_representer(data_type, multi_representer, Dumper=Dumper): + """ + Add a representer for the given type. + Multi-representer is a function accepting a Dumper instance + and an instance of the given data type or subtype + and producing the corresponding representation node. + """ + Dumper.add_multi_representer(data_type, multi_representer) + +class YAMLObjectMetaclass(type): + """ + The metaclass for YAMLObject. + """ + def __init__(cls, name, bases, kwds): + super(YAMLObjectMetaclass, cls).__init__(name, bases, kwds) + if 'yaml_tag' in kwds and kwds['yaml_tag'] is not None: + if isinstance(cls.yaml_loader, list): + for loader in cls.yaml_loader: + loader.add_constructor(cls.yaml_tag, cls.from_yaml) + else: + cls.yaml_loader.add_constructor(cls.yaml_tag, cls.from_yaml) + + cls.yaml_dumper.add_representer(cls, cls.to_yaml) + +class YAMLObject(metaclass=YAMLObjectMetaclass): + """ + An object that can dump itself to a YAML stream + and load itself from a YAML stream. + """ + + __slots__ = () # no direct instantiation, so allow immutable subclasses + + yaml_loader = [Loader, FullLoader, UnsafeLoader] + yaml_dumper = Dumper + + yaml_tag = None + yaml_flow_style = None + + @classmethod + def from_yaml(cls, loader, node): + """ + Convert a representation node to a Python object. + """ + return loader.construct_yaml_object(node, cls) + + @classmethod + def to_yaml(cls, dumper, data): + """ + Convert a Python object to a representation node. + """ + return dumper.represent_yaml_object(cls.yaml_tag, data, cls, + flow_style=cls.yaml_flow_style) + diff --git a/pipenv/patched/yaml3/composer.py b/pipenv/patched/yaml3/composer.py new file mode 100644 index 00000000..6d15cb40 --- /dev/null +++ b/pipenv/patched/yaml3/composer.py @@ -0,0 +1,139 @@ + +__all__ = ['Composer', 'ComposerError'] + +from .error import MarkedYAMLError +from .events import * +from .nodes import * + +class ComposerError(MarkedYAMLError): + pass + +class Composer: + + def __init__(self): + self.anchors = {} + + def check_node(self): + # Drop the STREAM-START event. + if self.check_event(StreamStartEvent): + self.get_event() + + # If there are more documents available? + return not self.check_event(StreamEndEvent) + + def get_node(self): + # Get the root node of the next document. + if not self.check_event(StreamEndEvent): + return self.compose_document() + + def get_single_node(self): + # Drop the STREAM-START event. + self.get_event() + + # Compose a document if the stream is not empty. + document = None + if not self.check_event(StreamEndEvent): + document = self.compose_document() + + # Ensure that the stream contains no more documents. + if not self.check_event(StreamEndEvent): + event = self.get_event() + raise ComposerError("expected a single document in the stream", + document.start_mark, "but found another document", + event.start_mark) + + # Drop the STREAM-END event. + self.get_event() + + return document + + def compose_document(self): + # Drop the DOCUMENT-START event. + self.get_event() + + # Compose the root node. + node = self.compose_node(None, None) + + # Drop the DOCUMENT-END event. + self.get_event() + + self.anchors = {} + return node + + def compose_node(self, parent, index): + if self.check_event(AliasEvent): + event = self.get_event() + anchor = event.anchor + if anchor not in self.anchors: + raise ComposerError(None, None, "found undefined alias %r" + % anchor, event.start_mark) + return self.anchors[anchor] + event = self.peek_event() + anchor = event.anchor + if anchor is not None: + if anchor in self.anchors: + raise ComposerError("found duplicate anchor %r; first occurrence" + % anchor, self.anchors[anchor].start_mark, + "second occurrence", event.start_mark) + self.descend_resolver(parent, index) + if self.check_event(ScalarEvent): + node = self.compose_scalar_node(anchor) + elif self.check_event(SequenceStartEvent): + node = self.compose_sequence_node(anchor) + elif self.check_event(MappingStartEvent): + node = self.compose_mapping_node(anchor) + self.ascend_resolver() + return node + + def compose_scalar_node(self, anchor): + event = self.get_event() + tag = event.tag + if tag is None or tag == '!': + tag = self.resolve(ScalarNode, event.value, event.implicit) + node = ScalarNode(tag, event.value, + event.start_mark, event.end_mark, style=event.style) + if anchor is not None: + self.anchors[anchor] = node + return node + + def compose_sequence_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(SequenceNode, None, start_event.implicit) + node = SequenceNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + index = 0 + while not self.check_event(SequenceEndEvent): + node.value.append(self.compose_node(node, index)) + index += 1 + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + + def compose_mapping_node(self, anchor): + start_event = self.get_event() + tag = start_event.tag + if tag is None or tag == '!': + tag = self.resolve(MappingNode, None, start_event.implicit) + node = MappingNode(tag, [], + start_event.start_mark, None, + flow_style=start_event.flow_style) + if anchor is not None: + self.anchors[anchor] = node + while not self.check_event(MappingEndEvent): + #key_event = self.peek_event() + item_key = self.compose_node(node, None) + #if item_key in node.value: + # raise ComposerError("while composing a mapping", start_event.start_mark, + # "found duplicate key", key_event.start_mark) + item_value = self.compose_node(node, item_key) + #node.value[item_key] = item_value + node.value.append((item_key, item_value)) + end_event = self.get_event() + node.end_mark = end_event.end_mark + return node + diff --git a/pipenv/patched/yaml3/constructor.py b/pipenv/patched/yaml3/constructor.py new file mode 100644 index 00000000..1948b125 --- /dev/null +++ b/pipenv/patched/yaml3/constructor.py @@ -0,0 +1,748 @@ + +__all__ = [ + 'BaseConstructor', + 'SafeConstructor', + 'FullConstructor', + 'UnsafeConstructor', + 'Constructor', + 'ConstructorError' +] + +from .error import * +from .nodes import * + +import collections.abc, datetime, base64, binascii, re, sys, types + +class ConstructorError(MarkedYAMLError): + pass + +class BaseConstructor: + + yaml_constructors = {} + yaml_multi_constructors = {} + + def __init__(self): + self.constructed_objects = {} + self.recursive_objects = {} + self.state_generators = [] + self.deep_construct = False + + def check_data(self): + # If there are more documents available? + return self.check_node() + + def check_state_key(self, key): + """Block special attributes/methods from being set in a newly created + object, to prevent user-controlled methods from being called during + deserialization""" + if self.get_state_keys_blacklist_regexp().match(key): + raise ConstructorError(None, None, + "blacklisted key '%s' in instance state found" % (key,), None) + + def get_data(self): + # Construct and return the next document. + if self.check_node(): + return self.construct_document(self.get_node()) + + def get_single_data(self): + # Ensure that the stream contains a single document and construct it. + node = self.get_single_node() + if node is not None: + return self.construct_document(node) + return None + + def construct_document(self, node): + data = self.construct_object(node) + while self.state_generators: + state_generators = self.state_generators + self.state_generators = [] + for generator in state_generators: + for dummy in generator: + pass + self.constructed_objects = {} + self.recursive_objects = {} + self.deep_construct = False + return data + + def construct_object(self, node, deep=False): + if node in self.constructed_objects: + return self.constructed_objects[node] + if deep: + old_deep = self.deep_construct + self.deep_construct = True + if node in self.recursive_objects: + raise ConstructorError(None, None, + "found unconstructable recursive node", node.start_mark) + self.recursive_objects[node] = None + constructor = None + tag_suffix = None + if node.tag in self.yaml_constructors: + constructor = self.yaml_constructors[node.tag] + else: + for tag_prefix in self.yaml_multi_constructors: + if tag_prefix is not None and node.tag.startswith(tag_prefix): + tag_suffix = node.tag[len(tag_prefix):] + constructor = self.yaml_multi_constructors[tag_prefix] + break + else: + if None in self.yaml_multi_constructors: + tag_suffix = node.tag + constructor = self.yaml_multi_constructors[None] + elif None in self.yaml_constructors: + constructor = self.yaml_constructors[None] + elif isinstance(node, ScalarNode): + constructor = self.__class__.construct_scalar + elif isinstance(node, SequenceNode): + constructor = self.__class__.construct_sequence + elif isinstance(node, MappingNode): + constructor = self.__class__.construct_mapping + if tag_suffix is None: + data = constructor(self, node) + else: + data = constructor(self, tag_suffix, node) + if isinstance(data, types.GeneratorType): + generator = data + data = next(generator) + if self.deep_construct: + for dummy in generator: + pass + else: + self.state_generators.append(generator) + self.constructed_objects[node] = data + del self.recursive_objects[node] + if deep: + self.deep_construct = old_deep + return data + + def construct_scalar(self, node): + if not isinstance(node, ScalarNode): + raise ConstructorError(None, None, + "expected a scalar node, but found %s" % node.id, + node.start_mark) + return node.value + + def construct_sequence(self, node, deep=False): + if not isinstance(node, SequenceNode): + raise ConstructorError(None, None, + "expected a sequence node, but found %s" % node.id, + node.start_mark) + return [self.construct_object(child, deep=deep) + for child in node.value] + + def construct_mapping(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + mapping = {} + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + if not isinstance(key, collections.abc.Hashable): + raise ConstructorError("while constructing a mapping", node.start_mark, + "found unhashable key", key_node.start_mark) + value = self.construct_object(value_node, deep=deep) + mapping[key] = value + return mapping + + def construct_pairs(self, node, deep=False): + if not isinstance(node, MappingNode): + raise ConstructorError(None, None, + "expected a mapping node, but found %s" % node.id, + node.start_mark) + pairs = [] + for key_node, value_node in node.value: + key = self.construct_object(key_node, deep=deep) + value = self.construct_object(value_node, deep=deep) + pairs.append((key, value)) + return pairs + + @classmethod + def add_constructor(cls, tag, constructor): + if not 'yaml_constructors' in cls.__dict__: + cls.yaml_constructors = cls.yaml_constructors.copy() + cls.yaml_constructors[tag] = constructor + + @classmethod + def add_multi_constructor(cls, tag_prefix, multi_constructor): + if not 'yaml_multi_constructors' in cls.__dict__: + cls.yaml_multi_constructors = cls.yaml_multi_constructors.copy() + cls.yaml_multi_constructors[tag_prefix] = multi_constructor + +class SafeConstructor(BaseConstructor): + + def construct_scalar(self, node): + if isinstance(node, MappingNode): + for key_node, value_node in node.value: + if key_node.tag == 'tag:yaml.org,2002:value': + return self.construct_scalar(value_node) + return super().construct_scalar(node) + + def flatten_mapping(self, node): + merge = [] + index = 0 + while index < len(node.value): + key_node, value_node = node.value[index] + if key_node.tag == 'tag:yaml.org,2002:merge': + del node.value[index] + if isinstance(value_node, MappingNode): + self.flatten_mapping(value_node) + merge.extend(value_node.value) + elif isinstance(value_node, SequenceNode): + submerge = [] + for subnode in value_node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing a mapping", + node.start_mark, + "expected a mapping for merging, but found %s" + % subnode.id, subnode.start_mark) + self.flatten_mapping(subnode) + submerge.append(subnode.value) + submerge.reverse() + for value in submerge: + merge.extend(value) + else: + raise ConstructorError("while constructing a mapping", node.start_mark, + "expected a mapping or list of mappings for merging, but found %s" + % value_node.id, value_node.start_mark) + elif key_node.tag == 'tag:yaml.org,2002:value': + key_node.tag = 'tag:yaml.org,2002:str' + index += 1 + else: + index += 1 + if merge: + node.value = merge + node.value + + def construct_mapping(self, node, deep=False): + if isinstance(node, MappingNode): + self.flatten_mapping(node) + return super().construct_mapping(node, deep=deep) + + def construct_yaml_null(self, node): + self.construct_scalar(node) + return None + + bool_values = { + 'yes': True, + 'no': False, + 'true': True, + 'false': False, + 'on': True, + 'off': False, + } + + def construct_yaml_bool(self, node): + value = self.construct_scalar(node) + return self.bool_values[value.lower()] + + def construct_yaml_int(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '') + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '0': + return 0 + elif value.startswith('0b'): + return sign*int(value[2:], 2) + elif value.startswith('0x'): + return sign*int(value[2:], 16) + elif value[0] == '0': + return sign*int(value, 8) + elif ':' in value: + digits = [int(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*int(value) + + inf_value = 1e300 + while inf_value != inf_value*inf_value: + inf_value *= inf_value + nan_value = -inf_value/inf_value # Trying to make a quiet NaN (like C99). + + def construct_yaml_float(self, node): + value = self.construct_scalar(node) + value = value.replace('_', '').lower() + sign = +1 + if value[0] == '-': + sign = -1 + if value[0] in '+-': + value = value[1:] + if value == '.inf': + return sign*self.inf_value + elif value == '.nan': + return self.nan_value + elif ':' in value: + digits = [float(part) for part in value.split(':')] + digits.reverse() + base = 1 + value = 0.0 + for digit in digits: + value += digit*base + base *= 60 + return sign*value + else: + return sign*float(value) + + def construct_yaml_binary(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + timestamp_regexp = re.compile( + r'''^(?P<year>[0-9][0-9][0-9][0-9]) + -(?P<month>[0-9][0-9]?) + -(?P<day>[0-9][0-9]?) + (?:(?:[Tt]|[ \t]+) + (?P<hour>[0-9][0-9]?) + :(?P<minute>[0-9][0-9]) + :(?P<second>[0-9][0-9]) + (?:\.(?P<fraction>[0-9]*))? + (?:[ \t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?) + (?::(?P<tz_minute>[0-9][0-9]))?))?)?$''', re.X) + + def construct_yaml_timestamp(self, node): + value = self.construct_scalar(node) + match = self.timestamp_regexp.match(node.value) + values = match.groupdict() + year = int(values['year']) + month = int(values['month']) + day = int(values['day']) + if not values['hour']: + return datetime.date(year, month, day) + hour = int(values['hour']) + minute = int(values['minute']) + second = int(values['second']) + fraction = 0 + tzinfo = None + if values['fraction']: + fraction = values['fraction'][:6] + while len(fraction) < 6: + fraction += '0' + fraction = int(fraction) + if values['tz_sign']: + tz_hour = int(values['tz_hour']) + tz_minute = int(values['tz_minute'] or 0) + delta = datetime.timedelta(hours=tz_hour, minutes=tz_minute) + if values['tz_sign'] == '-': + delta = -delta + tzinfo = datetime.timezone(delta) + elif values['tz']: + tzinfo = datetime.timezone.utc + return datetime.datetime(year, month, day, hour, minute, second, fraction, + tzinfo=tzinfo) + + def construct_yaml_omap(self, node): + # Note: we do not check for duplicate keys, because it's too + # CPU-expensive. + omap = [] + yield omap + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing an ordered map", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + omap.append((key, value)) + + def construct_yaml_pairs(self, node): + # Note: the same code as `construct_yaml_omap`. + pairs = [] + yield pairs + if not isinstance(node, SequenceNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a sequence, but found %s" % node.id, node.start_mark) + for subnode in node.value: + if not isinstance(subnode, MappingNode): + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a mapping of length 1, but found %s" % subnode.id, + subnode.start_mark) + if len(subnode.value) != 1: + raise ConstructorError("while constructing pairs", node.start_mark, + "expected a single mapping item, but found %d items" % len(subnode.value), + subnode.start_mark) + key_node, value_node = subnode.value[0] + key = self.construct_object(key_node) + value = self.construct_object(value_node) + pairs.append((key, value)) + + def construct_yaml_set(self, node): + data = set() + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_str(self, node): + return self.construct_scalar(node) + + def construct_yaml_seq(self, node): + data = [] + yield data + data.extend(self.construct_sequence(node)) + + def construct_yaml_map(self, node): + data = {} + yield data + value = self.construct_mapping(node) + data.update(value) + + def construct_yaml_object(self, node, cls): + data = cls.__new__(cls) + yield data + if hasattr(data, '__setstate__'): + state = self.construct_mapping(node, deep=True) + data.__setstate__(state) + else: + state = self.construct_mapping(node) + data.__dict__.update(state) + + def construct_undefined(self, node): + raise ConstructorError(None, None, + "could not determine a constructor for the tag %r" % node.tag, + node.start_mark) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:null', + SafeConstructor.construct_yaml_null) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:bool', + SafeConstructor.construct_yaml_bool) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:int', + SafeConstructor.construct_yaml_int) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:float', + SafeConstructor.construct_yaml_float) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:binary', + SafeConstructor.construct_yaml_binary) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:timestamp', + SafeConstructor.construct_yaml_timestamp) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:omap', + SafeConstructor.construct_yaml_omap) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:pairs', + SafeConstructor.construct_yaml_pairs) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:set', + SafeConstructor.construct_yaml_set) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:str', + SafeConstructor.construct_yaml_str) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:seq', + SafeConstructor.construct_yaml_seq) + +SafeConstructor.add_constructor( + 'tag:yaml.org,2002:map', + SafeConstructor.construct_yaml_map) + +SafeConstructor.add_constructor(None, + SafeConstructor.construct_undefined) + +class FullConstructor(SafeConstructor): + # 'extend' is blacklisted because it is used by + # construct_python_object_apply to add `listitems` to a newly generate + # python instance + def get_state_keys_blacklist(self): + return ['^extend$', '^__.*__$'] + + def get_state_keys_blacklist_regexp(self): + if not hasattr(self, 'state_keys_blacklist_regexp'): + self.state_keys_blacklist_regexp = re.compile('(' + '|'.join(self.get_state_keys_blacklist()) + ')') + return self.state_keys_blacklist_regexp + + def construct_python_str(self, node): + return self.construct_scalar(node) + + def construct_python_unicode(self, node): + return self.construct_scalar(node) + + def construct_python_bytes(self, node): + try: + value = self.construct_scalar(node).encode('ascii') + except UnicodeEncodeError as exc: + raise ConstructorError(None, None, + "failed to convert base64 data into ascii: %s" % exc, + node.start_mark) + try: + if hasattr(base64, 'decodebytes'): + return base64.decodebytes(value) + else: + return base64.decodestring(value) + except binascii.Error as exc: + raise ConstructorError(None, None, + "failed to decode base64 data: %s" % exc, node.start_mark) + + def construct_python_long(self, node): + return self.construct_yaml_int(node) + + def construct_python_complex(self, node): + return complex(self.construct_scalar(node)) + + def construct_python_tuple(self, node): + return tuple(self.construct_sequence(node)) + + def find_python_module(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python module", mark, + "expected non-empty name appended to the tag", mark) + if unsafe: + try: + __import__(name) + except ImportError as exc: + raise ConstructorError("while constructing a Python module", mark, + "cannot find module %r (%s)" % (name, exc), mark) + if name not in sys.modules: + raise ConstructorError("while constructing a Python module", mark, + "module %r is not imported" % name, mark) + return sys.modules[name] + + def find_python_name(self, name, mark, unsafe=False): + if not name: + raise ConstructorError("while constructing a Python object", mark, + "expected non-empty name appended to the tag", mark) + if '.' in name: + module_name, object_name = name.rsplit('.', 1) + else: + module_name = 'builtins' + object_name = name + if unsafe: + try: + __import__(module_name) + except ImportError as exc: + raise ConstructorError("while constructing a Python object", mark, + "cannot find module %r (%s)" % (module_name, exc), mark) + if module_name not in sys.modules: + raise ConstructorError("while constructing a Python object", mark, + "module %r is not imported" % module_name, mark) + module = sys.modules[module_name] + if not hasattr(module, object_name): + raise ConstructorError("while constructing a Python object", mark, + "cannot find %r in the module %r" + % (object_name, module.__name__), mark) + return getattr(module, object_name) + + def construct_python_name(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python name", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_name(suffix, node.start_mark) + + def construct_python_module(self, suffix, node): + value = self.construct_scalar(node) + if value: + raise ConstructorError("while constructing a Python module", node.start_mark, + "expected the empty value, but found %r" % value, node.start_mark) + return self.find_python_module(suffix, node.start_mark) + + def make_python_instance(self, suffix, node, + args=None, kwds=None, newobj=False, unsafe=False): + if not args: + args = [] + if not kwds: + kwds = {} + cls = self.find_python_name(suffix, node.start_mark) + if not (unsafe or isinstance(cls, type)): + raise ConstructorError("while constructing a Python instance", node.start_mark, + "expected a class, but found %r" % type(cls), + node.start_mark) + if newobj and isinstance(cls, type): + return cls.__new__(cls, *args, **kwds) + else: + return cls(*args, **kwds) + + def set_python_instance_state(self, instance, state, unsafe=False): + if hasattr(instance, '__setstate__'): + instance.__setstate__(state) + else: + slotstate = {} + if isinstance(state, tuple) and len(state) == 2: + state, slotstate = state + if hasattr(instance, '__dict__'): + if not unsafe and state: + for key in state.keys(): + self.check_state_key(key) + instance.__dict__.update(state) + elif state: + slotstate.update(state) + for key, value in slotstate.items(): + if not unsafe: + self.check_state_key(key) + setattr(instance, key, value) + + def construct_python_object(self, suffix, node): + # Format: + # !!python/object:module.name { ... state ... } + instance = self.make_python_instance(suffix, node, newobj=True) + yield instance + deep = hasattr(instance, '__setstate__') + state = self.construct_mapping(node, deep=deep) + self.set_python_instance_state(instance, state) + + def construct_python_object_apply(self, suffix, node, newobj=False): + # Format: + # !!python/object/apply # (or !!python/object/new) + # args: [ ... arguments ... ] + # kwds: { ... keywords ... } + # state: ... state ... + # listitems: [ ... listitems ... ] + # dictitems: { ... dictitems ... } + # or short format: + # !!python/object/apply [ ... arguments ... ] + # The difference between !!python/object/apply and !!python/object/new + # is how an object is created, check make_python_instance for details. + if isinstance(node, SequenceNode): + args = self.construct_sequence(node, deep=True) + kwds = {} + state = {} + listitems = [] + dictitems = {} + else: + value = self.construct_mapping(node, deep=True) + args = value.get('args', []) + kwds = value.get('kwds', {}) + state = value.get('state', {}) + listitems = value.get('listitems', []) + dictitems = value.get('dictitems', {}) + instance = self.make_python_instance(suffix, node, args, kwds, newobj) + if state: + self.set_python_instance_state(instance, state) + if listitems: + instance.extend(listitems) + if dictitems: + for key in dictitems: + instance[key] = dictitems[key] + return instance + + def construct_python_object_new(self, suffix, node): + return self.construct_python_object_apply(suffix, node, newobj=True) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/none', + FullConstructor.construct_yaml_null) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bool', + FullConstructor.construct_yaml_bool) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/str', + FullConstructor.construct_python_str) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/unicode', + FullConstructor.construct_python_unicode) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/bytes', + FullConstructor.construct_python_bytes) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/int', + FullConstructor.construct_yaml_int) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/long', + FullConstructor.construct_python_long) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/float', + FullConstructor.construct_yaml_float) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/complex', + FullConstructor.construct_python_complex) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/list', + FullConstructor.construct_yaml_seq) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/tuple', + FullConstructor.construct_python_tuple) + +FullConstructor.add_constructor( + 'tag:yaml.org,2002:python/dict', + FullConstructor.construct_yaml_map) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/name:', + FullConstructor.construct_python_name) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/module:', + FullConstructor.construct_python_module) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object:', + FullConstructor.construct_python_object) + +FullConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/new:', + FullConstructor.construct_python_object_new) + +class UnsafeConstructor(FullConstructor): + + def find_python_module(self, name, mark): + return super(UnsafeConstructor, self).find_python_module(name, mark, unsafe=True) + + def find_python_name(self, name, mark): + return super(UnsafeConstructor, self).find_python_name(name, mark, unsafe=True) + + def make_python_instance(self, suffix, node, args=None, kwds=None, newobj=False): + return super(UnsafeConstructor, self).make_python_instance( + suffix, node, args, kwds, newobj, unsafe=True) + + def set_python_instance_state(self, instance, state): + return super(UnsafeConstructor, self).set_python_instance_state( + instance, state, unsafe=True) + +UnsafeConstructor.add_multi_constructor( + 'tag:yaml.org,2002:python/object/apply:', + UnsafeConstructor.construct_python_object_apply) + +# Constructor is same as UnsafeConstructor. Need to leave this in place in case +# people have extended it directly. +class Constructor(UnsafeConstructor): + pass diff --git a/pipenv/patched/yaml3/cyaml.py b/pipenv/patched/yaml3/cyaml.py new file mode 100644 index 00000000..1e606c74 --- /dev/null +++ b/pipenv/patched/yaml3/cyaml.py @@ -0,0 +1,101 @@ + +__all__ = [ + 'CBaseLoader', 'CSafeLoader', 'CFullLoader', 'CUnsafeLoader', 'CLoader', + 'CBaseDumper', 'CSafeDumper', 'CDumper' +] + +from _yaml import CParser, CEmitter + +from .constructor import * + +from .serializer import * +from .representer import * + +from .resolver import * + +class CBaseLoader(CParser, BaseConstructor, BaseResolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class CSafeLoader(CParser, SafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class CFullLoader(CParser, FullConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class CUnsafeLoader(CParser, UnsafeConstructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + UnsafeConstructor.__init__(self) + Resolver.__init__(self) + +class CLoader(CParser, Constructor, Resolver): + + def __init__(self, stream): + CParser.__init__(self, stream) + Constructor.__init__(self) + Resolver.__init__(self) + +class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CSafeDumper(CEmitter, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class CDumper(CEmitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + CEmitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, encoding=encoding, + allow_unicode=allow_unicode, line_break=line_break, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml3/dumper.py b/pipenv/patched/yaml3/dumper.py new file mode 100644 index 00000000..6aadba55 --- /dev/null +++ b/pipenv/patched/yaml3/dumper.py @@ -0,0 +1,62 @@ + +__all__ = ['BaseDumper', 'SafeDumper', 'Dumper'] + +from .emitter import * +from .serializer import * +from .representer import * +from .resolver import * + +class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + SafeRepresenter.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + +class Dumper(Emitter, Serializer, Representer, Resolver): + + def __init__(self, stream, + default_style=None, default_flow_style=False, + canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None, + encoding=None, explicit_start=None, explicit_end=None, + version=None, tags=None, sort_keys=True): + Emitter.__init__(self, stream, canonical=canonical, + indent=indent, width=width, + allow_unicode=allow_unicode, line_break=line_break) + Serializer.__init__(self, encoding=encoding, + explicit_start=explicit_start, explicit_end=explicit_end, + version=version, tags=tags) + Representer.__init__(self, default_style=default_style, + default_flow_style=default_flow_style, sort_keys=sort_keys) + Resolver.__init__(self) + diff --git a/pipenv/patched/yaml3/emitter.py b/pipenv/patched/yaml3/emitter.py new file mode 100644 index 00000000..a664d011 --- /dev/null +++ b/pipenv/patched/yaml3/emitter.py @@ -0,0 +1,1137 @@ + +# Emitter expects events obeying the following grammar: +# stream ::= STREAM-START document* STREAM-END +# document ::= DOCUMENT-START node DOCUMENT-END +# node ::= SCALAR | sequence | mapping +# sequence ::= SEQUENCE-START node* SEQUENCE-END +# mapping ::= MAPPING-START (node node)* MAPPING-END + +__all__ = ['Emitter', 'EmitterError'] + +from .error import YAMLError +from .events import * + +class EmitterError(YAMLError): + pass + +class ScalarAnalysis: + def __init__(self, scalar, empty, multiline, + allow_flow_plain, allow_block_plain, + allow_single_quoted, allow_double_quoted, + allow_block): + self.scalar = scalar + self.empty = empty + self.multiline = multiline + self.allow_flow_plain = allow_flow_plain + self.allow_block_plain = allow_block_plain + self.allow_single_quoted = allow_single_quoted + self.allow_double_quoted = allow_double_quoted + self.allow_block = allow_block + +class Emitter: + + DEFAULT_TAG_PREFIXES = { + '!' : '!', + 'tag:yaml.org,2002:' : '!!', + } + + def __init__(self, stream, canonical=None, indent=None, width=None, + allow_unicode=None, line_break=None): + + # The stream should have the methods `write` and possibly `flush`. + self.stream = stream + + # Encoding can be overridden by STREAM-START. + self.encoding = None + + # Emitter is a state machine with a stack of states to handle nested + # structures. + self.states = [] + self.state = self.expect_stream_start + + # Current event and the event queue. + self.events = [] + self.event = None + + # The current indentation level and the stack of previous indents. + self.indents = [] + self.indent = None + + # Flow level. + self.flow_level = 0 + + # Contexts. + self.root_context = False + self.sequence_context = False + self.mapping_context = False + self.simple_key_context = False + + # Characteristics of the last emitted character: + # - current position. + # - is it a whitespace? + # - is it an indention character + # (indentation space, '-', '?', or ':')? + self.line = 0 + self.column = 0 + self.whitespace = True + self.indention = True + + # Whether the document requires an explicit document indicator + self.open_ended = False + + # Formatting details. + self.canonical = canonical + self.allow_unicode = allow_unicode + self.best_indent = 2 + if indent and 1 < indent < 10: + self.best_indent = indent + self.best_width = 80 + if width and width > self.best_indent*2: + self.best_width = width + self.best_line_break = '\n' + if line_break in ['\r', '\n', '\r\n']: + self.best_line_break = line_break + + # Tag prefixes. + self.tag_prefixes = None + + # Prepared anchor and tag. + self.prepared_anchor = None + self.prepared_tag = None + + # Scalar analysis and style. + self.analysis = None + self.style = None + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def emit(self, event): + self.events.append(event) + while not self.need_more_events(): + self.event = self.events.pop(0) + self.state() + self.event = None + + # In some cases, we wait for a few next events before emitting. + + def need_more_events(self): + if not self.events: + return True + event = self.events[0] + if isinstance(event, DocumentStartEvent): + return self.need_events(1) + elif isinstance(event, SequenceStartEvent): + return self.need_events(2) + elif isinstance(event, MappingStartEvent): + return self.need_events(3) + else: + return False + + def need_events(self, count): + level = 0 + for event in self.events[1:]: + if isinstance(event, (DocumentStartEvent, CollectionStartEvent)): + level += 1 + elif isinstance(event, (DocumentEndEvent, CollectionEndEvent)): + level -= 1 + elif isinstance(event, StreamEndEvent): + level = -1 + if level < 0: + return False + return (len(self.events) < count+1) + + def increase_indent(self, flow=False, indentless=False): + self.indents.append(self.indent) + if self.indent is None: + if flow: + self.indent = self.best_indent + else: + self.indent = 0 + elif not indentless: + self.indent += self.best_indent + + # States. + + # Stream handlers. + + def expect_stream_start(self): + if isinstance(self.event, StreamStartEvent): + if self.event.encoding and not hasattr(self.stream, 'encoding'): + self.encoding = self.event.encoding + self.write_stream_start() + self.state = self.expect_first_document_start + else: + raise EmitterError("expected StreamStartEvent, but got %s" + % self.event) + + def expect_nothing(self): + raise EmitterError("expected nothing, but got %s" % self.event) + + # Document handlers. + + def expect_first_document_start(self): + return self.expect_document_start(first=True) + + def expect_document_start(self, first=False): + if isinstance(self.event, DocumentStartEvent): + if (self.event.version or self.event.tags) and self.open_ended: + self.write_indicator('...', True) + self.write_indent() + if self.event.version: + version_text = self.prepare_version(self.event.version) + self.write_version_directive(version_text) + self.tag_prefixes = self.DEFAULT_TAG_PREFIXES.copy() + if self.event.tags: + handles = sorted(self.event.tags.keys()) + for handle in handles: + prefix = self.event.tags[handle] + self.tag_prefixes[prefix] = handle + handle_text = self.prepare_tag_handle(handle) + prefix_text = self.prepare_tag_prefix(prefix) + self.write_tag_directive(handle_text, prefix_text) + implicit = (first and not self.event.explicit and not self.canonical + and not self.event.version and not self.event.tags + and not self.check_empty_document()) + if not implicit: + self.write_indent() + self.write_indicator('---', True) + if self.canonical: + self.write_indent() + self.state = self.expect_document_root + elif isinstance(self.event, StreamEndEvent): + if self.open_ended: + self.write_indicator('...', True) + self.write_indent() + self.write_stream_end() + self.state = self.expect_nothing + else: + raise EmitterError("expected DocumentStartEvent, but got %s" + % self.event) + + def expect_document_end(self): + if isinstance(self.event, DocumentEndEvent): + self.write_indent() + if self.event.explicit: + self.write_indicator('...', True) + self.write_indent() + self.flush_stream() + self.state = self.expect_document_start + else: + raise EmitterError("expected DocumentEndEvent, but got %s" + % self.event) + + def expect_document_root(self): + self.states.append(self.expect_document_end) + self.expect_node(root=True) + + # Node handlers. + + def expect_node(self, root=False, sequence=False, mapping=False, + simple_key=False): + self.root_context = root + self.sequence_context = sequence + self.mapping_context = mapping + self.simple_key_context = simple_key + if isinstance(self.event, AliasEvent): + self.expect_alias() + elif isinstance(self.event, (ScalarEvent, CollectionStartEvent)): + self.process_anchor('&') + self.process_tag() + if isinstance(self.event, ScalarEvent): + self.expect_scalar() + elif isinstance(self.event, SequenceStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_sequence(): + self.expect_flow_sequence() + else: + self.expect_block_sequence() + elif isinstance(self.event, MappingStartEvent): + if self.flow_level or self.canonical or self.event.flow_style \ + or self.check_empty_mapping(): + self.expect_flow_mapping() + else: + self.expect_block_mapping() + else: + raise EmitterError("expected NodeEvent, but got %s" % self.event) + + def expect_alias(self): + if self.event.anchor is None: + raise EmitterError("anchor is not specified for alias") + self.process_anchor('*') + self.state = self.states.pop() + + def expect_scalar(self): + self.increase_indent(flow=True) + self.process_scalar() + self.indent = self.indents.pop() + self.state = self.states.pop() + + # Flow sequence handlers. + + def expect_flow_sequence(self): + self.write_indicator('[', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_sequence_item + + def expect_first_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator(']', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + def expect_flow_sequence_item(self): + if isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator(']', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + self.states.append(self.expect_flow_sequence_item) + self.expect_node(sequence=True) + + # Flow mapping handlers. + + def expect_flow_mapping(self): + self.write_indicator('{', True, whitespace=True) + self.flow_level += 1 + self.increase_indent(flow=True) + self.state = self.expect_first_flow_mapping_key + + def expect_first_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + self.write_indicator('}', False) + self.state = self.states.pop() + else: + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_key(self): + if isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.flow_level -= 1 + if self.canonical: + self.write_indicator(',', False) + self.write_indent() + self.write_indicator('}', False) + self.state = self.states.pop() + else: + self.write_indicator(',', False) + if self.canonical or self.column > self.best_width: + self.write_indent() + if not self.canonical and self.check_simple_key(): + self.states.append(self.expect_flow_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True) + self.states.append(self.expect_flow_mapping_value) + self.expect_node(mapping=True) + + def expect_flow_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + def expect_flow_mapping_value(self): + if self.canonical or self.column > self.best_width: + self.write_indent() + self.write_indicator(':', True) + self.states.append(self.expect_flow_mapping_key) + self.expect_node(mapping=True) + + # Block sequence handlers. + + def expect_block_sequence(self): + indentless = (self.mapping_context and not self.indention) + self.increase_indent(flow=False, indentless=indentless) + self.state = self.expect_first_block_sequence_item + + def expect_first_block_sequence_item(self): + return self.expect_block_sequence_item(first=True) + + def expect_block_sequence_item(self, first=False): + if not first and isinstance(self.event, SequenceEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + self.write_indicator('-', True, indention=True) + self.states.append(self.expect_block_sequence_item) + self.expect_node(sequence=True) + + # Block mapping handlers. + + def expect_block_mapping(self): + self.increase_indent(flow=False) + self.state = self.expect_first_block_mapping_key + + def expect_first_block_mapping_key(self): + return self.expect_block_mapping_key(first=True) + + def expect_block_mapping_key(self, first=False): + if not first and isinstance(self.event, MappingEndEvent): + self.indent = self.indents.pop() + self.state = self.states.pop() + else: + self.write_indent() + if self.check_simple_key(): + self.states.append(self.expect_block_mapping_simple_value) + self.expect_node(mapping=True, simple_key=True) + else: + self.write_indicator('?', True, indention=True) + self.states.append(self.expect_block_mapping_value) + self.expect_node(mapping=True) + + def expect_block_mapping_simple_value(self): + self.write_indicator(':', False) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + def expect_block_mapping_value(self): + self.write_indent() + self.write_indicator(':', True, indention=True) + self.states.append(self.expect_block_mapping_key) + self.expect_node(mapping=True) + + # Checkers. + + def check_empty_sequence(self): + return (isinstance(self.event, SequenceStartEvent) and self.events + and isinstance(self.events[0], SequenceEndEvent)) + + def check_empty_mapping(self): + return (isinstance(self.event, MappingStartEvent) and self.events + and isinstance(self.events[0], MappingEndEvent)) + + def check_empty_document(self): + if not isinstance(self.event, DocumentStartEvent) or not self.events: + return False + event = self.events[0] + return (isinstance(event, ScalarEvent) and event.anchor is None + and event.tag is None and event.implicit and event.value == '') + + def check_simple_key(self): + length = 0 + if isinstance(self.event, NodeEvent) and self.event.anchor is not None: + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + length += len(self.prepared_anchor) + if isinstance(self.event, (ScalarEvent, CollectionStartEvent)) \ + and self.event.tag is not None: + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(self.event.tag) + length += len(self.prepared_tag) + if isinstance(self.event, ScalarEvent): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + length += len(self.analysis.scalar) + return (length < 128 and (isinstance(self.event, AliasEvent) + or (isinstance(self.event, ScalarEvent) + and not self.analysis.empty and not self.analysis.multiline) + or self.check_empty_sequence() or self.check_empty_mapping())) + + # Anchor, Tag, and Scalar processors. + + def process_anchor(self, indicator): + if self.event.anchor is None: + self.prepared_anchor = None + return + if self.prepared_anchor is None: + self.prepared_anchor = self.prepare_anchor(self.event.anchor) + if self.prepared_anchor: + self.write_indicator(indicator+self.prepared_anchor, True) + self.prepared_anchor = None + + def process_tag(self): + tag = self.event.tag + if isinstance(self.event, ScalarEvent): + if self.style is None: + self.style = self.choose_scalar_style() + if ((not self.canonical or tag is None) and + ((self.style == '' and self.event.implicit[0]) + or (self.style != '' and self.event.implicit[1]))): + self.prepared_tag = None + return + if self.event.implicit[0] and tag is None: + tag = '!' + self.prepared_tag = None + else: + if (not self.canonical or tag is None) and self.event.implicit: + self.prepared_tag = None + return + if tag is None: + raise EmitterError("tag is not specified") + if self.prepared_tag is None: + self.prepared_tag = self.prepare_tag(tag) + if self.prepared_tag: + self.write_indicator(self.prepared_tag, True) + self.prepared_tag = None + + def choose_scalar_style(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.event.style == '"' or self.canonical: + return '"' + if not self.event.style and self.event.implicit[0]: + if (not (self.simple_key_context and + (self.analysis.empty or self.analysis.multiline)) + and (self.flow_level and self.analysis.allow_flow_plain + or (not self.flow_level and self.analysis.allow_block_plain))): + return '' + if self.event.style and self.event.style in '|>': + if (not self.flow_level and not self.simple_key_context + and self.analysis.allow_block): + return self.event.style + if not self.event.style or self.event.style == '\'': + if (self.analysis.allow_single_quoted and + not (self.simple_key_context and self.analysis.multiline)): + return '\'' + return '"' + + def process_scalar(self): + if self.analysis is None: + self.analysis = self.analyze_scalar(self.event.value) + if self.style is None: + self.style = self.choose_scalar_style() + split = (not self.simple_key_context) + #if self.analysis.multiline and split \ + # and (not self.style or self.style in '\'\"'): + # self.write_indent() + if self.style == '"': + self.write_double_quoted(self.analysis.scalar, split) + elif self.style == '\'': + self.write_single_quoted(self.analysis.scalar, split) + elif self.style == '>': + self.write_folded(self.analysis.scalar) + elif self.style == '|': + self.write_literal(self.analysis.scalar) + else: + self.write_plain(self.analysis.scalar, split) + self.analysis = None + self.style = None + + # Analyzers. + + def prepare_version(self, version): + major, minor = version + if major != 1: + raise EmitterError("unsupported YAML version: %d.%d" % (major, minor)) + return '%d.%d' % (major, minor) + + def prepare_tag_handle(self, handle): + if not handle: + raise EmitterError("tag handle must not be empty") + if handle[0] != '!' or handle[-1] != '!': + raise EmitterError("tag handle must start and end with '!': %r" % handle) + for ch in handle[1:-1]: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the tag handle: %r" + % (ch, handle)) + return handle + + def prepare_tag_prefix(self, prefix): + if not prefix: + raise EmitterError("tag prefix must not be empty") + chunks = [] + start = end = 0 + if prefix[0] == '!': + end = 1 + while end < len(prefix): + ch = prefix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?!:@&=+$,_.~*\'()[]': + end += 1 + else: + if start < end: + chunks.append(prefix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ord(ch)) + if start < end: + chunks.append(prefix[start:end]) + return ''.join(chunks) + + def prepare_tag(self, tag): + if not tag: + raise EmitterError("tag must not be empty") + if tag == '!': + return tag + handle = None + suffix = tag + prefixes = sorted(self.tag_prefixes.keys()) + for prefix in prefixes: + if tag.startswith(prefix) \ + and (prefix == '!' or len(prefix) < len(tag)): + handle = self.tag_prefixes[prefix] + suffix = tag[len(prefix):] + chunks = [] + start = end = 0 + while end < len(suffix): + ch = suffix[end] + if '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.~*\'()[]' \ + or (ch == '!' and handle != '!'): + end += 1 + else: + if start < end: + chunks.append(suffix[start:end]) + start = end = end+1 + data = ch.encode('utf-8') + for ch in data: + chunks.append('%%%02X' % ch) + if start < end: + chunks.append(suffix[start:end]) + suffix_text = ''.join(chunks) + if handle: + return '%s%s' % (handle, suffix_text) + else: + return '!<%s>' % suffix_text + + def prepare_anchor(self, anchor): + if not anchor: + raise EmitterError("anchor must not be empty") + for ch in anchor: + if not ('0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_'): + raise EmitterError("invalid character %r in the anchor: %r" + % (ch, anchor)) + return anchor + + def analyze_scalar(self, scalar): + + # Empty scalar is a special case. + if not scalar: + return ScalarAnalysis(scalar=scalar, empty=True, multiline=False, + allow_flow_plain=False, allow_block_plain=True, + allow_single_quoted=True, allow_double_quoted=True, + allow_block=False) + + # Indicators and special characters. + block_indicators = False + flow_indicators = False + line_breaks = False + special_characters = False + + # Important whitespace combinations. + leading_space = False + leading_break = False + trailing_space = False + trailing_break = False + break_space = False + space_break = False + + # Check document indicators. + if scalar.startswith('---') or scalar.startswith('...'): + block_indicators = True + flow_indicators = True + + # First character or preceded by a whitespace. + preceded_by_whitespace = True + + # Last character or followed by a whitespace. + followed_by_whitespace = (len(scalar) == 1 or + scalar[1] in '\0 \t\r\n\x85\u2028\u2029') + + # The previous character is a space. + previous_space = False + + # The previous character is a break. + previous_break = False + + index = 0 + while index < len(scalar): + ch = scalar[index] + + # Check for indicators. + if index == 0: + # Leading indicators are special characters. + if ch in '#,[]{}&*!|>\'\"%@`': + flow_indicators = True + block_indicators = True + if ch in '?:': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '-' and followed_by_whitespace: + flow_indicators = True + block_indicators = True + else: + # Some indicators cannot appear within a scalar as well. + if ch in ',?[]{}': + flow_indicators = True + if ch == ':': + flow_indicators = True + if followed_by_whitespace: + block_indicators = True + if ch == '#' and preceded_by_whitespace: + flow_indicators = True + block_indicators = True + + # Check for line breaks, special, and unicode characters. + if ch in '\n\x85\u2028\u2029': + line_breaks = True + if not (ch == '\n' or '\x20' <= ch <= '\x7E'): + if (ch == '\x85' or '\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD' + or '\U00010000' <= ch < '\U0010ffff') and ch != '\uFEFF': + unicode_characters = True + if not self.allow_unicode: + special_characters = True + else: + special_characters = True + + # Detect important whitespace combinations. + if ch == ' ': + if index == 0: + leading_space = True + if index == len(scalar)-1: + trailing_space = True + if previous_break: + break_space = True + previous_space = True + previous_break = False + elif ch in '\n\x85\u2028\u2029': + if index == 0: + leading_break = True + if index == len(scalar)-1: + trailing_break = True + if previous_space: + space_break = True + previous_space = False + previous_break = True + else: + previous_space = False + previous_break = False + + # Prepare for the next character. + index += 1 + preceded_by_whitespace = (ch in '\0 \t\r\n\x85\u2028\u2029') + followed_by_whitespace = (index+1 >= len(scalar) or + scalar[index+1] in '\0 \t\r\n\x85\u2028\u2029') + + # Let's decide what styles are allowed. + allow_flow_plain = True + allow_block_plain = True + allow_single_quoted = True + allow_double_quoted = True + allow_block = True + + # Leading and trailing whitespaces are bad for plain scalars. + if (leading_space or leading_break + or trailing_space or trailing_break): + allow_flow_plain = allow_block_plain = False + + # We do not permit trailing spaces for block scalars. + if trailing_space: + allow_block = False + + # Spaces at the beginning of a new line are only acceptable for block + # scalars. + if break_space: + allow_flow_plain = allow_block_plain = allow_single_quoted = False + + # Spaces followed by breaks, as well as special character are only + # allowed for double quoted scalars. + if space_break or special_characters: + allow_flow_plain = allow_block_plain = \ + allow_single_quoted = allow_block = False + + # Although the plain scalar writer supports breaks, we never emit + # multiline plain scalars. + if line_breaks: + allow_flow_plain = allow_block_plain = False + + # Flow indicators are forbidden for flow plain scalars. + if flow_indicators: + allow_flow_plain = False + + # Block indicators are forbidden for block plain scalars. + if block_indicators: + allow_block_plain = False + + return ScalarAnalysis(scalar=scalar, + empty=False, multiline=line_breaks, + allow_flow_plain=allow_flow_plain, + allow_block_plain=allow_block_plain, + allow_single_quoted=allow_single_quoted, + allow_double_quoted=allow_double_quoted, + allow_block=allow_block) + + # Writers. + + def flush_stream(self): + if hasattr(self.stream, 'flush'): + self.stream.flush() + + def write_stream_start(self): + # Write BOM if needed. + if self.encoding and self.encoding.startswith('utf-16'): + self.stream.write('\uFEFF'.encode(self.encoding)) + + def write_stream_end(self): + self.flush_stream() + + def write_indicator(self, indicator, need_whitespace, + whitespace=False, indention=False): + if self.whitespace or not need_whitespace: + data = indicator + else: + data = ' '+indicator + self.whitespace = whitespace + self.indention = self.indention and indention + self.column += len(data) + self.open_ended = False + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_indent(self): + indent = self.indent or 0 + if not self.indention or self.column > indent \ + or (self.column == indent and not self.whitespace): + self.write_line_break() + if self.column < indent: + self.whitespace = True + data = ' '*(indent-self.column) + self.column = indent + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_line_break(self, data=None): + if data is None: + data = self.best_line_break + self.whitespace = True + self.indention = True + self.line += 1 + self.column = 0 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + + def write_version_directive(self, version_text): + data = '%%YAML %s' % version_text + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + def write_tag_directive(self, handle_text, prefix_text): + data = '%%TAG %s %s' % (handle_text, prefix_text) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_line_break() + + # Scalar streams. + + def write_single_quoted(self, text, split=True): + self.write_indicator('\'', True) + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch is None or ch != ' ': + if start+1 == end and self.column > self.best_width and split \ + and start != 0 and end != len(text): + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029' or ch == '\'': + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch == '\'': + data = '\'\'' + self.column += 2 + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + 1 + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + self.write_indicator('\'', False) + + ESCAPE_REPLACEMENTS = { + '\0': '0', + '\x07': 'a', + '\x08': 'b', + '\x09': 't', + '\x0A': 'n', + '\x0B': 'v', + '\x0C': 'f', + '\x0D': 'r', + '\x1B': 'e', + '\"': '\"', + '\\': '\\', + '\x85': 'N', + '\xA0': '_', + '\u2028': 'L', + '\u2029': 'P', + } + + def write_double_quoted(self, text, split=True): + self.write_indicator('"', True) + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if ch is None or ch in '"\\\x85\u2028\u2029\uFEFF' \ + or not ('\x20' <= ch <= '\x7E' + or (self.allow_unicode + and ('\xA0' <= ch <= '\uD7FF' + or '\uE000' <= ch <= '\uFFFD'))): + if start < end: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + if ch in self.ESCAPE_REPLACEMENTS: + data = '\\'+self.ESCAPE_REPLACEMENTS[ch] + elif ch <= '\xFF': + data = '\\x%02X' % ord(ch) + elif ch <= '\uFFFF': + data = '\\u%04X' % ord(ch) + else: + data = '\\U%08X' % ord(ch) + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end+1 + if 0 < end < len(text)-1 and (ch == ' ' or start >= end) \ + and self.column+(end-start) > self.best_width and split: + data = text[start:end]+'\\' + if start < end: + start = end + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.write_indent() + self.whitespace = False + self.indention = False + if text[start] == ' ': + data = '\\' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + end += 1 + self.write_indicator('"', False) + + def determine_block_hints(self, text): + hints = '' + if text: + if text[0] in ' \n\x85\u2028\u2029': + hints += str(self.best_indent) + if text[-1] not in '\n\x85\u2028\u2029': + hints += '-' + elif len(text) == 1 or text[-2] in '\n\x85\u2028\u2029': + hints += '+' + return hints + + def write_folded(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('>'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + leading_space = True + spaces = False + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + if not leading_space and ch is not None and ch != ' ' \ + and text[start] == '\n': + self.write_line_break() + leading_space = (ch == ' ') + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + elif spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width: + self.write_indent() + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + spaces = (ch == ' ') + end += 1 + + def write_literal(self, text): + hints = self.determine_block_hints(text) + self.write_indicator('|'+hints, True) + if hints[-1:] == '+': + self.open_ended = True + self.write_line_break() + breaks = True + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if breaks: + if ch is None or ch not in '\n\x85\u2028\u2029': + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + if ch is not None: + self.write_indent() + start = end + else: + if ch is None or ch in '\n\x85\u2028\u2029': + data = text[start:end] + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + if ch is None: + self.write_line_break() + start = end + if ch is not None: + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 + + def write_plain(self, text, split=True): + if self.root_context: + self.open_ended = True + if not text: + return + if not self.whitespace: + data = ' ' + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + self.whitespace = False + self.indention = False + spaces = False + breaks = False + start = end = 0 + while end <= len(text): + ch = None + if end < len(text): + ch = text[end] + if spaces: + if ch != ' ': + if start+1 == end and self.column > self.best_width and split: + self.write_indent() + self.whitespace = False + self.indention = False + else: + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + elif breaks: + if ch not in '\n\x85\u2028\u2029': + if text[start] == '\n': + self.write_line_break() + for br in text[start:end]: + if br == '\n': + self.write_line_break() + else: + self.write_line_break(br) + self.write_indent() + self.whitespace = False + self.indention = False + start = end + else: + if ch is None or ch in ' \n\x85\u2028\u2029': + data = text[start:end] + self.column += len(data) + if self.encoding: + data = data.encode(self.encoding) + self.stream.write(data) + start = end + if ch is not None: + spaces = (ch == ' ') + breaks = (ch in '\n\x85\u2028\u2029') + end += 1 diff --git a/pipenv/patched/yaml3/error.py b/pipenv/patched/yaml3/error.py new file mode 100644 index 00000000..b796b4dc --- /dev/null +++ b/pipenv/patched/yaml3/error.py @@ -0,0 +1,75 @@ + +__all__ = ['Mark', 'YAMLError', 'MarkedYAMLError'] + +class Mark: + + def __init__(self, name, index, line, column, buffer, pointer): + self.name = name + self.index = index + self.line = line + self.column = column + self.buffer = buffer + self.pointer = pointer + + def get_snippet(self, indent=4, max_length=75): + if self.buffer is None: + return None + head = '' + start = self.pointer + while start > 0 and self.buffer[start-1] not in '\0\r\n\x85\u2028\u2029': + start -= 1 + if self.pointer-start > max_length/2-1: + head = ' ... ' + start += 5 + break + tail = '' + end = self.pointer + while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029': + end += 1 + if end-self.pointer > max_length/2-1: + tail = ' ... ' + end -= 5 + break + snippet = self.buffer[start:end] + return ' '*indent + head + snippet + tail + '\n' \ + + ' '*(indent+self.pointer-start+len(head)) + '^' + + def __str__(self): + snippet = self.get_snippet() + where = " in \"%s\", line %d, column %d" \ + % (self.name, self.line+1, self.column+1) + if snippet is not None: + where += ":\n"+snippet + return where + +class YAMLError(Exception): + pass + +class MarkedYAMLError(YAMLError): + + def __init__(self, context=None, context_mark=None, + problem=None, problem_mark=None, note=None): + self.context = context + self.context_mark = context_mark + self.problem = problem + self.problem_mark = problem_mark + self.note = note + + def __str__(self): + lines = [] + if self.context is not None: + lines.append(self.context) + if self.context_mark is not None \ + and (self.problem is None or self.problem_mark is None + or self.context_mark.name != self.problem_mark.name + or self.context_mark.line != self.problem_mark.line + or self.context_mark.column != self.problem_mark.column): + lines.append(str(self.context_mark)) + if self.problem is not None: + lines.append(self.problem) + if self.problem_mark is not None: + lines.append(str(self.problem_mark)) + if self.note is not None: + lines.append(self.note) + return '\n'.join(lines) + diff --git a/pipenv/patched/yaml3/events.py b/pipenv/patched/yaml3/events.py new file mode 100644 index 00000000..f79ad389 --- /dev/null +++ b/pipenv/patched/yaml3/events.py @@ -0,0 +1,86 @@ + +# Abstract classes. + +class Event(object): + def __init__(self, start_mark=None, end_mark=None): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in ['anchor', 'tag', 'implicit', 'value'] + if hasattr(self, key)] + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +class NodeEvent(Event): + def __init__(self, anchor, start_mark=None, end_mark=None): + self.anchor = anchor + self.start_mark = start_mark + self.end_mark = end_mark + +class CollectionStartEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, start_mark=None, end_mark=None, + flow_style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class CollectionEndEvent(Event): + pass + +# Implementations. + +class StreamStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndEvent(Event): + pass + +class DocumentStartEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None, version=None, tags=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + self.version = version + self.tags = tags + +class DocumentEndEvent(Event): + def __init__(self, start_mark=None, end_mark=None, + explicit=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.explicit = explicit + +class AliasEvent(NodeEvent): + pass + +class ScalarEvent(NodeEvent): + def __init__(self, anchor, tag, implicit, value, + start_mark=None, end_mark=None, style=None): + self.anchor = anchor + self.tag = tag + self.implicit = implicit + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class SequenceStartEvent(CollectionStartEvent): + pass + +class SequenceEndEvent(CollectionEndEvent): + pass + +class MappingStartEvent(CollectionStartEvent): + pass + +class MappingEndEvent(CollectionEndEvent): + pass + diff --git a/pipenv/patched/yaml3/loader.py b/pipenv/patched/yaml3/loader.py new file mode 100644 index 00000000..e90c1122 --- /dev/null +++ b/pipenv/patched/yaml3/loader.py @@ -0,0 +1,63 @@ + +__all__ = ['BaseLoader', 'FullLoader', 'SafeLoader', 'Loader', 'UnsafeLoader'] + +from .reader import * +from .scanner import * +from .parser import * +from .composer import * +from .constructor import * +from .resolver import * + +class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, BaseResolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + BaseConstructor.__init__(self) + BaseResolver.__init__(self) + +class FullLoader(Reader, Scanner, Parser, Composer, FullConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + FullConstructor.__init__(self) + Resolver.__init__(self) + +class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + SafeConstructor.__init__(self) + Resolver.__init__(self) + +class Loader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) + +# UnsafeLoader is the same as Loader (which is and was always unsafe on +# untrusted input). Use of either Loader or UnsafeLoader should be rare, since +# FullLoad should be able to load almost all YAML safely. Loader is left intact +# to ensure backwards compatibility. +class UnsafeLoader(Reader, Scanner, Parser, Composer, Constructor, Resolver): + + def __init__(self, stream): + Reader.__init__(self, stream) + Scanner.__init__(self) + Parser.__init__(self) + Composer.__init__(self) + Constructor.__init__(self) + Resolver.__init__(self) diff --git a/pipenv/patched/yaml3/nodes.py b/pipenv/patched/yaml3/nodes.py new file mode 100644 index 00000000..c4f070c4 --- /dev/null +++ b/pipenv/patched/yaml3/nodes.py @@ -0,0 +1,49 @@ + +class Node(object): + def __init__(self, tag, value, start_mark, end_mark): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + value = self.value + #if isinstance(value, list): + # if len(value) == 0: + # value = '<empty>' + # elif len(value) == 1: + # value = '<1 item>' + # else: + # value = '<%d items>' % len(value) + #else: + # if len(value) > 75: + # value = repr(value[:70]+u' ... ') + # else: + # value = repr(value) + value = repr(value) + return '%s(tag=%r, value=%s)' % (self.__class__.__name__, self.tag, value) + +class ScalarNode(Node): + id = 'scalar' + def __init__(self, tag, value, + start_mark=None, end_mark=None, style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + +class CollectionNode(Node): + def __init__(self, tag, value, + start_mark=None, end_mark=None, flow_style=None): + self.tag = tag + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + self.flow_style = flow_style + +class SequenceNode(CollectionNode): + id = 'sequence' + +class MappingNode(CollectionNode): + id = 'mapping' + diff --git a/pipenv/patched/yaml3/parser.py b/pipenv/patched/yaml3/parser.py new file mode 100644 index 00000000..13a5995d --- /dev/null +++ b/pipenv/patched/yaml3/parser.py @@ -0,0 +1,589 @@ + +# The following YAML grammar is LL(1) and is parsed by a recursive descent +# parser. +# +# stream ::= STREAM-START implicit_document? explicit_document* STREAM-END +# implicit_document ::= block_node DOCUMENT-END* +# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* +# block_node_or_indentless_sequence ::= +# ALIAS +# | properties (block_content | indentless_block_sequence)? +# | block_content +# | indentless_block_sequence +# block_node ::= ALIAS +# | properties block_content? +# | block_content +# flow_node ::= ALIAS +# | properties flow_content? +# | flow_content +# properties ::= TAG ANCHOR? | ANCHOR TAG? +# block_content ::= block_collection | flow_collection | SCALAR +# flow_content ::= flow_collection | SCALAR +# block_collection ::= block_sequence | block_mapping +# flow_collection ::= flow_sequence | flow_mapping +# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END +# indentless_sequence ::= (BLOCK-ENTRY block_node?)+ +# block_mapping ::= BLOCK-MAPPING_START +# ((KEY block_node_or_indentless_sequence?)? +# (VALUE block_node_or_indentless_sequence?)?)* +# BLOCK-END +# flow_sequence ::= FLOW-SEQUENCE-START +# (flow_sequence_entry FLOW-ENTRY)* +# flow_sequence_entry? +# FLOW-SEQUENCE-END +# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# flow_mapping ::= FLOW-MAPPING-START +# (flow_mapping_entry FLOW-ENTRY)* +# flow_mapping_entry? +# FLOW-MAPPING-END +# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? +# +# FIRST sets: +# +# stream: { STREAM-START } +# explicit_document: { DIRECTIVE DOCUMENT-START } +# implicit_document: FIRST(block_node) +# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR } +# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# block_sequence: { BLOCK-SEQUENCE-START } +# block_mapping: { BLOCK-MAPPING-START } +# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START BLOCK-ENTRY } +# indentless_sequence: { ENTRY } +# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START } +# flow_sequence: { FLOW-SEQUENCE-START } +# flow_mapping: { FLOW-MAPPING-START } +# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } +# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START KEY } + +__all__ = ['Parser', 'ParserError'] + +from .error import MarkedYAMLError +from .tokens import * +from .events import * +from .scanner import * + +class ParserError(MarkedYAMLError): + pass + +class Parser: + # Since writing a recursive-descendant parser is a straightforward task, we + # do not give many comments here. + + DEFAULT_TAGS = { + '!': '!', + '!!': 'tag:yaml.org,2002:', + } + + def __init__(self): + self.current_event = None + self.yaml_version = None + self.tag_handles = {} + self.states = [] + self.marks = [] + self.state = self.parse_stream_start + + def dispose(self): + # Reset the state attributes (to clear self-references) + self.states = [] + self.state = None + + def check_event(self, *choices): + # Check the type of the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + if self.current_event is not None: + if not choices: + return True + for choice in choices: + if isinstance(self.current_event, choice): + return True + return False + + def peek_event(self): + # Get the next event. + if self.current_event is None: + if self.state: + self.current_event = self.state() + return self.current_event + + def get_event(self): + # Get the next event and proceed further. + if self.current_event is None: + if self.state: + self.current_event = self.state() + value = self.current_event + self.current_event = None + return value + + # stream ::= STREAM-START implicit_document? explicit_document* STREAM-END + # implicit_document ::= block_node DOCUMENT-END* + # explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END* + + def parse_stream_start(self): + + # Parse the stream start. + token = self.get_token() + event = StreamStartEvent(token.start_mark, token.end_mark, + encoding=token.encoding) + + # Prepare the next state. + self.state = self.parse_implicit_document_start + + return event + + def parse_implicit_document_start(self): + + # Parse an implicit document. + if not self.check_token(DirectiveToken, DocumentStartToken, + StreamEndToken): + self.tag_handles = self.DEFAULT_TAGS + token = self.peek_token() + start_mark = end_mark = token.start_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=False) + + # Prepare the next state. + self.states.append(self.parse_document_end) + self.state = self.parse_block_node + + return event + + else: + return self.parse_document_start() + + def parse_document_start(self): + + # Parse any extra document end indicators. + while self.check_token(DocumentEndToken): + self.get_token() + + # Parse an explicit document. + if not self.check_token(StreamEndToken): + token = self.peek_token() + start_mark = token.start_mark + version, tags = self.process_directives() + if not self.check_token(DocumentStartToken): + raise ParserError(None, None, + "expected '<document start>', but found %r" + % self.peek_token().id, + self.peek_token().start_mark) + token = self.get_token() + end_mark = token.end_mark + event = DocumentStartEvent(start_mark, end_mark, + explicit=True, version=version, tags=tags) + self.states.append(self.parse_document_end) + self.state = self.parse_document_content + else: + # Parse the end of the stream. + token = self.get_token() + event = StreamEndEvent(token.start_mark, token.end_mark) + assert not self.states + assert not self.marks + self.state = None + return event + + def parse_document_end(self): + + # Parse the document end. + token = self.peek_token() + start_mark = end_mark = token.start_mark + explicit = False + if self.check_token(DocumentEndToken): + token = self.get_token() + end_mark = token.end_mark + explicit = True + event = DocumentEndEvent(start_mark, end_mark, + explicit=explicit) + + # Prepare the next state. + self.state = self.parse_document_start + + return event + + def parse_document_content(self): + if self.check_token(DirectiveToken, + DocumentStartToken, DocumentEndToken, StreamEndToken): + event = self.process_empty_scalar(self.peek_token().start_mark) + self.state = self.states.pop() + return event + else: + return self.parse_block_node() + + def process_directives(self): + self.yaml_version = None + self.tag_handles = {} + while self.check_token(DirectiveToken): + token = self.get_token() + if token.name == 'YAML': + if self.yaml_version is not None: + raise ParserError(None, None, + "found duplicate YAML directive", token.start_mark) + major, minor = token.value + if major != 1: + raise ParserError(None, None, + "found incompatible YAML document (version 1.* is required)", + token.start_mark) + self.yaml_version = token.value + elif token.name == 'TAG': + handle, prefix = token.value + if handle in self.tag_handles: + raise ParserError(None, None, + "duplicate tag handle %r" % handle, + token.start_mark) + self.tag_handles[handle] = prefix + if self.tag_handles: + value = self.yaml_version, self.tag_handles.copy() + else: + value = self.yaml_version, None + for key in self.DEFAULT_TAGS: + if key not in self.tag_handles: + self.tag_handles[key] = self.DEFAULT_TAGS[key] + return value + + # block_node_or_indentless_sequence ::= ALIAS + # | properties (block_content | indentless_block_sequence)? + # | block_content + # | indentless_block_sequence + # block_node ::= ALIAS + # | properties block_content? + # | block_content + # flow_node ::= ALIAS + # | properties flow_content? + # | flow_content + # properties ::= TAG ANCHOR? | ANCHOR TAG? + # block_content ::= block_collection | flow_collection | SCALAR + # flow_content ::= flow_collection | SCALAR + # block_collection ::= block_sequence | block_mapping + # flow_collection ::= flow_sequence | flow_mapping + + def parse_block_node(self): + return self.parse_node(block=True) + + def parse_flow_node(self): + return self.parse_node() + + def parse_block_node_or_indentless_sequence(self): + return self.parse_node(block=True, indentless_sequence=True) + + def parse_node(self, block=False, indentless_sequence=False): + if self.check_token(AliasToken): + token = self.get_token() + event = AliasEvent(token.value, token.start_mark, token.end_mark) + self.state = self.states.pop() + else: + anchor = None + tag = None + start_mark = end_mark = tag_mark = None + if self.check_token(AnchorToken): + token = self.get_token() + start_mark = token.start_mark + end_mark = token.end_mark + anchor = token.value + if self.check_token(TagToken): + token = self.get_token() + tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + elif self.check_token(TagToken): + token = self.get_token() + start_mark = tag_mark = token.start_mark + end_mark = token.end_mark + tag = token.value + if self.check_token(AnchorToken): + token = self.get_token() + end_mark = token.end_mark + anchor = token.value + if tag is not None: + handle, suffix = tag + if handle is not None: + if handle not in self.tag_handles: + raise ParserError("while parsing a node", start_mark, + "found undefined tag handle %r" % handle, + tag_mark) + tag = self.tag_handles[handle]+suffix + else: + tag = suffix + #if tag == '!': + # raise ParserError("while parsing a node", start_mark, + # "found non-specific tag '!'", tag_mark, + # "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag' and share your opinion.") + if start_mark is None: + start_mark = end_mark = self.peek_token().start_mark + event = None + implicit = (tag is None or tag == '!') + if indentless_sequence and self.check_token(BlockEntryToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark) + self.state = self.parse_indentless_sequence_entry + else: + if self.check_token(ScalarToken): + token = self.get_token() + end_mark = token.end_mark + if (token.plain and tag is None) or tag == '!': + implicit = (True, False) + elif tag is None: + implicit = (False, True) + else: + implicit = (False, False) + event = ScalarEvent(anchor, tag, implicit, token.value, + start_mark, end_mark, style=token.style) + self.state = self.states.pop() + elif self.check_token(FlowSequenceStartToken): + end_mark = self.peek_token().end_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_sequence_first_entry + elif self.check_token(FlowMappingStartToken): + end_mark = self.peek_token().end_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=True) + self.state = self.parse_flow_mapping_first_key + elif block and self.check_token(BlockSequenceStartToken): + end_mark = self.peek_token().start_mark + event = SequenceStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_sequence_first_entry + elif block and self.check_token(BlockMappingStartToken): + end_mark = self.peek_token().start_mark + event = MappingStartEvent(anchor, tag, implicit, + start_mark, end_mark, flow_style=False) + self.state = self.parse_block_mapping_first_key + elif anchor is not None or tag is not None: + # Empty scalars are allowed even if a tag or an anchor is + # specified. + event = ScalarEvent(anchor, tag, (implicit, False), '', + start_mark, end_mark) + self.state = self.states.pop() + else: + if block: + node = 'block' + else: + node = 'flow' + token = self.peek_token() + raise ParserError("while parsing a %s node" % node, start_mark, + "expected the node content, but found %r" % token.id, + token.start_mark) + return event + + # block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)* BLOCK-END + + def parse_block_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_sequence_entry() + + def parse_block_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, BlockEndToken): + self.states.append(self.parse_block_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_block_sequence_entry + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block collection", self.marks[-1], + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + # indentless_sequence ::= (BLOCK-ENTRY block_node?)+ + + def parse_indentless_sequence_entry(self): + if self.check_token(BlockEntryToken): + token = self.get_token() + if not self.check_token(BlockEntryToken, + KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_indentless_sequence_entry) + return self.parse_block_node() + else: + self.state = self.parse_indentless_sequence_entry + return self.process_empty_scalar(token.end_mark) + token = self.peek_token() + event = SequenceEndEvent(token.start_mark, token.start_mark) + self.state = self.states.pop() + return event + + # block_mapping ::= BLOCK-MAPPING_START + # ((KEY block_node_or_indentless_sequence?)? + # (VALUE block_node_or_indentless_sequence?)?)* + # BLOCK-END + + def parse_block_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_block_mapping_key() + + def parse_block_mapping_key(self): + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_value) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_value + return self.process_empty_scalar(token.end_mark) + if not self.check_token(BlockEndToken): + token = self.peek_token() + raise ParserError("while parsing a block mapping", self.marks[-1], + "expected <block end>, but found %r" % token.id, token.start_mark) + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_block_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(KeyToken, ValueToken, BlockEndToken): + self.states.append(self.parse_block_mapping_key) + return self.parse_block_node_or_indentless_sequence() + else: + self.state = self.parse_block_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_block_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + # flow_sequence ::= FLOW-SEQUENCE-START + # (flow_sequence_entry FLOW-ENTRY)* + # flow_sequence_entry? + # FLOW-SEQUENCE-END + # flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + # + # Note that while production rules for both flow_sequence_entry and + # flow_mapping_entry are equal, their interpretations are different. + # For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?` + # generate an inline mapping (set syntax). + + def parse_flow_sequence_first_entry(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_sequence_entry(first=True) + + def parse_flow_sequence_entry(self, first=False): + if not self.check_token(FlowSequenceEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow sequence", self.marks[-1], + "expected ',' or ']', but got %r" % token.id, token.start_mark) + + if self.check_token(KeyToken): + token = self.peek_token() + event = MappingStartEvent(None, None, True, + token.start_mark, token.end_mark, + flow_style=True) + self.state = self.parse_flow_sequence_entry_mapping_key + return event + elif not self.check_token(FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry) + return self.parse_flow_node() + token = self.get_token() + event = SequenceEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_sequence_entry_mapping_key(self): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_value + return self.process_empty_scalar(token.end_mark) + + def parse_flow_sequence_entry_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowSequenceEndToken): + self.states.append(self.parse_flow_sequence_entry_mapping_end) + return self.parse_flow_node() + else: + self.state = self.parse_flow_sequence_entry_mapping_end + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_sequence_entry_mapping_end + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_sequence_entry_mapping_end(self): + self.state = self.parse_flow_sequence_entry + token = self.peek_token() + return MappingEndEvent(token.start_mark, token.start_mark) + + # flow_mapping ::= FLOW-MAPPING-START + # (flow_mapping_entry FLOW-ENTRY)* + # flow_mapping_entry? + # FLOW-MAPPING-END + # flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)? + + def parse_flow_mapping_first_key(self): + token = self.get_token() + self.marks.append(token.start_mark) + return self.parse_flow_mapping_key(first=True) + + def parse_flow_mapping_key(self, first=False): + if not self.check_token(FlowMappingEndToken): + if not first: + if self.check_token(FlowEntryToken): + self.get_token() + else: + token = self.peek_token() + raise ParserError("while parsing a flow mapping", self.marks[-1], + "expected ',' or '}', but got %r" % token.id, token.start_mark) + if self.check_token(KeyToken): + token = self.get_token() + if not self.check_token(ValueToken, + FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_value) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_value + return self.process_empty_scalar(token.end_mark) + elif not self.check_token(FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_empty_value) + return self.parse_flow_node() + token = self.get_token() + event = MappingEndEvent(token.start_mark, token.end_mark) + self.state = self.states.pop() + self.marks.pop() + return event + + def parse_flow_mapping_value(self): + if self.check_token(ValueToken): + token = self.get_token() + if not self.check_token(FlowEntryToken, FlowMappingEndToken): + self.states.append(self.parse_flow_mapping_key) + return self.parse_flow_node() + else: + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(token.end_mark) + else: + self.state = self.parse_flow_mapping_key + token = self.peek_token() + return self.process_empty_scalar(token.start_mark) + + def parse_flow_mapping_empty_value(self): + self.state = self.parse_flow_mapping_key + return self.process_empty_scalar(self.peek_token().start_mark) + + def process_empty_scalar(self, mark): + return ScalarEvent(None, None, (True, False), '', mark, mark) + diff --git a/pipenv/patched/yaml3/reader.py b/pipenv/patched/yaml3/reader.py new file mode 100644 index 00000000..774b0219 --- /dev/null +++ b/pipenv/patched/yaml3/reader.py @@ -0,0 +1,185 @@ +# This module contains abstractions for the input stream. You don't have to +# looks further, there are no pretty code. +# +# We define two classes here. +# +# Mark(source, line, column) +# It's just a record and its only use is producing nice error messages. +# Parser does not use it for any other purposes. +# +# Reader(source, data) +# Reader determines the encoding of `data` and converts it to unicode. +# Reader provides the following methods and attributes: +# reader.peek(length=1) - return the next `length` characters +# reader.forward(length=1) - move the current position to `length` characters. +# reader.index - the number of the current character. +# reader.line, stream.column - the line and the column of the current character. + +__all__ = ['Reader', 'ReaderError'] + +from .error import YAMLError, Mark + +import codecs, re + +class ReaderError(YAMLError): + + def __init__(self, name, position, character, encoding, reason): + self.name = name + self.character = character + self.position = position + self.encoding = encoding + self.reason = reason + + def __str__(self): + if isinstance(self.character, bytes): + return "'%s' codec can't decode byte #x%02x: %s\n" \ + " in \"%s\", position %d" \ + % (self.encoding, ord(self.character), self.reason, + self.name, self.position) + else: + return "unacceptable character #x%04x: %s\n" \ + " in \"%s\", position %d" \ + % (self.character, self.reason, + self.name, self.position) + +class Reader(object): + # Reader: + # - determines the data encoding and converts it to a unicode string, + # - checks if characters are in allowed range, + # - adds '\0' to the end. + + # Reader accepts + # - a `bytes` object, + # - a `str` object, + # - a file-like object with its `read` method returning `str`, + # - a file-like object with its `read` method returning `unicode`. + + # Yeah, it's ugly and slow. + + def __init__(self, stream): + self.name = None + self.stream = None + self.stream_pointer = 0 + self.eof = True + self.buffer = '' + self.pointer = 0 + self.raw_buffer = None + self.raw_decode = None + self.encoding = None + self.index = 0 + self.line = 0 + self.column = 0 + if isinstance(stream, str): + self.name = "<unicode string>" + self.check_printable(stream) + self.buffer = stream+'\0' + elif isinstance(stream, bytes): + self.name = "<byte string>" + self.raw_buffer = stream + self.determine_encoding() + else: + self.stream = stream + self.name = getattr(stream, 'name', "<file>") + self.eof = False + self.raw_buffer = None + self.determine_encoding() + + def peek(self, index=0): + try: + return self.buffer[self.pointer+index] + except IndexError: + self.update(index+1) + return self.buffer[self.pointer+index] + + def prefix(self, length=1): + if self.pointer+length >= len(self.buffer): + self.update(length) + return self.buffer[self.pointer:self.pointer+length] + + def forward(self, length=1): + if self.pointer+length+1 >= len(self.buffer): + self.update(length+1) + while length: + ch = self.buffer[self.pointer] + self.pointer += 1 + self.index += 1 + if ch in '\n\x85\u2028\u2029' \ + or (ch == '\r' and self.buffer[self.pointer] != '\n'): + self.line += 1 + self.column = 0 + elif ch != '\uFEFF': + self.column += 1 + length -= 1 + + def get_mark(self): + if self.stream is None: + return Mark(self.name, self.index, self.line, self.column, + self.buffer, self.pointer) + else: + return Mark(self.name, self.index, self.line, self.column, + None, None) + + def determine_encoding(self): + while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2): + self.update_raw() + if isinstance(self.raw_buffer, bytes): + if self.raw_buffer.startswith(codecs.BOM_UTF16_LE): + self.raw_decode = codecs.utf_16_le_decode + self.encoding = 'utf-16-le' + elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE): + self.raw_decode = codecs.utf_16_be_decode + self.encoding = 'utf-16-be' + else: + self.raw_decode = codecs.utf_8_decode + self.encoding = 'utf-8' + self.update(1) + + NON_PRINTABLE = re.compile('[^\x09\x0A\x0D\x20-\x7E\x85\xA0-\uD7FF\uE000-\uFFFD\U00010000-\U0010ffff]') + def check_printable(self, data): + match = self.NON_PRINTABLE.search(data) + if match: + character = match.group() + position = self.index+(len(self.buffer)-self.pointer)+match.start() + raise ReaderError(self.name, position, ord(character), + 'unicode', "special characters are not allowed") + + def update(self, length): + if self.raw_buffer is None: + return + self.buffer = self.buffer[self.pointer:] + self.pointer = 0 + while len(self.buffer) < length: + if not self.eof: + self.update_raw() + if self.raw_decode is not None: + try: + data, converted = self.raw_decode(self.raw_buffer, + 'strict', self.eof) + except UnicodeDecodeError as exc: + character = self.raw_buffer[exc.start] + if self.stream is not None: + position = self.stream_pointer-len(self.raw_buffer)+exc.start + else: + position = exc.start + raise ReaderError(self.name, position, character, + exc.encoding, exc.reason) + else: + data = self.raw_buffer + converted = len(data) + self.check_printable(data) + self.buffer += data + self.raw_buffer = self.raw_buffer[converted:] + if self.eof: + self.buffer += '\0' + self.raw_buffer = None + break + + def update_raw(self, size=4096): + data = self.stream.read(size) + if self.raw_buffer is None: + self.raw_buffer = data + else: + self.raw_buffer += data + self.stream_pointer += len(data) + if not data: + self.eof = True diff --git a/pipenv/patched/yaml3/representer.py b/pipenv/patched/yaml3/representer.py new file mode 100644 index 00000000..3b0b192e --- /dev/null +++ b/pipenv/patched/yaml3/representer.py @@ -0,0 +1,389 @@ + +__all__ = ['BaseRepresenter', 'SafeRepresenter', 'Representer', + 'RepresenterError'] + +from .error import * +from .nodes import * + +import datetime, copyreg, types, base64, collections + +class RepresenterError(YAMLError): + pass + +class BaseRepresenter: + + yaml_representers = {} + yaml_multi_representers = {} + + def __init__(self, default_style=None, default_flow_style=False, sort_keys=True): + self.default_style = default_style + self.sort_keys = sort_keys + self.default_flow_style = default_flow_style + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent(self, data): + node = self.represent_data(data) + self.serialize(node) + self.represented_objects = {} + self.object_keeper = [] + self.alias_key = None + + def represent_data(self, data): + if self.ignore_aliases(data): + self.alias_key = None + else: + self.alias_key = id(data) + if self.alias_key is not None: + if self.alias_key in self.represented_objects: + node = self.represented_objects[self.alias_key] + #if node is None: + # raise RepresenterError("recursive objects are not allowed: %r" % data) + return node + #self.represented_objects[alias_key] = None + self.object_keeper.append(data) + data_types = type(data).__mro__ + if data_types[0] in self.yaml_representers: + node = self.yaml_representers[data_types[0]](self, data) + else: + for data_type in data_types: + if data_type in self.yaml_multi_representers: + node = self.yaml_multi_representers[data_type](self, data) + break + else: + if None in self.yaml_multi_representers: + node = self.yaml_multi_representers[None](self, data) + elif None in self.yaml_representers: + node = self.yaml_representers[None](self, data) + else: + node = ScalarNode(None, str(data)) + #if alias_key is not None: + # self.represented_objects[alias_key] = node + return node + + @classmethod + def add_representer(cls, data_type, representer): + if not 'yaml_representers' in cls.__dict__: + cls.yaml_representers = cls.yaml_representers.copy() + cls.yaml_representers[data_type] = representer + + @classmethod + def add_multi_representer(cls, data_type, representer): + if not 'yaml_multi_representers' in cls.__dict__: + cls.yaml_multi_representers = cls.yaml_multi_representers.copy() + cls.yaml_multi_representers[data_type] = representer + + def represent_scalar(self, tag, value, style=None): + if style is None: + style = self.default_style + node = ScalarNode(tag, value, style=style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + return node + + def represent_sequence(self, tag, sequence, flow_style=None): + value = [] + node = SequenceNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + for item in sequence: + node_item = self.represent_data(item) + if not (isinstance(node_item, ScalarNode) and not node_item.style): + best_style = False + value.append(node_item) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def represent_mapping(self, tag, mapping, flow_style=None): + value = [] + node = MappingNode(tag, value, flow_style=flow_style) + if self.alias_key is not None: + self.represented_objects[self.alias_key] = node + best_style = True + if hasattr(mapping, 'items'): + mapping = list(mapping.items()) + if self.sort_keys: + try: + mapping = sorted(mapping) + except TypeError: + pass + for item_key, item_value in mapping: + node_key = self.represent_data(item_key) + node_value = self.represent_data(item_value) + if not (isinstance(node_key, ScalarNode) and not node_key.style): + best_style = False + if not (isinstance(node_value, ScalarNode) and not node_value.style): + best_style = False + value.append((node_key, node_value)) + if flow_style is None: + if self.default_flow_style is not None: + node.flow_style = self.default_flow_style + else: + node.flow_style = best_style + return node + + def ignore_aliases(self, data): + return False + +class SafeRepresenter(BaseRepresenter): + + def ignore_aliases(self, data): + if data is None: + return True + if isinstance(data, tuple) and data == (): + return True + if isinstance(data, (str, bytes, bool, int, float)): + return True + + def represent_none(self, data): + return self.represent_scalar('tag:yaml.org,2002:null', 'null') + + def represent_str(self, data): + return self.represent_scalar('tag:yaml.org,2002:str', data) + + def represent_binary(self, data): + if hasattr(base64, 'encodebytes'): + data = base64.encodebytes(data).decode('ascii') + else: + data = base64.encodestring(data).decode('ascii') + return self.represent_scalar('tag:yaml.org,2002:binary', data, style='|') + + def represent_bool(self, data): + if data: + value = 'true' + else: + value = 'false' + return self.represent_scalar('tag:yaml.org,2002:bool', value) + + def represent_int(self, data): + return self.represent_scalar('tag:yaml.org,2002:int', str(data)) + + inf_value = 1e300 + while repr(inf_value) != repr(inf_value*inf_value): + inf_value *= inf_value + + def represent_float(self, data): + if data != data or (data == 0.0 and data == 1.0): + value = '.nan' + elif data == self.inf_value: + value = '.inf' + elif data == -self.inf_value: + value = '-.inf' + else: + value = repr(data).lower() + # Note that in some cases `repr(data)` represents a float number + # without the decimal parts. For instance: + # >>> repr(1e17) + # '1e17' + # Unfortunately, this is not a valid float representation according + # to the definition of the `!!float` tag. We fix this by adding + # '.0' before the 'e' symbol. + if '.' not in value and 'e' in value: + value = value.replace('e', '.0e', 1) + return self.represent_scalar('tag:yaml.org,2002:float', value) + + def represent_list(self, data): + #pairs = (len(data) > 0 and isinstance(data, list)) + #if pairs: + # for item in data: + # if not isinstance(item, tuple) or len(item) != 2: + # pairs = False + # break + #if not pairs: + return self.represent_sequence('tag:yaml.org,2002:seq', data) + #value = [] + #for item_key, item_value in data: + # value.append(self.represent_mapping(u'tag:yaml.org,2002:map', + # [(item_key, item_value)])) + #return SequenceNode(u'tag:yaml.org,2002:pairs', value) + + def represent_dict(self, data): + return self.represent_mapping('tag:yaml.org,2002:map', data) + + def represent_set(self, data): + value = {} + for key in data: + value[key] = None + return self.represent_mapping('tag:yaml.org,2002:set', value) + + def represent_date(self, data): + value = data.isoformat() + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_datetime(self, data): + value = data.isoformat(' ') + return self.represent_scalar('tag:yaml.org,2002:timestamp', value) + + def represent_yaml_object(self, tag, data, cls, flow_style=None): + if hasattr(data, '__getstate__'): + state = data.__getstate__() + else: + state = data.__dict__.copy() + return self.represent_mapping(tag, state, flow_style=flow_style) + + def represent_undefined(self, data): + raise RepresenterError("cannot represent an object", data) + +SafeRepresenter.add_representer(type(None), + SafeRepresenter.represent_none) + +SafeRepresenter.add_representer(str, + SafeRepresenter.represent_str) + +SafeRepresenter.add_representer(bytes, + SafeRepresenter.represent_binary) + +SafeRepresenter.add_representer(bool, + SafeRepresenter.represent_bool) + +SafeRepresenter.add_representer(int, + SafeRepresenter.represent_int) + +SafeRepresenter.add_representer(float, + SafeRepresenter.represent_float) + +SafeRepresenter.add_representer(list, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(tuple, + SafeRepresenter.represent_list) + +SafeRepresenter.add_representer(dict, + SafeRepresenter.represent_dict) + +SafeRepresenter.add_representer(set, + SafeRepresenter.represent_set) + +SafeRepresenter.add_representer(datetime.date, + SafeRepresenter.represent_date) + +SafeRepresenter.add_representer(datetime.datetime, + SafeRepresenter.represent_datetime) + +SafeRepresenter.add_representer(None, + SafeRepresenter.represent_undefined) + +class Representer(SafeRepresenter): + + def represent_complex(self, data): + if data.imag == 0.0: + data = '%r' % data.real + elif data.real == 0.0: + data = '%rj' % data.imag + elif data.imag > 0: + data = '%r+%rj' % (data.real, data.imag) + else: + data = '%r%rj' % (data.real, data.imag) + return self.represent_scalar('tag:yaml.org,2002:python/complex', data) + + def represent_tuple(self, data): + return self.represent_sequence('tag:yaml.org,2002:python/tuple', data) + + def represent_name(self, data): + name = '%s.%s' % (data.__module__, data.__name__) + return self.represent_scalar('tag:yaml.org,2002:python/name:'+name, '') + + def represent_module(self, data): + return self.represent_scalar( + 'tag:yaml.org,2002:python/module:'+data.__name__, '') + + def represent_object(self, data): + # We use __reduce__ API to save the data. data.__reduce__ returns + # a tuple of length 2-5: + # (function, args, state, listitems, dictitems) + + # For reconstructing, we calls function(*args), then set its state, + # listitems, and dictitems if they are not None. + + # A special case is when function.__name__ == '__newobj__'. In this + # case we create the object with args[0].__new__(*args). + + # Another special case is when __reduce__ returns a string - we don't + # support it. + + # We produce a !!python/object, !!python/object/new or + # !!python/object/apply node. + + cls = type(data) + if cls in copyreg.dispatch_table: + reduce = copyreg.dispatch_table[cls](data) + elif hasattr(data, '__reduce_ex__'): + reduce = data.__reduce_ex__(2) + elif hasattr(data, '__reduce__'): + reduce = data.__reduce__() + else: + raise RepresenterError("cannot represent an object", data) + reduce = (list(reduce)+[None]*5)[:5] + function, args, state, listitems, dictitems = reduce + args = list(args) + if state is None: + state = {} + if listitems is not None: + listitems = list(listitems) + if dictitems is not None: + dictitems = dict(dictitems) + if function.__name__ == '__newobj__': + function = args[0] + args = args[1:] + tag = 'tag:yaml.org,2002:python/object/new:' + newobj = True + else: + tag = 'tag:yaml.org,2002:python/object/apply:' + newobj = False + function_name = '%s.%s' % (function.__module__, function.__name__) + if not args and not listitems and not dictitems \ + and isinstance(state, dict) and newobj: + return self.represent_mapping( + 'tag:yaml.org,2002:python/object:'+function_name, state) + if not listitems and not dictitems \ + and isinstance(state, dict) and not state: + return self.represent_sequence(tag+function_name, args) + value = {} + if args: + value['args'] = args + if state or not isinstance(state, dict): + value['state'] = state + if listitems: + value['listitems'] = listitems + if dictitems: + value['dictitems'] = dictitems + return self.represent_mapping(tag+function_name, value) + + def represent_ordered_dict(self, data): + # Provide uniform representation across different Python versions. + data_type = type(data) + tag = 'tag:yaml.org,2002:python/object/apply:%s.%s' \ + % (data_type.__module__, data_type.__name__) + items = [[key, value] for key, value in data.items()] + return self.represent_sequence(tag, [items]) + +Representer.add_representer(complex, + Representer.represent_complex) + +Representer.add_representer(tuple, + Representer.represent_tuple) + +Representer.add_representer(type, + Representer.represent_name) + +Representer.add_representer(collections.OrderedDict, + Representer.represent_ordered_dict) + +Representer.add_representer(types.FunctionType, + Representer.represent_name) + +Representer.add_representer(types.BuiltinFunctionType, + Representer.represent_name) + +Representer.add_representer(types.ModuleType, + Representer.represent_module) + +Representer.add_multi_representer(object, + Representer.represent_object) + diff --git a/pipenv/patched/yaml3/resolver.py b/pipenv/patched/yaml3/resolver.py new file mode 100644 index 00000000..02b82e73 --- /dev/null +++ b/pipenv/patched/yaml3/resolver.py @@ -0,0 +1,227 @@ + +__all__ = ['BaseResolver', 'Resolver'] + +from .error import * +from .nodes import * + +import re + +class ResolverError(YAMLError): + pass + +class BaseResolver: + + DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str' + DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq' + DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map' + + yaml_implicit_resolvers = {} + yaml_path_resolvers = {} + + def __init__(self): + self.resolver_exact_paths = [] + self.resolver_prefix_paths = [] + + @classmethod + def add_implicit_resolver(cls, tag, regexp, first): + if not 'yaml_implicit_resolvers' in cls.__dict__: + implicit_resolvers = {} + for key in cls.yaml_implicit_resolvers: + implicit_resolvers[key] = cls.yaml_implicit_resolvers[key][:] + cls.yaml_implicit_resolvers = implicit_resolvers + if first is None: + first = [None] + for ch in first: + cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp)) + + @classmethod + def add_path_resolver(cls, tag, path, kind=None): + # Note: `add_path_resolver` is experimental. The API could be changed. + # `new_path` is a pattern that is matched against the path from the + # root to the node that is being considered. `node_path` elements are + # tuples `(node_check, index_check)`. `node_check` is a node class: + # `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None` + # matches any kind of a node. `index_check` could be `None`, a boolean + # value, a string value, or a number. `None` and `False` match against + # any _value_ of sequence and mapping nodes. `True` matches against + # any _key_ of a mapping node. A string `index_check` matches against + # a mapping value that corresponds to a scalar key which content is + # equal to the `index_check` value. An integer `index_check` matches + # against a sequence value with the index equal to `index_check`. + if not 'yaml_path_resolvers' in cls.__dict__: + cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy() + new_path = [] + for element in path: + if isinstance(element, (list, tuple)): + if len(element) == 2: + node_check, index_check = element + elif len(element) == 1: + node_check = element[0] + index_check = True + else: + raise ResolverError("Invalid path element: %s" % element) + else: + node_check = None + index_check = element + if node_check is str: + node_check = ScalarNode + elif node_check is list: + node_check = SequenceNode + elif node_check is dict: + node_check = MappingNode + elif node_check not in [ScalarNode, SequenceNode, MappingNode] \ + and not isinstance(node_check, str) \ + and node_check is not None: + raise ResolverError("Invalid node checker: %s" % node_check) + if not isinstance(index_check, (str, int)) \ + and index_check is not None: + raise ResolverError("Invalid index checker: %s" % index_check) + new_path.append((node_check, index_check)) + if kind is str: + kind = ScalarNode + elif kind is list: + kind = SequenceNode + elif kind is dict: + kind = MappingNode + elif kind not in [ScalarNode, SequenceNode, MappingNode] \ + and kind is not None: + raise ResolverError("Invalid node kind: %s" % kind) + cls.yaml_path_resolvers[tuple(new_path), kind] = tag + + def descend_resolver(self, current_node, current_index): + if not self.yaml_path_resolvers: + return + exact_paths = {} + prefix_paths = [] + if current_node: + depth = len(self.resolver_prefix_paths) + for path, kind in self.resolver_prefix_paths[-1]: + if self.check_resolver_prefix(depth, path, kind, + current_node, current_index): + if len(path) > depth: + prefix_paths.append((path, kind)) + else: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + for path, kind in self.yaml_path_resolvers: + if not path: + exact_paths[kind] = self.yaml_path_resolvers[path, kind] + else: + prefix_paths.append((path, kind)) + self.resolver_exact_paths.append(exact_paths) + self.resolver_prefix_paths.append(prefix_paths) + + def ascend_resolver(self): + if not self.yaml_path_resolvers: + return + self.resolver_exact_paths.pop() + self.resolver_prefix_paths.pop() + + def check_resolver_prefix(self, depth, path, kind, + current_node, current_index): + node_check, index_check = path[depth-1] + if isinstance(node_check, str): + if current_node.tag != node_check: + return + elif node_check is not None: + if not isinstance(current_node, node_check): + return + if index_check is True and current_index is not None: + return + if (index_check is False or index_check is None) \ + and current_index is None: + return + if isinstance(index_check, str): + if not (isinstance(current_index, ScalarNode) + and index_check == current_index.value): + return + elif isinstance(index_check, int) and not isinstance(index_check, bool): + if index_check != current_index: + return + return True + + def resolve(self, kind, value, implicit): + if kind is ScalarNode and implicit[0]: + if value == '': + resolvers = self.yaml_implicit_resolvers.get('', []) + else: + resolvers = self.yaml_implicit_resolvers.get(value[0], []) + resolvers += self.yaml_implicit_resolvers.get(None, []) + for tag, regexp in resolvers: + if regexp.match(value): + return tag + implicit = implicit[1] + if self.yaml_path_resolvers: + exact_paths = self.resolver_exact_paths[-1] + if kind in exact_paths: + return exact_paths[kind] + if None in exact_paths: + return exact_paths[None] + if kind is ScalarNode: + return self.DEFAULT_SCALAR_TAG + elif kind is SequenceNode: + return self.DEFAULT_SEQUENCE_TAG + elif kind is MappingNode: + return self.DEFAULT_MAPPING_TAG + +class Resolver(BaseResolver): + pass + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:bool', + re.compile(r'''^(?:yes|Yes|YES|no|No|NO + |true|True|TRUE|false|False|FALSE + |on|On|ON|off|Off|OFF)$''', re.X), + list('yYnNtTfFoO')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:float', + re.compile(r'''^(?:[-+]?(?:[0-9][0-9_]*)\.[0-9_]*(?:[eE][-+][0-9]+)? + |\.[0-9_]+(?:[eE][-+][0-9]+)? + |[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\.[0-9_]* + |[-+]?\.(?:inf|Inf|INF) + |\.(?:nan|NaN|NAN))$''', re.X), + list('-+0123456789.')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:int', + re.compile(r'''^(?:[-+]?0b[0-1_]+ + |[-+]?0[0-7_]+ + |[-+]?(?:0|[1-9][0-9_]*) + |[-+]?0x[0-9a-fA-F_]+ + |[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), + list('-+0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:merge', + re.compile(r'^(?:<<)$'), + ['<']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:null', + re.compile(r'''^(?: ~ + |null|Null|NULL + | )$''', re.X), + ['~', 'n', 'N', '']) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:timestamp', + re.compile(r'''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9] + |[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]? + (?:[Tt]|[ \t]+)[0-9][0-9]? + :[0-9][0-9] :[0-9][0-9] (?:\.[0-9]*)? + (?:[ \t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X), + list('0123456789')) + +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:value', + re.compile(r'^(?:=)$'), + ['=']) + +# The following resolver is only for documentation purposes. It cannot work +# because plain scalars cannot start with '!', '&', or '*'. +Resolver.add_implicit_resolver( + 'tag:yaml.org,2002:yaml', + re.compile(r'^(?:!|&|\*)$'), + list('!&*')) + diff --git a/pipenv/patched/yaml3/scanner.py b/pipenv/patched/yaml3/scanner.py new file mode 100644 index 00000000..7437ede1 --- /dev/null +++ b/pipenv/patched/yaml3/scanner.py @@ -0,0 +1,1435 @@ + +# Scanner produces tokens of the following types: +# STREAM-START +# STREAM-END +# DIRECTIVE(name, value) +# DOCUMENT-START +# DOCUMENT-END +# BLOCK-SEQUENCE-START +# BLOCK-MAPPING-START +# BLOCK-END +# FLOW-SEQUENCE-START +# FLOW-MAPPING-START +# FLOW-SEQUENCE-END +# FLOW-MAPPING-END +# BLOCK-ENTRY +# FLOW-ENTRY +# KEY +# VALUE +# ALIAS(value) +# ANCHOR(value) +# TAG(value) +# SCALAR(value, plain, style) +# +# Read comments in the Scanner code for more details. +# + +__all__ = ['Scanner', 'ScannerError'] + +from .error import MarkedYAMLError +from .tokens import * + +class ScannerError(MarkedYAMLError): + pass + +class SimpleKey: + # See below simple keys treatment. + + def __init__(self, token_number, required, index, line, column, mark): + self.token_number = token_number + self.required = required + self.index = index + self.line = line + self.column = column + self.mark = mark + +class Scanner: + + def __init__(self): + """Initialize the scanner.""" + # It is assumed that Scanner and Reader will have a common descendant. + # Reader do the dirty work of checking for BOM and converting the + # input data to Unicode. It also adds NUL to the end. + # + # Reader supports the following methods + # self.peek(i=0) # peek the next i-th character + # self.prefix(l=1) # peek the next l characters + # self.forward(l=1) # read the next l characters and move the pointer. + + # Had we reached the end of the stream? + self.done = False + + # The number of unclosed '{' and '['. `flow_level == 0` means block + # context. + self.flow_level = 0 + + # List of processed tokens that are not yet emitted. + self.tokens = [] + + # Add the STREAM-START token. + self.fetch_stream_start() + + # Number of tokens that were emitted through the `get_token` method. + self.tokens_taken = 0 + + # The current indentation level. + self.indent = -1 + + # Past indentation levels. + self.indents = [] + + # Variables related to simple keys treatment. + + # A simple key is a key that is not denoted by the '?' indicator. + # Example of simple keys: + # --- + # block simple key: value + # ? not a simple key: + # : { flow simple key: value } + # We emit the KEY token before all keys, so when we find a potential + # simple key, we try to locate the corresponding ':' indicator. + # Simple keys should be limited to a single line and 1024 characters. + + # Can a simple key start at the current position? A simple key may + # start: + # - at the beginning of the line, not counting indentation spaces + # (in block context), + # - after '{', '[', ',' (in the flow context), + # - after '?', ':', '-' (in the block context). + # In the block context, this flag also signifies if a block collection + # may start at the current position. + self.allow_simple_key = True + + # Keep track of possible simple keys. This is a dictionary. The key + # is `flow_level`; there can be no more that one possible simple key + # for each level. The value is a SimpleKey record: + # (token_number, required, index, line, column, mark) + # A simple key may start with ALIAS, ANCHOR, TAG, SCALAR(flow), + # '[', or '{' tokens. + self.possible_simple_keys = {} + + # Public methods. + + def check_token(self, *choices): + # Check if the next token is one of the given types. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + if not choices: + return True + for choice in choices: + if isinstance(self.tokens[0], choice): + return True + return False + + def peek_token(self): + # Return the next token, but do not delete if from the queue. + # Return None if no more tokens. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + return self.tokens[0] + else: + return None + + def get_token(self): + # Return the next token. + while self.need_more_tokens(): + self.fetch_more_tokens() + if self.tokens: + self.tokens_taken += 1 + return self.tokens.pop(0) + + # Private methods. + + def need_more_tokens(self): + if self.done: + return False + if not self.tokens: + return True + # The current token may be a potential simple key, so we + # need to look further. + self.stale_possible_simple_keys() + if self.next_possible_simple_key() == self.tokens_taken: + return True + + def fetch_more_tokens(self): + + # Eat whitespaces and comments until we reach the next token. + self.scan_to_next_token() + + # Remove obsolete possible simple keys. + self.stale_possible_simple_keys() + + # Compare the current indentation and column. It may add some tokens + # and decrease the current indentation level. + self.unwind_indent(self.column) + + # Peek the next character. + ch = self.peek() + + # Is it the end of stream? + if ch == '\0': + return self.fetch_stream_end() + + # Is it a directive? + if ch == '%' and self.check_directive(): + return self.fetch_directive() + + # Is it the document start? + if ch == '-' and self.check_document_start(): + return self.fetch_document_start() + + # Is it the document end? + if ch == '.' and self.check_document_end(): + return self.fetch_document_end() + + # TODO: support for BOM within a stream. + #if ch == '\uFEFF': + # return self.fetch_bom() <-- issue BOMToken + + # Note: the order of the following checks is NOT significant. + + # Is it the flow sequence start indicator? + if ch == '[': + return self.fetch_flow_sequence_start() + + # Is it the flow mapping start indicator? + if ch == '{': + return self.fetch_flow_mapping_start() + + # Is it the flow sequence end indicator? + if ch == ']': + return self.fetch_flow_sequence_end() + + # Is it the flow mapping end indicator? + if ch == '}': + return self.fetch_flow_mapping_end() + + # Is it the flow entry indicator? + if ch == ',': + return self.fetch_flow_entry() + + # Is it the block entry indicator? + if ch == '-' and self.check_block_entry(): + return self.fetch_block_entry() + + # Is it the key indicator? + if ch == '?' and self.check_key(): + return self.fetch_key() + + # Is it the value indicator? + if ch == ':' and self.check_value(): + return self.fetch_value() + + # Is it an alias? + if ch == '*': + return self.fetch_alias() + + # Is it an anchor? + if ch == '&': + return self.fetch_anchor() + + # Is it a tag? + if ch == '!': + return self.fetch_tag() + + # Is it a literal scalar? + if ch == '|' and not self.flow_level: + return self.fetch_literal() + + # Is it a folded scalar? + if ch == '>' and not self.flow_level: + return self.fetch_folded() + + # Is it a single quoted scalar? + if ch == '\'': + return self.fetch_single() + + # Is it a double quoted scalar? + if ch == '\"': + return self.fetch_double() + + # It must be a plain scalar then. + if self.check_plain(): + return self.fetch_plain() + + # No? It's an error. Let's produce a nice error message. + raise ScannerError("while scanning for the next token", None, + "found character %r that cannot start any token" % ch, + self.get_mark()) + + # Simple keys treatment. + + def next_possible_simple_key(self): + # Return the number of the nearest possible simple key. Actually we + # don't need to loop through the whole dictionary. We may replace it + # with the following code: + # if not self.possible_simple_keys: + # return None + # return self.possible_simple_keys[ + # min(self.possible_simple_keys.keys())].token_number + min_token_number = None + for level in self.possible_simple_keys: + key = self.possible_simple_keys[level] + if min_token_number is None or key.token_number < min_token_number: + min_token_number = key.token_number + return min_token_number + + def stale_possible_simple_keys(self): + # Remove entries that are no longer possible simple keys. According to + # the YAML specification, simple keys + # - should be limited to a single line, + # - should be no longer than 1024 characters. + # Disabling this procedure will allow simple keys of any length and + # height (may cause problems if indentation is broken though). + for level in list(self.possible_simple_keys): + key = self.possible_simple_keys[level] + if key.line != self.line \ + or self.index-key.index > 1024: + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + del self.possible_simple_keys[level] + + def save_possible_simple_key(self): + # The next token may start a simple key. We check if it's possible + # and save its position. This function is called for + # ALIAS, ANCHOR, TAG, SCALAR(flow), '[', and '{'. + + # Check if a simple key is required at the current position. + required = not self.flow_level and self.indent == self.column + + # The next token might be a simple key. Let's save it's number and + # position. + if self.allow_simple_key: + self.remove_possible_simple_key() + token_number = self.tokens_taken+len(self.tokens) + key = SimpleKey(token_number, required, + self.index, self.line, self.column, self.get_mark()) + self.possible_simple_keys[self.flow_level] = key + + def remove_possible_simple_key(self): + # Remove the saved possible key position at the current flow level. + if self.flow_level in self.possible_simple_keys: + key = self.possible_simple_keys[self.flow_level] + + if key.required: + raise ScannerError("while scanning a simple key", key.mark, + "could not find expected ':'", self.get_mark()) + + del self.possible_simple_keys[self.flow_level] + + # Indentation functions. + + def unwind_indent(self, column): + + ## In flow context, tokens should respect indentation. + ## Actually the condition should be `self.indent >= column` according to + ## the spec. But this condition will prohibit intuitively correct + ## constructions such as + ## key : { + ## } + #if self.flow_level and self.indent > column: + # raise ScannerError(None, None, + # "invalid indentation or unclosed '[' or '{'", + # self.get_mark()) + + # In the flow context, indentation is ignored. We make the scanner less + # restrictive then specification requires. + if self.flow_level: + return + + # In block context, we may need to issue the BLOCK-END tokens. + while self.indent > column: + mark = self.get_mark() + self.indent = self.indents.pop() + self.tokens.append(BlockEndToken(mark, mark)) + + def add_indent(self, column): + # Check if we need to increase indentation. + if self.indent < column: + self.indents.append(self.indent) + self.indent = column + return True + return False + + # Fetchers. + + def fetch_stream_start(self): + # We always add STREAM-START as the first token and STREAM-END as the + # last token. + + # Read the token. + mark = self.get_mark() + + # Add STREAM-START. + self.tokens.append(StreamStartToken(mark, mark, + encoding=self.encoding)) + + + def fetch_stream_end(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + self.possible_simple_keys = {} + + # Read the token. + mark = self.get_mark() + + # Add STREAM-END. + self.tokens.append(StreamEndToken(mark, mark)) + + # The steam is finished. + self.done = True + + def fetch_directive(self): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Scan and add DIRECTIVE. + self.tokens.append(self.scan_directive()) + + def fetch_document_start(self): + self.fetch_document_indicator(DocumentStartToken) + + def fetch_document_end(self): + self.fetch_document_indicator(DocumentEndToken) + + def fetch_document_indicator(self, TokenClass): + + # Set the current indentation to -1. + self.unwind_indent(-1) + + # Reset simple keys. Note that there could not be a block collection + # after '---'. + self.remove_possible_simple_key() + self.allow_simple_key = False + + # Add DOCUMENT-START or DOCUMENT-END. + start_mark = self.get_mark() + self.forward(3) + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_start(self): + self.fetch_flow_collection_start(FlowSequenceStartToken) + + def fetch_flow_mapping_start(self): + self.fetch_flow_collection_start(FlowMappingStartToken) + + def fetch_flow_collection_start(self, TokenClass): + + # '[' and '{' may start a simple key. + self.save_possible_simple_key() + + # Increase the flow level. + self.flow_level += 1 + + # Simple keys are allowed after '[' and '{'. + self.allow_simple_key = True + + # Add FLOW-SEQUENCE-START or FLOW-MAPPING-START. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_sequence_end(self): + self.fetch_flow_collection_end(FlowSequenceEndToken) + + def fetch_flow_mapping_end(self): + self.fetch_flow_collection_end(FlowMappingEndToken) + + def fetch_flow_collection_end(self, TokenClass): + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Decrease the flow level. + self.flow_level -= 1 + + # No simple keys after ']' or '}'. + self.allow_simple_key = False + + # Add FLOW-SEQUENCE-END or FLOW-MAPPING-END. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(TokenClass(start_mark, end_mark)) + + def fetch_flow_entry(self): + + # Simple keys are allowed after ','. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add FLOW-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(FlowEntryToken(start_mark, end_mark)) + + def fetch_block_entry(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a new entry? + if not self.allow_simple_key: + raise ScannerError(None, None, + "sequence entries are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-SEQUENCE-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockSequenceStartToken(mark, mark)) + + # It's an error for the block entry to occur in the flow context, + # but we let the parser detect this. + else: + pass + + # Simple keys are allowed after '-'. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add BLOCK-ENTRY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(BlockEntryToken(start_mark, end_mark)) + + def fetch_key(self): + + # Block context needs additional checks. + if not self.flow_level: + + # Are we allowed to start a key (not necessary a simple)? + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping keys are not allowed here", + self.get_mark()) + + # We may need to add BLOCK-MAPPING-START. + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after '?' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add KEY. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(KeyToken(start_mark, end_mark)) + + def fetch_value(self): + + # Do we determine a simple key? + if self.flow_level in self.possible_simple_keys: + + # Add KEY. + key = self.possible_simple_keys[self.flow_level] + del self.possible_simple_keys[self.flow_level] + self.tokens.insert(key.token_number-self.tokens_taken, + KeyToken(key.mark, key.mark)) + + # If this key starts a new block mapping, we need to add + # BLOCK-MAPPING-START. + if not self.flow_level: + if self.add_indent(key.column): + self.tokens.insert(key.token_number-self.tokens_taken, + BlockMappingStartToken(key.mark, key.mark)) + + # There cannot be two simple keys one after another. + self.allow_simple_key = False + + # It must be a part of a complex key. + else: + + # Block context needs additional checks. + # (Do we really need them? They will be caught by the parser + # anyway.) + if not self.flow_level: + + # We are allowed to start a complex value if and only if + # we can start a simple key. + if not self.allow_simple_key: + raise ScannerError(None, None, + "mapping values are not allowed here", + self.get_mark()) + + # If this value starts a new block mapping, we need to add + # BLOCK-MAPPING-START. It will be detected as an error later by + # the parser. + if not self.flow_level: + if self.add_indent(self.column): + mark = self.get_mark() + self.tokens.append(BlockMappingStartToken(mark, mark)) + + # Simple keys are allowed after ':' in the block context. + self.allow_simple_key = not self.flow_level + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Add VALUE. + start_mark = self.get_mark() + self.forward() + end_mark = self.get_mark() + self.tokens.append(ValueToken(start_mark, end_mark)) + + def fetch_alias(self): + + # ALIAS could be a simple key. + self.save_possible_simple_key() + + # No simple keys after ALIAS. + self.allow_simple_key = False + + # Scan and add ALIAS. + self.tokens.append(self.scan_anchor(AliasToken)) + + def fetch_anchor(self): + + # ANCHOR could start a simple key. + self.save_possible_simple_key() + + # No simple keys after ANCHOR. + self.allow_simple_key = False + + # Scan and add ANCHOR. + self.tokens.append(self.scan_anchor(AnchorToken)) + + def fetch_tag(self): + + # TAG could start a simple key. + self.save_possible_simple_key() + + # No simple keys after TAG. + self.allow_simple_key = False + + # Scan and add TAG. + self.tokens.append(self.scan_tag()) + + def fetch_literal(self): + self.fetch_block_scalar(style='|') + + def fetch_folded(self): + self.fetch_block_scalar(style='>') + + def fetch_block_scalar(self, style): + + # A simple key may follow a block scalar. + self.allow_simple_key = True + + # Reset possible simple key on the current level. + self.remove_possible_simple_key() + + # Scan and add SCALAR. + self.tokens.append(self.scan_block_scalar(style)) + + def fetch_single(self): + self.fetch_flow_scalar(style='\'') + + def fetch_double(self): + self.fetch_flow_scalar(style='"') + + def fetch_flow_scalar(self, style): + + # A flow scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after flow scalars. + self.allow_simple_key = False + + # Scan and add SCALAR. + self.tokens.append(self.scan_flow_scalar(style)) + + def fetch_plain(self): + + # A plain scalar could be a simple key. + self.save_possible_simple_key() + + # No simple keys after plain scalars. But note that `scan_plain` will + # change this flag if the scan is finished at the beginning of the + # line. + self.allow_simple_key = False + + # Scan and add SCALAR. May change `allow_simple_key`. + self.tokens.append(self.scan_plain()) + + # Checkers. + + def check_directive(self): + + # DIRECTIVE: ^ '%' ... + # The '%' indicator is already checked. + if self.column == 0: + return True + + def check_document_start(self): + + # DOCUMENT-START: ^ '---' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '---' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_document_end(self): + + # DOCUMENT-END: ^ '...' (' '|'\n') + if self.column == 0: + if self.prefix(3) == '...' \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return True + + def check_block_entry(self): + + # BLOCK-ENTRY: '-' (' '|'\n') + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_key(self): + + # KEY(flow context): '?' + if self.flow_level: + return True + + # KEY(block context): '?' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_value(self): + + # VALUE(flow context): ':' + if self.flow_level: + return True + + # VALUE(block context): ':' (' '|'\n') + else: + return self.peek(1) in '\0 \t\r\n\x85\u2028\u2029' + + def check_plain(self): + + # A plain scalar may start with any non-space character except: + # '-', '?', ':', ',', '[', ']', '{', '}', + # '#', '&', '*', '!', '|', '>', '\'', '\"', + # '%', '@', '`'. + # + # It may also start with + # '-', '?', ':' + # if it is followed by a non-space character. + # + # Note that we limit the last rule to the block context (except the + # '-' character) because we want the flow context to be space + # independent. + ch = self.peek() + return ch not in '\0 \t\r\n\x85\u2028\u2029-?:,[]{}#&*!|>\'\"%@`' \ + or (self.peek(1) not in '\0 \t\r\n\x85\u2028\u2029' + and (ch == '-' or (not self.flow_level and ch in '?:'))) + + # Scanners. + + def scan_to_next_token(self): + # We ignore spaces, line breaks and comments. + # If we find a line break in the block context, we set the flag + # `allow_simple_key` on. + # The byte order mark is stripped if it's the first character in the + # stream. We do not yet support BOM inside the stream as the + # specification requires. Any such mark will be considered as a part + # of the document. + # + # TODO: We need to make tab handling rules more sane. A good rule is + # Tabs cannot precede tokens + # BLOCK-SEQUENCE-START, BLOCK-MAPPING-START, BLOCK-END, + # KEY(block), VALUE(block), BLOCK-ENTRY + # So the checking code is + # if <TAB>: + # self.allow_simple_keys = False + # We also need to add the check for `allow_simple_keys == True` to + # `unwind_indent` before issuing BLOCK-END. + # Scanners for block, flow, and plain scalars need to be modified. + + if self.index == 0 and self.peek() == '\uFEFF': + self.forward() + found = False + while not found: + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + if self.scan_line_break(): + if not self.flow_level: + self.allow_simple_key = True + else: + found = True + + def scan_directive(self): + # See the specification for details. + start_mark = self.get_mark() + self.forward() + name = self.scan_directive_name(start_mark) + value = None + if name == 'YAML': + value = self.scan_yaml_directive_value(start_mark) + end_mark = self.get_mark() + elif name == 'TAG': + value = self.scan_tag_directive_value(start_mark) + end_mark = self.get_mark() + else: + end_mark = self.get_mark() + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + self.scan_directive_ignored_line(start_mark) + return DirectiveToken(name, value, start_mark, end_mark) + + def scan_directive_name(self, start_mark): + # See the specification for details. + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + return value + + def scan_yaml_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + major = self.scan_yaml_directive_number(start_mark) + if self.peek() != '.': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or '.', but found %r" % self.peek(), + self.get_mark()) + self.forward() + minor = self.scan_yaml_directive_number(start_mark) + if self.peek() not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a digit or ' ', but found %r" % self.peek(), + self.get_mark()) + return (major, minor) + + def scan_yaml_directive_number(self, start_mark): + # See the specification for details. + ch = self.peek() + if not ('0' <= ch <= '9'): + raise ScannerError("while scanning a directive", start_mark, + "expected a digit, but found %r" % ch, self.get_mark()) + length = 0 + while '0' <= self.peek(length) <= '9': + length += 1 + value = int(self.prefix(length)) + self.forward(length) + return value + + def scan_tag_directive_value(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + handle = self.scan_tag_directive_handle(start_mark) + while self.peek() == ' ': + self.forward() + prefix = self.scan_tag_directive_prefix(start_mark) + return (handle, prefix) + + def scan_tag_directive_handle(self, start_mark): + # See the specification for details. + value = self.scan_tag_handle('directive', start_mark) + ch = self.peek() + if ch != ' ': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_tag_directive_prefix(self, start_mark): + # See the specification for details. + value = self.scan_tag_uri('directive', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + return value + + def scan_directive_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a directive", start_mark, + "expected a comment or a line break, but found %r" + % ch, self.get_mark()) + self.scan_line_break() + + def scan_anchor(self, TokenClass): + # The specification does not restrict characters for anchors and + # aliases. This may lead to problems, for instance, the document: + # [ *alias, value ] + # can be interpreted in two ways, as + # [ "value" ] + # and + # [ *alias , "value" ] + # Therefore we restrict aliases to numbers and ASCII letters. + start_mark = self.get_mark() + indicator = self.peek() + if indicator == '*': + name = 'alias' + else: + name = 'anchor' + self.forward() + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if not length: + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + value = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch not in '\0 \t\r\n\x85\u2028\u2029?:,]}%@`': + raise ScannerError("while scanning an %s" % name, start_mark, + "expected alphabetic or numeric character, but found %r" + % ch, self.get_mark()) + end_mark = self.get_mark() + return TokenClass(value, start_mark, end_mark) + + def scan_tag(self): + # See the specification for details. + start_mark = self.get_mark() + ch = self.peek(1) + if ch == '<': + handle = None + self.forward(2) + suffix = self.scan_tag_uri('tag', start_mark) + if self.peek() != '>': + raise ScannerError("while parsing a tag", start_mark, + "expected '>', but found %r" % self.peek(), + self.get_mark()) + self.forward() + elif ch in '\0 \t\r\n\x85\u2028\u2029': + handle = None + suffix = '!' + self.forward() + else: + length = 1 + use_handle = False + while ch not in '\0 \r\n\x85\u2028\u2029': + if ch == '!': + use_handle = True + break + length += 1 + ch = self.peek(length) + handle = '!' + if use_handle: + handle = self.scan_tag_handle('tag', start_mark) + else: + handle = '!' + self.forward() + suffix = self.scan_tag_uri('tag', start_mark) + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a tag", start_mark, + "expected ' ', but found %r" % ch, self.get_mark()) + value = (handle, suffix) + end_mark = self.get_mark() + return TagToken(value, start_mark, end_mark) + + def scan_block_scalar(self, style): + # See the specification for details. + + if style == '>': + folded = True + else: + folded = False + + chunks = [] + start_mark = self.get_mark() + + # Scan the header. + self.forward() + chomping, increment = self.scan_block_scalar_indicators(start_mark) + self.scan_block_scalar_ignored_line(start_mark) + + # Determine the indentation level and go to the first non-empty line. + min_indent = self.indent+1 + if min_indent < 1: + min_indent = 1 + if increment is None: + breaks, max_indent, end_mark = self.scan_block_scalar_indentation() + indent = max(min_indent, max_indent) + else: + indent = min_indent+increment-1 + breaks, end_mark = self.scan_block_scalar_breaks(indent) + line_break = '' + + # Scan the inner part of the block scalar. + while self.column == indent and self.peek() != '\0': + chunks.extend(breaks) + leading_non_space = self.peek() not in ' \t' + length = 0 + while self.peek(length) not in '\0\r\n\x85\u2028\u2029': + length += 1 + chunks.append(self.prefix(length)) + self.forward(length) + line_break = self.scan_line_break() + breaks, end_mark = self.scan_block_scalar_breaks(indent) + if self.column == indent and self.peek() != '\0': + + # Unfortunately, folding rules are ambiguous. + # + # This is the folding according to the specification: + + if folded and line_break == '\n' \ + and leading_non_space and self.peek() not in ' \t': + if not breaks: + chunks.append(' ') + else: + chunks.append(line_break) + + # This is Clark Evans's interpretation (also in the spec + # examples): + # + #if folded and line_break == '\n': + # if not breaks: + # if self.peek() not in ' \t': + # chunks.append(' ') + # else: + # chunks.append(line_break) + #else: + # chunks.append(line_break) + else: + break + + # Chomp the tail. + if chomping is not False: + chunks.append(line_break) + if chomping is True: + chunks.extend(breaks) + + # We are done. + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + def scan_block_scalar_indicators(self, start_mark): + # See the specification for details. + chomping = None + increment = None + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + elif ch in '0123456789': + increment = int(ch) + if increment == 0: + raise ScannerError("while scanning a block scalar", start_mark, + "expected indentation indicator in the range 1-9, but found 0", + self.get_mark()) + self.forward() + ch = self.peek() + if ch in '+-': + if ch == '+': + chomping = True + else: + chomping = False + self.forward() + ch = self.peek() + if ch not in '\0 \r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected chomping or indentation indicators, but found %r" + % ch, self.get_mark()) + return chomping, increment + + def scan_block_scalar_ignored_line(self, start_mark): + # See the specification for details. + while self.peek() == ' ': + self.forward() + if self.peek() == '#': + while self.peek() not in '\0\r\n\x85\u2028\u2029': + self.forward() + ch = self.peek() + if ch not in '\0\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a block scalar", start_mark, + "expected a comment or a line break, but found %r" % ch, + self.get_mark()) + self.scan_line_break() + + def scan_block_scalar_indentation(self): + # See the specification for details. + chunks = [] + max_indent = 0 + end_mark = self.get_mark() + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() != ' ': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + else: + self.forward() + if self.column > max_indent: + max_indent = self.column + return chunks, max_indent, end_mark + + def scan_block_scalar_breaks(self, indent): + # See the specification for details. + chunks = [] + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + while self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + end_mark = self.get_mark() + while self.column < indent and self.peek() == ' ': + self.forward() + return chunks, end_mark + + def scan_flow_scalar(self, style): + # See the specification for details. + # Note that we loose indentation rules for quoted scalars. Quoted + # scalars don't need to adhere indentation because " and ' clearly + # mark the beginning and the end of them. Therefore we are less + # restrictive then the specification requires. We only need to check + # that document separators are not included in scalars. + if style == '"': + double = True + else: + double = False + chunks = [] + start_mark = self.get_mark() + quote = self.peek() + self.forward() + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + while self.peek() != quote: + chunks.extend(self.scan_flow_scalar_spaces(double, start_mark)) + chunks.extend(self.scan_flow_scalar_non_spaces(double, start_mark)) + self.forward() + end_mark = self.get_mark() + return ScalarToken(''.join(chunks), False, start_mark, end_mark, + style) + + ESCAPE_REPLACEMENTS = { + '0': '\0', + 'a': '\x07', + 'b': '\x08', + 't': '\x09', + '\t': '\x09', + 'n': '\x0A', + 'v': '\x0B', + 'f': '\x0C', + 'r': '\x0D', + 'e': '\x1B', + ' ': '\x20', + '\"': '\"', + '\\': '\\', + '/': '/', + 'N': '\x85', + '_': '\xA0', + 'L': '\u2028', + 'P': '\u2029', + } + + ESCAPE_CODES = { + 'x': 2, + 'u': 4, + 'U': 8, + } + + def scan_flow_scalar_non_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + length = 0 + while self.peek(length) not in '\'\"\\\0 \t\r\n\x85\u2028\u2029': + length += 1 + if length: + chunks.append(self.prefix(length)) + self.forward(length) + ch = self.peek() + if not double and ch == '\'' and self.peek(1) == '\'': + chunks.append('\'') + self.forward(2) + elif (double and ch == '\'') or (not double and ch in '\"\\'): + chunks.append(ch) + self.forward() + elif double and ch == '\\': + self.forward() + ch = self.peek() + if ch in self.ESCAPE_REPLACEMENTS: + chunks.append(self.ESCAPE_REPLACEMENTS[ch]) + self.forward() + elif ch in self.ESCAPE_CODES: + length = self.ESCAPE_CODES[ch] + self.forward() + for k in range(length): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "expected escape sequence of %d hexdecimal numbers, but found %r" % + (length, self.peek(k)), self.get_mark()) + code = int(self.prefix(length), 16) + chunks.append(chr(code)) + self.forward(length) + elif ch in '\r\n\x85\u2028\u2029': + self.scan_line_break() + chunks.extend(self.scan_flow_scalar_breaks(double, start_mark)) + else: + raise ScannerError("while scanning a double-quoted scalar", start_mark, + "found unknown escape character %r" % ch, self.get_mark()) + else: + return chunks + + def scan_flow_scalar_spaces(self, double, start_mark): + # See the specification for details. + chunks = [] + length = 0 + while self.peek(length) in ' \t': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch == '\0': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected end of stream", self.get_mark()) + elif ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + breaks = self.scan_flow_scalar_breaks(double, start_mark) + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + else: + chunks.append(whitespaces) + return chunks + + def scan_flow_scalar_breaks(self, double, start_mark): + # See the specification for details. + chunks = [] + while True: + # Instead of checking indentation, we check for document + # separators. + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + raise ScannerError("while scanning a quoted scalar", start_mark, + "found unexpected document separator", self.get_mark()) + while self.peek() in ' \t': + self.forward() + if self.peek() in '\r\n\x85\u2028\u2029': + chunks.append(self.scan_line_break()) + else: + return chunks + + def scan_plain(self): + # See the specification for details. + # We add an additional restriction for the flow context: + # plain scalars in the flow context cannot contain ',' or '?'. + # We also keep track of the `allow_simple_key` flag here. + # Indentation rules are loosed for the flow context. + chunks = [] + start_mark = self.get_mark() + end_mark = start_mark + indent = self.indent+1 + # We allow zero indentation for scalars, but then we need to check for + # document separators at the beginning of the line. + #if indent == 0: + # indent = 1 + spaces = [] + while True: + length = 0 + if self.peek() == '#': + break + while True: + ch = self.peek(length) + if ch in '\0 \t\r\n\x85\u2028\u2029' \ + or (ch == ':' and + self.peek(length+1) in '\0 \t\r\n\x85\u2028\u2029' + + (u',[]{}' if self.flow_level else u''))\ + or (self.flow_level and ch in ',?[]{}'): + break + length += 1 + if length == 0: + break + self.allow_simple_key = False + chunks.extend(spaces) + chunks.append(self.prefix(length)) + self.forward(length) + end_mark = self.get_mark() + spaces = self.scan_plain_spaces(indent, start_mark) + if not spaces or self.peek() == '#' \ + or (not self.flow_level and self.column < indent): + break + return ScalarToken(''.join(chunks), True, start_mark, end_mark) + + def scan_plain_spaces(self, indent, start_mark): + # See the specification for details. + # The specification is really confusing about tabs in plain scalars. + # We just forbid them completely. Do not use tabs in YAML! + chunks = [] + length = 0 + while self.peek(length) in ' ': + length += 1 + whitespaces = self.prefix(length) + self.forward(length) + ch = self.peek() + if ch in '\r\n\x85\u2028\u2029': + line_break = self.scan_line_break() + self.allow_simple_key = True + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + breaks = [] + while self.peek() in ' \r\n\x85\u2028\u2029': + if self.peek() == ' ': + self.forward() + else: + breaks.append(self.scan_line_break()) + prefix = self.prefix(3) + if (prefix == '---' or prefix == '...') \ + and self.peek(3) in '\0 \t\r\n\x85\u2028\u2029': + return + if line_break != '\n': + chunks.append(line_break) + elif not breaks: + chunks.append(' ') + chunks.extend(breaks) + elif whitespaces: + chunks.append(whitespaces) + return chunks + + def scan_tag_handle(self, name, start_mark): + # See the specification for details. + # For some strange reasons, the specification does not allow '_' in + # tag handles. I have allowed it anyway. + ch = self.peek() + if ch != '!': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length = 1 + ch = self.peek(length) + if ch != ' ': + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-_': + length += 1 + ch = self.peek(length) + if ch != '!': + self.forward(length) + raise ScannerError("while scanning a %s" % name, start_mark, + "expected '!', but found %r" % ch, self.get_mark()) + length += 1 + value = self.prefix(length) + self.forward(length) + return value + + def scan_tag_uri(self, name, start_mark): + # See the specification for details. + # Note: we do not check if URI is well-formed. + chunks = [] + length = 0 + ch = self.peek(length) + while '0' <= ch <= '9' or 'A' <= ch <= 'Z' or 'a' <= ch <= 'z' \ + or ch in '-;/?:@&=+$,_.!~*\'()[]%': + if ch == '%': + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + chunks.append(self.scan_uri_escapes(name, start_mark)) + else: + length += 1 + ch = self.peek(length) + if length: + chunks.append(self.prefix(length)) + self.forward(length) + length = 0 + if not chunks: + raise ScannerError("while parsing a %s" % name, start_mark, + "expected URI, but found %r" % ch, self.get_mark()) + return ''.join(chunks) + + def scan_uri_escapes(self, name, start_mark): + # See the specification for details. + codes = [] + mark = self.get_mark() + while self.peek() == '%': + self.forward() + for k in range(2): + if self.peek(k) not in '0123456789ABCDEFabcdef': + raise ScannerError("while scanning a %s" % name, start_mark, + "expected URI escape sequence of 2 hexdecimal numbers, but found %r" + % self.peek(k), self.get_mark()) + codes.append(int(self.prefix(2), 16)) + self.forward(2) + try: + value = bytes(codes).decode('utf-8') + except UnicodeDecodeError as exc: + raise ScannerError("while scanning a %s" % name, start_mark, str(exc), mark) + return value + + def scan_line_break(self): + # Transforms: + # '\r\n' : '\n' + # '\r' : '\n' + # '\n' : '\n' + # '\x85' : '\n' + # '\u2028' : '\u2028' + # '\u2029 : '\u2029' + # default : '' + ch = self.peek() + if ch in '\r\n\x85': + if self.prefix(2) == '\r\n': + self.forward(2) + else: + self.forward() + return '\n' + elif ch in '\u2028\u2029': + self.forward() + return ch + return '' diff --git a/pipenv/patched/yaml3/serializer.py b/pipenv/patched/yaml3/serializer.py new file mode 100644 index 00000000..fe911e67 --- /dev/null +++ b/pipenv/patched/yaml3/serializer.py @@ -0,0 +1,111 @@ + +__all__ = ['Serializer', 'SerializerError'] + +from .error import YAMLError +from .events import * +from .nodes import * + +class SerializerError(YAMLError): + pass + +class Serializer: + + ANCHOR_TEMPLATE = 'id%03d' + + def __init__(self, encoding=None, + explicit_start=None, explicit_end=None, version=None, tags=None): + self.use_encoding = encoding + self.use_explicit_start = explicit_start + self.use_explicit_end = explicit_end + self.use_version = version + self.use_tags = tags + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + self.closed = None + + def open(self): + if self.closed is None: + self.emit(StreamStartEvent(encoding=self.use_encoding)) + self.closed = False + elif self.closed: + raise SerializerError("serializer is closed") + else: + raise SerializerError("serializer is already opened") + + def close(self): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif not self.closed: + self.emit(StreamEndEvent()) + self.closed = True + + #def __del__(self): + # self.close() + + def serialize(self, node): + if self.closed is None: + raise SerializerError("serializer is not opened") + elif self.closed: + raise SerializerError("serializer is closed") + self.emit(DocumentStartEvent(explicit=self.use_explicit_start, + version=self.use_version, tags=self.use_tags)) + self.anchor_node(node) + self.serialize_node(node, None, None) + self.emit(DocumentEndEvent(explicit=self.use_explicit_end)) + self.serialized_nodes = {} + self.anchors = {} + self.last_anchor_id = 0 + + def anchor_node(self, node): + if node in self.anchors: + if self.anchors[node] is None: + self.anchors[node] = self.generate_anchor(node) + else: + self.anchors[node] = None + if isinstance(node, SequenceNode): + for item in node.value: + self.anchor_node(item) + elif isinstance(node, MappingNode): + for key, value in node.value: + self.anchor_node(key) + self.anchor_node(value) + + def generate_anchor(self, node): + self.last_anchor_id += 1 + return self.ANCHOR_TEMPLATE % self.last_anchor_id + + def serialize_node(self, node, parent, index): + alias = self.anchors[node] + if node in self.serialized_nodes: + self.emit(AliasEvent(alias)) + else: + self.serialized_nodes[node] = True + self.descend_resolver(parent, index) + if isinstance(node, ScalarNode): + detected_tag = self.resolve(ScalarNode, node.value, (True, False)) + default_tag = self.resolve(ScalarNode, node.value, (False, True)) + implicit = (node.tag == detected_tag), (node.tag == default_tag) + self.emit(ScalarEvent(alias, node.tag, implicit, node.value, + style=node.style)) + elif isinstance(node, SequenceNode): + implicit = (node.tag + == self.resolve(SequenceNode, node.value, True)) + self.emit(SequenceStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + index = 0 + for item in node.value: + self.serialize_node(item, node, index) + index += 1 + self.emit(SequenceEndEvent()) + elif isinstance(node, MappingNode): + implicit = (node.tag + == self.resolve(MappingNode, node.value, True)) + self.emit(MappingStartEvent(alias, node.tag, implicit, + flow_style=node.flow_style)) + for key, value in node.value: + self.serialize_node(key, node, None) + self.serialize_node(value, node, key) + self.emit(MappingEndEvent()) + self.ascend_resolver() + diff --git a/pipenv/patched/yaml3/tokens.py b/pipenv/patched/yaml3/tokens.py new file mode 100644 index 00000000..4d0b48a3 --- /dev/null +++ b/pipenv/patched/yaml3/tokens.py @@ -0,0 +1,104 @@ + +class Token(object): + def __init__(self, start_mark, end_mark): + self.start_mark = start_mark + self.end_mark = end_mark + def __repr__(self): + attributes = [key for key in self.__dict__ + if not key.endswith('_mark')] + attributes.sort() + arguments = ', '.join(['%s=%r' % (key, getattr(self, key)) + for key in attributes]) + return '%s(%s)' % (self.__class__.__name__, arguments) + +#class BOMToken(Token): +# id = '<byte order mark>' + +class DirectiveToken(Token): + id = '<directive>' + def __init__(self, name, value, start_mark, end_mark): + self.name = name + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class DocumentStartToken(Token): + id = '<document start>' + +class DocumentEndToken(Token): + id = '<document end>' + +class StreamStartToken(Token): + id = '<stream start>' + def __init__(self, start_mark=None, end_mark=None, + encoding=None): + self.start_mark = start_mark + self.end_mark = end_mark + self.encoding = encoding + +class StreamEndToken(Token): + id = '<stream end>' + +class BlockSequenceStartToken(Token): + id = '<block sequence start>' + +class BlockMappingStartToken(Token): + id = '<block mapping start>' + +class BlockEndToken(Token): + id = '<block end>' + +class FlowSequenceStartToken(Token): + id = '[' + +class FlowMappingStartToken(Token): + id = '{' + +class FlowSequenceEndToken(Token): + id = ']' + +class FlowMappingEndToken(Token): + id = '}' + +class KeyToken(Token): + id = '?' + +class ValueToken(Token): + id = ':' + +class BlockEntryToken(Token): + id = '-' + +class FlowEntryToken(Token): + id = ',' + +class AliasToken(Token): + id = '<alias>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class AnchorToken(Token): + id = '<anchor>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class TagToken(Token): + id = '<tag>' + def __init__(self, value, start_mark, end_mark): + self.value = value + self.start_mark = start_mark + self.end_mark = end_mark + +class ScalarToken(Token): + id = '<scalar>' + def __init__(self, value, plain, start_mark, end_mark, style=None): + self.value = value + self.plain = plain + self.start_mark = start_mark + self.end_mark = end_mark + self.style = style + diff --git a/pipenv/pipenv.1 b/pipenv/pipenv.1 index 62808744..7dd63f58 100644 --- a/pipenv/pipenv.1 +++ b/pipenv/pipenv.1 @@ -1,6 +1,6 @@ .\" Man page generated from reStructuredText. . -.TH "PIPENV" "1" "Oct 07, 2017" "8.2.7" "pipenv" +.TH "PIPENV" "1" "Jul 14, 2019" "2018.11.27.dev0" "pipenv" .SH NAME pipenv \- pipenv Documentation . @@ -38,39 +38,60 @@ level margin: \\n[rst2man-indent\\n[rst2man-indent-level]] .ce 0 .sp .sp -\fBPipenv\fP —\ the officially recommended Python packaging tool from \fI\%Python.org\fP, free (as in freedom). +\fBPipenv\fP is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first\-class citizen, in our world.\fP .sp -Pipenv is a tool that aims to bring the best of all packaging worlds (bundler, composer, npm, cargo, yarn, etc.) to the Python world. \fIWindows is a first-class citizen, in our world.\fP +It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever\-important \fBPipfile.lock\fP, which is used to produce deterministic builds. .sp -It automatically creates and manages a virtualenv for your projects, as well as adds/removes packages from your \fBPipfile\fP as you install/uninstall packages. It also generates the ever-important \fBPipfile.lock\fP, which is used to produce deterministic builds. +Pipenv is primarily meant to provide users and developers of applications with an easy method to setup a working environment. For the distinction between libraries and applications and the usage of \fBsetup.py\fP vs \fBPipfile\fP to define dependencies, see pipfile\-vs\-setuppy\&. +[image: a short animation of pipenv at work] +[image] .sp The problems that Pipenv seeks to solve are multi\-faceted: .INDENT 0.0 .IP \(bu 2 You no longer need to use \fBpip\fP and \fBvirtualenv\fP separately. They work together. .IP \(bu 2 -Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses the upcoming \fBPipfile\fP and \fBPipfile.lock\fP instead, which is superior for basic use cases. +Managing a \fBrequirements.txt\fP file \fI\%can be problematic\fP, so Pipenv uses \fBPipfile\fP and \fBPipfile.lock\fP to separate abstract dependency declarations from the last tested combination. .IP \(bu 2 Hashes are used everywhere, always. Security. Automatically expose security vulnerabilities. .IP \(bu 2 +Strongly encourage the use of the latest versions of dependencies to minimize security risks \fI\%arising from outdated components\fP\&. +.IP \(bu 2 Give you insight into your dependency graph (e.g. \fB$ pipenv graph\fP). .IP \(bu 2 Streamline development workflow by loading \fB\&.env\fP files. .UNINDENT -.SH INSTALL PIPENV TODAY! +.sp +You can quickly play with Pipenv right in your browser: +\fI\%Try in browser\fP.SH INSTALL PIPENV TODAY! +.sp +If you\(aqre on MacOS, you can install Pipenv easily with Homebrew. You can also use Linuxbrew on Linux using the same command: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pip install pipenv -✨🍰✨ +$ brew install pipenv .ft P .fi .UNINDENT .UNINDENT .sp -If you have excellent taste, there\(aqs also a \fI\%fancy installation method\fP\&. +Or, if you\(aqre using Fedora 28: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ sudo dnf install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Otherwise, refer to the installing\-pipenv chapter for instructions. +.sp +✨🍰✨ .SS Pipenv & Virtual Environments [image] .sp @@ -91,7 +112,7 @@ This guide is written for Python 3, however, these instructions should work fine on Python 2.7—if you are still using it, for some reason. .UNINDENT .UNINDENT -.SS ☤ Make sure you\(aqve got Python & pip +.SS ☤ Make sure you\(aqve got Python & pip .sp Before you go any further, make sure you have Python and that it\(aqs available from your command line. You can check this by simply running: @@ -149,9 +170,11 @@ pip 9.0.1 .UNINDENT .UNINDENT .sp -If you installed Python from source, with an installer from \fI\%python.org\fP, or -via \fI\%Homebrew\fP you should already have pip9. If you\(aqre on Linux and installed +If you installed Python from source, with an installer from \fI\%python.org\fP, via \fI\%Homebrew\fP or via \fI\%Linuxbrew\fP you should already have pip. If you\(aqre on Linux and installed using your OS package manager, you may have to \fI\%install pip\fP separately. +.sp +If you plan to install Pipenv using Homebrew or Linuxbrew you can skip this step. The +Homebrew/Linuxbrew installer takes care of pip for you. .SS ☤ Installing Pipenv .sp Pipenv is a dependency manager for Python projects. If you\(aqre familiar @@ -159,8 +182,42 @@ with Node.js\(aq \fI\%npm\fP or Ruby\(aqs \fI\%bundler\fP, it is similar in spir tools. While pip can install Python packages, Pipenv is recommended as it\(aqs a higher\-level tool that simplifies dependency management for common use cases. +.SS ☤ Homebrew Installation of Pipenv .sp -Use \fBpip\fP to install Pipenv: +\fI\%Homebrew\fP is a popular open\-source package management system for macOS. For Linux users, \fI\%Linuxbrew\fP is a Linux port of that. +.sp +Installing pipenv via Homebrew or Linuxbrew will keep pipenv and all of its dependencies in +an isolated virtual environment so it doesn\(aqt interfere with the rest of your +Python installation. +.sp +Once you have installed Homebrew or Linuxbrew simply run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew install pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ brew upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Pragmatic Installation of Pipenv +.sp +If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. +.sp +To install: .INDENT 0.0 .INDENT 3.5 .sp @@ -187,13 +244,40 @@ absolute path to your home directory) so you\(aqll need to add \fI\%modifying ~/.profile\fP\&. .sp On Windows you can find the user base binary directory by running -\fBpy \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with +\fBpython \-m site \-\-user\-site\fP and replacing \fBsite\-packages\fP with \fBScripts\fP\&. For example, this could return \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\esite\-packages\fP so you would need to set your \fBPATH\fP to include \fBC:\eUsers\eUsername\eAppData\eRoaming\ePython36\eScripts\fP\&. You can set your user \fBPATH\fP permanently in the \fI\%Control Panel\fP\&. You may need to log out for the \fBPATH\fP changes to take effect. +.sp +For more information, see the \fI\%user installs documentation\fP\&. +.UNINDENT +.UNINDENT +.sp +To upgrade pipenv at any time: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pip install \-\-user \-\-upgrade pipenv +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Crude Installation of Pipenv +.sp +If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ curl https://raw.githubusercontent.com/pypa/pipenv/master/get\-pipenv.py | python +.ft P +.fi .UNINDENT .UNINDENT .SS ☤ Installing packages for your project @@ -214,7 +298,7 @@ $ pipenv install requests .UNINDENT .sp Pipenv will install the excellent \fI\%Requests\fP library and create a \fBPipfile\fP -for you in your project\(aqs directory. The Pipfile is used to track which +for you in your project\(aqs directory. The \fBPipfile\fP is used to track which dependencies your project needs in case you need to re\-install them, such as when you share your project with others. You should get output similar to this (although the exact paths shown will vary): @@ -297,119 +381,799 @@ Your IP is 8.8.8.8 Using \fB$ pipenv run\fP ensures that your installed packages are available to your script. It\(aqs also possible to spawn a new shell that ensures all commands have access to your installed packages with \fB$ pipenv shell\fP\&. +.SS ☤ Virtualenv mapping caveat +.INDENT 0.0 +.IP \(bu 2 +Pipenv automatically maps projects to their specific virtualenvs. +.IP \(bu 2 +The virtualenv is stored globally with the name of the project’s root directory plus the hash of the full path to the project\(aqs root (e.g., \fBmy_project\-a3de50\fP). +.IP \(bu 2 +If you change your project\(aqs path, you break such a default mapping and pipenv will no longer be able to find and to use the project\(aqs virtualenv. +.IP \(bu 2 +You might want to set \fBexport PIPENV_VENV_IN_PROJECT=1\fP in your .bashrc/.zshrc (or any shell configuration file) for creating the virtualenv inside your project\(aqs directory, avoiding problems with subsequent path changes. +.UNINDENT .SS ☤ Next steps .sp Congratulations, you now know how to install and use Python packages! ✨ 🍰 ✨ -.SS ☤ Fancy Installation of Pipenv -.sp -To install pipenv in a fancy way, we recommend using \fI\%pipsi\fP\&. -.sp -Pipsi is a powerful tool which allows you to install Python scripts into isolated virtual environments. -.sp -To install pipsi, first run this: +.SS Release and Version History +.SS 2018.11.26 (2018\-11\-26) +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/mitsuhiko/pipsi/master/get\-pipsi.py | python -.ft P -.fi +.IP \(bu 2 +Environment variables are expanded correctly before running scripts on POSIX. \fI\%#3178\fP +.IP \(bu 2 +Pipenv will no longer disable user\-mode installation when the \fB\-\-system\fP flag is passed in. \fI\%#3222\fP +.IP \(bu 2 +Fixed an issue with attempting to render unicode output in non\-unicode locales. \fI\%#3223\fP +.IP \(bu 2 +Fixed a bug which could cause failures to occur when parsing python entries from global pyenv version files. \fI\%#3224\fP +.IP \(bu 2 +Fixed an issue which prevented the parsing of named extras sections from certain \fBsetup.py\fP files. \fI\%#3230\fP +.IP \(bu 2 +Correctly detect the virtualenv location inside an activated virtualenv. \fI\%#3231\fP +.IP \(bu 2 +Fixed a bug which caused spinner frames to be written to stdout during locking operations which could cause redirection pipes to fail. \fI\%#3239\fP +.IP \(bu 2 +Fixed a bug that editable pacakges can\(aqt be uninstalled correctly. \fI\%#3240\fP +.IP \(bu 2 +Corrected an issue with installation timeouts which caused dependency resolution to fail for longer duration resolution steps. \fI\%#3244\fP +.IP \(bu 2 +Adding normal pep 508 compatible markers is now fully functional when using VCS dependencies. \fI\%#3249\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP and \fBpythonfinder\fP for multiple bugfixes. \fI\%#3254\fP +.IP \(bu 2 +Pipenv will now ignore hashes when installing with \fB\-\-skip\-lock\fP\&. \fI\%#3255\fP +.IP \(bu 2 +Fixed an issue where pipenv could crash when multiple pipenv processes attempted to create the same directory. \fI\%#3257\fP +.IP \(bu 2 +Fixed an issue which sometimes prevented successful creation of project pipfiles. \fI\%#3260\fP +.IP \(bu 2 +\fBpipenv install\fP will now unset the \fBPYTHONHOME\fP environment variable when not combined with \fB\-\-system\fP\&. \fI\%#3261\fP +.IP \(bu 2 +Pipenv will ensure that warnings do not interfere with the resolution process by suppressing warnings\(aq usage of standard output and writing to standard error instead. \fI\%#3273\fP +.IP \(bu 2 +Fixed an issue which prevented variables from the environment, such as \fBPIPENV_DEV\fP or \fBPIPENV_SYSTEM\fP, from being parsed and implemented correctly. \fI\%#3278\fP +.IP \(bu 2 +Clear pythonfinder cache after Python install. \fI\%#3287\fP +.IP \(bu 2 +Fixed a race condition in hash resolution for dependencies for certain dependencies with missing cache entries or fresh Pipenv installs. \fI\%#3289\fP +.IP \(bu 2 +Pipenv will now respect top\-level pins over VCS dependency locks. \fI\%#3296\fP .UNINDENT -.UNINDENT -.sp -Follow the instructions, you\(aqll have to update your \fBPATH\fP\&. -.sp -Then, simply run: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi install pipenv -.ft P -.fi +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Update vendored dependencies to resolve resolution output parsing and python finding: +.INDENT 7.0 +.IP \(bu 2 +\fBpythonfinder 1.1.9 \-> 1.1.10\fP +.IP \(bu 2 +\fBrequirementslib 1.3.1 \-> 1.3.3\fP +.IP \(bu 2 +\fBvistir 0.2.3 \-> 0.2.5\fP \fI\%#3280\fP .UNINDENT .UNINDENT -.sp -To upgrade pipenv at any time: +.UNINDENT +.SS 2018.11.14 (2018\-11\-14) +.SS Features & Improvements .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pipsi upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Improved exceptions and error handling on failures. \fI\%#1977\fP +.IP \(bu 2 +Added persistent settings for all CLI flags via \fBPIPENV_{FLAG_NAME}\fP environment variables by enabling \fBauto_envvar_prefix=PIPENV\fP in click (implements PEEP\-0002). \fI\%#2200\fP +.IP \(bu 2 +Added improved messaging about available but skipped updates due to dependency conflicts when running \fBpipenv update \-\-outdated\fP\&. \fI\%#2411\fP +.IP \(bu 2 +Added environment variable \fIPIPENV_PYUP_API_KEY\fP to add ability +to override the bundled pyup.io API key. \fI\%#2825\fP +.IP \(bu 2 +Added additional output to \fBpipenv update \-\-outdated\fP to indicate that the operation succeded and all packages were already up to date. \fI\%#2828\fP +.IP \(bu 2 +Updated \fBcrayons\fP patch to enable colors on native powershell but swap native blue for magenta. \fI\%#3020\fP +.IP \(bu 2 +Added support for \fB\-\-bare\fP to \fBpipenv clean\fP, and fixed \fBpipenv sync \-\-bare\fP to actually reduce output. \fI\%#3041\fP +.IP \(bu 2 +Added windows\-compatible spinner via upgraded \fBvistir\fP dependency. \fI\%#3089\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Added support for python installations managed by \fBasdf\fP\&. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Improved runtime performance of no\-op commands such as \fBpipenv \-\-venv\fP by around 2/3. \fI\%#3158\fP +.IP \(bu 2 +Do not show error but success for running \fBpipenv uninstall \-\-all\fP in a fresh virtual environment. \fI\%#3170\fP +.IP \(bu 2 +Improved asynchronous installation and error handling via queued subprocess paralleization. \fI\%#3217\fP .UNINDENT -.sp -This will install \fBpipenv\fP in an isolated virtualenv, so it doesn\(aqt interfere with the rest of your Python installation! -.SS ☤ Pragmatic Installation of Pipenv -.sp -If you have a working installation of pip, and maintain certain "toolchain" type Python modules as global utilities in your user environment, pip \fI\%user installs\fP allow for installation into your home directory. Note that due to interaction between dependencies, you should limit tools installed in this way to basic building blocks for a Python workflow like virtualenv, pipenv, tox, and similar software. -.sp -To install: +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user pipenv -.ft P -.fi +.IP \(bu 2 +Remote non\-PyPI artifacts and local wheels and artifacts will now include their own hashes rather than including hashes from \fBPyPI\fP\&. \fI\%#2394\fP +.IP \(bu 2 +Non\-ascii characters will now be handled correctly when parsed by pipenv\(aqs \fBToML\fP parsers. \fI\%#2737\fP +.IP \(bu 2 +Updated \fBpipenv uninstall\fP to respect the \fB\-\-skip\-lock\fP argument. \fI\%#2848\fP +.IP \(bu 2 +Fixed a bug which caused uninstallation to sometimes fail to successfullly remove packages from \fBPipfiles\fP with comments on preceding or following lines. \fI\%#2885\fP, +\fI\%#3099\fP +.IP \(bu 2 +Pipenv will no longer fail when encountering python versions on Windows that have been uninstalled. \fI\%#2983\fP +.IP \(bu 2 +Fixed unnecessary extras are added when translating markers \fI\%#3026\fP +.IP \(bu 2 +Fixed a virtualenv creation issue which could cause new virtualenvs to inadvertently attempt to read and write to global site packages. \fI\%#3047\fP +.IP \(bu 2 +Fixed an issue with virtualenv path derivation which could cause errors, particularly for users on WSL bash. \fI\%#3055\fP +.IP \(bu 2 +Fixed a bug which caused \fBUnexpected EOF\fP errors to be thrown when \fBpip\fP was waiting for input from users who had put login credentials in environment variables. \fI\%#3088\fP +.IP \(bu 2 +Fixed a bug in \fBrequirementslib\fP which prevented successful installation from mercurial repositories. \fI\%#3090\fP +.IP \(bu 2 +Fixed random resource warnings when using pyenv or any other subprocess calls. \fI\%#3094\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Fixed a bug which sometimes prevented cloning and parsing \fBmercurial\fP requirements. \fI\%#3096\fP .UNINDENT +.IP \(bu 2 +Fixed an issue in \fBdelegator.py\fP related to subprocess calls when using \fBPopenSpawn\fP to stream output, which sometimes threw unexpected \fBEOF\fP errors. \fI\%#3102\fP, +\fI\%#3114\fP, +\fI\%#3117\fP +.IP \(bu 2 +Fix the path casing issue that makes \fIpipenv clean\fP fail on Windows \fI\%#3104\fP +.IP \(bu 2 +Pipenv will avoid leaving build artifacts in the current working directory. \fI\%#3106\fP +.IP \(bu 2 +Fixed issues with broken subprocess calls leaking resource handles and causing random and sporadic failures. \fI\%#3109\fP +.IP \(bu 2 +Fixed an issue which caused \fBpipenv clean\fP to sometimes clean packages from the base \fBsite\-packages\fP folder or fail entirely. \fI\%#3113\fP +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3121\fP +.IP \(bu 2 +Added additional logic for ignoring and replacing non\-ascii characters when formatting console output on non\-UTF\-8 systems. \fI\%#3131\fP +.IP \(bu 2 +Fix virtual environment discovery when \fIPIPENV_VENV_IN_PROJECT\fP is set, but the in\-project \fI\&.venv\fP is a file. \fI\%#3134\fP +.IP \(bu 2 +Hashes for remote and local non\-PyPI artifacts will now be included in \fBPipfile.lock\fP during resolution. \fI\%#3145\fP +.IP \(bu 2 +Fix project path hashing logic in purpose to prevent collisions of virtual environments. \fI\%#3151\fP +.IP \(bu 2 +Fix package installation when the virtual environment path contains parentheses. \fI\%#3158\fP +.IP \(bu 2 +Azure Pipelines YAML files are updated to use the latest syntax and product name. \fI\%#3164\fP +.IP \(bu 2 +Fixed new spinner success message to write only one success message during resolution. \fI\%#3183\fP +.IP \(bu 2 +Pipenv will now correctly respect the \fB\-\-pre\fP option when used with \fBpipenv install\fP\&. \fI\%#3185\fP +.IP \(bu 2 +Fix a bug where exception is raised when run pipenv graph in a project without created virtualenv \fI\%#3201\fP +.IP \(bu 2 +When sources are missing names, names will now be derived from the supplied URL. \fI\%#3216\fP .UNINDENT -.sp -For more information see the \fI\%user installs documentation\fP, but to add the installed cli tools from a pip user install to your path, add the output of: +.SS Vendored Libraries .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ python \-c "import site; import os; print(os.path.join(site.USER_BASE, \(aqbin\(aq))" -.ft P -.fi +.IP \(bu 2 +Updated \fBpythonfinder\fP to correct an issue with unnesting of nested paths when searching for python versions. \fI\%#3061\fP, +\fI\%#3121\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBcertifi 2018.08.24 => 2018.10.15\fP +.IP \(bu 2 +\fBurllib3 1.23 => 1.24\fP +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.0\fP +.IP \(bu 2 +\fBshellingham \(ga\(ga1.2.6 => 1.2.7\fP +.IP \(bu 2 +\fBtomlkit 0.4.4. => 0.4.6\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.1.8\fP +.IP \(bu 2 +\fBpythonfinder 0.1.2 => 0.1.3\fP +.IP \(bu 2 +\fBrequirementslib 1.1.9 => 1.1.10\fP +.IP \(bu 2 +\fBbackports.functools_lru_cache 1.5.0 (new)\fP +.IP \(bu 2 +\fBcursor 1.2.0 (new)\fP \fI\%#3089\fP .UNINDENT .UNINDENT -.sp -To upgrade pipenv at any time: +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBrequests 2.19.1 => 2.20.1\fP +.IP \(bu 2 +\fBtomlkit 0.4.46 => 0.5.2\fP +.IP \(bu 2 +\fBvistir 0.1.6 => 0.2.4\fP +.IP \(bu 2 +\fBpythonfinder 1.1.2 => 1.1.8\fP +.IP \(bu 2 +\fBrequirementslib 1.1.10 => 1.3.0\fP \fI\%#3096\fP +.UNINDENT +.UNINDENT +.IP \(bu 2 +Switch to \fBtomlkit\fP for parsing and writing. Drop \fBprettytoml\fP and \fBcontoml\fP from vendors. \fI\%#3191\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to aid in resolution of local and remote archives. \fI\%#3196\fP +.UNINDENT +.SS Improved Documentation .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ pip install \-\-user \-\-upgrade pipenv -.ft P -.fi +.IP \(bu 2 +Expanded development and testing documentation for contributors to get started. \fI\%#3074\fP .UNINDENT -.UNINDENT -.SS ☤ Crude Installation of Pipenv -.sp -If you don\(aqt even have pip installed, you can use this crude installation method, which will bootstrap your whole system: +.SS 2018.10.13 (2018\-10\-13) +.SS Bug Fixes .INDENT 0.0 -.INDENT 3.5 -.sp -.nf -.ft C -$ curl https://raw.githubusercontent.com/kennethreitz/pipenv/master/get\-pipenv.py | python -.ft P -.fi +.IP \(bu 2 +Fixed a bug in \fBpipenv clean\fP which caused global packages to sometimes be inadvertently targeted for cleanup. \fI\%#2849\fP +.IP \(bu 2 +Fix broken backport imports for vendored vistir. \fI\%#2950\fP, +\fI\%#2955\fP, +\fI\%#2961\fP +.IP \(bu 2 +Fixed a bug with importing local vendored dependencies when running \fBpipenv graph\fP\&. \fI\%#2952\fP +.IP \(bu 2 +Fixed a bug which caused executable discovery to fail when running inside a virtualenv. \fI\%#2957\fP +.IP \(bu 2 +Fix parsing of outline tables. \fI\%#2971\fP +.IP \(bu 2 +Fixed a bug which caused \fBverify_ssl\fP to fail to drop through to \fBpip install\fP correctly as \fBtrusted\-host\fP\&. \fI\%#2979\fP +.IP \(bu 2 +Fixed a bug which caused canonicalized package names to fail to resolve against PyPI. \fI\%#2989\fP +.IP \(bu 2 +Enhanced CI detection to detect Azure Devops builds. \fI\%#2993\fP +.IP \(bu 2 +Fixed a bug which prevented installing pinned versions which used redirection symbols from the command line. \fI\%#2998\fP +.IP \(bu 2 +Fixed a bug which prevented installing the local directory in non\-editable mode. \fI\%#3005\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.1.9\fP\&. \fI\%#2989\fP +.IP \(bu 2 +Upgraded \fBpythonfinder => 1.1.1\fP and \fBvistir => 0.1.7\fP\&. \fI\%#3007\fP +.UNINDENT +.SS 2018.10.9 (2018\-10\-09) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +Added environment variables \fIPIPENV_VERBOSE\fP and \fIPIPENV_QUIET\fP to control +output verbosity without needing to pass options. \fI\%#2527\fP +.IP \(bu 2 +Updated test\-pypi addon to better support json\-api access (forward compatibility). +Improved testing process for new contributors. \fI\%#2568\fP +.IP \(bu 2 +Greatly enhanced python discovery functionality: +.INDENT 2.0 +.IP \(bu 2 +Added pep514 (windows launcher/finder) support for python discovery. +.IP \(bu 2 +Introduced architecture discovery for python installations which support different architectures. \fI\%#2582\fP +.UNINDENT +.IP \(bu 2 +Added support for \fBpipenv shell\fP on msys and cygwin/mingw/git bash for Windows. \fI\%#2641\fP +.IP \(bu 2 +Enhanced resolution of editable and VCS dependencies. \fI\%#2643\fP +.IP \(bu 2 +Deduplicate and refactor CLI to use stateful arguments and object passing. See \fI\%this issue\fP for reference. \fI\%#2814\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Virtual environment activation for \fBrun\fP is revised to improve interpolation +with other Python discovery tools. \fI\%#2503\fP +.IP \(bu 2 +Improve terminal coloring to display better in Powershell. \fI\%#2511\fP +.IP \(bu 2 +Invoke \fBvirtualenv\fP directly for virtual environment creation, instead of depending on \fBpew\fP\&. \fI\%#2518\fP +.IP \(bu 2 +\fBpipenv \-\-help\fP will now include short help descriptions. \fI\%#2542\fP +.IP \(bu 2 +Add \fBCOMSPEC\fP to fallback option (along with \fBSHELL\fP and \fBPYENV_SHELL\fP) +if shell detection fails, improving robustness on Windows. \fI\%#2651\fP +.IP \(bu 2 +Fallback to shell mode if \fIrun\fP fails with Windows error 193 to handle non\-executable commands. This should improve usability on Windows, where some users run non\-executable files without specifying a command, relying on Windows file association to choose the current command. \fI\%#2718\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed a bug which prevented installation of editable requirements using \fBssh://\fP style urls \fI\%#1393\fP +.IP \(bu 2 +VCS Refs for locked local editable dependencies will now update appropriately to the latest hash when running \fBpipenv update\fP\&. \fI\%#1690\fP +.IP \(bu 2 +\fB\&.tar.gz\fP and \fB\&.zip\fP artifacts will now have dependencies installed even when they are missing from the lockfile. \fI\%#2173\fP +.IP \(bu 2 +The command line parser will now handle multiple \fB\-e/\-\-editable\fP dependencies properly via click\(aqs option parser to help mitigate future parsing issues. \fI\%#2279\fP +.IP \(bu 2 +Fixed the ability of pipenv to parse \fBdependency_links\fP from \fBsetup.py\fP when \fBPIP_PROCESS_DEPENDENCY_LINKS\fP is enabled. \fI\%#2434\fP +.IP \(bu 2 +Fixed a bug which could cause \fB\-i/\-\-index\fP arguments to sometimes be incorrectly picked up in packages. This is now handled in the command line parser. \fI\%#2494\fP +.IP \(bu 2 +Fixed non\-deterministic resolution issues related to changes to the internal package finder in \fBpip 10\fP\&. \fI\%#2499\fP, +\fI\%#2529\fP, +\fI\%#2589\fP, +\fI\%#2666\fP, +\fI\%#2767\fP, +\fI\%#2785\fP, +\fI\%#2795\fP, +\fI\%#2801\fP, +\fI\%#2824\fP, +\fI\%#2862\fP, +\fI\%#2879\fP, +\fI\%#2894\fP, +\fI\%#2933\fP +.IP \(bu 2 +Fix subshell invocation on Windows for Python 2. \fI\%#2515\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to throw a \fBTypeError\fP or to run into encoding issues when writing lockfiles on python 2. \fI\%#2561\fP +.IP \(bu 2 +Improve quoting logic for \fBpipenv run\fP so it works better with Windows +built\-in commands. \fI\%#2563\fP +.IP \(bu 2 +Fixed a bug related to parsing vcs requirements with both extras and subdirectory fragments. +Corrected an issue in the \fBrequirementslib\fP parser which led to some markers being discarded rather than evaluated. \fI\%#2564\fP +.IP \(bu 2 +Fixed multiple issues with finding the correct system python locations. \fI\%#2582\fP +.IP \(bu 2 +Catch JSON decoding error to prevent exception when the lock file is of +invalid format. \fI\%#2607\fP +.IP \(bu 2 +Fixed a rare bug which could sometimes cause errors when installing packages with custom sources. \fI\%#2610\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +Fixed an issue which prevented passing multiple \fB\-\-ignore\fP parameters to \fBpipenv check\fP\&. \fI\%#2632\fP +.IP \(bu 2 +Fixed a bug which caused attempted hashing of \fBssh://\fP style URIs which could cause failures during installation of private ssh repositories. +\- Corrected path conversion issues which caused certain editable VCS paths to be converted to \fBssh://\fP URIs improperly. \fI\%#2639\fP +.IP \(bu 2 +Fixed a bug which caused paths to be formatted incorrectly when using \fBpipenv shell\fP in bash for windows. \fI\%#2641\fP +.IP \(bu 2 +Dependency links to private repositories defined via \fBssh://\fP schemes will now install correctly and skip hashing as long as \fBPIP_PROCESS_DEPENDENCY_LINKS=1\fP\&. \fI\%#2643\fP +.IP \(bu 2 +Fixed a bug which sometimes caused pipenv to parse the \fBtrusted_host\fP argument to pip incorrectly when parsing source URLs which specify \fBverify_ssl = false\fP\&. \fI\%#2656\fP +.IP \(bu 2 +Prevent crashing when a virtual environment in \fBWORKON_HOME\fP is faulty. \fI\%#2676\fP +.IP \(bu 2 +Fixed virtualenv creation failure when a .venv file is present in the project root. \fI\%#2680\fP +.IP \(bu 2 +Fixed a bug which could cause the \fB\-e/\-\-editable\fP argument on a dependency to be accidentally parsed as a dependency itself. \fI\%#2714\fP +.IP \(bu 2 +Correctly pass \fIverbose\fP and \fIdebug\fP flags to the resolver subprocess so it generates appropriate output. This also resolves a bug introduced by the fix to #2527. \fI\%#2732\fP +.IP \(bu 2 +All markers are now included in \fBpipenv lock \-\-requirements\fP output. \fI\%#2748\fP +.IP \(bu 2 +Fixed a bug in marker resolution which could cause duplicate and non\-deterministic markers. \fI\%#2760\fP +.IP \(bu 2 +Fixed a bug in the dependency resolver which caused regular issues when handling \fBsetup.py\fP based dependency resolution. \fI\%#2766\fP +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2802\fP, .UNINDENT .UNINDENT .sp -Congratulations, you now have pip and Pipenv installed! +\fI\%#2867\fP, +\fI\%#2880\fP +.IP \(bu 2 +Fixed a bug where \fIpipenv\fP crashes when the \fIWORKON_HOME\fP directory does not exist. \fI\%#2877\fP +.IP \(bu 2 +Fixed pip is not loaded from pipenv\(aqs patched one but the system one \fI\%#2912\fP +.IP \(bu 2 +Fixed various bugs related to \fBpip 18.1\fP release which prevented locking, installation, and syncing, and dumping to a \fBrequirements.txt\fP file. \fI\%#2924\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Pew is no longer vendored. Entry point \fBpewtwo\fP, packages \fBpipenv.pew\fP and +\fBpipenv.patched.pew\fP are removed. \fI\%#2521\fP +.IP \(bu 2 +Update \fBpythonfinder\fP to major release \fB1.0.0\fP for integration. \fI\%#2582\fP +.IP \(bu 2 +Update requirementslib to fix a bug which could raise an \fBUnboundLocalError\fP when parsing malformed VCS URIs. \fI\%#2617\fP +.IP \(bu 2 +.INDENT 2.0 +.IP \(bu 2 +Vendored new libraries \fBvistir\fP and \fBpip\-shims\fP, \fBtomlkit\fP, \fBmodutil\fP, and \fBplette\fP\&. +.IP \(bu 2 +Update vendored libraries: +\- \fBscandir\fP to \fB1.9.0\fP +\- \fBclick\-completion\fP to \fB0.4.1\fP +\- \fBsemver\fP to \fB2.8.1\fP +\- \fBshellingham\fP to \fB1.2.4\fP +\- \fBpytoml\fP to \fB0.1.18\fP +\- \fBcertifi\fP to \fB2018.8.24\fP +\- \fBptyprocess\fP to \fB0.6.0\fP +\- \fBrequirementslib\fP to \fB1.1.5\fP +\- \fBpythonfinder\fP to \fB1.0.2\fP +\- \fBpipdeptree\fP to \fB0.13.0\fP +\- \fBpython\-dotenv\fP to \fB0.9.1\fP \fI\%#2639\fP +.UNINDENT +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Updated vendored dependencies: +.INDENT 7.0 +.IP \(bu 2 +\fBpip\-tools\fP (updated and patched to latest w/ \fBpip 18.0\fP compatibilty) +.IP \(bu 2 +\fBpip 10.0.1 => 18.0\fP +.IP \(bu 2 +\fBclick 6.7 => 7.0\fP +.IP \(bu 2 +\fBtoml 0.9.4 => 0.10.0\fP +.IP \(bu 2 +\fBpyparsing 2.2.0 => 2.2.2\fP +.IP \(bu 2 +\fBdelegator 0.1.0 => 0.1.1\fP +.IP \(bu 2 +\fBattrs 18.1.0 => 18.2.0\fP +.IP \(bu 2 +\fBdistlib 0.2.7 => 0.2.8\fP +.IP \(bu 2 +\fBpackaging 17.1.0 => 18.0\fP +.IP \(bu 2 +\fBpassa 0.2.0 => 0.3.1\fP +.IP \(bu 2 +\fBpip_shims 0.1.2 => 0.3.1\fP +.IP \(bu 2 +\fBplette 0.1.1 => 0.2.2\fP +.IP \(bu 2 +\fBpythonfinder 1.0.2 => 1.1.0\fP +.IP \(bu 2 +\fBpytoml 0.1.18 => 0.1.19\fP +.IP \(bu 2 +\fBrequirementslib 1.1.16 => 1.1.17\fP +.IP \(bu 2 +\fBshellingham 1.2.4 => 1.2.6\fP +.IP \(bu 2 +\fBtomlkit 0.4.2 => 0.4.4\fP +.IP \(bu 2 +\fBvistir 0.1.4 => 0.1.6\fP \fI\%#2902\fP, +.UNINDENT +.UNINDENT +.sp +\fI\%#2935\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Simplified the test configuration process. \fI\%#2568\fP +.IP \(bu 2 +Updated documentation to use working fortune cookie addon. \fI\%#2644\fP +.IP \(bu 2 +Added additional information about troubleshooting \fBpipenv shell\fP by using the the \fB$PIPENV_SHELL\fP environment variable. \fI\%#2671\fP +.IP \(bu 2 +Added a link to \fBPEP\-440\fP version specifiers in the documentation for additional detail. \fI\%#2674\fP +.IP \(bu 2 +Added simple example to README.md for installing from git. \fI\%#2685\fP +.IP \(bu 2 +Stopped recommending \fI\-\-system\fP for Docker contexts. \fI\%#2762\fP +.IP \(bu 2 +Fixed the example url for doing "pipenv install \-e +some\-repo\-url#egg=something", it was missing the "egg=" in the fragment +identifier. \fI\%#2792\fP +.IP \(bu 2 +Fixed link to the "be cordial" essay in the contribution documentation. \fI\%#2793\fP +.IP \(bu 2 +Clarify \fIpipenv install\fP documentation \fI\%#2844\fP +.IP \(bu 2 +Replace reference to uservoice with PEEP\-000 \fI\%#2909\fP +.UNINDENT +.SS 2018.7.1 (2018\-07\-01) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP, +\fI\%#2434\fP, +\fI\%#2436\fP +.IP \(bu 2 +Added new flag \fBpipenv \-\-support\fP to replace the diagnostic command \fBpython \-m pipenv.help\fP\&. \fI\%#2477\fP, +\fI\%#2478\fP +.IP \(bu 2 +Improved import times and CLI runtimes with minor tweaks. \fI\%#2485\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Fixed an ongoing bug which sometimes resolved incompatible versions into lockfiles. \fI\%#1901\fP +.IP \(bu 2 +Fixed a bug which caused errors when creating virtualenvs which contained leading dash characters. \fI\%#2415\fP +.IP \(bu 2 +Fixed a logic error which caused \fB\-\-deploy \-\-system\fP to overwrite editable vcs packages in the pipfile before installing, which caused any installation to fail by default. \fI\%#2417\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Added support for python 3.7 via a few small compatibility / bugfixes. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Pipenv will now ensure that its internal package names registry files are written with unicode strings. \fI\%#2450\fP +.IP \(bu 2 +Fixed a bug causing requirements input as relative paths to be output as absolute paths or URIs. +Fixed a bug affecting normalization of \fBgit+git@host\fP uris. \fI\%#2453\fP +.IP \(bu 2 +Pipenv will now always use \fBpathlib2\fP for \fBPath\fP based filesystem interactions by default on \fBpython<3.5\fP\&. \fI\%#2454\fP +.IP \(bu 2 +Fixed a bug which prevented passing proxy PyPI indexes set with \fB\-\-pypi\-mirror\fP from being passed to pip during virtualenv creation, which could cause the creation to freeze in some cases. \fI\%#2462\fP +.IP \(bu 2 +Using the \fBpython \-m pipenv.help\fP command will now use proper encoding for the host filesystem to avoid encoding issues. \fI\%#2466\fP +.IP \(bu 2 +The new \fBjinja2\fP templates for \fBclick_completion\fP will now be included in pipenv source distributions. \fI\%#2479\fP +.IP \(bu 2 +Resolved a long\-standing issue with re\-using previously generated \fBInstallRequirement\fP objects for resolution which could cause \fBPKG\-INFO\fP file information to be deleted, raising a \fBTypeError\fP\&. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +All calls to \fBpipenv shell\fP are now implemented from the ground up using \fI\%shellingham\fP, a custom library which was purpose built to handle edge cases and shell detection. \fI\%#2371\fP +.IP \(bu 2 +Updated requirementslib to fix an issue with properly quoting markers in VCS requirements. \fI\%#2419\fP +.IP \(bu 2 +Installed new vendored jinja2 templates for \fBclick\-completion\fP which were causing template errors for users with completion enabled. \fI\%#2422\fP +.IP \(bu 2 +Add patch to \fBprettytoml\fP to support Python 3.7. \fI\%#2426\fP +.IP \(bu 2 +Patched \fBprettytoml.AbstractTable._enumerate_items\fP to handle \fBStopIteration\fP errors in preparation of release of python 3.7. \fI\%#2427\fP +.IP \(bu 2 +Fixed an issue reading package names from \fBsetup.py\fP files in projects which imported utilities such as \fBversioneer\fP\&. \fI\%#2433\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to version \fB1.0.9\fP \fI\%#2453\fP +.IP \(bu 2 +Unraveled a lot of old, unnecessary patches to \fBpip\-tools\fP which were causing non\-deterministic resolution errors. \fI\%#2480\fP +.IP \(bu 2 +Resolved an issue parsing usernames from private PyPI URIs in \fBPipfiles\fP by updating \fBrequirementslib\fP\&. \fI\%#2484\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Added instructions for installing using Fedora\(aqs official repositories. \fI\%#2404\fP +.UNINDENT +.SS 2018.6.25 (2018\-06\-25) +.SS Features & Improvements +.INDENT 0.0 +.IP \(bu 2 +Pipenv\-created virtualenvs will now be associated with a \fB\&.project\fP folder +(features can be implemented on top of this later or users may choose to use +\fBpipenv\-pipes\fP to take full advantage of this.) \fI\%#1861\fP +.IP \(bu 2 +Virtualenv names will now appear in prompts for most Windows users. \fI\%#2167\fP +.IP \(bu 2 +Added support for cmder shell paths with spaces. \fI\%#2168\fP +.IP \(bu 2 +Added nested JSON output to the \fBpipenv graph\fP command. \fI\%#2199\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +PyPI mirror URLs can now be set to override instances of PyPI urls by passing +the \fB\-\-pypi\-mirror\fP argument from the command line or setting the +\fBPIPENV_PYPI_MIRROR\fP environment variable. \fI\%#2281\fP +.IP \(bu 2 +Virtualenv activation lines will now avoid being written to some shell +history files. \fI\%#2287\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.IP \(bu 2 +Added support for mounted drives via UNC paths. \fI\%#2331\fP +.IP \(bu 2 +Added support for Windows Subsystem for Linux bash shell detection. \fI\%#2363\fP +.IP \(bu 2 +Pipenv will now generate hashes much more quickly by resolving them in a +single pass during locking. \fI\%#2384\fP +.IP \(bu 2 +\fBpipenv run\fP will now avoid spawning additional \fBCOMSPEC\fP instances to +run commands in when possible. \fI\%#2385\fP +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#2388\fP +.IP \(bu 2 +\fBpipenv check\fP now may take multiple of the additional argument +\fB\-\-ignore\fP which takes a parameter \fBcve_id\fP for the purpose of ignoring +specific CVEs. \fI\%#2408\fP +.UNINDENT +.SS Behavior Changes +.INDENT 0.0 +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP markers +.. warning:: This could cause an issue if you have an out of date \fBPipfile\fP +which lowercases the comparison value (e.g. \fBcpython\fP instead of +\fBCPython\fP). \fI\%#2123\fP +.IP \(bu 2 +Pipenv will now only search for \fBrequirements.txt\fP files when creating new +projects, and during that time only if the user doesn\(aqt specify packages to +pass in. \fI\%#2309\fP +.UNINDENT +.SS Bug Fixes +.INDENT 0.0 +.IP \(bu 2 +Massive internal improvements to requirements parsing codebase, resolver, and +error messaging. \fI\%#1962\fP, +\fI\%#2186\fP, +\fI\%#2263\fP, +\fI\%#2312\fP +.IP \(bu 2 +Pipenv will now parse & capitalize \fBplatform_python_implementation\fP +markers. \fI\%#2123\fP +.IP \(bu 2 +Fixed a bug with parsing and grouping old\-style \fBsetup.py\fP extras during +resolution \fI\%#2142\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to throw unhelpful exceptions when running +against empty or non\-existent environments. \fI\%#2161\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-system\fP to incorrectly abort when users were in +a virtualenv. \fI\%#2181\fP +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Fixed a regression which led to direct invocations of \fBvirtualenv\fP, rather +than calling it by module. \fI\%#2198\fP +.IP \(bu 2 +Locking will now pin the correct VCS ref during \fBpipenv update\fP runs. +Running \fBpipenv update\fP with a new vcs ref specified in the \fBPipfile\fP +will now properly obtain, resolve, and install the specified dependency at +the specified ref. \fI\%#2209\fP +.IP \(bu 2 +\fBpipenv clean\fP will now correctly ignore comments from \fBpip freeze\fP when +cleaning the environment. \fI\%#2262\fP +.IP \(bu 2 +Resolution bugs causing packages for incompatible python versions to be +locked have been fixed. \fI\%#2267\fP +.IP \(bu 2 +Fixed a bug causing pipenv graph to fail to display sometimes. \fI\%#2268\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Windows executable discovery now leverages \fBos.pathext\fP\&. \fI\%#2298\fP +.IP \(bu 2 +Fixed a bug which caused \fB\-\-deploy \-\-system\fP to inadvertently create a +virtualenv before failing. \fI\%#2301\fP +.IP \(bu 2 +Fixed an issue which led to a failure to unquote special characters in file +and wheel paths. \fI\%#2302\fP +.IP \(bu 2 +VCS dependencies are now manually obtained only if they do not match the +requested ref. \fI\%#2304\fP +.IP \(bu 2 +Added error handling functionality to properly cope with single\-digit +\fBRequires\-Python\fP metatdata with no specifiers. \fI\%#2377\fP +.IP \(bu 2 +\fBpipenv update\fP will now always run the resolver and lock before ensuring +your dependencies are in sync with your lockfile. \fI\%#2379\fP +.IP \(bu 2 +Resolved a bug in our patched resolvers which could cause nondeterministic +resolution failures in certain conditions. Running \fBpipenv install\fP with no +arguments in a project with only a \fBPipfile\fP will now correctly lock first +for dependency resolution before installing. \fI\%#2384\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.UNINDENT +.SS Improved Documentation +.INDENT 0.0 +.IP \(bu 2 +Update documentation wording to clarify Pipenv\(aqs overall role in the packaging ecosystem. \fI\%#2194\fP +.IP \(bu 2 +Added contribution documentation and guidelines. \fI\%#2205\fP +.IP \(bu 2 +Added instructions for supervisord compatibility. \fI\%#2215\fP +.IP \(bu 2 +Fixed broken links to development philosophy and contribution documentation. \fI\%#2248\fP +.UNINDENT +.SS Vendored Libraries +.INDENT 0.0 +.IP \(bu 2 +Removed vendored \fBcacert.pem\fP which could cause issues for some users with +custom certificate settings. \fI\%#2193\fP +.IP \(bu 2 +Dropped vendored pip 9 and vendored, patched, and migrated to pip 10. Updated +patched piptools version. \fI\%#2255\fP +.IP \(bu 2 +Updated \fBrequirementslib\fP to fix a bug in pipfile parsing affecting +relative path conversions. \fI\%#2269\fP +.IP \(bu 2 +Added custom shell detection library \fBshellingham\fP, a port of our changes +to \fBpew\fP\&. \fI\%#2363\fP +.IP \(bu 2 +Patched \fBpython\-dotenv\fP to ensure that environment variables always get +encoded to the filesystem encoding. \fI\%#2386\fP +.IP \(bu 2 +Updated vendored libraries. The following vendored libraries were updated: +.INDENT 2.0 +.IP \(bu 2 +distlib from version \fB0.2.6\fP to \fB0.2.7\fP\&. +.IP \(bu 2 +jinja2 from version \fB2.9.5\fP to \fB2.10\fP\&. +.IP \(bu 2 +pathlib2 from version \fB2.1.0\fP to \fB2.3.2\fP\&. +.IP \(bu 2 +parse from version \fB2.8.0\fP to \fB2.8.4\fP\&. +.IP \(bu 2 +pexpect from version \fB2.5.2\fP to \fB2.6.0\fP\&. +.IP \(bu 2 +requests from version \fB2.18.4\fP to \fB2.19.1\fP\&. +.IP \(bu 2 +idna from version \fB2.6\fP to \fB2.7\fP\&. +.IP \(bu 2 +certifi from version \fB2018.1.16\fP to \fB2018.4.16\fP\&. +.IP \(bu 2 +packaging from version \fB16.8\fP to \fB17.1\fP\&. +.IP \(bu 2 +six from version \fB1.10.0\fP to \fB1.11.0\fP\&. +.IP \(bu 2 +requirementslib from version \fB0.2.0\fP to \fB1.0.1\fP\&. +.UNINDENT +.sp +In addition, scandir was vendored and patched to avoid importing host system binaries when falling back to pathlib2. \fI\%#2368\fP +.UNINDENT .SH USER TESTIMONIALS .INDENT 0.0 .TP -\fBJannis Leidel\fP, former pip maintainer— -\fIPipenv is the porcelain I always wanted to build for pip9. It fits my brain and mostly replaces virtualenvwrapper and manual pip calls for me. Use it.\fP +\fBDavid Gang\fP— +\fIThis package manager is really awesome. For the first time I know exactly what my dependencies are which I installed and what the transitive dependencies are. Combined with the fact that installs are deterministic, makes this package manager first class, like cargo\fP\&. .TP \fBJustin Myles Holmes\fP— \fIPipenv is finally an abstraction meant to engage the mind instead of merely the filesystem.\fP -.TP -\fBIsaac Sanders\fP— -\fIPipenv is literally the best thing about my day today. Thanks, Kenneth!\fP .UNINDENT .SH ☤ PIPENV FEATURES .INDENT 0.0 @@ -448,11 +1212,11 @@ Otherwise, whatever virtualenv defaults to will be the default. .SS Other Commands .INDENT 0.0 .IP \(bu 2 -\fBgraph\fP will show you a dependency graph, of your installed dependencies. +\fBgraph\fP will show you a dependency graph of your installed dependencies. .IP \(bu 2 -\fBshell\fP will spawn a shell with the virtualenv activated. +\fBshell\fP will spawn a shell with the virtualenv activated. This shell can be deactivated by using \fBexit\fP\&. .IP \(bu 2 -\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP). +\fBrun\fP will run a given command from the virtualenv, with any arguments forwarded (e.g. \fB$ pipenv run python\fP or \fB$ pipenv run pip freeze\fP). .IP \(bu 2 \fBcheck\fP checks for security vulnerabilities and asserts that PEP 508 requirements are being met by the current environment. .UNINDENT @@ -577,6 +1341,89 @@ pytest = "*" .fi .UNINDENT .UNINDENT +.SS ☤ General Recommendations & Version Control +.INDENT 0.0 +.IP \(bu 2 +Generally, keep both \fBPipfile\fP and \fBPipfile.lock\fP in version control. +.IP \(bu 2 +Do not keep \fBPipfile.lock\fP in version control if multiple versions of Python are being targeted. +.IP \(bu 2 +Specify your target Python version in your \fIPipfile\fP\(aqs \fB[requires]\fP section. Ideally, you should only have one target Python version, as this is a deployment tool. +.IP \(bu 2 +\fBpipenv install\fP is fully compatible with \fBpip install\fP syntax, for which the full documentation can be found \fI\%here\fP\&. +.IP \(bu 2 +Note that the \fBPipfile\fP uses the \fI\%TOML Spec\fP\&. +.UNINDENT +.SS ☤ Example Pipenv Workflow +.sp +Clone / create project repository: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ cd myproject +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Install from Pipfile, if there is one: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Or, add a package to your new project: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install <package> +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will create a \fBPipfile\fP if one doesn\(aqt exist. If one does exist, it will automatically be edited with the new package you provided. +.sp +Next, activate the Pipenv shell: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv shell +$ python \-\-version +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will spawn a new shell subprocess, which can be deactivated by using \fBexit\fP\&. +.SS ☤ Example Pipenv Upgrade Workflow +.INDENT 0.0 +.IP \(bu 2 +Find out what\(aqs changed upstream: \fB$ pipenv update \-\-outdated\fP\&. +.IP \(bu 2 +.INDENT 2.0 +.TP +.B Upgrade packages, two options: +.INDENT 7.0 +.IP a. 3 +Want to upgrade everything? Just do \fB$ pipenv update\fP\&. +.IP b. 3 +Want to upgrade packages one\-at\-a\-time? \fB$ pipenv update <pkg>\fP for each outdated package. +.UNINDENT +.UNINDENT +.UNINDENT .SS ☤ Importing from requirements.txt .sp If you only have a \fBrequirements.txt\fP file available when running \fBpipenv install\fP, @@ -584,23 +1431,71 @@ pipenv will automatically import the contents of this file and create a \fBPipfi .sp You can also specify \fB$ pipenv install \-r path/to/requirements.txt\fP to import a requirements file. .sp -Note, that when importing a requirements file, they often have version numbers pinned, which you likely won\(aqt want -in your \fBPipfile\fP, so you\(aqll have to manually update your \fBPipfile\fP afterwards to reflect this. +If your requirements file has version numbers pinned, you\(aqll likely want to edit the new \fBPipfile\fP +to remove those, and let \fBpipenv\fP keep track of pinning. If you want to keep the pinned versions +in your \fBPipfile.lock\fP for now, run \fBpipenv lock \-\-keep\-outdated\fP\&. Make sure to +\fI\%upgrade\fP soon! .SS ☤ Specifying Versions of a Package .sp -To tell pipenv to install a specific version of a library, the usage is simple: +You can specify versions of a package using the \fI\%Semantic Versioning scheme\fP +(i.e. \fBmajor.minor.micro\fP). +.sp +For example, to install requests you can use: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install requests==2.13.0 +$ pipenv install requests~=1.2 # equivalent to requests~=1.2.0 .ft P .fi .UNINDENT .UNINDENT .sp +Pipenv will install version \fB1.2\fP and any minor update, but not \fB2.0\fP\&. +.sp This will update your \fBPipfile\fP to reflect this requirement, automatically. +.sp +In general, Pipenv uses the same specifier format as pip. However, note that according to \fI\%PEP 440\fP , you can\(aqt use versions containing a hyphen or a plus sign. +.sp +To make inclusive or exclusive version comparisons you can use: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests>=1.4" # will install a version equal or larger than 1.4.0 +$ pipenv install "requests<=2.13" # will install a version equal or lower than 2.13.0 +$ pipenv install "requests>2.19" # will install 2.19.1 but not 2.19.0 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The use of double quotes around the package and version specification (i.e. \fB"requests>2.19"\fP) is highly recommended +to avoid issues with \fI\%Input and output redirection\fP +in Unix\-based operating systems. +.UNINDENT +.UNINDENT +.sp +The use of \fB~=\fP is preferred over the \fB==\fP identifier as the latter prevents pipenv from updating the packages: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install "requests~=2.2" # locks the major version of the package (this is equivalent to using ==2.*) +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +To avoid installing a specific version you can use the \fB!=\fP identifier. +.sp +For an in depth explanation of the valid identifiers and more complex use cases check \fI\%the relevant section of PEP\-440\fP\&. .SS ☤ Specifying Versions of Python .sp To create a new virtualenv, using a specific version of Python you have installed (and @@ -665,11 +1560,16 @@ python_version = "3.6" .UNINDENT .UNINDENT .sp -Note the inclusion of \fB[requires] python_version = "3.6"\fP\&. This specifies that your application requires this version +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +The inclusion of \fB[requires] python_version = "3.6"\fP specifies that your application requires this version of Python, and will be used automatically when running \fBpipenv install\fP against this \fBPipfile\fP in the future (e.g. on other machines). If this is not true, feel free to simply remove this section. +.UNINDENT +.UNINDENT .sp -If you don\(aqt specify a Python version on the command-line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected +If you don\(aqt specify a Python version on the command–line, either the \fB[requires]\fP \fBpython_full_version\fP or \fBpython_version\fP will be selected automatically, falling back to whatever your system\(aqs default \fBpython\fP installation is, at time of execution. .SS ☤ Editable Dependencies (e.g. \fB\-e .\fP ) .sp @@ -680,17 +1580,25 @@ the current working directory when working on packages: .sp .nf .ft C -$ pipenv install \(aq\-e .\(aq \-\-dev +$ pipenv install \-\-dev \-e . $ cat Pipfile +\&... [dev\-packages] "e1839a8" = {path = ".", editable = true} +\&... .ft P .fi .UNINDENT .UNINDENT .sp -Note that all sub\-dependencies will get added to the \fBPipfile.lock\fP as well. +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +All sub\-dependencies will get added to the \fBPipfile.lock\fP as well. Sub\-dependencies are \fBnot\fP added to the +\fBPipfile.lock\fP if you leave the \fB\-e\fP option out. +.UNINDENT +.UNINDENT .SS ☤ Environment Management with Pipenv .sp The three primary commands you\(aqll use in managing your pipenv environment are @@ -745,7 +1653,7 @@ is unique. .UNINDENT .INDENT 0.0 .IP \(bu 2 -\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile.lock\fP\&. +\fB\-\-dev\fP — Install both \fBdevelop\fP and \fBdefault\fP packages from \fBPipfile\fP\&. .IP \(bu 2 \fB\-\-system\fP — Use the system \fBpip\fP command rather than the one from your virtualenv. .IP \(bu 2 @@ -758,13 +1666,16 @@ is unique. .SS $ pipenv uninstall .sp \fB$ pipenv uninstall\fP supports all of the parameters in \fI\%pipenv install\fP, -as well as one additional, \fB\-\-all\fP\&. +as well as two additional options, \fB\-\-all\fP and \fB\-\-all\-dev\fP\&. .INDENT 0.0 .INDENT 3.5 .INDENT 0.0 .IP \(bu 2 \fB\-\-all\fP — This parameter will purge all files from the virtual environment, but leave the Pipfile untouched. +.IP \(bu 2 +\fB\-\-all\-dev\fP — This parameter will remove all of the development packages from +the virtual environment, and remove them from the Pipfile. .UNINDENT .UNINDENT .UNINDENT @@ -797,22 +1708,50 @@ You should do this for your shell too, in your \fB~/.profile\fP or \fB~/.bashrc\ The shell launched in interactive mode. This means that if your shell reads its configuration from a specific file for interactive mode (e.g. bash by default looks for a \fB~/.bashrc\fP configuration file for interactive mode), then you\(aqll need to modify (or create) this file. .UNINDENT .UNINDENT +.sp +If you experience issues with \fB$ pipenv shell\fP, just check the \fBPIPENV_SHELL\fP environment variable, which \fB$ pipenv shell\fP will use if available. For detail, see configuration\-with\-environment\-variables\&. .SS ☤ A Note about VCS Dependencies .sp -Pipenv will resolve the sub-dependencies of VCS dependencies, but only if they are editable, like so: +You can install packages with pipenv from git and other version control systems using URLs formatted according to the following rule: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -[packages] -requests = {git = "https://github.com/requests/requests.git", editable=true} +<vcs_type>+<scheme>://<location>/<user_or_organization>/<repository>@<branch_or_tag>#egg=<package_name> .ft P .fi .UNINDENT .UNINDENT .sp -If editable is not true, sub-dependencies will not get resolved. +The only optional section is the \fB@<branch_or_tag>\fP section. When using git over SSH, you may use the shorthand vcs and scheme alias \fBgit+git@<location>:<user_or_organization>/<repository>@<branch_or_tag>#<package_name>\fP\&. Note that this is translated to \fBgit+ssh://git@<location>\fP when parsed. +.sp +Note that it is \fBstrongly recommended\fP that you install any version\-controlled dependencies in editable mode, using \fBpipenv install \-e\fP, in order to ensure that dependency resolution can be performed with an up to date copy of the repository each time it is performed, and that it includes all known dependencies. +.sp +Below is an example usage which installs the git repository located at \fBhttps://github.com/requests/requests.git\fP from tag \fBv2.20.1\fP as package name \fBrequests\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests +Creating a Pipfile for this project... +Installing \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... +[...snipped...] +Adding \-e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile\(aqs [packages]... +[...] + +$ cat Pipfile +[packages] +requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Valid values for \fB<vcs_type>\fP include \fBgit\fP, \fBbzr\fP, \fBsvn\fP, and \fBhg\fP\&. Valid values for \fB<scheme>\fP include \fBhttp\fP, \fBhttps\fP, \fBssh\fP, and \fBfile\fP\&. In specific cases you also have access to other schemes: \fBsvn\fP may be combined with \fBsvn\fP as a scheme, and \fBbzr\fP can be combined with \fBsftp\fP and \fBlp\fP\&. +.sp +You can read more about pip\(aqs implementation of VCS support \fI\%here\fP\&. For more information about other options available when specifying VCS dependencies, please check the \fI\%Pipfile spec\fP\&. .SS ☤ Pipfile.lock Security Features .sp \fBPipfile.lock\fP takes advantage of some great new security improvements in \fBpip\fP\&. @@ -824,10 +1763,28 @@ We highly recommend approaching deployments with promoting projects from a devel environment into production. You can use \fBpipenv lock\fP to compile your dependencies on your development environment and deploy the compiled \fBPipfile.lock\fP to all of your production environments for reproducible builds. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +If you\(aqd like a \fBrequirements.txt\fP output of the lockfile, run \fB$ pipenv lock \-r\fP\&. +This will include all hashes, however (which is great!). To get a \fBrequirements.txt\fP +without hashes, use \fB$ pipenv run pip freeze\fP\&. +.UNINDENT +.UNINDENT .SS Advanced Usage of Pipenv [image] .sp This document covers some of Pipenv\(aqs more glorious and advanced features. +.SS ☤ Caveats +.INDENT 0.0 +.IP \(bu 2 +Dependencies of wheels provided in a \fBPipfile\fP will not be captured by \fB$ pipenv lock\fP\&. +.IP \(bu 2 +There are some known issues with using private indexes, related to hashing. We\(aqre actively working to solve this problem. You may have great luck with this, however. +.IP \(bu 2 +Installation is intended to be as deterministic as possible —\ use the \fB\-\-sequential\fP flag to increase this, if experiencing issues. +.UNINDENT .SS ☤ Specifying Package Indexes .sp If you\(aqd like a specific package to be installed with a specific package index, you can do the following: @@ -858,6 +1815,63 @@ records = "*" .UNINDENT .sp Very fancy. +.SS ☤ Using a PyPI Mirror +.sp +If you\(aqd like to override the default PyPI index urls with the url for a PyPI mirror, you can use the following: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-pypi\-mirror <mirror_url> + +$ pipenv update \-\-pypi\-mirror <mirror_url> + +$ pipenv sync \-\-pypi\-mirror <mirror_url> + +$ pipenv lock \-\-pypi\-mirror <mirror_url> + +$ pipenv uninstall \-\-pypi\-mirror <mirror_url> +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Alternatively, you can set the \fBPIPENV_PYPI_MIRROR\fP environment variable. +.SS ☤ Injecting credentials into Pipfiles via environment variables +.sp +Pipenv will expand environment variables (if defined) in your Pipfile. Quite +useful if you need to authenticate to a private PyPI: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://$USERNAME:${PASSWORD}@mypypi.example.com/simple" +verify_ssl = true +name = "pypi" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Luckily \- pipenv will hash your Pipfile \fIbefore\fP expanding environment +variables (and, helpfully, will substitute the environment variables again when +you install from the lock file \- so no need to commit any secrets! Woo!) +.sp +If your credentials contain a special character, surround the references to the environment variables with quotation marks. For example, if your password contain a double quotation mark, surround the password variable with single quotation marks. Otherwise, you may get a \fBValueError, "No closing quotation"\fP error while installing dependencies. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://$USERNAME:\(aq${PASSWORD}\(aq@mypypi.example.com/simple" +.ft P +.fi +.UNINDENT +.UNINDENT .SS ☤ Specifying Basically Anything .sp If you\(aqd like to specify that a specific package only be installed on certain systems, @@ -876,7 +1890,7 @@ name = "pypi" [packages] requests = "*" -pywinusb = {version = "*", os_name = "== \(aqwindows\(aq"} +pywinusb = {version = "*", sys_platform = "== \(aqwin32\(aq"} .ft P .fi .UNINDENT @@ -902,9 +1916,45 @@ unittest2 = {version = ">=1.0,<3.0", markers="python_version < \(aq2.7.9\(aq or .UNINDENT .sp Magic. Pure, unadulterated magic. -.SS ☤ Deploying System Dependencies +.SS ☤ Using pipenv for Deployments .sp -You can tell Pipenv to install things into its parent system with the \fB\-\-system\fP flag: +You may want to use \fBpipenv\fP as part of a deployment process. +.sp +You can enforce that your \fBPipfile.lock\fP is up to date using the \fB\-\-deploy\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-\-deploy +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will fail a build if the \fBPipfile.lock\fP is out–of–date, instead of generating a new one. +.sp +Or you can install packages exactly as specified in \fBPipfile.lock\fP using the \fBsync\fP command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv sync +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBpipenv install \-\-ignore\-pipfile\fP is nearly equivalent to \fBpipenv sync\fP, but \fBpipenv sync\fP will \fInever\fP attempt to re\-lock your dependencies as it is considered an atomic operation. \fBpipenv install\fP by default does attempt to re\-lock unless using the \fB\-\-deploy\fP flag. +.UNINDENT +.UNINDENT +.SS Deploying System Dependencies +.sp +You can tell Pipenv to install a Pipfile\(aqs contents into its parent system with the \fB\-\-system\fP flag: .INDENT 0.0 .INDENT 3.5 .sp @@ -916,24 +1966,35 @@ $ pipenv install \-\-system .UNINDENT .UNINDENT .sp -This is useful for Docker containers, and deployment infrastructure (e.g. Heroku does this). +This is useful for managing the system Python, and deployment infrastructure (e.g. Heroku does this). +.SS ☤ Pipenv and Other Python Distributions .sp -Also useful for deployment is the \fB\-\-deploy\fP flag: +To use Pipenv with a third\-party Python distribution (e.g. Anaconda), you simply provide the path to the Python binary: .INDENT 0.0 .INDENT 3.5 .sp .nf .ft C -$ pipenv install \-\-system \-\-deploy +$ pipenv install \-\-python=/path/to/python .ft P .fi .UNINDENT .UNINDENT .sp -This will fail a build if the \fBPipfile.lock\fP is out-of-date, instead of generating a new one. +Anaconda uses Conda to manage packages. To reuse Conda–installed Python packages, use the \fB\-\-site\-packages\fP flag: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv \-\-python=/path/to/python \-\-site\-packages +.ft P +.fi +.UNINDENT +.UNINDENT .SS ☤ Generating a \fBrequirements.txt\fP .sp -You can convert a \fBPipfile\fP and \fBPipenv.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of hashes, extras, and other goodies we have included. +You can convert a \fBPipfile\fP and \fBPipfile.lock\fP into a \fBrequirements.txt\fP file very easily, and get all the benefits of extras and other goodies we have included. .sp Let\(aqs take this \fBPipfile\fP: .INDENT 0.0 @@ -959,11 +2020,42 @@ And generate a \fBrequirements.txt\fP out of it: .nf .ft C $ pipenv lock \-r -chardet==3.0.4 \-\-hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691 \-\-hash=sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae -requests==2.18.4 \-\-hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b \-\-hash=sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e -certifi==2017.7.27.1 \-\-hash=sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704 \-\-hash=sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5 -idna==2.6 \-\-hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 \-\-hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f -urllib3==1.22 \-\-hash=sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b \-\-hash=sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f +chardet==3.0.4 +requests==2.18.4 +certifi==2017.7.27.1 +idna==2.6 +urllib3==1.22 +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +If you wish to generate a \fBrequirements.txt\fP with only the development requirements you can do that too! Let\(aqs take the following \fBPipfile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true + +[dev\-packages] +pytest = {version="*"} +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And generate a \fBrequirements.txt\fP out of it: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv lock \-r \-\-dev +py==1.4.34 +pytest==3.2.3 .ft P .fi .UNINDENT @@ -1025,26 +2117,50 @@ hardened for production use and should be used only as a development aid. .UNINDENT .sp ✨🍰✨ -.SS ☤ Code Style Checking .sp -Pipenv has \fI\%Flake 8\fP built into it. You can check the style of your code like so, without installing anything: +\fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 +In order to enable this functionality while maintaining its permissive +copyright license, \fIpipenv\fP embeds an API client key for the backend +Safety API operated by pyup.io rather than including a full copy of the +CC\-BY\-NC\-SA licensed Safety\-DB database. This embedded client key is +shared across all \fIpipenv check\fP users, and hence will be subject to +API access throttling based on overall usage rather than individual +client usage. .sp -.nf -.ft C -$ cat t.py -import requests - -$ pipenv check \-\-style t.py -t.py:1:1: F401 \(aqrequests\(aq imported but unused -t.py:1:16: W292 no newline at end of file -.ft P -.fi +You can also use your own safety API key by setting the +environment variable \fBPIPENV_PYUP_API_KEY\fP\&. .UNINDENT .UNINDENT +.SS ☤ Community Integrations .sp -Super useful :) +There are a range of community\-maintained plugins and extensions available for a range of editors and IDEs, as well as +different products which integrate with Pipenv projects: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Heroku\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%Platform.sh\fP (Cloud Hosting) +.IP \(bu 2 +\fI\%PyUp\fP (Security Notification) +.IP \(bu 2 +\fI\%Emacs\fP (Editor Integration) +.IP \(bu 2 +\fI\%Fish Shell\fP (Automatic \fB$ pipenv shell\fP!) +.IP \(bu 2 +\fI\%VS Code\fP (Editor Integration) +.IP \(bu 2 +\fI\%PyCharm\fP (Editor Integration) +.UNINDENT +.sp +Works in progress: +.INDENT 0.0 +.IP \(bu 2 +\fI\%Sublime Text\fP (Editor Integration) +.IP \(bu 2 +Mysterious upcoming Google Cloud product (Cloud Hosting) +.UNINDENT .SS ☤ Open a Module in Your Editor .sp Pipenv allows you to open any Python module that is installed (including ones in your codebase), with the \fB$ pipenv open\fP command: @@ -1070,7 +2186,7 @@ This allows you to easily read the code you\(aqre consuming, instead of looking \fBNOTE:\fP .INDENT 0.0 .INDENT 3.5 -The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using Sublime Text, for example, you\(aqll want to \fBexport EDITOR=subl\fP (once you\(aqve installed the command\-line utility). +The standard \fBEDITOR\fP environment variable is used for this. If you\(aqre using VS Code, for example, you\(aqll want to \fBexport EDITOR=code\fP (if you\(aqre on macOS you will want to \fI\%install the command\fP on to your \fBPATH\fP first). .UNINDENT .UNINDENT .SS ☤ Automatic Python Installation @@ -1157,38 +2273,327 @@ $ PIPENV_DOTENV_LOCATION=/path/to/.env pipenv shell .fi .UNINDENT .UNINDENT +.sp +To prevent pipenv from loading the \fB\&.env\fP file, set the \fBPIPENV_DONT_LOAD_ENV\fP environment variable: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ PIPENV_DONT_LOAD_ENV=1 pipenv shell +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Custom Script Shortcuts +.sp +Pipenv supports creating custom shortcuts in the (optional) \fB[scripts]\fP section of your Pipfile. +.sp +You can then run \fBpipenv run <shortcut name>\fP in your terminal to run the command in the +context of your pipenv virtual environment even if you have not activated the pipenv shell first. +.sp +For example, in your Pipfile: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[scripts] +printspam = "python \-c \e"print(\(aqI am a silly example, no one would need to do this\(aq)\e"" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +And then in your terminal: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv run printspam +I am a silly example, no one would need to do this +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Commands that expect arguments will also work. +For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv run echospam "indeed" +I am really a very silly example indeed +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ Support for Environment Variables +.sp +Pipenv supports the usage of environment variables in place of authentication fragments +in your Pipfile. These will only be parsed if they are present in the \fB[[source]]\fP +section. For example: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[[source]] +url = "https://${PYPI_USERNAME}:${PYPI_PASSWORD}@my_private_repo.example.com/simple" +verify_ssl = true +name = "pypi" + +[dev\-packages] + +[packages] +requests = {version="*", index="home"} +maya = {version="*", index="pypi"} +records = "*" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +Environment variables may be specified as \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. +.sp +On Windows, \fB%MY_ENVAR%\fP is supported in addition to \fB${MY_ENVAR}\fP or \fB$MY_ENVAR\fP\&. .SS ☤ Configuration With Environment Variables .sp -\fBpipenv\fP comes with a handful of options that can be enabled via shell environment +Pipenv comes with a handful of options that can be enabled via shell environment variables. To activate them, simply create the variable in your shell and pipenv will detect it. .INDENT 0.0 -.INDENT 3.5 +.TP +.B pipenv.environments.PIPENV_CACHE_DIR = \(aq/Users/fming/Library/Caches/pipenv\(aq +Location for Pipenv to store it\(aqs package cache. +.sp +Default is to use appdir\(aqs user cache directory. +.UNINDENT .INDENT 0.0 -.IP \(bu 2 -\fBPIPENV_DEFAULT_PYTHON_VERSION\fP — Use this version of Python when creating new virtual environments, by default (e.g. \fB3.6\fP). -.IP \(bu 2 -\fBPIPENV_SHELL_FANCY\fP — Always use fancy mode when invoking \fBpipenv shell\fP\&. -.IP \(bu 2 -\fBPIPENV_VENV_IN_PROJECT\fP — If set, use \fB\&.venv\fP in your project directory -instead of the global location. -.IP \(bu 2 -\fBPIPENV_COLORBLIND\fP — Disable terminal colors, for some reason. -.IP \(bu 2 -\fBPIPENV_NOSPIN\fP — Disable terminal spinner, for cleaner logs. Automatically set in CI environments. -.IP \(bu 2 -\fBPIPENV_MAX_DEPTH\fP — Set to an integer for the maximum number of directories to recursively -search for a Pipfile. -.IP \(bu 2 -\fBPIPENV_TIMEOUT\fP — Set to an integer for the max number of seconds Pipenv will -wait for virtualenv creation to complete. Defaults to 120 seconds. -.IP \(bu 2 -\fBPIPENV_IGNORE_VIRTUALENVS\fP — Set to disable automatically using an activated virtualenv over -the current project\(aqs own virtual environment. -.UNINDENT +.TP +.B pipenv.environments.PIPENV_COLORBLIND = False +If set, disable terminal colors. +.sp +Some people don\(aqt like colors in their terminals, for some reason. Default is +to show colors. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DEFAULT_PYTHON_VERSION = None +Use this Python version when creating new virtual environments by default. +.sp +This can be set to a version string, e.g. \fB3.6\fP, or a path. Default is to use +whatever Python Pipenv is installed under (i.e. \fBsys.executable\fP). Command +line flags (e.g. \fB\-\-python\fP, \fB\-\-three\fP, and \fB\-\-two\fP) are prioritized over +this configuration. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_LOAD_ENV = False +If set, Pipenv does not load the \fB\&.env\fP file. +.sp +Default is to load \fB\&.env\fP for \fBrun\fP and \fBshell\fP commands. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DONT_USE_PYENV = False +If set, Pipenv does not attempt to install Python with pyenv. +.sp +Default is to install Python automatically via pyenv when needed, if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_DOTENV_LOCATION = None +If set, Pipenv loads the \fB\&.env\fP file at the specified location. +.sp +Default is to load \fB\&.env\fP from the project root, if found. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_EMULATOR = \(aq\(aq +If set, the terminal emulator\(aqs name for \fBpipenv shell\fP to use. +.sp +Default is to detect emulators automatically. This should be set if your +emulator, e.g. Cmder, cannot be detected correctly. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_HIDE_EMOJIS = False +Disable emojis in output. +.sp +Default is to show emojis. This is automatically set on Windows. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_IGNORE_VIRTUALENVS = False +If set, Pipenv will always assign a virtual environment for this project. +.sp +By default, Pipenv tries to detect whether it is run inside a virtual +environment, and reuses it if possible. This is usually the desired behavior, +and enables the user to use any user\-built environments with Pipenv. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_INSTALL_TIMEOUT = 900 +Max number of seconds to wait for package installation. +.sp +Defaults to 900 (15 minutes), a very long arbitrary time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_DEPTH = 4 +Maximum number of directories to recursively search for a Pipfile. +.sp +Default is 3. See also \fBPIPENV_NO_INHERIT\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_RETRIES = 0 +Specify how many retries Pipenv should attempt for network requests. +.sp +Default is 0. Automatically set to 1 on CI environments for robust testing. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_ROUNDS = 16 +Tells Pipenv how many rounds of resolving to do for Pip\-Tools. +.sp +Default is 16, an arbitrary number that works most of the time. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_MAX_SUBPROCESS = 8 +How many subprocesses should Pipenv use when installing. +.sp +Default is 16, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NOSPIN = False +If set, disable terminal spinner. +.sp +This can make the logs cleaner. Automatically set on Windows, and in CI +environments. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_NO_INHERIT = False +Tell Pipenv not to inherit parent directories. +.sp +This is useful for deployment to avoid using the wrong current directory. +Overwrites \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PIPFILE = None +If set, this specifies a custom Pipfile location. +.sp +When running pipenv from a location other than the same directory where the +Pipfile is located, instruct pipenv to find the Pipfile in the location +specified by this environment variable. +.sp +Default is to find Pipfile automatically in the current and parent directories. +See also \fBPIPENV_MAX_DEPTH\fP\&. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_PYPI_MIRROR = \(aqhttps://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple\(aq +If set, tells pipenv to override PyPI index urls with a mirror. +.sp +Default is to not mirror PyPI, i.e. use the real one, pypi.org. The +\fB\-\-pypi\-mirror\fP command line flag overwrites this. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_RESOLVE_VCS = False +Tells Pipenv whether to resolve all VCS dependencies in full. +.sp +As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full. +To retain this behavior and avoid handling any conflicts that arise from the new +approach, you may set this to \(aq0\(aq, \(aqoff\(aq, or \(aqfalse\(aq. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL = \(aq/bin/zsh\(aq +An absolute path to the preferred shell for \fBpipenv shell\fP\&. +.sp +Default is to detect automatically what shell is currently in use. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SHELL_FANCY = False +If set, always use fancy mode when invoking \fBpipenv shell\fP\&. +.sp +Default is to use the compatibility shell if possible. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SKIP_LOCK = False +If set, Pipenv won\(aqt lock dependencies automatically. +.sp +This might be desirable if a project has large number of dependencies, +because locking is an inherently slow operation. +.sp +Default is to lock dependencies and update \fBPipfile.lock\fP on each run. +.sp +NOTE: This only affects the \fBinstall\fP and \fBuninstall\fP commands. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_SPINNER = \(aqdots\(aq +Sets the default spinner type. +.sp +Spinners are identitcal to the node.js spinners and can be found at +\fI\%https://github.com/sindresorhus/cli\-spinners\fP +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_TIMEOUT = 120 +Max number of seconds Pipenv will wait for virtualenv creation to complete. +.sp +Default is 120 seconds, an arbitrary number that seems to work. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_VENV_IN_PROJECT = False +If set, creates \fB\&.venv\fP in your project directory. +.sp +Default is to create new virtual environments in a global location. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIPENV_YES = False +If set, Pipenv automatically assumes "yes" at all prompts. +.sp +Default is to prompt the user for an answer if the current command line session +if interactive. +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.PIP_EXISTS_ACTION = \(aqw\(aq +Specifies the value for pip\(aqs \-\-exists\-action option +.sp +Defaullts to (w)ipe +.UNINDENT +.INDENT 0.0 +.TP +.B pipenv.environments.is_in_virtualenv() +Check virtualenv membership dynamically +.INDENT 7.0 +.TP +.B Returns +True or false depending on whether we are in a regular virtualenv or not +.TP +.B Return type +bool .UNINDENT .UNINDENT .sp +If you\(aqd like to set these environment variables on a per\-project basis, I recommend utilizing the fantastic \fI\%direnv\fP project, in order to do so. +.sp Also note that \fI\%pip itself supports environment variables\fP, if you need additional customization. .sp For example: @@ -1204,8 +2609,9 @@ $ PIP_INSTALL_OPTION="\-\- \-DCMAKE_BUILD_TYPE=Release" pipenv install \-e . .UNINDENT .SS ☤ Custom Virtual Environment Location .sp -Pipenv will automatically honor the \fBWORKON_HOME\fP environment -variable, if you have it set —\ so you can tell pipenv to store your virtual environments wherever you want, e.g.: +Pipenv automatically honors the \fBWORKON_HOME\fP environment variable, if you +have it set —\ so you can tell pipenv to store your virtual environments +wherever you want, e.g.: .INDENT 0.0 .INDENT 3.5 .sp @@ -1241,7 +2647,7 @@ python: \- "3.4" \- "3.5" \- "3.6" - \- "3.7dev" + \- "3.7\-dev" # command to install dependencies install: "make" @@ -1265,7 +2671,7 @@ init: pipenv install \-\-dev test: - pipenv run py.test tests + pipenv run pytest tests .ft P .fi .UNINDENT @@ -1286,12 +2692,11 @@ envlist = flake8\-py3, py26, py27, py33, py34, py35, py36, pypy deps = pipenv commands= pipenv install \-\-dev - pipenv run py.test tests + pipenv run pytest tests [testenv:flake8\-py3] basepython = python3.4 commands= - {[testenv]deps} pipenv install \-\-dev pipenv run flake8 \-\-version pipenv run flake8 setup.py docs project test @@ -1299,6 +2704,22 @@ commands= .fi .UNINDENT .UNINDENT +.sp +Pipenv will automatically use the virtualenv provided by \fBtox\fP\&. If \fBpipenv install \-\-dev\fP installs e.g. \fBpytest\fP, then installed command \fBpytest\fP will be present in given virtualenv and can be called directly by \fBpytest tests\fP instead of \fBpipenv run pytest tests\fP\&. +.sp +You might also want to add \fB\-\-ignore\-pipfile\fP to \fBpipenv install\fP, as to +not accidentally modify the lock\-file on each test run. This causes Pipenv +to ignore changes to the \fBPipfile\fP and (more importantly) prevents it from +adding the current environment to \fBPipfile.lock\fP\&. This might be important as +the current environment (i.e. the virtualenv provisioned by tox) will usually +contain the current project (which may or may not be desired) and additional +dependencies from \fBtox\fP\(aqs \fBdeps\fP directive. The initial provisioning may +alternatively be disabled by adding \fBskip_install = True\fP to tox.ini. +.sp +This method requires you to be explicit about updating the lock\-file, which is +probably a good idea in any case. +.sp +A 3rd party plugin, \fI\%tox\-pipenv\fP is also available to use Pipenv natively with tox. .SS ☤ Shell Completion .sp To enable completion in fish, add this to your config: @@ -1313,6 +2734,18 @@ eval (pipenv \-\-completion) .UNINDENT .UNINDENT .sp +Alternatively, with bash or zsh, add this to your config: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +eval "$(pipenv \-\-completion)" +.ft P +.fi +.UNINDENT +.UNINDENT +.sp Magic shell completions are now enabled! .sp ✨🍰✨ @@ -1348,7 +2781,1335 @@ $ PIP_IGNORE_INSTALLED=1 pipenv install \-\-dev .fi .UNINDENT .UNINDENT +.SS ☤ Pipfile vs setup.py +.sp +There is a subtle but very important distinction to be made between \fBapplications\fP and \fBlibraries\fP\&. This is a very common source of confusion in the Python community. +.sp +Libraries provide reusable functionality to other libraries and applications (let\(aqs use the umbrella term \fBprojects\fP here). They are required to work alongside other libraries, all with their own set of subdependencies. They define \fBabstract dependencies\fP\&. To avoid version conflicts in subdependencies of different libraries within a project, libraries should never ever pin dependency versions. Although they may specify lower or (less frequently) upper bounds, if they rely on some specific feature/fix/bug. Library dependencies are specified via \fBinstall_requires\fP in \fBsetup.py\fP\&. +.sp +Libraries are ultimately meant to be used in some \fBapplication\fP\&. Applications are different in that they usually are not depended on by other projects. They are meant to be deployed into some specific environment and only then should the exact versions of all their dependencies and subdependencies be made concrete. To make this process easier is currently the main goal of Pipenv. +.sp +To summarize: +.INDENT 0.0 +.IP \(bu 2 +For libraries, define \fBabstract dependencies\fP via \fBinstall_requires\fP in \fBsetup.py\fP\&. The decision of which version exactly to be installed and where to obtain that dependency is not yours to make! +.IP \(bu 2 +For applications, define \fBdependencies and where to get them\fP in the \fIPipfile\fP and use this file to update the set of \fBconcrete dependencies\fP in \fBPipfile.lock\fP\&. This file defines a specific idempotent environment that is known to work for your project. The \fBPipfile.lock\fP is your source of truth. The \fBPipfile\fP is a convenience for you to create that lock\-file, in that it allows you to still remain somewhat vague about the exact version of a dependency to be used. Pipenv is there to help you define a working conflict\-free set of specific dependency\-versions, which would otherwise be a very tedious task. +.IP \(bu 2 +Of course, \fBPipfile\fP and Pipenv are still useful for library developers, as they can be used to define a development or test environment. +.IP \(bu 2 +And, of course, there are projects for which the distinction between library and application isn\(aqt that clear. In that case, use \fBinstall_requires\fP alongside Pipenv and \fBPipfile\fP\&. +.UNINDENT +.sp +You can also do this: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ pipenv install \-e . +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will tell Pipenv to lock all your \fBsetup.py\fP–declared dependencies. +.SS ☤ Changing Pipenv\(aqs Cache Location +.sp +You can force Pipenv to use a different cache location by setting the environment variable \fBPIPENV_CACHE_DIR\fP to the location you wish. This is useful in the same situations that you would change \fBPIP_CACHE_DIR\fP to a different directory. +.SS ☤ Changing Default Python Versions +.sp +By default, Pipenv will initialize a project using whatever version of python the python3 is. Besides starting a project with the \fB\-\-three\fP or \fB\-\-two\fP flags, you can also use \fBPIPENV_DEFAULT_PYTHON_VERSION\fP to specify what version to use when starting a project when \fB\-\-three\fP or \fB\-\-two\fP aren\(aqt used. +.SS Frequently Encountered Pipenv Problems +.sp +Pipenv is constantly being improved by volunteers, but is still a very young +project with limited resources, and has some quirks that needs to be dealt +with. We need everyone’s help (including yours!). +.sp +Here are some common questions people have using Pipenv. Please take a look +below and see if they resolve your problem. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +\fBMake sure you’re running the newest Pipenv version first!\fP +.UNINDENT +.UNINDENT +.SS ☤ Your dependencies could not be resolved +.sp +Make sure your dependencies actually \fIdo\fP resolve. If you’re confident they +are, you may need to clear your resolver cache. Run the following command: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock \-\-clear +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +and try again. +.sp +If this does not work, try manually deleting the whole cache directory. It is +usually one of the following locations: +.INDENT 0.0 +.IP \(bu 2 +\fB~/Library/Caches/pipenv\fP (macOS) +.IP \(bu 2 +\fB%LOCALAPPDATA%\epipenv\epipenv\eCache\fP (Windows) +.IP \(bu 2 +\fB~/.cache/pipenv\fP (other operating systems) +.UNINDENT +.sp +Pipenv does not install prereleases (i.e. a version with an alpha/beta/etc. +suffix, such as \fI1.0b1\fP) by default. You will need to pass the \fB\-\-pre\fP flag +in your command, or set +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[pipenv] +allow_prereleases = true +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +in your Pipfile. +.SS ☤ No module named <module name> +.sp +This is usually a result of mixing Pipenv with system packages. We \fIstrongly\fP +recommend installing Pipenv in an isolated environment. Uninstall all existing +Pipenv installations, and see installing\-pipenv to choose one of the +recommended way to install Pipenv instead. +.SS ☤ My pyenv\-installed Python is not found +.sp +Make sure you have \fBPYENV_ROOT\fP set correctly. Pipenv only supports CPython +distributions, with version name like \fB3.6.4\fP or similar. +.SS ☤ Pipenv does not respect pyenv’s global and local Python versions +.sp +Pipenv by default uses the Python it is installed against to create the +virtualenv. You can set the \fB\-\-python\fP option, or +\fB$PYENV_ROOT/shims/python\fP to let it consult pyenv when choosing the +interpreter. See specifying_versions for more information. +.sp +If you want Pipenv to automatically “do the right thing”, you can set the +environment variable \fBPIPENV_PYTHON\fP to \fB$PYENV_ROOT/shims/python\fP\&. This +will make Pipenv use pyenv’s active Python version to create virtual +environments by default. +.SS ☤ ValueError: unknown locale: UTF\-8 +.sp +macOS has a bug in its locale detection that prevents us from detecting your +shell encoding correctly. This can also be an issue on other systems if the +locale variables do not specify an encoding. +.sp +The workaround is to set the following two environment variables to a standard +localization format: +.INDENT 0.0 +.IP \(bu 2 +\fBLC_ALL\fP +.IP \(bu 2 +\fBLANG\fP +.UNINDENT +.sp +For Bash, for example, you can add the following to your \fB~/.bash_profile\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +export LC_ALL=\(aqen_US.UTF\-8\(aq +export LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For Zsh, the file to edit is \fB~/.zshrc\fP\&. +.sp +\fBNOTE:\fP +.INDENT 0.0 +.INDENT 3.5 +You can change both the \fBen_US\fP and \fBUTF\-8\fP part to the +language/locale and encoding you use. +.UNINDENT +.UNINDENT +.SS ☤ /bin/pip: No such file or directory +.sp +This may be related to your locale setting. See \fI\%☤ ValueError: unknown locale: UTF\-8\fP +for a possible solution. +.SS ☤ \fBshell\fP does not show the virtualenv’s name in prompt +.sp +This is intentional. You can do it yourself with either shell plugins, or +clever \fBPS1\fP configuration. If you really want it back, use +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell \-c +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +instead (not available on Windows). +.SS ☤ Pipenv does not respect dependencies in setup.py +.sp +No, it does not, intentionally. Pipfile and setup.py serve different purposes, +and should not consider each other by default. See pipfile\-vs\-setuppy +for more information. +.SS ☤ Using \fBpipenv run\fP in Supervisor program +.sp +When you configure a supervisor program\(aqs \fBcommand\fP with \fBpipenv run ...\fP, you +need to set locale enviroment variables properly to make it work. +.sp +Add this line under \fB[supervisord]\fP section in \fB/etc/supervisor/supervisord.conf\fP: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +[supervisord] +environment=LC_ALL=\(aqen_US.UTF\-8\(aq,LANG=\(aqen_US.UTF\-8\(aq +.ft P +.fi +.UNINDENT +.UNINDENT +.SS ☤ An exception is raised during \fBLocking dependencies…\fP +.sp +Run \fBpipenv lock \-\-clear\fP and try again. The lock sequence caches results +to speed up subsequent runs. The cache may contain faulty results if a bug +causes the format to corrupt, even after the bug is fixed. \fB\-\-clear\fP flushes +the cache, and therefore removes the bad results. +.SH CONTRIBUTION GUIDES +.SS Development Philosophy +.sp +Pipenv is an open but opinionated tool, created by an open but opinionated developer. +.SS Management Style +.sp +\fI\%Kenneth Reitz\fP is the BDFL. He has final say in any decision related to the Pipenv project. Kenneth is responsible for the direction and form of the library, as well as its presentation. In addition to making decisions based on technical merit, he is responsible for making decisions based on the development philosophy of Pipenv. +.sp +\fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, and \fI\%Nate Prewitt\fP are the core contributors. +They are responsible for triaging bug reports, reviewing pull requests and ensuring that Kenneth is kept up to speed with developments around the library. +The day\-to\-day managing of the project is done by the core contributors. They are responsible for making judgements about whether or not a feature request is +likely to be accepted by Kenneth. +.SS Values +.INDENT 0.0 +.IP \(bu 2 +Simplicity is always better than functionality. +.IP \(bu 2 +Listen to everyone, then disregard it. +.IP \(bu 2 +The API is all that matters. Everything else is secondary. +.IP \(bu 2 +Fit the 90% use\-case. Ignore the nay\-sayers. +.UNINDENT +.SS Contributing to Pipenv +.sp +If you\(aqre reading this, you\(aqre probably interested in contributing to Pipenv. +Thank you very much! Open source projects live\-and\-die based on the support +they receive from others, and the fact that you\(aqre even considering +contributing to the Pipenv project is \fIvery\fP generous of you. +.sp +This document lays out guidelines and advice for contributing to this project. +If you\(aqre thinking of contributing, please start by reading this document and +getting a feel for how contributing to this project works. If you have any +questions, feel free to reach out to either \fI\%Dan Ryan\fP, \fI\%Tzu\-ping Chung\fP, +or \fI\%Nate Prewitt\fP, the primary maintainers. +.sp +The guide is split into sections based on the type of contribution you\(aqre +thinking of making, with a section that covers general guidelines for all +contributors. +.SS General Guidelines +.SS Be Cordial +.INDENT 0.0 +.INDENT 3.5 +\fBBe cordial or be on your way\fP\&. \fI—Kenneth Reitz\fP +.UNINDENT +.UNINDENT +.sp +Pipenv has one very important rule governing all forms of contribution, +including reporting bugs or requesting features. This golden rule is +"\fI\%be cordial or be on your way\fP". +.sp +\fBAll contributions are welcome\fP, as long as +everyone involved is treated with respect. +.SS Get Early Feedback +.sp +If you are contributing, do not feel the need to sit on your contribution until +it is perfectly polished and complete. It helps everyone involved for you to +seek feedback as early as you possibly can. Submitting an early, unfinished +version of your contribution for feedback in no way prejudices your chances of +getting that contribution accepted, and can save you from putting a lot of work +into a contribution that is not suitable for the project. +.SS Contribution Suitability +.sp +Our project maintainers have the last word on whether or not a contribution is +suitable for Pipenv. All contributions will be considered carefully, but from +time to time, contributions will be rejected because they do not suit the +current goals or needs of the project. +.sp +If your contribution is rejected, don\(aqt despair! As long as you followed these +guidelines, you will have a much better chance of getting your next +contribution accepted. +.SS Questions +.sp +The GitHub issue tracker is for \fIbug reports\fP and \fIfeature requests\fP\&. Please do +not use it to ask questions about how to use Pipenv. These questions should +instead be directed to \fI\%Stack Overflow\fP\&. Make sure that your question is tagged +with the \fBpipenv\fP tag when asking it on Stack Overflow, to ensure that it is +answered promptly and accurately. +.SS Code Contributions +.SS Steps for Submitting Code +.sp +When contributing code, you\(aqll want to follow this checklist: +.INDENT 0.0 +.IP 1. 3 +Understand our \fI\%development philosophy\fP\&. +.IP 2. 3 +Fork the repository on GitHub. +.IP 3. 3 +Set up your \fI\%Development Setup\fP +.IP 4. 3 +Run the tests (\fI\%Testing\fP) to confirm they all pass on your system. +If they don\(aqt, you\(aqll need to investigate why they fail. If you\(aqre unable +to diagnose this yourself, raise it as a bug report by following the guidelines +in this document: \fI\%Bug Reports\fP\&. +.IP 5. 3 +Write tests that demonstrate your bug or feature. Ensure that they fail. +.IP 6. 3 +Make your change. +.IP 7. 3 +Run the entire test suite again, confirming that all tests pass \fIincluding +the ones you just added\fP\&. +.IP 8. 3 +Send a GitHub Pull Request to the main repository\(aqs \fBmaster\fP branch. +GitHub Pull Requests are the expected method of code collaboration on this +project. +.UNINDENT +.sp +The following sub\-sections go into more detail on some of the points above. +.SS Development Setup +.sp +To get your development environment setup, run: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pip install \-e . +pipenv install \-\-dev +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will install the repo version of Pipenv and then install the development +dependencies. Once that has completed, you can start developing. +.sp +The repo version of Pipenv must be installed over other global versions to +resolve conflicts with the \fBpipenv\fP folder being implicitly added to \fBsys.path\fP\&. +See \fI\%pypa/pipenv#2557\fP for more details. +.SS Testing +.sp +Tests are written in \fBpytest\fP style and can be run very simply: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pytest +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +This will run all Pipenv tests, which can take awhile. To run a subset of the +tests, the standard pytest filters are available, such as: +.INDENT 0.0 +.IP \(bu 2 +provide a directory or file: \fBpytest tests/unit\fP or \fBpytest tests/unit/test_cmdparse.py\fP +.IP \(bu 2 +provide a keyword expression: \fBpytest \-k test_lock_editable_vcs_without_install\fP +.IP \(bu 2 +provide a nodeid: \fBpytest tests/unit/test_cmdparse.py::test_parse\fP +.IP \(bu 2 +provide a test marker: \fBpytest \-m lock\fP +.UNINDENT +.SS Code Review +.sp +Contributions will not be merged until they\(aqve been code reviewed. You should +implement any code review feedback unless you strongly object to it. In the +event that you object to the code review feedback, you should make your case +clearly and calmly. If, after doing so, the feedback is judged to still apply, +you must either apply the feedback or withdraw your contribution. +.SS Package Index +.sp +To speed up testing, tests that rely on a package index for locking and +installing use a local server that contains vendored packages in the +\fBtests/pypi\fP directory. Each vendored package should have it\(aqs own folder +containing the necessary releases. When adding a release for a package, it is +easiest to use either the \fB\&.tar.gz\fP or universal wheels (ex: \fBpy2.py3\-none\fP). If +a \fB\&.tar.gz\fP or universal wheel is not available, add wheels for all available +architectures and platforms. +.SS Documentation Contributions +.sp +Documentation improvements are always welcome! The documentation files live in +the \fBdocs/\fP directory of the codebase. They\(aqre written in +\fI\%reStructuredText\fP, and use \fI\%Sphinx\fP to generate the full suite of +documentation. +.sp +When contributing documentation, please do your best to follow the style of the +documentation files. This means a soft\-limit of 79 characters wide in your text +files and a semi\-formal, yet friendly and approachable, prose style. +.sp +When presenting Python code, use single\-quoted strings (\fB\(aqhello\(aq\fP instead of +\fB"hello"\fP). +.SS Bug Reports +.sp +Bug reports are hugely important! They are recorded as \fI\%GitHub issues\fP\&. Please +be aware of the following things when filing bug reports: +.INDENT 0.0 +.IP 1. 3 +Avoid raising duplicate issues. \fIPlease\fP use the GitHub issue search feature +to check whether your bug report or feature request has been mentioned in +the past. Duplicate bug reports and feature requests are a huge maintenance +burden on the limited resources of the project. If it is clear from your +report that you would have struggled to find the original, that\(aqs ok, but +if searching for a selection of words in your issue title would have found +the duplicate then the issue will likely be closed extremely abruptly. +.IP 2. 3 +When filing bug reports about exceptions or tracebacks, please include the +\fIcomplete\fP traceback. Partial tracebacks, or just the exception text, are +not helpful. Issues that do not contain complete tracebacks may be closed +without warning. +.IP 3. 3 +Make sure you provide a suitable amount of information to work with. This +means you should provide: +.INDENT 3.0 +.IP \(bu 2 +Guidance on \fBhow to reproduce the issue\fP\&. Ideally, this should be a +\fIsmall\fP code sample that can be run immediately by the maintainers. +Failing that, let us know what you\(aqre doing, how often it happens, what +environment you\(aqre using, etc. Be thorough: it prevents us needing to ask +further questions. +.IP \(bu 2 +Tell us \fBwhat you expected to happen\fP\&. When we run your example code, +what are we expecting to happen? What does "success" look like for your +code? +.IP \(bu 2 +Tell us \fBwhat actually happens\fP\&. It\(aqs not helpful for you to say "it +doesn\(aqt work" or "it fails". Tell us \fIhow\fP it fails: do you get an +exception? A hang? The packages installed seem incorrect? +How was the actual result different from your expected result? +.IP \(bu 2 +Tell us \fBwhat version of Pipenv you\(aqre using\fP, and +\fBhow you installed it\fP\&. Different versions of Pipenv behave +differently and have different bugs, and some distributors of Pipenv +ship patches on top of the code we supply. +.UNINDENT +.sp +If you do not provide all of these things, it will take us much longer to +fix your problem. If we ask you to clarify these and you never respond, we +will close your issue without fixing it. +.UNINDENT +.SS Run the tests +.sp +Three ways of running the tests are as follows: +.INDENT 0.0 +.IP 1. 3 +\fBmake test\fP (which uses \fBdocker\fP) +.IP 2. 3 +\fB\&./run\-tests.sh\fP or \fBrun\-tests.bat\fP +.IP 3. 3 +Using pipenv: +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +$ git clone https://github.com/pypa/pipenv.git +$ cd pipenv +$ git submodule sync && git submodule update \-\-init \-\-recursive +$ pipenv install \-\-dev +$ pipenv run pytest +.ft P +.fi +.UNINDENT +.UNINDENT +.sp +For the last two, it is important that your environment is setup correctly, and +this may take some work, for example, on a specific Mac installation, the following +steps may be needed: +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +# Make sure the tests can access github +if [ "$SSH_AGENT_PID" = "" ] +then + eval \(gassh\-agent\(ga + ssh\-add +fi + +# Use unix like utilities, installed with brew, +# e.g. brew install coreutils +for d in /usr/local/opt/*/libexec/gnubin /usr/local/opt/python/libexec/bin +do + [[ ":$PATH:" != *":$d:"* ]] && PATH="$d:${PATH}" +done + +export PATH + +# PIP_FIND_LINKS currently breaks test_uninstall.py +unset PIP_FIND_LINKS +.ft P +.fi +.UNINDENT +.UNINDENT .SH ☤ PIPENV USAGE +.SS pipenv +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-where +Output project home information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-venv +Output virtualenv information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-py +Output Python interpreter information. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-envs +Output Environment Variable options. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-rm +Remove the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-completion +Output completion (to be eval\(aqd). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-man +Display manpage. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-support +Output diagnostic information for use in GitHub issues. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-site\-packages +Enable site\-packages for the virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-version +Show the version and exit. +.UNINDENT +.SS check +.sp +Checks for security vulnerabilities and against PEP 508 markers provided in Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv check [OPTIONS] [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-unused <unused> +Given a code path, show potentially unused dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-ignore <ignore> +Ignore specified vulnerability during safety checks. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS clean +.sp +Uninstalls all packages not specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv clean [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +Just output unneeded packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.SS graph +.sp +Displays currently\-installed dependency graph information. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv graph [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json +Output JSON. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-json\-tree +Output JSON in nested tree. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-reverse +Reversed dependency graph. +.UNINDENT +.SS install +.sp +Installs provided packages and adds them to Pipfile, or (if no packages are given), installs all packages from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv install [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-system +System pip management. +.UNINDENT +.INDENT 0.0 +.TP +.B \-c, \-\-code <code> +Install packages automatically discovered from import statements. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-deploy +Abort if the Pipfile.lock is out\-of\-date, or Python version is wrong. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable <editable> +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements <requirements> +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url <extra_index_url> +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index <index> +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-i\fP +.UNINDENT +.UNINDENT +.UNINDENT +.SS lock +.sp +Generates Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv lock [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-r, \-\-requirements +Generate output in requirements.txt format. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +.SS open +.sp +View a given module in your editor. +.sp +This uses the EDITOR environment variable. You can temporarily override it, +for example: +.INDENT 0.0 +.INDENT 3.5 +EDITOR=atom pipenv open requests +.UNINDENT +.UNINDENT +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv open [OPTIONS] MODULE +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B MODULE +Required argument +.UNINDENT +.SS run +.sp +Spawns a command installed into the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv run [OPTIONS] COMMAND [ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B COMMAND +Required argument +.UNINDENT +.INDENT 0.0 +.TP +.B ARGS +Optional argument(s) +.UNINDENT +.SS shell +.sp +Spawns a shell within the virtualenv. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv shell [OPTIONS] [SHELL_ARGS]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-fancy +Run in shell in fancy mode (for elegantly configured shells). +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-anyway +Always spawn a subshell, even if one is already spawned. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B SHELL_ARGS +Optional argument(s) +.UNINDENT +.SS sync +.sp +Installs all packages specified in Pipfile.lock. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv sync [OPTIONS] +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +.SS uninstall +.sp +Un\-installs a provided package and removes it from Pipfile. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv uninstall [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-all\-dev +Un\-install all package from [dev\-packages]. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-all +Purge all package(s) from virtualenv. Does not edit Pipfile. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable <editable> +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-skip\-lock +Skip locking mechanisms and use the Pipfile instead during operation. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIPENV_SKIP_LOCK +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-\-skip\-lock\fP +.UNINDENT +.UNINDENT +.UNINDENT +.SS update +.sp +Runs lock, then sync. +.INDENT 0.0 +.INDENT 3.5 +.sp +.nf +.ft C +pipenv update [OPTIONS] [PACKAGES]... +.ft P +.fi +.UNINDENT +.UNINDENT +Options.INDENT 0.0 +.TP +.B \-\-bare +Minimal output. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-outdated +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-dry\-run +List out\-of\-date dependencies. +.UNINDENT +.INDENT 0.0 +.TP +.B \-e, \-\-editable <editable> +An editable python package URL or path, often to a VCS repo. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-ignore\-pipfile +Ignore Pipfile when installing, using the Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-selective\-upgrade +Update specified packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-r, \-\-requirements <requirements> +Import a requirements.txt file. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-extra\-index\-url <extra_index_url> +URLs to the extra PyPI compatible indexes to query for package lookups. +.UNINDENT +.INDENT 0.0 +.TP +.B \-i, \-\-index <index> +Target PyPI\-compatible package index url. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-sequential +Install dependencies one\-at\-a\-time, instead of concurrently. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-keep\-outdated +Keep out\-dated dependencies from being updated in Pipfile.lock. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pre +Allow pre\-releases. +.UNINDENT +.INDENT 0.0 +.TP +.B \-d, \-\-dev +Install both develop and default packages. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-python <python> +Specify which version of Python virtualenv should use. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-three, \-\-two +Use Python 3/2 when creating virtualenv. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-clear +Clears caches (pipenv, pip, and pip\-tools). +.UNINDENT +.INDENT 0.0 +.TP +.B \-v, \-\-verbose +Verbose mode. +.UNINDENT +.INDENT 0.0 +.TP +.B \-\-pypi\-mirror <pypi_mirror> +Specify a PyPI mirror. +.UNINDENT +Arguments.INDENT 0.0 +.TP +.B PACKAGES +Optional argument(s) +.UNINDENT +Environment variables.INDENT 0.0 +.TP +.B PIP_EXTRA_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-\-extra\-index\-url\fP +.UNINDENT +.UNINDENT +.UNINDENT +.INDENT 0.0 +.TP +.B PIP_INDEX_URL +.INDENT 7.0 +.INDENT 3.5 +Provide a default for \fI\%\-i\fP +.UNINDENT +.UNINDENT +.UNINDENT .INDENT 0.0 .IP \(bu 2 genindex @@ -1360,6 +4121,6 @@ search .SH AUTHOR Kenneth Reitz .SH COPYRIGHT -2017. A <a href="http://kennethreitz.com/pages/open-projects.html">Kenneth Reitz</a> Project +2017. A project founded by <a href="http://kennethreitz.com/pages/open-projects.html">Kenneth Reitz</a> .\" Generated by docutils manpage writer. . diff --git a/pipenv/progress.py b/pipenv/progress.py index 1328e4d5..8968150c 100644 --- a/pipenv/progress.py +++ b/pipenv/progress.py @@ -21,25 +21,20 @@ from .environments import PIPENV_COLORBLIND, PIPENV_HIDE_EMOJIS STREAM = sys.stderr MILL_TEMPLATE = "%s %s %i/%i\r" DOTS_CHAR = "." -if os.name != "nt": - if PIPENV_HIDE_EMOJIS: - if PIPENV_COLORBLIND: - BAR_FILLED_CHAR = "=" - BAR_EMPTY_CHAR = "-" - else: - BAR_FILLED_CHAR = str(crayons.green("=", bold=True)) - BAR_EMPTY_CHAR = str(crayons.black("-")) +if PIPENV_HIDE_EMOJIS: + if PIPENV_COLORBLIND: + BAR_FILLED_CHAR = "=" + BAR_EMPTY_CHAR = "-" else: - if PIPENV_COLORBLIND: - BAR_FILLED_CHAR = "▉" - BAR_EMPTY_CHAR = " " - else: - BAR_FILLED_CHAR = str(crayons.green("▉", bold=True)) - BAR_EMPTY_CHAR = str(crayons.black("▉")) - + BAR_FILLED_CHAR = str(crayons.green("=", bold=True)) + BAR_EMPTY_CHAR = str(crayons.black("-")) else: - BAR_FILLED_CHAR = "=" - BAR_EMPTY_CHAR = "-" + if PIPENV_COLORBLIND: + BAR_FILLED_CHAR = "▉" + BAR_EMPTY_CHAR = " " + else: + BAR_FILLED_CHAR = str(crayons.green("▉", bold=True)) + BAR_EMPTY_CHAR = str(crayons.black("▉")) if (sys.version_info[0] >= 3) and (os.name != "nt"): BAR_TEMPLATE = u" %s%s%s %i/%i — {0}\r".format(crayons.black("%s")) diff --git a/pipenv/project.py b/pipenv/project.py index edbcff87..4cb2ff9c 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -15,45 +15,47 @@ import toml import tomlkit import vistir -from first import first - import pipfile import pipfile.api -from cached_property import cached_property +from .vendor.cached_property import cached_property from .cmdparse import Script from .environment import Environment from .environments import ( PIPENV_DEFAULT_PYTHON_VERSION, PIPENV_IGNORE_VIRTUALENVS, PIPENV_MAX_DEPTH, PIPENV_PIPFILE, PIPENV_PYTHON, PIPENV_TEST_INDEX, PIPENV_VENV_IN_PROJECT, - is_in_virtualenv + is_in_virtualenv, is_type_checking ) +from .vendor.requirementslib.models.utils import get_default_pyproject_backend from .utils import ( cleanup_toml, convert_toml_outline_tables, find_requirements, get_canonical_names, get_url_name, get_workon_home, is_editable, is_installable_file, is_star, is_valid_url, is_virtual_environment, looks_like_dir, normalize_drive, pep423_name, proper_case, python_version, - safe_expandvars + safe_expandvars, get_pipenv_dist ) +if is_type_checking(): + from typing import Dict, Text, Union + TSource = Dict[Text, Union[Text, bool]] + def _normalized(p): if p is None: return None loc = vistir.compat.Path(p) - if not loc.is_absolute(): - try: - loc = loc.resolve() - except OSError: - loc = loc.absolute() + try: + loc = loc.resolve() + except OSError: + loc = loc.absolute() # Recase the path properly on Windows. From https://stackoverflow.com/a/35229734/5043728 if os.name == 'nt': matches = glob.glob(re.sub(r'([^:/\\])(?=[/\\]|$)', r'[\1]', str(loc))) path_str = matches and matches[0] or str(loc) else: path_str = str(loc) - return normalize_drive(path_str) + return normalize_drive(os.path.abspath(path_str)) DEFAULT_NEWLINES = u"\n" @@ -98,6 +100,9 @@ if PIPENV_PIPFILE: else: PIPENV_PIPFILE = _normalized(PIPENV_PIPFILE) + # Overwrite environment variable so that subprocesses can get the correct path. + # See https://github.com/pypa/pipenv/issues/3584 + os.environ['PIPENV_PIPFILE'] = PIPENV_PIPFILE # (path, file contents) => TOMLFile # keeps track of pipfiles that we've seen so we do not need to re-parse 'em _pipfile_cache = {} @@ -202,14 +207,12 @@ class Project(object): # First exclude anything that is a vcs entry either in the key or value if not ( any(is_vcs(i) for i in [k, v]) - or # Then exclude any installable files that are not directories # Because pip-tools can resolve setup.py for example - any(is_installable_file(i) for i in [k, v]) - or + or any(is_installable_file(i) for i in [k, v]) # Then exclude any URLs because they need to be editable also # Things that are excluded can only be 'shallow resolved' - any(is_valid_url(i) for i in [k, v]) + or any(is_valid_url(i) for i in [k, v]) ): ps.update({k: v}) return ps @@ -222,7 +225,7 @@ class Project(object): @property def pipfile_exists(self): - return bool(self.pipfile_location) + return os.path.isfile(self.pipfile_location) @property def required_python_version(self): @@ -237,11 +240,7 @@ class Project(object): @property def project_directory(self): - if self.pipfile_location is not None: - return os.path.abspath(os.path.join(self.pipfile_location, os.pardir)) - - else: - return None + return os.path.abspath(os.path.join(self.pipfile_location, os.pardir)) @property def requirements_exists(self): @@ -255,8 +254,7 @@ class Project(object): @property def virtualenv_exists(self): - # TODO: Decouple project from existence of Pipfile. - if self.pipfile_exists and os.path.exists(self.virtualenv_location): + if os.path.exists(self.virtualenv_location): if os.name == "nt": extra = ["Scripts", "activate.bat"] else: @@ -335,12 +333,16 @@ class Project(object): if not self._environment: prefix = self.virtualenv_location is_venv = is_in_virtualenv() - sources = self.sources if self.sources else [DEFAULT_SOURCE,] + sources = self.sources if self.sources else [DEFAULT_SOURCE] self._environment = Environment( prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, project=self ) - self._environment.add_dist("pipenv") + pipenv_dist = get_pipenv_dist(pkg="pipenv") + if pipenv_dist: + self._environment.extend_dists(pipenv_dist) + else: + self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): @@ -415,8 +417,10 @@ class Project(object): def virtualenv_location(self): # if VIRTUAL_ENV is set, use that. virtualenv_env = os.getenv("VIRTUAL_ENV") - if ("PIPENV_ACTIVE" not in os.environ and - not PIPENV_IGNORE_VIRTUALENVS and virtualenv_env): + if ( + "PIPENV_ACTIVE" not in os.environ + and not PIPENV_IGNORE_VIRTUALENVS and virtualenv_env + ): return virtualenv_env if not self._virtualenv_location: # Use cached version, if available. @@ -470,7 +474,7 @@ class Project(object): try: loc = pipfile.Pipfile.find(max_depth=PIPENV_MAX_DEPTH) except RuntimeError: - loc = None + loc = "Pipfile" self._pipfile_location = _normalized(loc) return self._pipfile_location @@ -499,6 +503,8 @@ class Project(object): def read_pipfile(self): # Open the pipfile, read it into memory. + if not self.pipfile_exists: + return "" with io.open(self.pipfile_location) as f: contents = f.read() self._pipfile_newlines = preferred_newlines(f) @@ -525,18 +531,18 @@ class Project(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": get_default_pyproject_backend(), } self._build_system = build_system @property def build_requires(self): - return self._build_system.get("requires", []) + return self._build_system.get("requires", ["setuptools>=40.8.0", "wheel"]) @property def build_backend(self): - return self._build_system.get("build-backend", None) + return self._build_system.get("build-backend", get_default_pyproject_backend()) @property def settings(self): @@ -603,10 +609,8 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" - # section = "{0}-editable".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(k) or is_editable(v) } @@ -615,10 +619,8 @@ class Project(object): def _get_vcs_packages(self, dev=False): from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" - # section = "{0}-vcs".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } @@ -659,11 +661,6 @@ class Project(object): """Returns a list of dev-packages, for pip-tools to consume.""" return self._build_package_list("dev-packages") - def touch_pipfile(self): - """Simply touches the Pipfile, for later use.""" - with open("Pipfile", "a"): - os.utime("Pipfile", None) - @property def pipfile_is_empty(self): if not self.pipfile_exists: @@ -680,7 +677,6 @@ class Project(object): ConfigOptionParser, make_option_group, index_group ) - name = self.name if self.name is not None else "Pipfile" config_parser = ConfigOptionParser(name=self.name) config_parser.add_option_group(make_option_group(index_group, config_parser)) install = config_parser.option_groups[0] @@ -689,7 +685,7 @@ class Project(object): .lstrip("\n") .split("\n") ) - sources = [DEFAULT_SOURCE,] + sources = [DEFAULT_SOURCE] for i, index in enumerate(indexes): if not index: continue @@ -727,7 +723,7 @@ class Project(object): if "verify_ssl" not in source: source["verify_ssl"] = "https://" in source["url"] if not isinstance(source["verify_ssl"], bool): - source["verify_ssl"] = source["verify_ssl"].lower() == "true" + source["verify_ssl"] = str(source["verify_ssl"]).lower() == "true" return source def get_or_create_lockfile(self, from_pipfile=False): @@ -757,7 +753,7 @@ class Project(object): if not sources: sources = self.pipfile_sources elif not isinstance(sources, list): - sources = [sources,] + sources = [sources] lockfile_dict["_meta"]["sources"] = [ self.populate_source(s) for s in sources ] @@ -776,7 +772,7 @@ class Project(object): else: sources = [dict(source) for source in self.parsed_pipfile["source"]] if not isinstance(sources, list): - sources = [sources,] + sources = [sources] return { "hash": {"sha256": self.calculate_pipfile_hash()}, "pipfile-spec": PIPFILE_SPEC_CURRENT, @@ -834,7 +830,7 @@ class Project(object): @property def pipfile_sources(self): - if "source" not in self.parsed_pipfile: + if self.pipfile_is_empty or "source" not in self.parsed_pipfile: return [DEFAULT_SOURCE] # We need to make copies of the source info so we don't # accidentally modify the cache. See #2100 where values are @@ -855,8 +851,13 @@ class Project(object): else: return self.pipfile_sources + @property + def index_urls(self): + return [src.get("url") for src in self.sources] + def find_source(self, source): - """given a source, find it. + """ + Given a source, find it. source can be a url or an index name. """ @@ -869,23 +870,34 @@ class Project(object): source = self.get_source(url=source) return source - def get_source(self, name=None, url=None): + def get_source(self, name=None, url=None, refresh=False): + from .utils import is_url_equal + def find_source(sources, name=None, url=None): source = None if name: - source = [s for s in sources if s.get("name") == name] + source = next(iter( + s for s in sources if "name" in s and s["name"] == name + ), None) elif url: - source = [s for s in sources if url.startswith(s.get("url"))] - if source: - return first(source) + source = next(iter( + s for s in sources + if "url" in s and is_url_equal(url, s.get("url", "")) + ), None) + if source is not None: + return source - found_source = find_source(self.sources, name=name, url=url) - if found_source: - return found_source - found_source = find_source(self.pipfile_sources, name=name, url=url) - if found_source: - return found_source - raise SourceNotFound(name or url) + sources = (self.sources, self.pipfile_sources) + if refresh: + self.clear_pipfile_cache() + sources = reversed(sources) + found = next( + iter(find_source(source, name=name, url=url) for source in sources), None + ) + target = next(iter(t for t in (name, url) if t is not None)) + if found is None: + raise SourceNotFound(target) + return found def get_package_name_in_pipfile(self, package_name, dev=False): """Get the equivalent package name in pipfile""" @@ -930,17 +942,17 @@ class Project(object): # Don't re-capitalize file URLs or VCSs. if not isinstance(package, Requirement): package = Requirement.from_line(package.strip()) - _, converted = package.pipfile_entry + req_name, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - name = self.get_package_name_in_pipfile(package.name, dev) + name = self.get_package_name_in_pipfile(req_name, dev) if name and is_star(converted): # Skip for wildcard version return # Add the package to the group. - p[key][name or pep423_name(package.name)] = converted + p[key][name or pep423_name(req_name)] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ed474fb4..cd04fccb 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- + +from __future__ import absolute_import, print_function import json import logging import os @@ -7,12 +10,48 @@ import sys os.environ["PIP_PYTHON_PATH"] = str(sys.executable) -def _patch_path(): +def find_site_path(pkg, site_dir=None): + import pkg_resources + if site_dir is not None: + site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:]) + for dist in working_set: + root = dist.location + base_name = dist.project_name if dist.project_name else dist.key + name = None + if "top_level.txt" in dist.metadata_listdir(""): + name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None) + if name is None: + name = pkg_resources.safe_name(base_name).replace("-", "_") + if not any(pkg == _ for _ in [base_name, name]): + continue + path_options = [name, "{0}.py".format(name)] + path_options = [os.path.join(root, p) for p in path_options if p is not None] + path = next(iter(p for p in path_options if os.path.exists(p)), None) + if path is not None: + return (dist, path) + return (None, None) + + +def _patch_path(pipenv_site=None): import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): + pipenv_dist = None + if pipenv_site is not None: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site) + else: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir) + if pipenv_dist is not None: + pipenv_dist.activate() + else: + site.addsitedir(next(iter( + sitedir for sitedir in (pipenv_site, pipenv_site_dir) + if sitedir is not None + ), None)) + if pipenv_path is not None: + pipenv_libdir = pipenv_path + for _dir in ("vendor", "patched", pipenv_libdir): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) @@ -22,10 +61,16 @@ def get_parser(): parser.add_argument("--pre", action="store_true", default=False) parser.add_argument("--clear", action="store_true", default=False) parser.add_argument("--verbose", "-v", action="count", default=False) + parser.add_argument("--dev", action="store_true", default=False) parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--parse-only", action="store_true", default=False) + parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store", + default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", - default=os.environ.get("PIPENV_REQ_DIR")) + default=os.environ.get("PIPENV_REQ_DIR")) + parser.add_argument("--write", metavar="write", action="store", + default=os.environ.get("PIPENV_RESOLVER_FILE")) parser.add_argument("packages", nargs="*") return parser @@ -41,17 +86,636 @@ def handle_parsed_args(parsed): logging.getLogger("notpip").setLevel(logging.DEBUG) elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) + os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose) if "PIPENV_PACKAGES" in os.environ: parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def _main(pre, clear, verbose, system, requirements_dir, packages): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = str(sys.executable) +class Entry(object): + """A resolved entry from a resolver run""" + def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=False): + super(Entry, self).__init__() + from pipenv.vendor.requirementslib.models.utils import tomlkit_value_to_python + self.name = name + if isinstance(entry_dict, dict): + self.entry_dict = self.clean_initial_dict(entry_dict) + else: + self.entry_dict = entry_dict + self.project = project + section = "develop" if dev else "default" + pipfile_section = "dev-packages" if dev else "packages" + self.dev = dev + self.pipfile = tomlkit_value_to_python( + project.parsed_pipfile.get(pipfile_section, {}) + ) + self.lockfile = project.lockfile_content.get(section, {}) + self.pipfile_dict = self.pipfile.get(self.pipfile_name, {}) + if self.dev and self.name in project.lockfile_content.get("default", {}): + self.lockfile_dict = project.lockfile_content["default"][name] + else: + self.lockfile_dict = self.lockfile.get(name, entry_dict) + self.resolver = resolver + self.reverse_deps = reverse_deps + self._original_markers = None + self._markers = None + self._entry = None + self._lockfile_entry = None + self._pipfile_entry = None + self._parent_deps = [] + self._flattened_parents = [] + self._requires = None + self._deptree = None + self._parents_in_pipfile = [] + + @staticmethod + def make_requirement(name=None, entry=None, from_ireq=False): + from pipenv.vendor.requirementslib.models.requirements import Requirement + if from_ireq: + return Requirement.from_ireq(entry) + return Requirement.from_pipfile(name, entry) + + @classmethod + def clean_initial_dict(cls, entry_dict): + if not entry_dict.get("version", "").startswith("=="): + entry_dict["version"] = cls.clean_specifier(entry_dict.get("version", "")) + if "name" in entry_dict: + del entry_dict["name"] + return entry_dict + + @classmethod + def parse_pyparsing_exprs(cls, expr_iterable): + from pipenv.vendor.pyparsing import Literal, MatchFirst + keys = [] + expr_list = [] + expr = expr_iterable.copy() + if isinstance(expr, Literal) or ( + expr.__class__.__name__ == Literal.__name__ + ): + keys.append(expr.match) + elif isinstance(expr, MatchFirst) or ( + expr.__class__.__name__ == MatchFirst.__name__ + ): + expr_list = expr.exprs + elif isinstance(expr, list): + expr_list = expr + if expr_list: + for part in expr_list: + keys.extend(cls.parse_pyparsing_exprs(part)) + return keys + + @classmethod + def get_markers_from_dict(cls, entry_dict): + from pipenv.vendor.packaging import markers as packaging_markers + from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + marker_keys = cls.parse_pyparsing_exprs(packaging_markers.VARIABLE) + markers = set() + keys_in_dict = [k for k in marker_keys if k in entry_dict] + markers = { + normalize_marker_str("{k} {v}".format(k=k, v=entry_dict.pop(k))) + for k in keys_in_dict + } + if "markers" in entry_dict: + markers.add(normalize_marker_str(entry_dict["markers"])) + if None in markers: + markers.remove(None) + if markers: + entry_dict["markers"] = " and ".join(list(markers)) + else: + markers = None + return markers, entry_dict + + @property + def markers(self): + self._markers, self.entry_dict = self.get_markers_from_dict(self.entry_dict) + return self._markers + + @markers.setter + def markers(self, markers): + if not markers: + marker_str = self.marker_to_str(markers) + if marker_str: + self._entry = self.entry.merge_markers(marker_str) + self._markers = self.marker_to_str(self._entry.markers) + entry_dict = self.entry_dict.copy() + entry_dict["markers"] = self.marker_to_str(self._entry.markers) + self.entry_dict = entry_dict + + @property + def original_markers(self): + original_markers, lockfile_dict = self.get_markers_from_dict( + self.lockfile_dict + ) + self.lockfile_dict = lockfile_dict + self._original_markers = self.marker_to_str(original_markers) + return self._original_markers + + @staticmethod + def marker_to_str(marker): + from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + if not marker: + return None + from pipenv.vendor import six + from pipenv.vendor.vistir.compat import Mapping + marker_str = None + if isinstance(marker, Mapping): + marker_dict, _ = Entry.get_markers_from_dict(marker) + if marker_dict: + marker_str = "{0}".format(marker_dict.popitem()[1]) + elif isinstance(marker, (list, set, tuple)): + marker_str = " and ".join([normalize_marker_str(m) for m in marker if m]) + elif isinstance(marker, six.string_types): + marker_str = "{0}".format(normalize_marker_str(marker)) + if isinstance(marker_str, six.string_types): + return marker_str + return None + + def get_cleaned_dict(self, keep_outdated=False): + if keep_outdated and self.is_updated: + self.validate_constraints() + self.ensure_least_updates_possible() + elif not keep_outdated: + self.validate_constraints() + if self.entry.extras != self.lockfile_entry.extras: + entry_extras = list(self.entry.extras) + if self.lockfile_entry.extras: + entry_extras.extend(list(self.lockfile_entry.extras)) + self._entry.req.extras = entry_extras + self.entry_dict["extras"] = self.entry.extras + if self.original_markers and not self.markers: + original_markers = self.marker_to_str(self.original_markers) + self.markers = original_markers + self.entry_dict["markers"] = self.marker_to_str(original_markers) + entry_hashes = set(self.entry.hashes) + locked_hashes = set(self.lockfile_entry.hashes) + if entry_hashes != locked_hashes and not self.is_updated: + self.entry_dict["hashes"] = list(entry_hashes | locked_hashes) + self.entry_dict["name"] = self.name + if "version" in self.entry_dict: + self.entry_dict["version"] = self.strip_version(self.entry_dict["version"]) + _, self.entry_dict = self.get_markers_from_dict(self.entry_dict) + return self.entry_dict + + @property + def lockfile_entry(self): + if self._lockfile_entry is None: + self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict) + return self._lockfile_entry + + @lockfile_entry.setter + def lockfile_entry(self, entry): + self._lockfile_entry = entry + + @property + def pipfile_entry(self): + if self._pipfile_entry is None: + self._pipfile_entry = self.make_requirement(self.pipfile_name, self.pipfile_dict) + return self._pipfile_entry + + @property + def entry(self): + if self._entry is None: + self._entry = self.make_requirement(self.name, self.entry_dict) + return self._entry + + @property + def normalized_name(self): + return self.entry.normalized_name + + @property + def pipfile_name(self): + return self.project.get_package_name_in_pipfile(self.name, dev=self.dev) + + @property + def is_in_pipfile(self): + return True if self.pipfile_name else False + + @property + def pipfile_packages(self): + return self.project.pipfile_package_names["dev" if self.dev else "default"] + + def create_parent(self, name, specifier="*"): + parent = self.create(name, specifier, self.project, self.resolver, + self.reverse_deps, self.dev) + parent._deptree = self.deptree + return parent + + @property + def deptree(self): + if not self._deptree: + self._deptree = self.project.environment.get_package_requirements() + return self._deptree + + @classmethod + def create(cls, name, entry_dict, project, resolver, reverse_deps=None, dev=False): + return cls(name, entry_dict, project, resolver, reverse_deps, dev) + + @staticmethod + def clean_specifier(specifier): + from pipenv.vendor.packaging.specifiers import Specifier + if not any(specifier.startswith(k) for k in Specifier._operators.keys()): + if specifier.strip().lower() in ["any", "<any>", "*"]: + return "*" + specifier = "=={0}".format(specifier) + elif specifier.startswith("==") and specifier.count("=") > 3: + specifier = "=={0}".format(specifier.lstrip("=")) + return specifier + + @staticmethod + def strip_version(specifier): + from pipenv.vendor.packaging.specifiers import Specifier + op = next(iter( + k for k in Specifier._operators.keys() if specifier.startswith(k) + ), None) + if op: + specifier = specifier[len(op):] + while op: + op = next(iter( + k for k in Specifier._operators.keys() if specifier.startswith(k) + ), None) + if op: + specifier = specifier[len(op):] + return specifier + + @property + def parent_deps(self): + if not self._parent_deps: + self._parent_deps = self.get_parent_deps(unnest=False) + return self._parent_deps + + @property + def flattened_parents(self): + if not self._flattened_parents: + self._flattened_parents = self.get_parent_deps(unnest=True) + return self._flattened_parents + + @property + def parents_in_pipfile(self): + if not self._parents_in_pipfile: + self._parents_in_pipfile = [ + p for p in self.flattened_parents + if p.normalized_name in self.pipfile_packages + ] + return self._parents_in_pipfile + + @property + def is_updated(self): + return self.entry.specifiers != self.lockfile_entry.specifiers + + @property + def requirements(self): + if not self._requires: + self._requires = next(iter( + self.project.environment.get_package_requirements(self.name) + ), {}) + return self._requires + + @property + def updated_version(self): + version = self.entry.specifiers + return self.strip_version(version) + + @property + def updated_specifier(self): + # type: () -> str + return self.entry.specifiers + + @property + def original_specifier(self): + # type: () -> str + return self.lockfile_entry.specifiers + + @property + def original_version(self): + if self.original_specifier: + return self.strip_version(self.original_specifier) + return None + + def validate_specifiers(self): + if self.is_in_pipfile and not self.pipfile_entry.editable: + return self.pipfile_entry.requirement.specifier.contains(self.updated_version) + return True + + def get_dependency(self, name): + if self.requirements: + return next(iter( + dep for dep in self.requirements.get("dependencies", []) + if dep and dep.get("package_name", "") == name + ), {}) + return {} + + def get_parent_deps(self, unnest=False): + from pipenv.vendor.packaging.specifiers import Specifier + parents = [] + for spec in self.reverse_deps.get(self.normalized_name, {}).get("parents", set()): + spec_match = next(iter(c for c in Specifier._operators if c in spec), None) + name = spec + parent = None + if spec_match is not None: + spec_index = spec.index(spec_match) + specifier = self.clean_specifier(spec[spec_index:len(spec_match)]).strip() + name_start = spec_index + len(spec_match) + name = spec[name_start:].strip() + parent = self.create_parent(name, specifier) + else: + name = spec + parent = self.create_parent(name) + if parent is not None: + parents.append(parent) + if not unnest or parent.pipfile_name is not None: + continue + if self.reverse_deps.get(parent.normalized_name, {}).get("parents", set()): + parents.extend(parent.flattened_parents) + return parents + + def ensure_least_updates_possible(self): + """ + Mutate the current entry to ensure that we are making the smallest amount of + changes possible to the existing lockfile -- this will keep the old locked + versions of packages if they satisfy new constraints. + + :return: None + """ + constraints = self.get_constraints() + can_use_original = True + can_use_updated = True + satisfied_by_versions = set() + for constraint in constraints: + if not constraint.specifier.contains(self.original_version): + self.can_use_original = False + if not constraint.specifier.contains(self.updated_version): + self.can_use_updated = False + satisfied_by_value = getattr(constraint, "satisfied_by", None) + if satisfied_by_value: + satisfied_by = "{0}".format( + self.clean_specifier(str(satisfied_by_value.version)) + ) + satisfied_by_versions.add(satisfied_by) + if can_use_original: + self.entry_dict = self.lockfile_dict.copy() + elif can_use_updated: + if len(satisfied_by_versions) == 1: + self.entry_dict["version"] = next(iter( + sat_by for sat_by in satisfied_by_versions if sat_by + ), None) + hashes = None + if self.lockfile_entry.specifiers == satisfied_by: + ireq = self.lockfile_entry.as_ireq() + if not self.lockfile_entry.hashes and self.resolver._should_include_hash(ireq): + hashes = self.resolver.get_hash(ireq) + else: + hashes = self.lockfile_entry.hashes + else: + if self.resolver._should_include_hash(constraint): + hashes = self.resolver.get_hash(constraint) + if hashes: + self.entry_dict["hashes"] = list(hashes) + self._entry.hashes = frozenset(hashes) + else: + # check for any parents, since they depend on this and the current + # installed versions are not compatible with the new version, so + # we will need to update the top level dependency if possible + self.check_flattened_parents() + + def get_constraints(self): + """ + Retrieve all of the relevant constraints, aggregated from the pipfile, resolver, + and parent dependencies and their respective conflict resolution where possible. + + :return: A set of **InstallRequirement** instances representing constraints + :rtype: Set + """ + constraints = { + c for c in self.resolver.parsed_constraints + if c and c.name == self.entry.name + } + pipfile_constraint = self.get_pipfile_constraint() + if pipfile_constraint and not (self.pipfile_entry.editable or pipfile_constraint.editable): + constraints.add(pipfile_constraint) + return constraints + + def get_pipfile_constraint(self): + """ + Retrieve the version constraint from the pipfile if it is specified there, + otherwise check the constraints of the parent dependencies and their conflicts. + + :return: An **InstallRequirement** instance representing a version constraint + """ + if self.is_in_pipfile: + return self.pipfile_entry.as_ireq() + return self.constraint_from_parent_conflicts() + + def constraint_from_parent_conflicts(self): + """ + Given a resolved entry with multiple parent dependencies with different + constraints, searches for the resolution that satisfies all of the parent + constraints. + + :return: A new **InstallRequirement** satisfying all parent constraints + :raises: :exc:`~pipenv.exceptions.DependencyConflict` if resolution is impossible + """ + # ensure that we satisfy the parent dependencies of this dep + parent_dependencies = set() + has_mismatch = False + can_use_original = True + for p in self.parent_deps: + # updated dependencies should be satisfied since they were resolved already + if p.is_updated: + continue + # parents with no requirements can't conflict + if not p.requirements: + continue + entry_ref = p.get_dependency(self.name) + required = entry_ref.get("required_version", "*") + required = self.clean_specifier(required) + parent_requires = self.make_requirement(self.name, required) + parent_dependencies.add("{0} => {1} ({2})".format(p.name, self.name, required)) + # use pre=True here or else prereleases dont satisfy constraints + if parent_requires.requirement.specifier and ( + not parent_requires.requirement.specifier.contains(self.original_version, prereleases=True) + ): + can_use_original = False + if parent_requires.requirement.specifier and ( + not parent_requires.requirement.specifier.contains(self.updated_version, prereleases=True) + ): + if not self.entry.editable and self.updated_version != self.original_version: + has_mismatch = True + if has_mismatch and not can_use_original: + from pipenv.exceptions import DependencyConflict + msg = ( + "Cannot resolve {0} ({1}) due to conflicting parent dependencies: " + "\n\t{2}".format( + self.name, self.updated_version, "\n\t".join(parent_dependencies) + ) + ) + raise DependencyConflict(msg) + elif can_use_original: + return self.lockfile_entry.as_ireq() + return self.entry.as_ireq() + + def validate_constraints(self): + """ + Retrieves the full set of available constraints and iterate over them, validating + that they exist and that they are not causing unresolvable conflicts. + + :return: True if the constraints are satisfied by the resolution provided + :raises: :exc:`pipenv.exceptions.DependencyConflict` if the constraints dont exist + """ + constraints = self.get_constraints() + for constraint in constraints: + try: + constraint.check_if_exists(False) + except Exception: + from pipenv.exceptions import DependencyConflict + from pipenv.environments import is_verbose + if is_verbose(): + print("Tried constraint: {0!r}".format(constraint), file=sys.stderr) + msg = ( + "Cannot resolve conflicting version {0}{1} while {2}{3} is " + "locked.".format( + self.name, self.updated_specifier, self.old_name, self.old_specifiers + ) + ) + raise DependencyConflict(msg) + return True + + def check_flattened_parents(self): + for parent in self.parents_in_pipfile: + if not parent.updated_specifier: + continue + if not parent.validate_specifiers(): + from pipenv.exceptions import DependencyConflict + msg = ( + "Cannot resolve conflicting versions: (Root: {0}) {1}{2} (Pipfile) " + "Incompatible with {3}{4} (resolved)\n".format( + self.name, parent.pipfile_name, + parent.pipfile_entry.requirement.specifiers, parent.name, + parent.updated_specifiers + ) + ) + raise DependencyConflict(msg) + + def __getattribute__(self, key): + result = None + old_version = ["was_", "had_", "old_"] + new_version = ["is_", "has_", "new_"] + if any(key.startswith(v) for v in new_version): + entry = Entry.__getattribute__(self, "entry") + try: + keystart = key.index("_") + 1 + try: + result = getattr(entry, key[keystart:]) + except AttributeError: + result = getattr(entry, key) + except AttributeError: + result = super(Entry, self).__getattribute__(key) + return result + if any(key.startswith(v) for v in old_version): + lockfile_entry = Entry.__getattribute__(self, "lockfile_entry") + try: + keystart = key.index("_") + 1 + try: + result = getattr(lockfile_entry, key[keystart:]) + except AttributeError: + result = getattr(lockfile_entry, key) + except AttributeError: + result = super(Entry, self).__getattribute__(key) + return result + return super(Entry, self).__getattribute__(key) + + +def clean_results(results, resolver, project, dev=False): + from pipenv.utils import translate_markers + if not project.lockfile_exists: + return results + lockfile = project.lockfile_content + section = "develop" if dev else "default" + reverse_deps = project.environment.reverse_dependencies() + new_results = [r for r in results if r["name"] not in lockfile[section]] + for result in results: + name = result.get("name") + entry_dict = result.copy() + entry = Entry(name, entry_dict, project, resolver, reverse_deps=reverse_deps, dev=dev) + entry_dict = translate_markers(entry.get_cleaned_dict(keep_outdated=False)) + new_results.append(entry_dict) + return new_results + + +def clean_outdated(results, resolver, project, dev=False): + if not project.lockfile_exists: + return results + lockfile = project.lockfile_content + section = "develop" if dev else "default" + reverse_deps = project.environment.reverse_dependencies() + new_results = [r for r in results if r["name"] not in lockfile[section]] + for result in results: + name = result.get("name") + entry_dict = result.copy() + entry = Entry(name, entry_dict, project, resolver, reverse_deps=reverse_deps, dev=dev) + # The old entry was editable but this one isnt; prefer the old one + # TODO: Should this be the case for all locking? + if entry.was_editable and not entry.is_editable: + continue + lockfile_entry = lockfile[section].get(name, None) + if not lockfile_entry: + alternate_section = "develop" if not dev else "default" + if name in lockfile[alternate_section]: + lockfile_entry = lockfile[alternate_section][name] + if lockfile_entry and not entry.is_updated: + old_markers = next(iter(m for m in ( + entry.lockfile_entry.markers, lockfile_entry.get("markers", None) + ) if m is not None), None) + new_markers = entry_dict.get("markers", None) + if old_markers: + old_markers = Entry.marker_to_str(old_markers) + if old_markers and not new_markers: + entry.markers = old_markers + elif new_markers and not old_markers: + del entry.entry_dict["markers"] + entry._entry.req.req.marker = None + entry._entry.markers = None + # if the entry has not changed versions since the previous lock, + # don't introduce new markers since that is more restrictive + # if entry.has_markers and not entry.had_markers and not entry.is_updated: + # do make sure we retain the original markers for entries that are not changed + entry_dict = entry.get_cleaned_dict(keep_outdated=True) + new_results.append(entry_dict) + return new_results + + +def parse_packages(packages, pre, clear, system, requirements_dir=None): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor.vistir.contextmanagers import cd, temp_path + from pipenv.utils import parse_indexes + parsed_packages = [] + for package in packages: + indexes, trusted_hosts, line = parse_indexes(package) + line = " ".join(line) + pf = dict() + req = Requirement.from_line(line) + if not req.name: + with temp_path(), cd(req.req.setup_info.base_dir): + sys.path.insert(0, req.req.setup_info.base_dir) + req.req._setup_info.get_info() + req.update_name_from_path(req.req.setup_info.base_dir) + try: + name, entry = req.pipfile_entry + except Exception: + continue + else: + if name is not None and entry is not None: + pf[name] = entry + parsed_packages.append(pf) + print("RESULTS:") + if parsed_packages: + print(json.dumps(parsed_packages)) + else: + print(json.dumps([])) + + +def resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources - pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) if "PIPENV_PYPI_MIRROR" in os.environ @@ -59,6 +723,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): ) def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): + from pipenv.patched.piptools import logging as piptools_logging + piptools_logging.log.verbosity = 1 if verbose else 0 return resolve_deps( packages, which, @@ -76,7 +742,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): if pypi_mirror_source else project.pipfile_sources ) - results = resolve( + keep_outdated = os.environ.get("PIPENV_KEEP_OUTDATED", False) + results, resolver = resolve( packages, pre=pre, project=project, @@ -85,43 +752,56 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): system=system, requirements_dir=requirements_dir, ) - print("RESULTS:") - if results: - print(json.dumps(results)) + if keep_outdated: + results = clean_outdated(results, resolver, project) else: - print(json.dumps([])) + results = clean_results(results, resolver, project) + if write: + with open(write, "w") as fh: + if not results: + json.dump([], fh) + else: + json.dump(results, fh) + else: + print("RESULTS:") + if results: + print(json.dumps(results)) + else: + print(json.dumps([])) + + +def _main(pre, clear, verbose, system, write, requirements_dir, packages, parse_only=False): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) + else: + resolve_packages(pre, clear, verbose, system, write, requirements_dir, packages) def main(): - _patch_path() - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.simplefilter("ignore", category=ResourceWarning) - import io - import six - if six.PY3: - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper - else: - from pipenv._compat import force_encoding - force_encoding() - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") - os.environ["PYTHONIOENCODING"] = str("utf-8") parser = get_parser() parsed, remaining = parser.parse_known_args() - # sys.argv = remaining + _patch_path(pipenv_site=parsed.pipenv_site) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + from pipenv.vendor.vistir.misc import replace_with_text_stream + warnings.simplefilter("ignore", category=ResourceWarning) + replace_with_text_stream("stdout") + replace_with_text_stream("stderr") + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") + os.environ["PYTHONUNBUFFERED"] = str("1") parsed = handle_parsed_args(parsed) - _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, parsed.write, + parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) if __name__ == "__main__": - _patch_path() - from pipenv.vendor import colorama - colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index 1fc4fd2d..13fba133 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1,38 +1,42 @@ # -*- coding: utf-8 -*- +from __future__ import print_function + import contextlib import errno import logging import os +import posixpath import re import shutil +import signal import stat import sys import warnings - from contextlib import contextmanager from distutils.spawn import find_executable import six import toml -import tomlkit - from click import echo as click_echo -from first import first -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # noqa -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # noqa -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # noqa -from six.moves import Mapping, Sequence, Set from six.moves.urllib.parse import urlparse -from urllib3 import util as urllib3_util -from vistir.compat import ResourceWarning -from vistir.misc import fs_str import crayons import parse +import tomlkit from . import environments -from .exceptions import PipenvUsageError +from .exceptions import PipenvCmdError, PipenvUsageError, RequirementError, ResolutionFailure from .pep508checker import lookup +from .vendor.packaging.markers import Marker +from .vendor.urllib3 import util as urllib3_util +from .vendor.vistir.compat import Mapping, ResourceWarning, Sequence, Set, lru_cache +from .vendor.vistir.misc import fs_str, run + +if environments.MYPY_RUNNING: + from typing import Tuple, Dict, Any, List, Union, Optional, Text + from .vendor.requirementslib.models.requirements import Requirement, Line + from .vendor.requirementslib.models.pipfile import Pipfile + from .project import Project, TSource logging.basicConfig(level=logging.ERROR) @@ -41,7 +45,7 @@ specifiers = [k for k in lookup.keys()] # List of version control systems we support. VCS_LIST = ("git", "svn", "hg", "bzr") SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") -requests_session = None +requests_session = None # type: ignore def _get_requests_session(): @@ -85,7 +89,11 @@ def cleanup_toml(tml): def convert_toml_outline_tables(parsed): """Converts all outline tables to inline tables.""" def convert_tomlkit_table(section): - for key, value in section._body: + if isinstance(section, tomlkit.items.Table): + body = section.value._body + else: + body = section._body + for key, value in body: if not key: continue if hasattr(value, "keys") and not isinstance(value, tomlkit.items.InlineTable): @@ -113,6 +121,47 @@ def convert_toml_outline_tables(parsed): return parsed +def run_command(cmd, *args, **kwargs): + """ + Take an input command and run it, handling exceptions and error codes and returning + its stdout and stderr. + + :param cmd: The list of command and arguments. + :type cmd: list + :returns: A 2-tuple of the output and error from the command + :rtype: Tuple[str, str] + :raises: exceptions.PipenvCmdError + """ + + from pipenv.vendor import delegator + from ._compat import decode_for_output + from .cmdparse import Script + catch_exceptions = kwargs.pop("catch_exceptions", True) + if isinstance(cmd, (six.string_types, list, tuple)): + cmd = Script.parse(cmd) + if not isinstance(cmd, Script): + raise TypeError("Command input must be a string, list or tuple") + if "env" not in kwargs: + kwargs["env"] = os.environ.copy() + kwargs["env"]["PYTHONIOENCODING"] = "UTF-8" + try: + cmd_string = cmd.cmdify() + except TypeError: + click_echo("Error turning command into string: {0}".format(cmd), err=True) + sys.exit(1) + if environments.is_verbose(): + click_echo("Running command: $ {0}".format(cmd_string, err=True)) + c = delegator.run(cmd_string, *args, **kwargs) + return_code = c.return_code + if environments.is_verbose(): + click_echo("Command output: {0}".format( + crayons.blue(decode_for_output(c.out)) + ), err=True) + if not c.ok and catch_exceptions: + raise PipenvCmdError(cmd_string, c.out, c.err, return_code) + return c + + def parse_python_version(output): """Parse a Python version output returned by `python --version`. @@ -210,8 +259,10 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["-i", package_url]) # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): + url_parts = urllib3_util.parse_url(package_url) + url_port = ":{0}".format(url_parts.port) if url_parts.port else "" pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(package_url).host] + ["--trusted-host", "{0}{1}".format(url_parts.host, url_port)] ) # Add additional sources as extra indexes. if len(sources) > 1: @@ -222,44 +273,117 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args.extend(["--extra-index-url", url]) # Trust the host if it's not verified. if not source.get("verify_ssl", True): + url_parts = urllib3_util.parse_url(url) + url_port = ":{0}".format(url_parts.port) if url_parts.port else "" pip_args.extend( - ["--trusted-host", urllib3_util.parse_url(url).host] + ["--trusted-host", "{0}{1}".format(url_parts.host, url_port)] ) return pip_args -def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): - from .vendor.requirementslib.models.requirements import Requirement - constraints = [] - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - constraints.append(req.constraint_line) - if url: - source = first( - s for s in sources if s.get("url") and url.startswith(s["url"])) - if source: - index_lookup[req.name] = source.get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - return constraints +def get_project_index(index=None, trusted_hosts=None, project=None): + # type: (Optional[Union[str, TSource]], Optional[List[str]], Optional[Project]) -> TSource + from .project import SourceNotFound + if not project: + from .core import project + if trusted_hosts is None: + trusted_hosts = [] + if isinstance(index, Mapping): + return project.find_source(index.get("url")) + try: + source = project.find_source(index) + except SourceNotFound: + index_url = urllib3_util.parse_url(index) + src_name = project.src_name_from_url(index) + verify_ssl = index_url.host not in trusted_hosts + source = {"url": index, "verify_ssl": verify_ssl, "name": src_name} + return source + + +def get_source_list( + index=None, # type: Optional[Union[str, TSource]] + extra_indexes=None, # type: Optional[List[str]] + trusted_hosts=None, # type: Optional[List[str]] + pypi_mirror=None, # type: Optional[str] + project=None, # type: Optional[Project] +): + # type: (...) -> List[TSource] + sources = [] # type: List[TSource] + if not project: + from .core import project + if index: + sources.append(get_project_index(index)) + if extra_indexes: + if isinstance(extra_indexes, six.string_types): + extra_indexes = [extra_indexes] + for source in extra_indexes: + extra_src = get_project_index(source) + if not sources or extra_src["url"] != sources[0]["url"]: + sources.append(extra_src) + else: + for source in project.pipfile_sources: + if not sources or source["url"] != sources[0]["url"]: + sources.append(source) + if not sources: + sources = project.pipfile_sources[:] + if pypi_mirror: + sources = [ + create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source + for source in sources + ] + return sources + + +def get_indexes_from_requirement(req, project=None, index=None, extra_indexes=None, trusted_hosts=None, pypi_mirror=None): + # type: (Requirement, Optional[Project], Optional[Text], Optional[List[Text]], Optional[List[Text]], Optional[Text]) -> Tuple[TSource, List[TSource], List[Text]] + if not project: + from .core import project + index_sources = [] # type: List[TSource] + if not trusted_hosts: + trusted_hosts = [] # type: List[Text] + if extra_indexes is None: + extra_indexes = [] + project_indexes = project.pipfile_sources[:] + indexes = [] + if req.index: + indexes.append(req.index) + if getattr(req, "extra_indexes", None): + if not isinstance(req.extra_indexes, list): + indexes.append(req.extra_indexes) + else: + indexes.extend(req.extra_indexes) + indexes.extend(project_indexes) + if len(indexes) > 1: + index, extra_indexes = indexes[0], indexes[1:] + index_sources = get_source_list(index=index, extra_indexes=extra_indexes, trusted_hosts=trusted_hosts, pypi_mirror=pypi_mirror, project=project) + if len(index_sources) > 1: + index_source, extra_index_sources = index_sources[0], index_sources[1:] + else: + index_source, extra_index_sources = index_sources[0], [] + return index_source, extra_index_sources + + +@lru_cache() +def get_pipenv_sitedir(): + # type: () -> Optional[str] + import pkg_resources + site_dir = next( + iter(d for d in pkg_resources.working_set if d.key.lower() == "pipenv"), None + ) + if site_dir is not None: + return site_dir.location + return None class Resolver(object): - def __init__(self, constraints, req_dir, project, sources, clear=False, pre=False): + def __init__( + self, constraints, req_dir, project, sources, index_lookup=None, + markers_lookup=None, skipped=None, clear=False, pre=False + ): from pipenv.patched.piptools import logging as piptools_logging if environments.is_verbose(): logging.log.verbose = True - piptools_logging.log.verbose = True + piptools_logging.log.verbosity = environments.PIPENV_VERBOSITY self.initial_constraints = constraints self.req_dir = req_dir self.project = project @@ -269,6 +393,11 @@ class Resolver(object): self.clear = clear self.pre = pre self.results = None + self.markers_lookup = markers_lookup if markers_lookup is not None else {} + self.index_lookup = index_lookup if index_lookup is not None else {} + self.skipped = skipped if skipped is not None else {} + self.markers = {} + self.requires_python_markers = {} self._pip_args = None self._constraints = None self._parsed_constraints = None @@ -286,16 +415,266 @@ class Resolver(object): "sources={self.sources})>".format(self=self) ) - def _get_pip_command(self): - from pip_shims.shims import Command + @staticmethod + @lru_cache() + def _get_pip_command(): + from .vendor.pip_shims.shims import InstallCommand + return InstallCommand() - class PipCommand(Command): - """Needed for pip-tools.""" + @classmethod + def get_metadata( + cls, + deps, # type: List[str] + index_lookup, # type: Dict[str, str] + markers_lookup, # type: Dict[str, str] + project, # type: Project + sources, # type: Dict[str, str] + req_dir=None, # type: Optional[str] + pre=False, # type: bool + clear=False, # type: bool + ): + # type: (...) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]], Dict[str, str], Dict[str, str]] + constraints = set() # type: Set[str] + skipped = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if not req_dir: + from .vendor.vistir.path import create_tracked_tempdir + req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-reqdir") + transient_resolver = cls( + [], req_dir, project, sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, clear=clear, pre=pre + ) + for dep in deps: + if not dep: + continue + req, req_idx, markers_idx = cls.parse_line( + dep, index_lookup=index_lookup, markers_lookup=markers_lookup, project=project + ) + index_lookup.update(req_idx) + markers_lookup.update(markers_idx) + # Add dependencies of any file (e.g. wheels/tarballs), source, or local + # directories into the initial constraint pool to be resolved with the + # rest of the dependencies, while adding the files/vcs deps/paths themselves + # to the lockfile directly + constraint_update, lockfile_update = cls.get_deps_from_req( + req, resolver=transient_resolver + ) + constraints |= constraint_update + skipped.update(lockfile_update) + return constraints, skipped, index_lookup, markers_lookup - name = "PipCommand" + @classmethod + def parse_line( + cls, + line, # type: str + index_lookup=None, # type: Dict[str, str] + markers_lookup=None, # type: Dict[str, str] + project=None # type: Optional[Project] + ): + # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]] + from .vendor.requirementslib.models.requirements import Requirement + from .vendor.requirementslib.models.utils import DIRECT_URL_RE + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if project is None: + from .project import Project + project = Project() + url = None + indexes, trusted_hosts, remainder = parse_indexes(line) + if indexes: + url = indexes[0] + line = " ".join(remainder) + req = None # type: Requirement + try: + req = Requirement.from_line(line) + except ValueError: + direct_url = DIRECT_URL_RE.match(line) + if direct_url: + line = "{0}#egg={1}".format(line, direct_url.groupdict()["name"]) + try: + req = Requirement.from_line(line) + except ValueError: + raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line)) + else: + raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line)) + if url: + try: + index_lookup[req.normalized_name] = project.get_source( + url=url, refresh=True).get("name") + except TypeError: + pass + try: + req.normalized_name + except TypeError: + raise RequirementError(req=req) + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers: + markers_lookup[req.normalized_name] = req.markers.replace('"', "'") + return req, index_lookup, markers_lookup - from pipenv.patched.piptools.scripts.compile import get_pip_command - return get_pip_command() + @classmethod + def get_deps_from_line(cls, line): + # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + req, _, _ = cls.parse_line(line) + return cls.get_deps_from_req(req) + + @classmethod + def get_deps_from_req(cls, req, resolver=None): + # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name + from .vendor.requirementslib.models.requirements import Requirement + from requirementslib.utils import is_installable_dir + # TODO: this is way too complex, refactor this + constraints = set() # type: Set[str] + locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if (req.is_file_or_url or req.is_vcs) and not req.is_wheel: + # for local packages with setup.py files and potential direct url deps: + if req.is_vcs: + req_list, lockfile = get_vcs_deps(reqs=[req]) + req = next(iter(req for req in req_list if req is not None), req_list) + entry = lockfile[pep423_name(req.normalized_name)] + else: + _, entry = req.pipfile_entry + parsed_line = req.req.parsed_line # type: Line + setup_info = None # type: Any + try: + name = req.normalized_name + except TypeError: + raise RequirementError(req=req) + setup_info = req.req.setup_info + setup_info.get_info() + locked_deps[pep423_name(name)] = entry + requirements = [] + # Allow users to toggle resolution off for non-editable VCS packages + # but leave it on for local, installable folders on the filesystem + if environments.PIPENV_RESOLVE_VCS or ( + req.editable or parsed_line.is_wheel or ( + req.is_file_or_url and parsed_line.is_local + and is_installable_dir(parsed_line.path) + ) + ): + requirements = [v for v in getattr(setup_info, "requires", {}).values()] + for r in requirements: + if getattr(r, "url", None) and not getattr(r, "editable", False): + if r is not None: + if not r.url: + continue + line = _requirement_to_str_lowercase_name(r) + new_req, _, _ = cls.parse_line(line) + if r.marker and not r.marker.evaluate(): + new_constraints = {} + _, new_entry = req.pipfile_entry + new_lock = { + pep423_name(new_req.normalized_name): new_entry + } + else: + new_constraints, new_lock = cls.get_deps_from_req( + new_req, resolver + ) + locked_deps.update(new_lock) + constraints |= new_constraints + # if there is no marker or there is a valid marker, add the constraint line + elif r and (not r.marker or (r.marker and r.marker.evaluate())): + line = _requirement_to_str_lowercase_name(r) + constraints.add(line) + # ensure the top level entry remains as provided + # note that we shouldn't pin versions for editable vcs deps + if (not req.is_vcs or (req.is_vcs and not req.editable)): + if req.specifiers: + locked_deps[name]["version"] = req.specifiers + elif parsed_line.setup_info and parsed_line.setup_info.version: + locked_deps[name]["version"] = "=={}".format( + parsed_line.setup_info.version + ) + # if not req.is_vcs: + locked_deps.update({name: entry}) + if req.is_vcs and req.editable: + constraints.add(req.constraint_line) + if req.is_file_or_url and req.req.is_local and req.editable and ( + req.req.setup_path is not None and os.path.exists(req.req.setup_path)): + constraints.add(req.constraint_line) + else: + # if the dependency isn't installable, don't add it to constraints + # and instead add it directly to the lock + if req and req.requirement and ( + req.requirement.marker and not req.requirement.marker.evaluate() + ): + pypi = resolver.repository if resolver else None + best_match = pypi.find_best_match(req.ireq) if pypi else None + if best_match: + hashes = resolver.collect_hashes(best_match) if resolver else [] + new_req = Requirement.from_ireq(best_match) + new_req = new_req.add_hashes(hashes) + name, entry = new_req.pipfile_entry + locked_deps[pep423_name(name)] = translate_markers(entry) + return constraints, locked_deps + constraints.add(req.constraint_line) + return constraints, locked_deps + return constraints, locked_deps + + @classmethod + def create( + cls, + deps, # type: List[str] + index_lookup=None, # type: Dict[str, str] + markers_lookup=None, # type: Dict[str, str] + project=None, # type: Project + sources=None, # type: List[str] + req_dir=None, # type: str + clear=False, # type: bool + pre=False # type: bool + ): + # type: (...) -> "Resolver" + from pipenv.vendor.vistir.path import create_tracked_tempdir + if not req_dir: + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if project is None: + from pipenv.core import project + project = project + if sources is None: + sources = project.sources + constraints, skipped, index_lookup, markers_lookup = cls.get_metadata( + deps, index_lookup, markers_lookup, project, sources, req_dir=req_dir, + pre=pre, clear=clear + ) + return Resolver( + constraints, req_dir, project, sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, skipped=skipped, clear=clear, pre=pre + ) + + @classmethod + def from_pipfile(cls, project=None, pipfile=None, dev=False, pre=False, clear=False): + # type: (Optional[Project], Optional[Pipfile], bool, bool, bool) -> "Resolver" + from pipenv.vendor.vistir.path import create_tracked_tempdir + if not project: + from pipenv.core import project + if not pipfile: + pipfile = project._pipfile + req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + index_lookup, markers_lookup = {}, {} + deps = set() + if dev: + deps.update(set([req.as_line() for req in pipfile.dev_packages])) + deps.update(set([req.as_line() for req in pipfile.packages])) + constraints, skipped, index_lookup, markers_lookup = cls.get_metadata( + list(deps), index_lookup, markers_lookup, project, project.sources, + req_dir=req_dir, pre=pre, clear=clear + ) + return Resolver( + constraints, req_dir, project, project.sources, index_lookup=index_lookup, + markers_lookup=markers_lookup, skipped=skipped, clear=clear, pre=pre + ) @property def pip_command(self): @@ -303,16 +682,29 @@ class Resolver(object): self._pip_command = self._get_pip_command() return self._pip_command - def prepare_pip_args(self): + def prepare_pip_args(self, use_pep517=True, build_isolation=True): pip_args = [] if self.sources: pip_args = prepare_pip_source_args(self.sources, pip_args) + if not use_pep517: + pip_args.append("--no-use-pep517") + if not build_isolation: + pip_args.append("--no-build-isolation") + pip_args.extend(["--cache-dir", environments.PIPENV_CACHE_DIR]) return pip_args @property def pip_args(self): + use_pep517 = False if ( + os.environ.get("PIP_NO_USE_PEP517", None) is not None + ) else (True if os.environ.get("PIP_USE_PEP517", None) is not None else None) + build_isolation = False if ( + os.environ.get("PIP_NO_BUILD_ISOLATION", None) is not None + ) else (True if os.environ.get("PIP_BUILD_ISOLATION", None) is not None else None) if self._pip_args is None: - self._pip_args = self.prepare_pip_args() + self._pip_args = self.prepare_pip_args( + use_pep517=use_pep517, build_isolation=build_isolation + ) return self._pip_args def prepare_constraint_file(self): @@ -324,8 +716,13 @@ class Resolver(object): dir=self.req_dir, delete=False, ) + skip_args = ("build-isolation", "use-pep517", "cache-dir") + args_to_add = [ + arg for arg in self.pip_args + if not any(bad_arg in arg for bad_arg in skip_args) + ] if self.sources: - requirementstxt_sources = " ".join(self.pip_args) if self.pip_args else "" + requirementstxt_sources = " ".join(args_to_add) if args_to_add else "" requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") constraints_file.write(u"{0}\n".format(requirementstxt_sources)) constraints = self.initial_constraints @@ -345,16 +742,16 @@ class Resolver(object): pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) pip_options.cache_dir = environments.PIPENV_CACHE_DIR self._pip_options = pip_options - if environments.is_verbose(): - click_echo( - crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True - ) return self._pip_options @property def session(self): if self._session is None: self._session = self.pip_command._build_session(self.pip_options) + # if environments.is_verbose(): + # click_echo( + # crayons.blue("Using pip: {0}".format(" ".join(self.pip_args))), err=True + # ) return self._session @property @@ -362,7 +759,8 @@ class Resolver(object): if self._repository is None: from pipenv.patched.piptools.repositories.pypi import PyPIRepository self._repository = PyPIRepository( - pip_options=self.pip_options, use_json=False, session=self.session + self.pip_args, use_json=False, session=self.session, + build_isolation=self.pip_options.build_isolation ) return self._repository @@ -401,22 +799,108 @@ class Resolver(object): from pipenv.patched.piptools.exceptions import NoCandidateFound from pipenv.patched.piptools.cache import CorruptCacheError from .exceptions import CacheError, ResolutionFailure - try: - results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) - except CorruptCacheError as e: - if environments.PIPENV_IS_CI or self.clear: - if self._retry_attempts < 3: - self.get_resolver(clear=True, pre=self.pre) - self._retry_attempts += 1 - self.resolve() + with temp_environ(): + os.environ["PIP_NO_USE_PEP517"] = str("") + try: + results = self.resolver.resolve(max_rounds=environments.PIPENV_MAX_ROUNDS) + except CorruptCacheError as e: + if environments.PIPENV_IS_CI or self.clear: + if self._retry_attempts < 3: + self.get_resolver(clear=True, pre=self.pre) + self._retry_attempts += 1 + self.resolve() + else: + raise CacheError(e.path) + except (NoCandidateFound, DistributionNotFound, HTTPError) as e: + raise ResolutionFailure(message=str(e)) else: - raise CacheError(e.path) - except (NoCandidateFound, DistributionNotFound, HTTPError) as e: - raise ResolutionFailure(message=str(e)) - else: - self.results = results - self.resolved_tree.update(results) - return self.resolved_tree + self.results = results + self.resolved_tree.update(results) + return self.resolved_tree + + @lru_cache(maxsize=1024) + def fetch_candidate(self, ireq): + candidates = self.repository.find_all_candidates(ireq.name) + matched_version = next(iter(sorted( + ireq.specifier.filter((c.version for c in candidates), True), reverse=True) + ), None) + if matched_version: + matched_candidate = next(iter( + c for c in candidates if c.version == matched_version + )) + return matched_candidate + return None + + def resolve_constraints(self): + from .vendor.requirementslib.models.markers import marker_from_specifier + new_tree = set() + for result in self.resolved_tree: + if result.markers: + self.markers[result.name] = result.markers + else: + candidate = self.fetch_candidate(result) + requires_python = getattr(candidate, "requires_python", None) + if requires_python: + marker = marker_from_specifier(candidate.requires_python) + self.markers[result.name] = marker + result.markers = marker + if result.req: + result.req.marker = marker + new_tree.add(result) + self.resolved_tree = new_tree + + @classmethod + def prepend_hash_types(cls, checksums): + cleaned_checksums = [] + for checksum in checksums: + if not checksum: + continue + if not checksum.startswith("sha256:"): + checksum = "sha256:{0}".format(checksum) + cleaned_checksums.append(checksum) + return cleaned_checksums + + def collect_hashes(self, ireq): + from .vendor.requests import ConnectionError + collected_hashes = [] + if ireq in self.hashes: + collected_hashes += list(self.hashes.get(ireq, [])) + if self._should_include_hash(ireq): + try: + hash_map = self.get_hash(ireq) + collected_hashes += list(hash_map) + except (ValueError, KeyError, IndexError, ConnectionError): + pass + elif any( + "python.org" in source["url"] or "pypi.org" in source["url"] + for source in self.sources + ): + pkg_url = "https://pypi.org/pypi/{0}/json".format(ireq.name) + session = _get_requests_session() + try: + # Grab the hashes from the new warehouse API. + r = session.get(pkg_url, timeout=10) + api_releases = r.json()["releases"] + cleaned_releases = {} + for api_version, api_info in api_releases.items(): + api_version = clean_pkg_version(api_version) + cleaned_releases[api_version] = api_info + version = "" + if ireq.specifier: + spec = next(iter(s for s in list(ireq.specifier._specs)), None) + if spec: + version = spec.version + for release in cleaned_releases[version]: + collected_hashes.append(release["digests"]["sha256"]) + collected_hashes = self.prepend_hash_types(collected_hashes) + except (ValueError, KeyError, ConnectionError): + if environments.is_verbose(): + click_echo( + "{0}: Error generating hash for {1}".format( + crayons.red("Warning", bold=True), ireq.name + ), err=True + ) + return collected_hashes @staticmethod def _should_include_hash(ireq): @@ -437,38 +921,139 @@ class Resolver(object): # We also don't want to try to hash directories as this will fail # as these are editable deps and are not hashable. - if (ireq.link.scheme == "file" and - Path(to_native_string(url_to_path(ireq.link.url))).is_dir()): + if ( + ireq.link.scheme == "file" + and Path(to_native_string(url_to_path(ireq.link.url))).is_dir() + ): return False return True + def get_hash(self, ireq, ireq_hashes=None): + """ + Retrieve hashes for a specific ``InstallRequirement`` instance. + + :param ireq: An ``InstallRequirement`` to retrieve hashes for + :type ireq: :class:`~pip_shims.InstallRequirement` + :return: A set of hashes. + :rtype: Set + """ + + # We _ALWAYS MUST PRIORITIZE_ the inclusion of hashes from local sources + # PLEASE *DO NOT MODIFY THIS* TO CHECK WHETHER AN IREQ ALREADY HAS A HASH + # RESOLVED. The resolver will pull hashes from PyPI and only from PyPI. + # The entire purpose of this approach is to include missing hashes. + # This fixes a race condition in resolution for missing dependency caches + # see pypa/pipenv#3289 + if not self._should_include_hash(ireq): + return add_to_set(set(), ireq_hashes) + elif self._should_include_hash(ireq) and ( + not ireq_hashes or ireq.link.scheme == "file" + ): + if not ireq_hashes: + ireq_hashes = set() + new_hashes = self.resolver.repository._hash_cache.get_hash(ireq.link) + ireq_hashes = add_to_set(ireq_hashes, new_hashes) + else: + ireq_hashes = set(ireq_hashes) + # The _ONLY CASE_ where we flat out set the value is if it isn't present + # It's a set, so otherwise we *always* need to do a union update + if ireq not in self.hashes: + return ireq_hashes + else: + return self.hashes[ireq] | ireq_hashes + def resolve_hashes(self): if self.results is not None: resolved_hashes = self.resolver.resolve_hashes(self.results) for ireq, ireq_hashes in resolved_hashes.items(): - # We _ALWAYS MUST PRIORITIZE_ the inclusion of hashes from local sources - # PLEASE *DO NOT MODIFY THIS* TO CHECK WHETHER AN IREQ ALREADY HAS A HASH - # RESOLVED. The resolver will pull hashes from PyPI and only from PyPI. - # The entire purpose of this approach is to include missing hashes. - # This fixes a race condition in resolution for missing dependency caches - # see pypa/pipenv#3289 - if self._should_include_hash(ireq) and ( - not ireq_hashes or ireq.link.scheme == "file" - ): - if not ireq_hashes: - ireq_hashes = set() - new_hashes = self.resolver.repository._hash_cache.get_hash(ireq.link) - add_to_set(ireq_hashes, new_hashes) - else: - ireq_hashes = set(ireq_hashes) - # The _ONLY CASE_ where we flat out set the value is if it isn't present - # It's a set, so otherwise we *always* need to do a union update - if ireq not in self.hashes: - self.hashes[ireq] = ireq_hashes - else: - self.hashes[ireq] |= ireq_hashes + self.hashes[ireq] = self.get_hash(ireq, ireq_hashes=ireq_hashes) return self.hashes + def _clean_skipped_result(self, req, value): + ref = None + if req.is_vcs: + ref = req.commit_hash + ireq = req.as_ireq() + entry = value.copy() + entry["name"] = req.name + if entry.get("editable", False) and entry.get("version"): + del entry["version"] + ref = ref if ref is not None else entry.get("ref") + if ref: + entry["ref"] = ref + if self._should_include_hash(ireq): + collected_hashes = self.collect_hashes(ireq) + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + return req.name, entry + + def clean_results(self): + from pipenv.vendor.requirementslib.models.requirements import Requirement + reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in self.resolved_tree] + results = {} + for req, ireq in reqs: + if (req.vcs and req.editable and not req.is_direct_url): + continue + collected_hashes = self.collect_hashes(ireq) + req = req.add_hashes(collected_hashes) + if not collected_hashes and self._should_include_hash(ireq): + discovered_hashes = self.hashes.get(ireq, set()) | self.get_hash(ireq) + if discovered_hashes: + req = req.add_hashes(discovered_hashes) + self.hashes[ireq] = collected_hashes = discovered_hashes + if collected_hashes: + collected_hashes = sorted(set(collected_hashes)) + name, entry = format_requirement_for_lockfile( + req, self.markers_lookup, self.index_lookup, collected_hashes + ) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + for k in list(self.skipped.keys()): + req = Requirement.from_pipfile(k, self.skipped[k]) + name, entry = self._clean_skipped_result(req, self.skipped[k]) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + results = list(results.values()) + return results + + +def format_requirement_for_lockfile(req, markers_lookup, index_lookup, hashes=None): + if req.specifiers: + version = str(req.get_version()) + else: + version = None + index = index_lookup.get(req.normalized_name) + markers = markers_lookup.get(req.normalized_name) + req.index = index + name, pf_entry = req.pipfile_entry + name = pep423_name(req.name) + entry = {} + if isinstance(pf_entry, six.string_types): + entry["version"] = pf_entry.lstrip("=") + else: + entry.update(pf_entry) + if version is not None: + entry["version"] = version + if req.line_instance.is_direct_url: + entry["file"] = req.req.uri + if hashes: + entry["hashes"] = sorted(set(hashes)) + entry["name"] = name + if index: # and index != next(iter(project.sources), {}).get("name"): + entry.update({"index": index}) + if markers: + entry.update({"markers": markers}) + entry = translate_markers(entry) + if req.vcs or req.editable and entry.get("index"): + del entry["index"] + return name, entry + def _show_warning(message, category, filename, lineno, line): warnings.showwarning(message=message, category=category, filename=filename, @@ -493,74 +1078,84 @@ def actually_resolve_deps( warning_list = [] with warnings.catch_warnings(record=True) as warning_list: - constraints = get_resolver_metadata( - deps, index_lookup, markers_lookup, project, sources, + resolver = Resolver.create( + deps, index_lookup, markers_lookup, project, sources, req_dir, clear, pre ) - resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) - resolved_tree = resolver.resolve() + resolver.resolve() hashes = resolver.resolve_hashes() - + resolver.resolve_constraints() + results = resolver.clean_results() for warning in warning_list: _show_warning(warning.message, warning.category, warning.filename, warning.lineno, warning.line) - return (resolved_tree, hashes, markers_lookup, resolver) + return (results, hashes, resolver.markers_lookup, resolver, resolver.skipped) @contextlib.contextmanager def create_spinner(text, nospin=None, spinner_name=None): - import vistir.spin + from .vendor.vistir import spin + from .vendor.vistir.misc import fs_str if not spinner_name: spinner_name = environments.PIPENV_SPINNER if nospin is None: nospin = environments.PIPENV_NOSPIN - with vistir.spin.create_spinner( - spinner_name=spinner_name, - start_text=vistir.compat.fs_str(text), - nospin=nospin, write_to_stdout=False + with spin.create_spinner( + spinner_name=spinner_name, + start_text=fs_str(text), + nospin=nospin, write_to_stdout=False ) as sp: yield sp def resolve(cmd, sp): - from .vendor import delegator + import delegator from .cmdparse import Script from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor.vistir.compat import to_native_string + from .vendor.vistir.misc import echo EOF.__module__ = "pexpect.exceptions" from ._compat import decode_output c = delegator.run(Script.parse(cmd).cmdify(), block=False, env=os.environ.copy()) + if environments.is_verbose(): + c.subprocess.logfile = sys.stderr _out = decode_output("") result = None out = to_native_string("") while True: + result = None try: result = c.expect(u"\n", timeout=environments.PIPENV_INSTALL_TIMEOUT) - except (EOF, TIMEOUT): + except TIMEOUT: pass - if result is None: + except EOF: break - _out = c.subprocess.before - if _out is not None: + except KeyboardInterrupt: + c.kill() + break + if result: + _out = c.subprocess.before _out = decode_output("{0}".format(_out)) out += _out - sp.text = to_native_string("{0}".format(_out[:100])) - if environments.is_verbose(): - if _out is not None: - sp._hide_cursor() - sp.write(_out.rstrip()) - sp._show_cursor() + # sp.text = to_native_string("{0}".format(_out[:100])) + if environments.is_verbose(): + sp.hide_and_write(out.splitlines()[-1].rstrip()) + else: + break c.block() if c.return_code != 0: sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( "Locking Failed!" )) - click_echo(c.out.strip(), err=True) - click_echo(c.err.strip(), err=True) + echo(c.out.strip(), err=True) + if not environments.is_verbose(): + echo(out, err=True) sys.exit(c.return_code) + if environments.is_verbose(): + echo(c.err.strip(), err=True) return c -def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): +def get_locked_dep(dep, pipfile_section, prefer_pipfile=True): # the prefer pipfile flag is not used yet, but we are introducing # it now for development purposes # TODO: Is this implementation clear? How can it be improved? @@ -570,8 +1165,11 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): "pipfile_entry": None } if isinstance(dep, Mapping) and dep.get("name", ""): - name_options = [dep["name"], pep423_name(dep["name"])] - name = next(iter(k for k in name_options if k in pipfile_section), None) + dep_name = pep423_name(dep["name"]) + name = next(iter( + k for k in pipfile_section.keys() + if pep423_name(k) == dep_name + ), None) entry = pipfile_section[name] if name else None if entry: @@ -581,24 +1179,34 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): version = entry.get("version", "") if entry else "" else: version = entry if entry else "" - lockfile_version = lockfile_entry.get("version", "") + lockfile_name, lockfile_dict = lockfile_entry.copy().popitem() + lockfile_version = lockfile_dict.get("version", "") # Keep pins from the lockfile if prefer_pipfile and lockfile_version != version and version.startswith("=="): - lockfile_version = version + lockfile_dict["version"] = version + lockfile_entry[lockfile_name] = lockfile_dict return lockfile_entry def prepare_lockfile(results, pipfile, lockfile): - from .vendor.requirementslib.utils import is_vcs + # from .vendor.requirementslib.utils import is_vcs for dep in results: + if not dep: + continue # Merge in any relevant information from the pipfile entry, including # markers, normalized names, URL info, etc that we may have dropped during lock - if not is_vcs(dep): - lockfile_entry = get_locked_dep(dep, pipfile) - name = next(iter(k for k in lockfile_entry.keys())) - current_entry = lockfile.get(name) - if not current_entry or not is_vcs(current_entry): - lockfile.update(lockfile_entry) + # if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + name = next(iter(k for k in lockfile_entry.keys())) + current_entry = lockfile.get(name) + if current_entry: + if not isinstance(current_entry, Mapping): + lockfile[name] = lockfile_entry[name] + else: + lockfile[name].update(lockfile_entry[name]) + lockfile[name] = translate_markers(lockfile[name]) + else: + lockfile[name] = lockfile_entry[name] return lockfile @@ -612,41 +1220,56 @@ def venv_resolve_deps( pypi_mirror=None, dev=False, pipfile=None, - lockfile=None + lockfile=None, + keep_outdated=False ): + """ + Resolve dependencies for a pipenv project, acts as a portal to the target environment. + + Regardless of whether a virtual environment is present or not, this will spawn + a subproces which is isolated to the target environment and which will perform + dependency resolution. This function reads the output of that call and mutates + the provided lockfile accordingly, returning nothing. + + :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve. + :param Callable which: [description] + :param project: The pipenv Project instance to use during resolution + :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False + :param Optional[bool] clear: Whether to clear the cache during resolution, defaults to False + :param Optional[bool] allow_global: Whether to use *sys.executable* as the python binary, defaults to False + :param Optional[str] pypi_mirror: A URL to substitute any time *pypi.org* is encountered, defaults to None + :param Optional[bool] dev: Whether to target *dev-packages* or not, defaults to False + :param pipfile: A Pipfile section to operate on, defaults to None + :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]] + :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None + :param bool keep_outdated: Whether to retain outdated dependencies and resolve with them in mind, defaults to False + :raises RuntimeError: Raised on resolution failure + :return: Nothing + :rtype: None + """ + from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError, NamedTemporaryFile from .vendor.vistir.path import create_tracked_tempdir from . import resolver + from ._compat import decode_for_output import json - vcs_deps = [] - vcs_lockfile = {} results = [] - pipfile_section = "dev_packages" if dev else "packages" + pipfile_section = "dev-packages" if dev else "packages" lockfile_section = "develop" if dev else "default" - vcs_section = "vcs_{0}".format(pipfile_section) - vcs_deps = getattr(project, vcs_section, {}) - if not deps and not vcs_deps: - return {} + if not deps: + if not project.pipfile_exists: + return None + deps = project.parsed_pipfile.get(pipfile_section, {}) + if not deps: + return None if not pipfile: - pipfile = getattr(project, pipfile_section, None) + pipfile = getattr(project, pipfile_section, {}) if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - if vcs_deps: - with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=allow_global, - dev=dev, - ) - vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] - lockfile[lockfile_section].update(vcs_lockfile) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -657,49 +1280,59 @@ def venv_resolve_deps( cmd.append("--clear") if allow_global: cmd.append("--system") + if dev: + cmd.append("--dev") + target_file = NamedTemporaryFile(prefix="resolver", suffix=".json", delete=False) + target_file.close() + cmd.extend(["--write", make_posix(target_file.name)]) with temp_environ(): - os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} - os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) + os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - with create_spinner(text=fs_str("Locking...")) as sp: - c = resolve(cmd, sp) - results = c.out - if vcs_deps: - with temp_environ(): - os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) - sp.text = to_native_string("Locking VCS Dependencies...") - vcs_c = resolve(cmd, sp) - vcs_results, vcs_err = vcs_c.out, vcs_c.err - else: - vcs_results, vcs_err = "", "" - sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) - outputs = [results, vcs_results] - if environments.is_verbose(): - for output in outputs: - click_echo(output.split("RESULTS:")[0], err=True) - try: - results = json.loads(results.split("RESULTS:")[1].strip()) - if vcs_results: - # For vcs dependencies, treat the initial pass at locking (i.e. checkout) - # as the pipfile entry because it gets us an actual ref to use - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) - vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) + pipenv_site_dir = get_pipenv_sitedir() + if pipenv_site_dir is not None: + os.environ["PIPENV_SITE_DIR"] = pipenv_site_dir else: - vcs_results = [] - + os.environ.pop("PIPENV_SITE_DIR", None) + if keep_outdated: + os.environ["PIPENV_KEEP_OUTDATED"] = fs_str("1") + with create_spinner(text=decode_for_output("Locking...")) as sp: + # This conversion is somewhat slow on local and file-type requirements since + # we now download those requirements / make temporary folders to perform + # dependency resolution on them, so we are including this step inside the + # spinner context manager for the UX improvement + sp.write(decode_for_output("Building requirements...")) + deps = convert_deps_to_pip( + deps, project, r=False, include_index=True + ) + constraints = set(deps) + os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) + sp.write(decode_for_output("Resolving dependencies...")) + c = resolve(cmd, sp) + results = c.out.strip() + if c.ok: + sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) + else: + sp.red.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!")) + click_echo("Output: {0}".format(c.out.strip()), err=True) + click_echo("Error: {0}".format(c.err.strip()), err=True) + try: + with open(target_file.name, "r") as fh: + results = json.load(fh) except (IndexError, JSONDecodeError): - for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: - click_echo(out.strip(), err=True) - click_echo(err.strip(), err=True) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) + if os.path.exists(target_file.name): + os.unlink(target_file.name) raise RuntimeError("There was a problem with locking.") - lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) - for k, v in vcs_lockfile.items(): - if k in getattr(project, vcs_section, {}) or k not in lockfile[lockfile_section]: - lockfile[lockfile_section][k].update(v) + if os.path.exists(target_file.name): + os.unlink(target_file.name) + if lockfile_section not in lockfile: + lockfile[lockfile_section] = {} + prepare_lockfile(results, pipfile, lockfile[lockfile_section]) def resolve_deps( @@ -716,9 +1349,6 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .vendor.requests.exceptions import ConnectionError - from .vendor.requirementslib.models.requirements import Requirement - index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) @@ -726,8 +1356,9 @@ def resolve_deps( os.environ["PIP_SRC"] = project.virtualenv_src_location backup_python_path = sys.executable results = [] + resolver = None if not deps: - return results + return results, resolver # First (proper) attempt: req_dir = req_dir if req_dir else os.environ.get("req_dir", None) if not req_dir: @@ -735,7 +1366,7 @@ def resolve_deps( req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + results, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -747,9 +1378,9 @@ def resolve_deps( ) except RuntimeError: # Don't exit here, like usual. - resolved_tree = None + results = None # Second (last-resort) attempt: - if resolved_tree is None: + if results is None: with HackedPythonVersion( python_version=".".join([str(s) for s in sys.version_info[:3]]), python_path=backup_python_path, @@ -757,7 +1388,7 @@ def resolve_deps( try: # Attempt to resolve again, with different Python version information, # particularly for particularly particular packages. - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + results, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -769,64 +1400,7 @@ def resolve_deps( ) except RuntimeError: sys.exit(1) - for result in resolved_tree: - if not result.editable: - req = Requirement.from_ireq(result) - name = pep423_name(req.name) - version = str(req.get_version()) - index = index_lookup.get(result.name) - req.index = index - collected_hashes = [] - if result in hashes: - collected_hashes = list(hashes.get(result)) - elif any( - "python.org" in source["url"] or "pypi.org" in source["url"] - for source in sources - ): - pkg_url = "https://pypi.org/pypi/{0}/json".format(name) - session = _get_requests_session() - try: - # Grab the hashes from the new warehouse API. - r = session.get(pkg_url, timeout=10) - api_releases = r.json()["releases"] - cleaned_releases = {} - for api_version, api_info in api_releases.items(): - api_version = clean_pkg_version(api_version) - cleaned_releases[api_version] = api_info - for release in cleaned_releases[version]: - collected_hashes.append(release["digests"]["sha256"]) - collected_hashes = ["sha256:" + s for s in collected_hashes] - except (ValueError, KeyError, ConnectionError): - if environments.is_verbose(): - click_echo( - "{0}: Error generating hash for {1}".format( - crayons.red("Warning", bold=True), name - ), err=True - ) - # # Collect un-collectable hashes (should work with devpi). - # try: - # collected_hashes = collected_hashes + list( - # list(resolver.resolve_hashes([result]).items())[0][1] - # ) - # except (ValueError, KeyError, ConnectionError, IndexError): - # if verbose: - # print('Error generating hash for {}'.format(name)) - req.hashes = sorted(set(collected_hashes)) - name, _entry = req.pipfile_entry - entry = {} - if isinstance(_entry, six.string_types): - entry["version"] = _entry.lstrip("=") - else: - entry.update(_entry) - entry["version"] = version - entry["name"] = name - # if index: - # d.update({"index": index}) - if markers_lookup.get(result.name): - entry.update({"markers": markers_lookup.get(result.name)}) - entry = translate_markers(entry) - results.append(entry) - return results + return results, resolver def is_star(val): @@ -845,7 +1419,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): dependencies = [] for dep_name, dep in deps.items(): - indexes = project.pipfile_sources if hasattr(project, "pipfile_sources") else [] + if project: + project.clear_pipfile_cache() + indexes = getattr(project, "pipfile_sources", []) if project is not None else [] new_dep = Requirement.from_pipfile(dep_name, dep) if new_dep.index: include_index = True @@ -1003,53 +1579,6 @@ def proper_case(package_name): return good_name -def split_section(input_file, section_suffix, test_function): - """ - Split a pipfile or a lockfile section out by section name and test function - - :param dict input_file: A dictionary containing either a pipfile or lockfile - :param str section_suffix: A string of the name of the section - :param func test_function: A test function to test against the value in the key/value pair - - >>> split_section(my_lockfile, 'vcs', is_vcs) - { - 'default': { - "six": { - "hashes": [ - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb", - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9" - ], - "version": "==1.11.0" - } - }, - 'default-vcs': { - "e1839a8": { - "editable": true, - "path": "." - } - } - } - """ - pipfile_sections = ("packages", "dev-packages") - lockfile_sections = ("default", "develop") - if any(section in input_file for section in pipfile_sections): - sections = pipfile_sections - elif any(section in input_file for section in lockfile_sections): - sections = lockfile_sections - else: - # return the original file if we can't find any pipfile or lockfile sections - return input_file - - for section in sections: - split_dict = {} - entries = input_file.get(section, {}) - for k in list(entries.keys()): - if test_function(entries.get(k)): - split_dict[k] = entries.pop(k) - input_file["-".join([section, section_suffix])] = split_dict - return input_file - - def get_windows_path(*args): """Sanitize a path for windows environments @@ -1101,7 +1630,7 @@ def get_canonical_names(packages): if not isinstance(packages, Sequence): if not isinstance(packages, six.string_types): return packages - packages = [packages,] + packages = [packages] return set([canonicalize_name(pkg) for pkg in packages if pkg]) @@ -1134,15 +1663,14 @@ def walk_up(bottom): def find_requirements(max_depth=3): - """Returns the path of a Pipfile in parent directories.""" + """Returns the path of a requirements.txt file in parent directories.""" i = 0 for c, d, f in walk_up(os.getcwd()): i += 1 if i < max_depth: - if "requirements.txt": - r = os.path.join(c, "requirements.txt") - if os.path.isfile(r): - return r + r = os.path.join(c, "requirements.txt") + if os.path.isfile(r): + return r raise RuntimeError("No requirements.txt found!") @@ -1287,7 +1815,7 @@ def handle_remove_readonly(func, path, exc): warnings.warn(default_warning_message.format(path), ResourceWarning) return - raise + raise exc def escape_cmd(cmd): @@ -1305,37 +1833,53 @@ def safe_expandvars(value): def get_vcs_deps( - project, - which=None, - clear=False, - pre=False, - allow_global=False, + project=None, dev=False, pypi_mirror=None, + packages=None, + reqs=None ): from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" - reqs = [] + if reqs is None: + reqs = [] lockfile = {} - try: - packages = getattr(project, section) - except AttributeError: - return [], [] - for pkg_name, pkg_pipfile in packages.items(): - requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) + if not reqs: + if not project and not packages: + raise ValueError( + "Must supply either a project or a pipfile section to lock vcs dependencies." + ) + if not packages: + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()] + result = [] + for requirement in reqs: name = requirement.normalized_name commit_hash = None if requirement.is_vcs: try: - with locked_repository(requirement) as repo: + with temp_path(), locked_repository(requirement) as repo: + from pipenv.vendor.requirementslib.models.requirements import Requirement + # from distutils.sysconfig import get_python_lib + # sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] commit_hash = repo.get_commit_hash() + name = requirement.normalized_name + version = requirement._specifiers = "=={0}".format(requirement.req.setup_info.version) lockfile[name] = requirement.pipfile_entry[1] lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + result.append(requirement) + version = requirement.specifiers + if not version and requirement.specifiers: + version = requirement.specifiers + if version: + lockfile[name]['version'] = version except OSError: continue - return reqs, lockfile + return result, lockfile def translate_markers(pipfile_entry): @@ -1351,23 +1895,24 @@ def translate_markers(pipfile_entry): """ if not isinstance(pipfile_entry, Mapping): raise TypeError("Entry is not a pipfile formatted mapping.") - from .vendor.distlib.markers import DEFAULT_CONTEXT as marker_context - from .vendor.packaging.markers import Marker + from .vendor.packaging.markers import default_environment from .vendor.vistir.misc import dedup - allowed_marker_keys = ["markers"] + [k for k in marker_context.keys()] + allowed_marker_keys = ["markers"] + list(default_environment().keys()) provided_keys = list(pipfile_entry.keys()) if hasattr(pipfile_entry, "keys") else [] - pipfile_markers = [k for k in provided_keys if k in allowed_marker_keys] + pipfile_markers = set(provided_keys) & set(allowed_marker_keys) new_pipfile = dict(pipfile_entry).copy() marker_set = set() if "markers" in new_pipfile: - marker = str(Marker(new_pipfile.pop("markers"))) - if 'extra' not in marker: - marker_set.add(marker) + marker_str = new_pipfile.pop("markers") + if marker_str: + marker = str(Marker(marker_str)) + if 'extra' not in marker: + marker_set.add(marker) for m in pipfile_markers: entry = "{0}".format(pipfile_entry[m]) if m != "markers": - marker_set.add(str(Marker("{0}{1}".format(m, entry)))) + marker_set.add(str(Marker("{0} {1}".format(m, entry)))) new_pipfile.pop(m) if marker_set: new_pipfile["markers"] = str(Marker(" or ".join( @@ -1378,35 +1923,50 @@ def translate_markers(pipfile_entry): def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): + from .vendor.requirementslib.utils import is_vcs name = pep423_name(dep["name"]) + lockfile = {} # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - lockfile = {"version": "=={0}".format(dep["version"])} - for key in ["hashes", "index", "extras"]: + if "version" in dep and dep["version"] and not dep.get("editable", False): + version = "{0}".format(dep["version"]) + if not version.startswith("=="): + version = "=={0}".format(version) + lockfile["version"] = version + if is_vcs(dep): + ref = dep.get("ref", None) + if ref is not None: + lockfile["ref"] = ref + vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None) + if vcs_type: + lockfile[vcs_type] = dep[vcs_type] + if "subdirectory" in dep: + lockfile["subdirectory"] = dep["subdirectory"] + for key in ["hashes", "index", "extras", "editable"]: if key in dep: lockfile[key] = dep[key] # In case we lock a uri or a file when the user supplied a path # remove the uri or file keys from the entry and keep the path - if pipfile_entry and any(k in pipfile_entry for k in ["file", "path"]): - fs_key = next((k for k in ["path", "file"] if k in pipfile_entry), None) - lockfile_key = next((k for k in ["uri", "file", "path"] if k in lockfile), None) - if fs_key != lockfile_key: - try: - del lockfile[lockfile_key] - except KeyError: - # pass when there is no lock file, usually because it's the first time - pass - lockfile[fs_key] = pipfile_entry[fs_key] + fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) + pipfile_fs_key = None + if pipfile_entry: + pipfile_fs_key = next(iter(k for k in ["path", "file"] if k in pipfile_entry), None) + if fs_key and pipfile_fs_key and fs_key != pipfile_fs_key: + lockfile[pipfile_fs_key] = pipfile_entry[pipfile_fs_key] + elif fs_key is not None: + lockfile[fs_key] = dep[fs_key] # If a package is **PRESENT** in the pipfile but has no markers, make sure we # **NEVER** include markers in the lockfile - if "markers" in dep: + if "markers" in dep and dep.get("markers", "").strip(): # First, handle the case where there is no top level dependency in the pipfile if not is_top_level: - try: - lockfile["markers"] = translate_markers(dep)["markers"] - except TypeError: - pass + translated = translate_markers(dep).get("markers", "").strip() + if translated: + try: + lockfile["markers"] = translated + except TypeError: + pass # otherwise make sure we are prioritizing whatever the pipfile says about the markers # If the pipfile says nothing, then we should put nothing in the lockfile else: @@ -1501,11 +2061,11 @@ def parse_indexes(line): ) parser.add_argument( "--extra-index-url", "--extra-index", - metavar="extra_indexes",action="append", + metavar="extra_indexes", action="append", ) parser.add_argument("--trusted-host", metavar="trusted_hosts", action="append") args, remainder = parser.parse_known_args(line.split()) - index = [] if not args.index else [args.index,] + index = [] if not args.index else [args.index] extra_indexes = [] if not args.extra_index_url else args.extra_index_url indexes = index + extra_indexes trusted_hosts = args.trusted_host if args.trusted_host else [] @@ -1536,3 +2096,203 @@ def add_to_set(original_set, element): original_set |= set(element) else: original_set.add(element) + return original_set + + +def is_url_equal(url, other_url): + # type: (str, str) -> bool + """ + Compare two urls by scheme, host, and path, ignoring auth + + :param str url: The initial URL to compare + :param str url: Second url to compare to the first + :return: Whether the URLs are equal without **auth**, **query**, and **fragment** + :rtype: bool + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://user2:pass2@mydomain.com/some/path") + True + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://mydomain.com/some?some_query") + False + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(url)) + if not isinstance(other_url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(other_url)) + parsed_url = urllib3_util.parse_url(url) + parsed_other_url = urllib3_util.parse_url(other_url) + unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url + unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url + return unparsed == unparsed_other + + +@lru_cache() +def make_posix(path): + # type: (str) -> str + """ + Convert a path with possible windows-style separators to a posix-style path + (with **/** separators instead of **\\** separators). + + :param Text path: A path to convert. + :return: A converted posix-style path + :rtype: Text + + >>> make_posix("c:/users/user/venvs/some_venv\\Lib\\site-packages") + "c:/users/user/venvs/some_venv/Lib/site-packages" + + >>> make_posix("c:\\users\\user\\venvs\\some_venv") + "c:/users/user/venvs/some_venv" + """ + if not isinstance(path, six.string_types): + raise TypeError("Expected a string for path, received {0!r}...".format(path)) + starts_with_sep = path.startswith(os.path.sep) + separated = normalize_path(path).split(os.path.sep) + if isinstance(separated, (list, tuple)): + path = posixpath.join(*separated) + if starts_with_sep: + path = "/{0}".format(path) + return path + + +def get_pipenv_dist(pkg="pipenv", pipenv_site=None): + from .resolver import find_site_path + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + if pipenv_site is None: + pipenv_site = os.path.dirname(pipenv_libdir) + pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) + return pipenv_dist + + +def find_python(finder, line=None): + """ + Given a `pythonfinder.Finder` instance and an optional line, find a corresponding python + + :param finder: A :class:`pythonfinder.Finder` instance to use for searching + :type finder: :class:pythonfinder.Finder` + :param str line: A version, path, name, or nothing, defaults to None + :return: A path to python + :rtype: str + """ + + if line and not isinstance(line, six.string_types): + raise TypeError( + "Invalid python search type: expected string, received {0!r}".format(line) + ) + if line and os.path.isabs(line): + if os.name == "nt": + line = make_posix(line) + return line + if not finder: + from pipenv.vendor.pythonfinder import Finder + finder = Finder(global_search=True) + if not line: + result = next(iter(finder.find_all_python_versions()), None) + elif line and line[0].isdigit() or re.match(r'[\d\.]+', line): + result = finder.find_python_version(line) + else: + result = finder.find_python_version(name=line) + if not result: + result = finder.which(line) + if not result and not line.startswith("python"): + line = "python{0}".format(line) + result = find_python(finder, line) + if not result: + result = next(iter(finder.find_all_python_versions()), None) + if result: + if not isinstance(result, six.string_types): + return result.path.as_posix() + return result + return + + +def is_python_command(line): + """ + Given an input, checks whether the input is a request for python or notself. + + This can be a version, a python runtime name, or a generic 'python' or 'pythonX.Y' + + :param str line: A potential request to find python + :returns: Whether the line is a python lookup + :rtype: bool + """ + + if not isinstance(line, six.string_types): + raise TypeError("Not a valid command to check: {0!r}".format(line)) + + from pipenv.vendor.pythonfinder.utils import PYTHON_IMPLEMENTATIONS + is_version = re.match(r'[\d\.]+', line) + if (line.startswith("python") or is_version + or any(line.startswith(v) for v in PYTHON_IMPLEMENTATIONS)): + return True + # we are less sure about this but we can guess + if line.startswith("py"): + return True + return False + + +# def make_marker_from_specifier(spec): +# # type: (str) -> Optional[Marker] +# """Given a python version specifier, create a marker + +# :param spec: A specifier +# :type spec: str +# :return: A new marker +# :rtype: Optional[:class:`packaging.marker.Marker`] +# """ +# from .vendor.packaging.markers import Marker +# from .vendor.packaging.specifiers import SpecifierSet, Specifier +# from .vendor.requirementslib.models.markers import cleanup_pyspecs, format_pyversion +# if not any(spec.startswith(k) for k in Specifier._operators.keys()): +# if spec.strip().lower() in ["any", "<any>", "*"]: +# return None +# spec = "=={0}".format(spec) +# elif spec.startswith("==") and spec.count("=") > 3: +# spec = "=={0}".format(spec.lstrip("=")) +# if not spec: +# return None +# marker_segments = [] +# print(spec) +# for marker_segment in cleanup_pyspecs(spec): +# print(marker_segment) +# marker_segments.append(format_pyversion(marker_segment)) +# marker_str = " and ".join(marker_segments) +# return Marker(marker_str) + + +@contextlib.contextmanager +def interrupt_handled_subprocess( + cmd, verbose=False, return_object=True, write_to_stdout=False, combine_stderr=True, + block=True, nospin=True, env=None +): + """Given a :class:`subprocess.Popen` instance, wrap it in exception handlers. + + Terminates the subprocess when and if a `SystemExit` or `KeyboardInterrupt` are + processed. + + Arguments: + :param str cmd: A command to run + :param bool verbose: Whether to run with verbose mode enabled, default False + :param bool return_object: Whether to return a subprocess instance or a 2-tuple, default True + :param bool write_to_stdout: Whether to write directly to stdout, default False + :param bool combine_stderr: Whether to combine stdout and stderr, default True + :param bool block: Whether the subprocess should be a blocking subprocess, default True + :param bool nospin: Whether to suppress the spinner with the subprocess, default True + :param Optional[Dict[str, str]] env: A dictionary to merge into the subprocess environment + :return: A subprocess, wrapped in exception handlers, as a context manager + :rtype: :class:`subprocess.Popen` obj: An instance of a running subprocess + """ + obj = run( + cmd, verbose=verbose, return_object=True, write_to_stdout=False, + combine_stderr=False, block=True, nospin=True, env=env, + ) + try: + yield obj + except (SystemExit, KeyboardInterrupt): + if os.name == "nt": + os.kill(obj.pid, signal.CTRL_BREAK_EVENT) + else: + os.kill(obj.pid, signal.SIGINT) + obj.wait() + raise diff --git a/pipenv/vendor/attr/__init__.py b/pipenv/vendor/attr/__init__.py index debfd57b..9ff4d47f 100644 --- a/pipenv/vendor/attr/__init__.py +++ b/pipenv/vendor/attr/__init__.py @@ -16,9 +16,11 @@ from ._make import ( make_class, validate, ) +from ._version_info import VersionInfo -__version__ = "18.2.0" +__version__ = "19.3.0" +__version_info__ = VersionInfo._from_version_string(__version__) __title__ = "attrs" __description__ = "Classes Without Boilerplate" @@ -37,6 +39,7 @@ s = attributes = attrs ib = attr = attrib dataclass = partial(attrs, auto_attribs=True) # happy Easter ;) + __all__ = [ "Attribute", "Factory", diff --git a/pipenv/vendor/attr/__init__.pyi b/pipenv/vendor/attr/__init__.pyi index 492fb85e..38f16f06 100644 --- a/pipenv/vendor/attr/__init__.pyi +++ b/pipenv/vendor/attr/__init__.pyi @@ -20,12 +20,27 @@ from . import filters as filters from . import converters as converters from . import validators as validators +from ._version_info import VersionInfo + +__version__: str +__version_info__: VersionInfo +__title__: str +__description__: str +__url__: str +__uri__: str +__author__: str +__email__: str +__license__: str +__copyright__: str + _T = TypeVar("_T") _C = TypeVar("_C", bound=type) -_ValidatorType = Callable[[Any, Attribute, _T], Any] +_ValidatorType = Callable[[Any, Attribute[_T], _T], Any] _ConverterType = Callable[[Any], _T] -_FilterType = Callable[[Attribute, Any], bool] +_FilterType = Callable[[Attribute[_T], _T], bool] +_ReprType = Callable[[Any], str] +_ReprArgType = Union[bool, _ReprType] # FIXME: in reality, if multiple validators are passed they must be in a list or tuple, # but those are invariant and so would prevent subtypes of _ValidatorType from working # when passed in a list or tuple. @@ -49,18 +64,16 @@ class Attribute(Generic[_T]): name: str default: Optional[_T] validator: Optional[_ValidatorType[_T]] - repr: bool + repr: _ReprArgType cmp: bool + eq: bool + order: bool hash: Optional[bool] init: bool converter: Optional[_ConverterType[_T]] metadata: Dict[Any, Any] type: Optional[Type[_T]] kw_only: bool - def __lt__(self, x: Attribute) -> bool: ... - def __le__(self, x: Attribute) -> bool: ... - def __gt__(self, x: Attribute) -> bool: ... - def __ge__(self, x: Attribute) -> bool: ... # NOTE: We had several choices for the annotation to use for type arg: # 1) Type[_T] @@ -89,16 +102,17 @@ class Attribute(Generic[_T]): def attrib( default: None = ..., validator: None = ..., - repr: bool = ..., - cmp: bool = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., - convert: None = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: None = ..., converter: None = ..., factory: None = ..., kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> Any: ... # This form catches an explicit None or no default and infers the type from the other arguments. @@ -106,16 +120,17 @@ def attrib( def attrib( default: None = ..., validator: Optional[_ValidatorArgType[_T]] = ..., - repr: bool = ..., - cmp: bool = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., - convert: Optional[_ConverterType[_T]] = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: Optional[Type[_T]] = ..., converter: Optional[_ConverterType[_T]] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> _T: ... # This form catches an explicit default argument. @@ -123,16 +138,17 @@ def attrib( def attrib( default: _T, validator: Optional[_ValidatorArgType[_T]] = ..., - repr: bool = ..., - cmp: bool = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., - convert: Optional[_ConverterType[_T]] = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: Optional[Type[_T]] = ..., converter: Optional[_ConverterType[_T]] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> _T: ... # This form covers type=non-Type: e.g. forward references (str), Any @@ -140,16 +156,17 @@ def attrib( def attrib( default: Optional[_T] = ..., validator: Optional[_ValidatorArgType[_T]] = ..., - repr: bool = ..., - cmp: bool = ..., + repr: _ReprArgType = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., - convert: Optional[_ConverterType[_T]] = ..., metadata: Optional[Mapping[Any, Any]] = ..., type: object = ..., converter: Optional[_ConverterType[_T]] = ..., factory: Optional[Callable[[], _T]] = ..., kw_only: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> Any: ... @overload def attrs( @@ -157,7 +174,7 @@ def attrs( these: Optional[Dict[str, Any]] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., - cmp: bool = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., @@ -167,6 +184,9 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> _C: ... @overload def attrs( @@ -174,7 +194,7 @@ def attrs( these: Optional[Dict[str, Any]] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., - cmp: bool = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., @@ -184,14 +204,17 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> Callable[[_C], _C]: ... # TODO: add support for returning NamedTuple from the mypy plugin -class _Fields(Tuple[Attribute, ...]): - def __getattr__(self, name: str) -> Attribute: ... +class _Fields(Tuple[Attribute[Any], ...]): + def __getattr__(self, name: str) -> Attribute[Any]: ... def fields(cls: type) -> _Fields: ... -def fields_dict(cls: type) -> Dict[str, Attribute]: ... +def fields_dict(cls: type) -> Dict[str, Attribute[Any]]: ... def validate(inst: Any) -> None: ... # TODO: add support for returning a proper attrs class from the mypy plugin @@ -202,7 +225,7 @@ def make_class( bases: Tuple[type, ...] = ..., repr_ns: Optional[str] = ..., repr: bool = ..., - cmp: bool = ..., + cmp: Optional[bool] = ..., hash: Optional[bool] = ..., init: bool = ..., slots: bool = ..., @@ -212,6 +235,9 @@ def make_class( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., + eq: Optional[bool] = ..., + order: Optional[bool] = ..., ) -> type: ... # _funcs -- @@ -223,7 +249,7 @@ def make_class( def asdict( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., + filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., ) -> Dict[str, Any]: ... @@ -232,8 +258,8 @@ def asdict( def astuple( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., - tuple_factory: Type[Sequence] = ..., + filter: Optional[_FilterType[Any]] = ..., + tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... def has(cls: type) -> bool: ... diff --git a/pipenv/vendor/attr/_compat.py b/pipenv/vendor/attr/_compat.py index 5bb06593..a915db8e 100644 --- a/pipenv/vendor/attr/_compat.py +++ b/pipenv/vendor/attr/_compat.py @@ -20,6 +20,7 @@ else: if PY2: from UserDict import IterableUserDict + from collections import Mapping, Sequence # We 'bundle' isclass instead of using inspect as importing inspect is # fairly expensive (order of 10-15 ms for a modern machine in 2016) @@ -89,8 +90,28 @@ if PY2: res.data.update(d) # We blocked update, so we have to do it like this. return res + def just_warn(*args, **kw): # pragma: nocover + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ -else: + +else: # Python 3 and later. + from collections.abc import Mapping, Sequence # noqa + + def just_warn(*args, **kw): + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + warnings.warn( + "Running interpreter doesn't sufficiently support code object " + "introspection. Some features like bare super() or accessing " + "__class__ will not work with slotted classes.", + RuntimeWarning, + stacklevel=2, + ) def isclass(klass): return isinstance(klass, type) @@ -104,60 +125,106 @@ else: return types.MappingProxyType(dict(d)) -def import_ctypes(): - """ - Moved into a function for testability. - """ - import ctypes - - return ctypes - - -if not PY2: - - def just_warn(*args, **kw): - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - warnings.warn( - "Missing ctypes. Some features like bare super() or accessing " - "__class__ will not work with slots classes.", - RuntimeWarning, - stacklevel=2, - ) - - -else: - - def just_warn(*args, **kw): # pragma: nocover - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - - def make_set_closure_cell(): + """Return a function of two arguments (cell, value) which sets + the value stored in the closure cell `cell` to `value`. """ - Moved into a function for testability. - """ + # pypy makes this easy. (It also supports the logic below, but + # why not do the easy/fast thing?) if PYPY: # pragma: no cover def set_closure_cell(cell, value): cell.__setstate__((value,)) - else: - try: - ctypes = import_ctypes() + return set_closure_cell - set_closure_cell = ctypes.pythonapi.PyCell_Set - set_closure_cell.argtypes = (ctypes.py_object, ctypes.py_object) - set_closure_cell.restype = ctypes.c_int - except Exception: - # We try best effort to set the cell, but sometimes it's not - # possible. For example on Jython or on GAE. - set_closure_cell = just_warn - return set_closure_cell + # Otherwise gotta do it the hard way. + + # Create a function that will set its first cellvar to `value`. + def set_first_cellvar_to(value): + x = value + return + + # This function will be eliminated as dead code, but + # not before its reference to `x` forces `x` to be + # represented as a closure cell rather than a local. + def force_x_to_be_a_cell(): # pragma: no cover + return x + + try: + # Extract the code object and make sure our assumptions about + # the closure behavior are correct. + if PY2: + co = set_first_cellvar_to.func_code + else: + co = set_first_cellvar_to.__code__ + if co.co_cellvars != ("x",) or co.co_freevars != (): + raise AssertionError # pragma: no cover + + # Convert this code object to a code object that sets the + # function's first _freevar_ (not cellvar) to the argument. + if sys.version_info >= (3, 8): + # CPython 3.8+ has an incompatible CodeType signature + # (added a posonlyargcount argument) but also added + # CodeType.replace() to do this without counting parameters. + set_first_freevar_code = co.replace( + co_cellvars=co.co_freevars, co_freevars=co.co_cellvars + ) + else: + args = [co.co_argcount] + if not PY2: + args.append(co.co_kwonlyargcount) + args.extend( + [ + co.co_nlocals, + co.co_stacksize, + co.co_flags, + co.co_code, + co.co_consts, + co.co_names, + co.co_varnames, + co.co_filename, + co.co_name, + co.co_firstlineno, + co.co_lnotab, + # These two arguments are reversed: + co.co_cellvars, + co.co_freevars, + ] + ) + set_first_freevar_code = types.CodeType(*args) + + def set_closure_cell(cell, value): + # Create a function using the set_first_freevar_code, + # whose first closure cell is `cell`. Calling it will + # change the value of that cell. + setter = types.FunctionType( + set_first_freevar_code, {}, "setter", (), (cell,) + ) + # And call it to set the cell. + setter(value) + + # Make sure it works on this interpreter: + def make_func_with_cell(): + x = None + + def func(): + return x # pragma: no cover + + return func + + if PY2: + cell = make_func_with_cell().func_closure[0] + else: + cell = make_func_with_cell().__closure__[0] + set_closure_cell(cell, 100) + if cell.cell_contents != 100: + raise AssertionError # pragma: no cover + + except Exception: + return just_warn + else: + return set_closure_cell set_closure_cell = make_set_closure_cell() diff --git a/pipenv/vendor/attr/_funcs.py b/pipenv/vendor/attr/_funcs.py index b61d2394..c077e428 100644 --- a/pipenv/vendor/attr/_funcs.py +++ b/pipenv/vendor/attr/_funcs.py @@ -24,7 +24,7 @@ def asdict( ``attrs``-decorated. :param callable filter: A callable whose return code determines whether an attribute or element is included (``True``) or dropped (``False``). Is - called with the :class:`attr.Attribute` as the first argument and the + called with the `attr.Attribute` as the first argument and the value as the second argument. :param callable dict_factory: A callable to produce dictionaries from. For example, to produce ordered dictionaries instead of normal Python @@ -130,7 +130,7 @@ def astuple( ``attrs``-decorated. :param callable filter: A callable whose return code determines whether an attribute or element is included (``True``) or dropped (``False``). Is - called with the :class:`attr.Attribute` as the first argument and the + called with the `attr.Attribute` as the first argument and the value as the second argument. :param callable tuple_factory: A callable to produce tuples from. For example, to produce lists instead of tuples. @@ -219,7 +219,7 @@ def has(cls): :param type cls: Class to introspect. :raise TypeError: If *cls* is not a class. - :rtype: :class:`bool` + :rtype: bool """ return getattr(cls, "__attrs_attrs__", None) is not None @@ -239,7 +239,7 @@ def assoc(inst, **changes): class. .. deprecated:: 17.1.0 - Use :func:`evolve` instead. + Use `evolve` instead. """ import warnings diff --git a/pipenv/vendor/attr/_make.py b/pipenv/vendor/attr/_make.py index f7fd05e7..46f9c54e 100644 --- a/pipenv/vendor/attr/_make.py +++ b/pipenv/vendor/attr/_make.py @@ -1,10 +1,10 @@ from __future__ import absolute_import, division, print_function import copy -import hashlib import linecache import sys import threading +import uuid import warnings from operator import itemgetter @@ -42,6 +42,9 @@ _hash_cache_field = "_attrs_cached_hash" _empty_metadata_singleton = metadata_proxy({}) +# Unique object for unequivocal getattr() defaults. +_sentinel = object() + class _Nothing(object): """ @@ -71,15 +74,16 @@ def attrib( default=NOTHING, validator=None, repr=True, - cmp=True, + cmp=None, hash=None, init=True, - convert=None, metadata=None, type=None, converter=None, factory=None, kw_only=False, + eq=None, + order=None, ): """ Create a new attribute on a class. @@ -87,30 +91,30 @@ def attrib( .. warning:: Does *not* do anything unless the class is also decorated with - :func:`attr.s`! + `attr.s`! :param default: A value that is used if an ``attrs``-generated ``__init__`` is used and no value is passed while instantiating or the attribute is excluded using ``init=False``. - If the value is an instance of :class:`Factory`, its callable will be + If the value is an instance of `Factory`, its callable will be used to construct a new value (useful for mutable data types like lists or dicts). If a default is not set (or set manually to ``attr.NOTHING``), a value - *must* be supplied when instantiating; otherwise a :exc:`TypeError` + *must* be supplied when instantiating; otherwise a `TypeError` will be raised. The default can also be set using decorator notation as shown below. - :type default: Any value. + :type default: Any value :param callable factory: Syntactic sugar for ``default=attr.Factory(callable)``. - :param validator: :func:`callable` that is called by ``attrs``-generated + :param validator: `callable` that is called by ``attrs``-generated ``__init__`` methods after the instance has been initialized. They - receive the initialized instance, the :class:`Attribute`, and the + receive the initialized instance, the `Attribute`, and the passed value. The return value is *not* inspected so the validator has to throw an @@ -120,18 +124,29 @@ def attrib( all pass. Validators can be globally disabled and re-enabled using - :func:`get_run_validators`. + `get_run_validators`. The validator can also be set using decorator notation as shown below. :type validator: ``callable`` or a ``list`` of ``callable``\\ s. - :param bool repr: Include this attribute in the generated ``__repr__`` - method. - :param bool cmp: Include this attribute in the generated comparison methods - (``__eq__`` et al). + :param repr: Include this attribute in the generated ``__repr__`` + method. If ``True``, include the attribute; if ``False``, omit it. By + default, the built-in ``repr()`` function is used. To override how the + attribute value is formatted, pass a ``callable`` that takes a single + value and returns a string. Note that the resulting string is used + as-is, i.e. it will be used directly *instead* of calling ``repr()`` + (the default). + :type repr: a ``bool`` or a ``callable`` to use a custom function. + :param bool eq: If ``True`` (default), include this attribute in the + generated ``__eq__`` and ``__ne__`` methods that check two instances + for equality. + :param bool order: If ``True`` (default), include this attributes in the + generated ``__lt__``, ``__le__``, ``__gt__`` and ``__ge__`` methods. + :param bool cmp: Setting to ``True`` is equivalent to setting ``eq=True, + order=True``. Deprecated in favor of *eq* and *order*. :param hash: Include this attribute in the generated ``__hash__`` - method. If ``None`` (default), mirror *cmp*'s value. This is the + method. If ``None`` (default), mirror *eq*'s value. This is the correct behavior according the Python spec. Setting this value to anything else than ``None`` is *discouraged*. :type hash: ``bool`` or ``None`` @@ -139,13 +154,13 @@ def attrib( method. It is possible to set this to ``False`` and set a default value. In that case this attributed is unconditionally initialized with the specified default value or factory. - :param callable converter: :func:`callable` that is called by + :param callable converter: `callable` that is called by ``attrs``-generated ``__init__`` methods to converter attribute's value to the desired format. It is given the passed-in value, and the returned value will be used as the new value of the attribute. The value is converted before being passed to the validator, if any. :param metadata: An arbitrary mapping, to be used by third-party - components. See :ref:`extending_metadata`. + components. See `extending_metadata`. :param type: The type of the attribute. In Python 3.6 or greater, the preferred method to specify the type is using a variable annotation (see `PEP 526 <https://www.python.org/dev/peps/pep-0526/>`_). @@ -155,7 +170,7 @@ def attrib( Please note that ``attrs`` doesn't do anything with this metadata by itself. You can use it as part of your own code or for - :doc:`static type checking <types>`. + `static type checking <types>`. :param kw_only: Make this attribute keyword-only (Python 3+) in the generated ``__init__`` (if ``init`` is ``False``, this parameter is ignored). @@ -164,7 +179,7 @@ def attrib( .. versionadded:: 16.3.0 *metadata* .. versionchanged:: 17.1.0 *validator* can be a ``list`` now. .. versionchanged:: 17.1.0 - *hash* is ``None`` and therefore mirrors *cmp* by default. + *hash* is ``None`` and therefore mirrors *eq* by default. .. versionadded:: 17.3.0 *type* .. deprecated:: 17.4.0 *convert* .. versionadded:: 17.4.0 *converter* as a replacement for the deprecated @@ -172,26 +187,18 @@ def attrib( .. versionadded:: 18.1.0 ``factory=f`` is syntactic sugar for ``default=attr.Factory(f)``. .. versionadded:: 18.2.0 *kw_only* + .. versionchanged:: 19.2.0 *convert* keyword argument removed + .. versionchanged:: 19.2.0 *repr* also accepts a custom callable. + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* """ + eq, order = _determine_eq_order(cmp, eq, order) + if hash is not None and hash is not True and hash is not False: raise TypeError( "Invalid value for hash. Must be True, False, or None." ) - if convert is not None: - if converter is not None: - raise RuntimeError( - "Can't pass both `convert` and `converter`. " - "Please use `converter` only." - ) - warnings.warn( - "The `convert` argument is deprecated in favor of `converter`. " - "It will be removed after 2019/01.", - DeprecationWarning, - stacklevel=2, - ) - converter = convert - if factory is not None: if default is not NOTHING: raise ValueError( @@ -209,13 +216,15 @@ def attrib( default=default, validator=validator, repr=repr, - cmp=cmp, + cmp=None, hash=hash, init=init, converter=converter, metadata=metadata, type=type, kw_only=kw_only, + eq=eq, + order=order, ) @@ -385,39 +394,20 @@ def _transform_attrs(cls, these, auto_attribs, kw_only): attrs = AttrsClass(base_attrs + own_attrs) + # Mandatory vs non-mandatory attr order only matters when they are part of + # the __init__ signature and when they aren't kw_only (which are moved to + # the end and can be mandatory or non-mandatory in any order, as they will + # be specified as keyword args anyway). Check the order of those attrs: had_default = False - was_kw_only = False - for a in attrs: - if ( - was_kw_only is False - and had_default is True - and a.default is NOTHING - and a.init is True - and a.kw_only is False - ): + for a in (a for a in attrs if a.init is not False and a.kw_only is False): + if had_default is True and a.default is NOTHING: raise ValueError( "No mandatory attributes allowed after an attribute with a " "default value or factory. Attribute in question: %r" % (a,) ) - elif ( - had_default is False - and a.default is not NOTHING - and a.init is not False - and - # Keyword-only attributes without defaults can be specified - # after keyword-only attributes with defaults. - a.kw_only is False - ): + + if had_default is False and a.default is not NOTHING: had_default = True - if was_kw_only is True and a.kw_only is False: - raise ValueError( - "Non keyword-only attributes are not allowed after a " - "keyword-only attribute. Attribute in question: {a!r}".format( - a=a - ) - ) - if was_kw_only is False and a.init is True and a.kw_only is True: - was_kw_only = True return _Attributes((attrs, base_attrs, base_attr_map)) @@ -454,6 +444,7 @@ class _ClassBuilder(object): "_has_post_init", "_delete_attribs", "_base_attr_map", + "_is_exc", ) def __init__( @@ -466,6 +457,7 @@ class _ClassBuilder(object): auto_attribs, kw_only, cache_hash, + is_exc, ): attrs, base_attrs, base_map = _transform_attrs( cls, these, auto_attribs, kw_only @@ -483,6 +475,7 @@ class _ClassBuilder(object): self._cache_hash = cache_hash self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) self._delete_attribs = not bool(these) + self._is_exc = is_exc self._cls_dict["__attrs_attrs__"] = self._attrs @@ -516,7 +509,7 @@ class _ClassBuilder(object): for name in self._attr_names: if ( name not in base_names - and getattr(cls, name, None) is not None + and getattr(cls, name, _sentinel) is not _sentinel ): try: delattr(cls, name) @@ -530,6 +523,26 @@ class _ClassBuilder(object): for name, value in self._cls_dict.items(): setattr(cls, name, value) + # Attach __setstate__. This is necessary to clear the hash code + # cache on deserialization. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for dict classes. + # For slotted classes, see similar code in _create_slots_class . + if self._cache_hash: + existing_set_state_method = getattr(cls, "__setstate__", None) + if existing_set_state_method: + raise NotImplementedError( + "Currently you cannot use hash caching if " + "you specify your own __setstate__ method." + "See https://github.com/python-attrs/attrs/issues/494 ." + ) + + def cache_hash_set_state(chss_self, _): + # clear hash code cache + setattr(chss_self, _hash_cache_field, None) + + setattr(cls, "__setstate__", cache_hash_set_state) + return cls def _create_slots_class(self): @@ -582,6 +595,8 @@ class _ClassBuilder(object): """ return tuple(getattr(self, name) for name in state_attr_names) + hash_caching_enabled = self._cache_hash + def slots_setstate(self, state): """ Automatically created by attrs. @@ -589,6 +604,13 @@ class _ClassBuilder(object): __bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in zip(state_attr_names, state): __bound_setattr(name, value) + # Clearing the hash code cache on deserialization is needed + # because hash codes can change from run to run. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for slotted classes. + # For dict classes, see similar code in _patch_original_class . + if hash_caching_enabled: + __bound_setattr(_hash_cache_field, None) # slots and frozen require __getstate__/__setstate__ to work cd["__getstate__"] = slots_getstate @@ -645,7 +667,10 @@ class _ClassBuilder(object): def add_hash(self): self._cls_dict["__hash__"] = self._add_method_dunders( _make_hash( - self._attrs, frozen=self._frozen, cache_hash=self._cache_hash + self._cls, + self._attrs, + frozen=self._frozen, + cache_hash=self._cache_hash, ) ) @@ -654,24 +679,35 @@ class _ClassBuilder(object): def add_init(self): self._cls_dict["__init__"] = self._add_method_dunders( _make_init( + self._cls, self._attrs, self._has_post_init, self._frozen, self._slots, self._cache_hash, self._base_attr_map, + self._is_exc, ) ) return self - def add_cmp(self): + def add_eq(self): cd = self._cls_dict - cd["__eq__"], cd["__ne__"], cd["__lt__"], cd["__le__"], cd[ - "__gt__" - ], cd["__ge__"] = ( - self._add_method_dunders(meth) for meth in _make_cmp(self._attrs) + cd["__eq__"], cd["__ne__"] = ( + self._add_method_dunders(meth) + for meth in _make_eq(self._cls, self._attrs) + ) + + return self + + def add_order(self): + cd = self._cls_dict + + cd["__lt__"], cd["__le__"], cd["__gt__"], cd["__ge__"] = ( + self._add_method_dunders(meth) + for meth in _make_order(self._cls, self._attrs) ) return self @@ -695,12 +731,45 @@ class _ClassBuilder(object): return method +_CMP_DEPRECATION = ( + "The usage of `cmp` is deprecated and will be removed on or after " + "2021-06-01. Please use `eq` and `order` instead." +) + + +def _determine_eq_order(cmp, eq, order): + """ + Validate the combination of *cmp*, *eq*, and *order*. Derive the effective + values of eq and order. + """ + if cmp is not None and any((eq is not None, order is not None)): + raise ValueError("Don't mix `cmp` with `eq' and `order`.") + + # cmp takes precedence due to bw-compatibility. + if cmp is not None: + warnings.warn(_CMP_DEPRECATION, DeprecationWarning, stacklevel=3) + + return cmp, cmp + + # If left None, equality is on and ordering mirrors equality. + if eq is None: + eq = True + + if order is None: + order = eq + + if eq is False and order is True: + raise ValueError("`order` can only be True if `eq` is True too.") + + return eq, order + + def attrs( maybe_cls=None, these=None, repr_ns=None, repr=True, - cmp=True, + cmp=None, hash=None, init=True, slots=False, @@ -710,13 +779,16 @@ def attrs( auto_attribs=False, kw_only=False, cache_hash=False, + auto_exc=False, + eq=None, + order=None, ): r""" A class decorator that adds `dunder <https://wiki.python.org/moin/DunderAlias>`_\ -methods according to the - specified attributes using :func:`attr.ib` or the *these* argument. + specified attributes using `attr.ib` or the *these* argument. - :param these: A dictionary of name to :func:`attr.ib` mappings. This is + :param these: A dictionary of name to `attr.ib` mappings. This is useful to avoid the definition of your attributes within the class body because you can't (e.g. if you want to add ``__repr__`` methods to Django models) or don't want to. @@ -724,12 +796,12 @@ def attrs( If *these* is not ``None``, ``attrs`` will *not* search the class body for attributes and will *not* remove any attributes from it. - If *these* is an ordered dict (:class:`dict` on Python 3.6+, - :class:`collections.OrderedDict` otherwise), the order is deduced from + If *these* is an ordered dict (`dict` on Python 3.6+, + `collections.OrderedDict` otherwise), the order is deduced from the order of the attributes inside *these*. Otherwise the order of the definition of the attributes is used. - :type these: :class:`dict` of :class:`str` to :func:`attr.ib` + :type these: `dict` of `str` to `attr.ib` :param str repr_ns: When using nested classes, there's no way in Python 2 to automatically detect that. Therefore it's possible to set the @@ -738,18 +810,29 @@ def attrs( representation of ``attrs`` attributes.. :param bool str: Create a ``__str__`` method that is identical to ``__repr__``. This is usually not necessary except for - :class:`Exception`\ s. - :param bool cmp: Create ``__eq__``, ``__ne__``, ``__lt__``, ``__le__``, - ``__gt__``, and ``__ge__`` methods that compare the class as if it were - a tuple of its ``attrs`` attributes. But the attributes are *only* - compared, if the types of both classes are *identical*! + `Exception`\ s. + :param bool eq: If ``True`` or ``None`` (default), add ``__eq__`` and + ``__ne__`` methods that check two instances for equality. + + They compare the instances as if they were tuples of their ``attrs`` + attributes, but only iff the types of both classes are *identical*! + :type eq: `bool` or `None` + :param bool order: If ``True``, add ``__lt__``, ``__le__``, ``__gt__``, + and ``__ge__`` methods that behave like *eq* above and allow instances + to be ordered. If ``None`` (default) mirror value of *eq*. + :type order: `bool` or `None` + :param cmp: Setting to ``True`` is equivalent to setting ``eq=True, + order=True``. Deprecated in favor of *eq* and *order*, has precedence + over them for backward-compatibility though. Must not be mixed with + *eq* or *order*. + :type cmp: `bool` or `None` :param hash: If ``None`` (default), the ``__hash__`` method is generated - according how *cmp* and *frozen* are set. + according how *eq* and *frozen* are set. 1. If *both* are True, ``attrs`` will generate a ``__hash__`` for you. - 2. If *cmp* is True and *frozen* is False, ``__hash__`` will be set to + 2. If *eq* is True and *frozen* is False, ``__hash__`` will be set to None, marking it unhashable (which it is). - 3. If *cmp* is False, ``__hash__`` will be left untouched meaning the + 3. If *eq* is False, ``__hash__`` will be left untouched meaning the ``__hash__`` method of the base class will be used (if base class is ``object``, this means it will fall back to id-based hashing.). @@ -758,29 +841,29 @@ def attrs( didn't freeze it programmatically) by passing ``True`` or not. Both of these cases are rather special and should be used carefully. - See the `Python documentation \ - <https://docs.python.org/3/reference/datamodel.html#object.__hash__>`_ - and the `GitHub issue that led to the default behavior \ - <https://github.com/python-attrs/attrs/issues/136>`_ for more details. + See our documentation on `hashing`, Python's documentation on + `object.__hash__`, and the `GitHub issue that led to the default \ + behavior <https://github.com/python-attrs/attrs/issues/136>`_ for more + details. :type hash: ``bool`` or ``None`` :param bool init: Create a ``__init__`` method that initializes the ``attrs`` attributes. Leading underscores are stripped for the argument name. If a ``__attrs_post_init__`` method exists on the class, it will be called after the class is fully initialized. - :param bool slots: Create a slots_-style class that's more - memory-efficient. See :ref:`slots` for further ramifications. + :param bool slots: Create a `slotted class <slotted classes>` that's more + memory-efficient. :param bool frozen: Make instances immutable after initialization. If someone attempts to modify a frozen instance, - :exc:`attr.exceptions.FrozenInstanceError` is raised. + `attr.exceptions.FrozenInstanceError` is raised. Please note: 1. This is achieved by installing a custom ``__setattr__`` method - on your class so you can't implement an own one. + on your class, so you can't implement your own. 2. True immutability is impossible in Python. - 3. This *does* have a minor a runtime performance :ref:`impact + 3. This *does* have a minor a runtime performance `impact <how-frozen>` when initializing new instances. In other words: ``__init__`` is slightly slower with ``frozen=True``. @@ -789,24 +872,24 @@ def attrs( circumvent that limitation by using ``object.__setattr__(self, "attribute_name", value)``. - .. _slots: https://docs.python.org/3/reference/datamodel.html#slots :param bool weakref_slot: Make instances weak-referenceable. This has no effect unless ``slots`` is also enabled. :param bool auto_attribs: If True, collect `PEP 526`_-annotated attributes (Python 3.6 and later only) from the class body. In this case, you **must** annotate every field. If ``attrs`` - encounters a field that is set to an :func:`attr.ib` but lacks a type - annotation, an :exc:`attr.exceptions.UnannotatedAttributeError` is + encounters a field that is set to an `attr.ib` but lacks a type + annotation, an `attr.exceptions.UnannotatedAttributeError` is raised. Use ``field_name: typing.Any = attr.ib(...)`` if you don't want to set a type. If you assign a value to those attributes (e.g. ``x: int = 42``), that value becomes the default value like if it were passed using - ``attr.ib(default=42)``. Passing an instance of :class:`Factory` also + ``attr.ib(default=42)``. Passing an instance of `Factory` also works as expected. - Attributes annotated as :data:`typing.ClassVar` are **ignored**. + Attributes annotated as `typing.ClassVar`, and attributes that are + neither annotated nor set to an `attr.ib` are **ignored**. .. _`PEP 526`: https://www.python.org/dev/peps/pep-0526/ :param bool kw_only: Make all attributes keyword-only (Python 3+) @@ -815,10 +898,23 @@ def attrs( :param bool cache_hash: Ensure that the object's hash code is computed only once and stored on the object. If this is set to ``True``, hashing must be either explicitly or implicitly enabled for this - class. If the hash code is cached, then no attributes of this - class which participate in hash code computation may be mutated - after object creation. + class. If the hash code is cached, avoid any reassignments of + fields involved in hash code computation or mutations of the objects + those fields point to after object creation. If such changes occur, + the behavior of the object's hash code is undefined. + :param bool auto_exc: If the class subclasses `BaseException` + (which implicitly includes any subclass of any exception), the + following happens to behave like a well-behaved Python exceptions + class: + - the values for *eq*, *order*, and *hash* are ignored and the + instances compare and hash by the instance's ids (N.B. ``attrs`` will + *not* remove existing implementations of ``__hash__`` or the equality + methods. It just won't add own ones.), + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the ``args`` + attribute, + - the value of *str* is ignored leaving ``__str__`` to base classes. .. versionadded:: 16.0.0 *slots* .. versionadded:: 16.1.0 *frozen* @@ -833,17 +929,27 @@ def attrs( .. versionadded:: 18.2.0 *weakref_slot* .. deprecated:: 18.2.0 ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now raise a - :class:`DeprecationWarning` if the classes compared are subclasses of + `DeprecationWarning` if the classes compared are subclasses of each other. ``__eq`` and ``__ne__`` never tried to compared subclasses to each other. + .. versionchanged:: 19.2.0 + ``__lt__``, ``__le__``, ``__gt__``, and ``__ge__`` now do not consider + subclasses comparable anymore. .. versionadded:: 18.2.0 *kw_only* .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* + .. deprecated:: 19.2.0 *cmp* Removal on or after 2021-06-01. + .. versionadded:: 19.2.0 *eq* and *order* """ + eq, order = _determine_eq_order(cmp, eq, order) def wrap(cls): + if getattr(cls, "__class__", None) is None: raise TypeError("attrs only works with new-style classes.") + is_exc = auto_exc is True and issubclass(cls, BaseException) + builder = _ClassBuilder( cls, these, @@ -853,30 +959,37 @@ def attrs( auto_attribs, kw_only, cache_hash, + is_exc, ) if repr is True: builder.add_repr(repr_ns) if str is True: builder.add_str() - if cmp is True: - builder.add_cmp() + if eq is True and not is_exc: + builder.add_eq() + if order is True and not is_exc: + builder.add_order() if hash is not True and hash is not False and hash is not None: # Can't use `hash in` because 1 == True for example. raise TypeError( "Invalid value for hash. Must be True, False, or None." ) - elif hash is False or (hash is None and cmp is False): + elif hash is False or (hash is None and eq is False) or is_exc: + # Don't do anything. Should fall back to __object__'s __hash__ + # which is by id. if cache_hash: raise TypeError( "Invalid value for cache_hash. To use hash caching," " hashing must be either explicitly or implicitly " "enabled." ) - elif hash is True or (hash is None and cmp is True and frozen is True): + elif hash is True or (hash is None and eq is True and frozen is True): + # Build a __hash__ if told so, or if it's safe. builder.add_hash() else: + # Raise TypeError on attempts to hash. if cache_hash: raise TypeError( "Invalid value for cache_hash. To use hash caching," @@ -942,19 +1055,44 @@ def _attrs_to_tuple(obj, attrs): return tuple(getattr(obj, a.name) for a in attrs) -def _make_hash(attrs, frozen, cache_hash): +def _generate_unique_filename(cls, func_name): + """ + Create a "filename" suitable for a function being generated. + """ + unique_id = uuid.uuid4() + extra = "" + count = 1 + + while True: + unique_filename = "<attrs generated {0} {1}.{2}{3}>".format( + func_name, + cls.__module__, + getattr(cls, "__qualname__", cls.__name__), + extra, + ) + # To handle concurrency we essentially "reserve" our spot in + # the linecache with a dummy line. The caller can then + # set this value correctly. + cache_line = (1, None, (str(unique_id),), unique_filename) + if ( + linecache.cache.setdefault(unique_filename, cache_line) + == cache_line + ): + return unique_filename + + # Looks like this spot is taken. Try again. + count += 1 + extra = "-{0}".format(count) + + +def _make_hash(cls, attrs, frozen, cache_hash): attrs = tuple( - a - for a in attrs - if a.hash is True or (a.hash is None and a.cmp is True) + a for a in attrs if a.hash is True or (a.hash is None and a.eq is True) ) tab = " " - # We cache the generated hash methods for the same kinds of attributes. - sha1 = hashlib.sha1() - sha1.update(repr(attrs).encode("utf-8")) - unique_filename = "<attrs generated hash %s>" % (sha1.hexdigest(),) + unique_filename = _generate_unique_filename(cls, "hash") type_hash = hash(unique_filename) method_lines = ["def __hash__(self):"] @@ -1011,7 +1149,7 @@ def _add_hash(cls, attrs): """ Add a hash method to *cls*. """ - cls.__hash__ = _make_hash(attrs, frozen=False, cache_hash=False) + cls.__hash__ = _make_hash(cls, attrs, frozen=False, cache_hash=False) return cls @@ -1027,19 +1165,10 @@ def __ne__(self, other): return not result -WARNING_CMP_ISINSTANCE = ( - "Comparision of subclasses using __%s__ is deprecated and will be removed " - "in 2019." -) +def _make_eq(cls, attrs): + attrs = [a for a in attrs if a.eq] - -def _make_cmp(attrs): - attrs = [a for a in attrs if a.cmp] - - # We cache the generated eq methods for the same kinds of attributes. - sha1 = hashlib.sha1() - sha1.update(repr(attrs).encode("utf-8")) - unique_filename = "<attrs generated eq %s>" % (sha1.hexdigest(),) + unique_filename = _generate_unique_filename(cls, "eq") lines = [ "def __eq__(self, other):", " if other.__class__ is not self.__class__:", @@ -1072,8 +1201,11 @@ def _make_cmp(attrs): script.splitlines(True), unique_filename, ) - eq = locs["__eq__"] - ne = __ne__ + return locs["__eq__"], __ne__ + + +def _make_order(cls, attrs): + attrs = [a for a in attrs if a.order] def attrs_to_tuple(obj): """ @@ -1085,67 +1217,49 @@ def _make_cmp(attrs): """ Automatically created by attrs. """ - if isinstance(other, self.__class__): - if other.__class__ is not self.__class__: - warnings.warn( - WARNING_CMP_ISINSTANCE % ("lt",), DeprecationWarning - ) + if other.__class__ is self.__class__: return attrs_to_tuple(self) < attrs_to_tuple(other) - else: - return NotImplemented + + return NotImplemented def __le__(self, other): """ Automatically created by attrs. """ - if isinstance(other, self.__class__): - if other.__class__ is not self.__class__: - warnings.warn( - WARNING_CMP_ISINSTANCE % ("le",), DeprecationWarning - ) + if other.__class__ is self.__class__: return attrs_to_tuple(self) <= attrs_to_tuple(other) - else: - return NotImplemented + + return NotImplemented def __gt__(self, other): """ Automatically created by attrs. """ - if isinstance(other, self.__class__): - if other.__class__ is not self.__class__: - warnings.warn( - WARNING_CMP_ISINSTANCE % ("gt",), DeprecationWarning - ) + if other.__class__ is self.__class__: return attrs_to_tuple(self) > attrs_to_tuple(other) - else: - return NotImplemented + + return NotImplemented def __ge__(self, other): """ Automatically created by attrs. """ - if isinstance(other, self.__class__): - if other.__class__ is not self.__class__: - warnings.warn( - WARNING_CMP_ISINSTANCE % ("ge",), DeprecationWarning - ) + if other.__class__ is self.__class__: return attrs_to_tuple(self) >= attrs_to_tuple(other) - else: - return NotImplemented - return eq, ne, __lt__, __le__, __gt__, __ge__ + return NotImplemented + + return __lt__, __le__, __gt__, __ge__ -def _add_cmp(cls, attrs=None): +def _add_eq(cls, attrs=None): """ - Add comparison methods to *cls*. + Add equality methods to *cls* with *attrs*. """ if attrs is None: attrs = cls.__attrs_attrs__ - cls.__eq__, cls.__ne__, cls.__lt__, cls.__le__, cls.__gt__, cls.__ge__ = _make_cmp( # noqa - attrs - ) + cls.__eq__, cls.__ne__ = _make_eq(cls, attrs) return cls @@ -1155,9 +1269,17 @@ _already_repring = threading.local() def _make_repr(attrs, ns): """ - Make a repr method for *attr_names* adding *ns* to the full name. + Make a repr method that includes relevant *attrs*, adding *ns* to the full + name. """ - attr_names = tuple(a.name for a in attrs if a.repr) + + # Figure out which attributes to include, and which function to use to + # format them. The a.repr value can be either bool or a custom callable. + attr_names_with_reprs = tuple( + (a.name, repr if a.repr is True else a.repr) + for a in attrs + if a.repr is not False + ) def __repr__(self): """ @@ -1189,12 +1311,14 @@ def _make_repr(attrs, ns): try: result = [class_name, "("] first = True - for name in attr_names: + for name, attr_repr in attr_names_with_reprs: if first: first = False else: result.append(", ") - result.extend((name, "=", repr(getattr(self, name, NOTHING)))) + result.extend( + (name, "=", attr_repr(getattr(self, name, NOTHING))) + ) return "".join(result) + ")" finally: working_set.remove(id(self)) @@ -1213,25 +1337,26 @@ def _add_repr(cls, ns=None, attrs=None): return cls -def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): +def _make_init( + cls, attrs, post_init, frozen, slots, cache_hash, base_attr_map, is_exc +): attrs = [a for a in attrs if a.init or a.default is not NOTHING] - # We cache the generated init methods for the same kinds of attributes. - sha1 = hashlib.sha1() - sha1.update(repr(attrs).encode("utf-8")) - unique_filename = "<attrs generated init {0}>".format(sha1.hexdigest()) + unique_filename = _generate_unique_filename(cls, "init") script, globs, annotations = _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ) locs = {} bytecode = compile(script, unique_filename, "exec") attr_dict = dict((a.name, a) for a in attrs) globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + if frozen is True: # Save the lookup overhead in __init__ if we need to circumvent # immutability. globs["_cached_setattr"] = _obj_setattr + eval(bytecode, globs, locs) # In order of debuggers like PDB being able to step through the code, @@ -1245,24 +1370,10 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): __init__ = locs["__init__"] __init__.__annotations__ = annotations + return __init__ -def _add_init(cls, frozen): - """ - Add a __init__ method to *cls*. If *frozen* is True, make it immutable. - """ - cls.__init__ = _make_init( - cls.__attrs_attrs__, - getattr(cls, "__attrs_post_init__", False), - frozen, - _is_slot_cls(cls), - cache_hash=False, - base_attr_map={}, - ) - return cls - - def fields(cls): """ Return the tuple of ``attrs`` attributes for a class. @@ -1276,7 +1387,7 @@ def fields(cls): :raise attr.exceptions.NotAnAttrsClassError: If *cls* is not an ``attrs`` class. - :rtype: tuple (with name accessors) of :class:`attr.Attribute` + :rtype: tuple (with name accessors) of `attr.Attribute` .. versionchanged:: 16.2.0 Returned tuple allows accessing the fields by name. @@ -1303,7 +1414,7 @@ def fields_dict(cls): class. :rtype: an ordered dict where keys are attribute names and values are - :class:`attr.Attribute`\\ s. This will be a :class:`dict` if it's + `attr.Attribute`\\ s. This will be a `dict` if it's naturally ordered like on Python 3.6+ or an :class:`~collections.OrderedDict` otherwise. @@ -1348,7 +1459,7 @@ def _is_slot_attr(a_name, base_attr_map): def _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ): """ Return a script of an initializer for *attrs* and a dict of globals. @@ -1597,6 +1708,13 @@ def _attrs_to_init_script( init_hash_cache = "self.%s = %s" lines.append(init_hash_cache % (_hash_cache_field, "None")) + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join("self." + a.name for a in attrs if a.init) + + lines.append("BaseException.__init__(self, %s)" % (vals,)) + args = ", ".join(args) if kw_only_args: if PY2: @@ -1626,9 +1744,10 @@ class Attribute(object): :attribute name: The name of the attribute. - Plus *all* arguments of :func:`attr.ib`. + Plus *all* arguments of `attr.ib` (except for ``factory`` + which is only syntactic sugar for ``default=Factory(...)``. - For the version history of the fields, see :func:`attr.ib`. + For the version history of the fields, see `attr.ib`. """ __slots__ = ( @@ -1636,7 +1755,8 @@ class Attribute(object): "default", "validator", "repr", - "cmp", + "eq", + "order", "hash", "init", "metadata", @@ -1651,39 +1771,29 @@ class Attribute(object): default, validator, repr, - cmp, + cmp, # XXX: unused, remove along with other cmp code. hash, init, - convert=None, metadata=None, type=None, converter=None, kw_only=False, + eq=None, + order=None, ): + eq, order = _determine_eq_order(cmp, eq, order) + # Cache this descriptor here to speed things up later. bound_setattr = _obj_setattr.__get__(self, Attribute) # Despite the big red warning, people *do* instantiate `Attribute` # themselves. - if convert is not None: - if converter is not None: - raise RuntimeError( - "Can't pass both `convert` and `converter`. " - "Please use `converter` only." - ) - warnings.warn( - "The `convert` argument is deprecated in favor of `converter`." - " It will be removed after 2019/01.", - DeprecationWarning, - stacklevel=2, - ) - converter = convert - bound_setattr("name", name) bound_setattr("default", default) bound_setattr("validator", validator) bound_setattr("repr", repr) - bound_setattr("cmp", cmp) + bound_setattr("eq", eq) + bound_setattr("order", order) bound_setattr("hash", hash) bound_setattr("init", init) bound_setattr("converter", converter) @@ -1701,16 +1811,6 @@ class Attribute(object): def __setattr__(self, name, value): raise FrozenInstanceError() - @property - def convert(self): - warnings.warn( - "The `convert` attribute is deprecated in favor of `converter`. " - "It will be removed after 2019/01.", - DeprecationWarning, - stacklevel=2, - ) - return self.converter - @classmethod def from_counting_attr(cls, name, ca, type=None): # type holds the annotated value. deal with conflicts: @@ -1729,7 +1829,6 @@ class Attribute(object): "validator", "default", "type", - "convert", ) # exclude methods and deprecated alias } return cls( @@ -1737,9 +1836,19 @@ class Attribute(object): validator=ca._validator, default=ca._default, type=type, + cmp=None, **inst_dict ) + @property + def cmp(self): + """ + Simulate the presence of a cmp attribute and warn. + """ + warnings.warn(_CMP_DEPRECATION, DeprecationWarning, stacklevel=2) + + return self.eq and self.order + # Don't use attr.assoc since fields(Attribute) doesn't work def _assoc(self, **changes): """ @@ -1787,16 +1896,17 @@ _a = [ default=NOTHING, validator=None, repr=True, - cmp=True, + cmp=None, + eq=True, + order=False, hash=(name != "metadata"), init=True, ) for name in Attribute.__slots__ - if name != "convert" # XXX: remove once `convert` is gone ] Attribute = _add_hash( - _add_cmp(_add_repr(Attribute, attrs=_a), attrs=_a), + _add_eq(_add_repr(Attribute, attrs=_a), attrs=_a), attrs=[a for a in _a if a.hash], ) @@ -1814,7 +1924,8 @@ class _CountingAttr(object): "counter", "_default", "repr", - "cmp", + "eq", + "order", "hash", "init", "metadata", @@ -1829,22 +1940,34 @@ class _CountingAttr(object): default=NOTHING, validator=None, repr=True, - cmp=True, + cmp=None, hash=True, init=True, kw_only=False, + eq=True, + order=False, + ) + for name in ( + "counter", + "_default", + "repr", + "eq", + "order", + "hash", + "init", ) - for name in ("counter", "_default", "repr", "cmp", "hash", "init") ) + ( Attribute( name="metadata", default=None, validator=None, repr=True, - cmp=True, + cmp=None, hash=False, init=True, kw_only=False, + eq=True, + order=False, ), ) cls_counter = 0 @@ -1854,13 +1977,15 @@ class _CountingAttr(object): default, validator, repr, - cmp, + cmp, # XXX: unused, remove along with cmp hash, init, converter, metadata, type, kw_only, + eq, + order, ): _CountingAttr.cls_counter += 1 self.counter = _CountingAttr.cls_counter @@ -1871,7 +1996,8 @@ class _CountingAttr(object): else: self._validator = validator self.repr = repr - self.cmp = cmp + self.eq = eq + self.order = order self.hash = hash self.init = init self.converter = converter @@ -1911,7 +2037,7 @@ class _CountingAttr(object): return meth -_CountingAttr = _add_cmp(_add_repr(_CountingAttr)) +_CountingAttr = _add_eq(_add_repr(_CountingAttr)) @attrs(slots=True, init=False, hash=True) @@ -1919,7 +2045,7 @@ class Factory(object): """ Stores a factory callable. - If passed as the default value to :func:`attr.ib`, the factory is used to + If passed as the default value to `attr.ib`, the factory is used to generate a new value. :param callable factory: A callable that takes either none or exactly one @@ -1952,15 +2078,15 @@ def make_class(name, attrs, bases=(object,), **attributes_arguments): :param attrs: A list of names or a dictionary of mappings of names to attributes. - If *attrs* is a list or an ordered dict (:class:`dict` on Python 3.6+, - :class:`collections.OrderedDict` otherwise), the order is deduced from + If *attrs* is a list or an ordered dict (`dict` on Python 3.6+, + `collections.OrderedDict` otherwise), the order is deduced from the order of the names or attributes inside *attrs*. Otherwise the order of the definition of the attributes is used. - :type attrs: :class:`list` or :class:`dict` + :type attrs: `list` or `dict` :param tuple bases: Classes that the new class will subclass. - :param attributes_arguments: Passed unmodified to :func:`attr.s`. + :param attributes_arguments: Passed unmodified to `attr.s`. :return: A new class with *attrs*. :rtype: type @@ -1992,6 +2118,14 @@ def make_class(name, attrs, bases=(object,), **attributes_arguments): except (AttributeError, ValueError): pass + # We do it here for proper warnings with meaningful stacklevel. + cmp = attributes_arguments.pop("cmp", None) + attributes_arguments["eq"], attributes_arguments[ + "order" + ] = _determine_eq_order( + cmp, attributes_arguments.get("eq"), attributes_arguments.get("order") + ) + return _attrs(these=cls_dict, **attributes_arguments)(type_) diff --git a/pipenv/vendor/attr/_version_info.py b/pipenv/vendor/attr/_version_info.py new file mode 100644 index 00000000..014e78a1 --- /dev/null +++ b/pipenv/vendor/attr/_version_info.py @@ -0,0 +1,85 @@ +from __future__ import absolute_import, division, print_function + +from functools import total_ordering + +from ._funcs import astuple +from ._make import attrib, attrs + + +@total_ordering +@attrs(eq=False, order=False, slots=True, frozen=True) +class VersionInfo(object): + """ + A version object that can be compared to tuple of length 1--4: + + >>> attr.VersionInfo(19, 1, 0, "final") <= (19, 2) + True + >>> attr.VersionInfo(19, 1, 0, "final") < (19, 1, 1) + True + >>> vi = attr.VersionInfo(19, 2, 0, "final") + >>> vi < (19, 1, 1) + False + >>> vi < (19,) + False + >>> vi == (19, 2,) + True + >>> vi == (19, 2, 1) + False + + .. versionadded:: 19.2 + """ + + year = attrib(type=int) + minor = attrib(type=int) + micro = attrib(type=int) + releaselevel = attrib(type=str) + + @classmethod + def _from_version_string(cls, s): + """ + Parse *s* and return a _VersionInfo. + """ + v = s.split(".") + if len(v) == 3: + v.append("final") + + return cls( + year=int(v[0]), minor=int(v[1]), micro=int(v[2]), releaselevel=v[3] + ) + + def _ensure_tuple(self, other): + """ + Ensure *other* is a tuple of a valid length. + + Returns a possibly transformed *other* and ourselves as a tuple of + the same length as *other*. + """ + + if self.__class__ is other.__class__: + other = astuple(other) + + if not isinstance(other, tuple): + raise NotImplementedError + + if not (1 <= len(other) <= 4): + raise NotImplementedError + + return astuple(self)[: len(other)], other + + def __eq__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + return us == them + + def __lt__(self, other): + try: + us, them = self._ensure_tuple(other) + except NotImplementedError: + return NotImplemented + + # Since alphabetically "dev0" < "final" < "post1" < "post2", we don't + # have to do anything special with releaselevel for now. + return us < them diff --git a/pipenv/vendor/attr/_version_info.pyi b/pipenv/vendor/attr/_version_info.pyi new file mode 100644 index 00000000..45ced086 --- /dev/null +++ b/pipenv/vendor/attr/_version_info.pyi @@ -0,0 +1,9 @@ +class VersionInfo: + @property + def year(self) -> int: ... + @property + def minor(self) -> int: ... + @property + def micro(self) -> int: ... + @property + def releaselevel(self) -> str: ... diff --git a/pipenv/vendor/attr/converters.py b/pipenv/vendor/attr/converters.py index 37c4a07a..85928978 100644 --- a/pipenv/vendor/attr/converters.py +++ b/pipenv/vendor/attr/converters.py @@ -32,14 +32,14 @@ def default_if_none(default=NOTHING, factory=None): result of *factory*. :param default: Value to be used if ``None`` is passed. Passing an instance - of :class:`attr.Factory` is supported, however the ``takes_self`` option + of `attr.Factory` is supported, however the ``takes_self`` option is *not*. :param callable factory: A callable that takes not parameters whose result is used if ``None`` is passed. :raises TypeError: If **neither** *default* or *factory* is passed. :raises TypeError: If **both** *default* and *factory* are passed. - :raises ValueError: If an instance of :class:`attr.Factory` is passed with + :raises ValueError: If an instance of `attr.Factory` is passed with ``takes_self=True``. .. versionadded:: 18.2.0 diff --git a/pipenv/vendor/attr/exceptions.py b/pipenv/vendor/attr/exceptions.py index b12e41e9..d1b76185 100644 --- a/pipenv/vendor/attr/exceptions.py +++ b/pipenv/vendor/attr/exceptions.py @@ -6,7 +6,7 @@ class FrozenInstanceError(AttributeError): A frozen/immutable instance has been attempted to be modified. It mirrors the behavior of ``namedtuples`` by using the same error message - and subclassing :exc:`AttributeError`. + and subclassing `AttributeError`. .. versionadded:: 16.1.0 """ @@ -55,3 +55,20 @@ class PythonTooOldError(RuntimeError): .. versionadded:: 18.2.0 """ + + +class NotCallableError(TypeError): + """ + A ``attr.ib()`` requiring a callable has been set with a value + that is not callable. + + .. versionadded:: 19.2.0 + """ + + def __init__(self, msg, value): + super(TypeError, self).__init__(msg, value) + self.msg = msg + self.value = value + + def __str__(self): + return str(self.msg) diff --git a/pipenv/vendor/attr/exceptions.pyi b/pipenv/vendor/attr/exceptions.pyi index 48fffcc1..736fde2e 100644 --- a/pipenv/vendor/attr/exceptions.pyi +++ b/pipenv/vendor/attr/exceptions.pyi @@ -1,3 +1,5 @@ +from typing import Any + class FrozenInstanceError(AttributeError): msg: str = ... @@ -5,3 +7,9 @@ class AttrsAttributeNotFoundError(ValueError): ... class NotAnAttrsClassError(ValueError): ... class DefaultAlreadySetError(RuntimeError): ... class UnannotatedAttributeError(RuntimeError): ... +class PythonTooOldError(RuntimeError): ... + +class NotCallableError(TypeError): + msg: str = ... + value: Any = ... + def __init__(self, msg: str, value: Any) -> None: ... diff --git a/pipenv/vendor/attr/filters.py b/pipenv/vendor/attr/filters.py index f1c69b8b..dc47e8fa 100644 --- a/pipenv/vendor/attr/filters.py +++ b/pipenv/vendor/attr/filters.py @@ -1,5 +1,5 @@ """ -Commonly useful filters for :func:`attr.asdict`. +Commonly useful filters for `attr.asdict`. """ from __future__ import absolute_import, division, print_function @@ -23,9 +23,9 @@ def include(*what): Whitelist *what*. :param what: What to whitelist. - :type what: :class:`list` of :class:`type` or :class:`attr.Attribute`\\ s + :type what: `list` of `type` or `attr.Attribute`\\ s - :rtype: :class:`callable` + :rtype: `callable` """ cls, attrs = _split_what(what) @@ -40,9 +40,9 @@ def exclude(*what): Blacklist *what*. :param what: What to blacklist. - :type what: :class:`list` of classes or :class:`attr.Attribute`\\ s. + :type what: `list` of classes or `attr.Attribute`\\ s. - :rtype: :class:`callable` + :rtype: `callable` """ cls, attrs = _split_what(what) diff --git a/pipenv/vendor/attr/filters.pyi b/pipenv/vendor/attr/filters.pyi index a618140c..68368fe2 100644 --- a/pipenv/vendor/attr/filters.pyi +++ b/pipenv/vendor/attr/filters.pyi @@ -1,5 +1,5 @@ -from typing import Union +from typing import Union, Any from . import Attribute, _FilterType -def include(*what: Union[type, Attribute]) -> _FilterType: ... -def exclude(*what: Union[type, Attribute]) -> _FilterType: ... +def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... +def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... diff --git a/pipenv/vendor/attr/validators.py b/pipenv/vendor/attr/validators.py index f12d0aa5..839d310c 100644 --- a/pipenv/vendor/attr/validators.py +++ b/pipenv/vendor/attr/validators.py @@ -4,10 +4,23 @@ Commonly useful validators. from __future__ import absolute_import, division, print_function +import re + from ._make import _AndValidator, and_, attrib, attrs +from .exceptions import NotCallableError -__all__ = ["and_", "in_", "instance_of", "optional", "provides"] +__all__ = [ + "and_", + "deep_iterable", + "deep_mapping", + "in_", + "instance_of", + "is_callable", + "matches_re", + "optional", + "provides", +] @attrs(repr=False, slots=True, hash=True) @@ -40,20 +53,92 @@ class _InstanceOfValidator(object): def instance_of(type): """ - A validator that raises a :exc:`TypeError` if the initializer is called + A validator that raises a `TypeError` if the initializer is called with a wrong type for this particular attribute (checks are performed using - :func:`isinstance` therefore it's also valid to pass a tuple of types). + `isinstance` therefore it's also valid to pass a tuple of types). :param type: The type to check for. :type type: type or tuple of types :raises TypeError: With a human readable error message, the attribute - (of type :class:`attr.Attribute`), the expected type, and the value it + (of type `attr.Attribute`), the expected type, and the value it got. """ return _InstanceOfValidator(type) +@attrs(repr=False, frozen=True) +class _MatchesReValidator(object): + regex = attrib() + flags = attrib() + match_func = attrib() + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not self.match_func(value): + raise ValueError( + "'{name}' must match regex {regex!r}" + " ({value!r} doesn't)".format( + name=attr.name, regex=self.regex.pattern, value=value + ), + attr, + self.regex, + value, + ) + + def __repr__(self): + return "<matches_re validator for pattern {regex!r}>".format( + regex=self.regex + ) + + +def matches_re(regex, flags=0, func=None): + r""" + A validator that raises `ValueError` if the initializer is called + with a string that doesn't match *regex*. + + :param str regex: a regex string to match against + :param int flags: flags that will be passed to the underlying re function + (default 0) + :param callable func: which underlying `re` function to call (options + are `re.fullmatch`, `re.search`, `re.match`, default + is ``None`` which means either `re.fullmatch` or an emulation of + it on Python 2). For performance reasons, they won't be used directly + but on a pre-`re.compile`\ ed pattern. + + .. versionadded:: 19.2.0 + """ + fullmatch = getattr(re, "fullmatch", None) + valid_funcs = (fullmatch, None, re.search, re.match) + if func not in valid_funcs: + raise ValueError( + "'func' must be one of %s." + % ( + ", ".join( + sorted( + e and e.__name__ or "None" for e in set(valid_funcs) + ) + ), + ) + ) + + pattern = re.compile(regex, flags) + if func is re.match: + match_func = pattern.match + elif func is re.search: + match_func = pattern.search + else: + if fullmatch: + match_func = pattern.fullmatch + else: + pattern = re.compile(r"(?:{})\Z".format(regex), flags) + match_func = pattern.match + + return _MatchesReValidator(pattern, flags, match_func) + + @attrs(repr=False, slots=True, hash=True) class _ProvidesValidator(object): interface = attrib() @@ -81,7 +166,7 @@ class _ProvidesValidator(object): def provides(interface): """ - A validator that raises a :exc:`TypeError` if the initializer is called + A validator that raises a `TypeError` if the initializer is called with an object that does not provide the requested *interface* (checks are performed using ``interface.providedBy(value)`` (see `zope.interface <https://zopeinterface.readthedocs.io/en/latest/>`_). @@ -89,7 +174,7 @@ def provides(interface): :param zope.interface.Interface interface: The interface to check for. :raises TypeError: With a human readable error message, the attribute - (of type :class:`attr.Attribute`), the expected interface, and the + (of type `attr.Attribute`), the expected interface, and the value it got. """ return _ProvidesValidator(interface) @@ -119,7 +204,7 @@ def optional(validator): :param validator: A validator (or a list of validators) that is used for non-``None`` values. - :type validator: callable or :class:`list` of callables. + :type validator: callable or `list` of callables. .. versionadded:: 15.1.0 .. versionchanged:: 17.1.0 *validator* can be a list of validators. @@ -136,7 +221,7 @@ class _InValidator(object): def __call__(self, inst, attr, value): try: in_options = value in self.options - except TypeError as e: # e.g. `1 in "abc"` + except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: @@ -154,17 +239,140 @@ class _InValidator(object): def in_(options): """ - A validator that raises a :exc:`ValueError` if the initializer is called + A validator that raises a `ValueError` if the initializer is called with a value that does not belong in the options provided. The check is performed using ``value in options``. :param options: Allowed options. - :type options: list, tuple, :class:`enum.Enum`, ... + :type options: list, tuple, `enum.Enum`, ... :raises ValueError: With a human readable error message, the attribute (of - type :class:`attr.Attribute`), the expected options, and the value it + type `attr.Attribute`), the expected options, and the value it got. .. versionadded:: 17.1.0 """ return _InValidator(options) + + +@attrs(repr=False, slots=False, hash=True) +class _IsCallableValidator(object): + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + message = ( + "'{name}' must be callable " + "(got {value!r} that is a {actual!r})." + ) + raise NotCallableError( + msg=message.format( + name=attr.name, value=value, actual=value.__class__ + ), + value=value, + ) + + def __repr__(self): + return "<is_callable validator>" + + +def is_callable(): + """ + A validator that raises a `attr.exceptions.NotCallableError` if the + initializer is called with a value for this particular attribute + that is not callable. + + .. versionadded:: 19.1.0 + + :raises `attr.exceptions.NotCallableError`: With a human readable error + message containing the attribute (`attr.Attribute`) name, + and the value it got. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, hash=True) +class _DeepIterable(object): + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else " {iterable!r}".format(iterable=self.iterable_validator) + ) + return ( + "<deep_iterable validator for{iterable_identifier}" + " iterables of {member!r}>" + ).format( + iterable_identifier=iterable_identifier, + member=self.member_validator, + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + :param member_validator: Validator to apply to iterable members + :param iterable_validator: Validator to apply to iterable itself + (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, hash=True) +class _DeepMapping(object): + key_validator = attrib(validator=is_callable()) + value_validator = attrib(validator=is_callable()) + mapping_validator = attrib(default=None, validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + self.key_validator(inst, attr, key) + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return ( + "<deep_mapping validator for objects mapping {key!r} to {value!r}>" + ).format(key=self.key_validator, value=self.value_validator) + + +def deep_mapping(key_validator, value_validator, mapping_validator=None): + """ + A validator that performs deep validation of a dictionary. + + :param key_validator: Validator to apply to dictionary keys + :param value_validator: Validator to apply to dictionary values + :param mapping_validator: Validator to apply to top-level mapping + attribute (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepMapping(key_validator, value_validator, mapping_validator) diff --git a/pipenv/vendor/attr/validators.pyi b/pipenv/vendor/attr/validators.pyi index abbaedf1..9a22abb1 100644 --- a/pipenv/vendor/attr/validators.pyi +++ b/pipenv/vendor/attr/validators.pyi @@ -1,14 +1,66 @@ -from typing import Container, List, Union, TypeVar, Type, Any, Optional, Tuple +from typing import ( + Container, + List, + Union, + TypeVar, + Type, + Any, + Optional, + Tuple, + Iterable, + Mapping, + Callable, + Match, + AnyStr, + overload, +) from . import _ValidatorType _T = TypeVar("_T") +_T1 = TypeVar("_T1") +_T2 = TypeVar("_T2") +_T3 = TypeVar("_T3") +_I = TypeVar("_I", bound=Iterable) +_K = TypeVar("_K") +_V = TypeVar("_V") +_M = TypeVar("_M", bound=Mapping) +# To be more precise on instance_of use some overloads. +# If there are more than 3 items in the tuple then we fall back to Any +@overload +def instance_of(type: Type[_T]) -> _ValidatorType[_T]: ... +@overload +def instance_of(type: Tuple[Type[_T]]) -> _ValidatorType[_T]: ... +@overload def instance_of( - type: Union[Tuple[Type[_T], ...], Type[_T]] -) -> _ValidatorType[_T]: ... + type: Tuple[Type[_T1], Type[_T2]] +) -> _ValidatorType[Union[_T1, _T2]]: ... +@overload +def instance_of( + type: Tuple[Type[_T1], Type[_T2], Type[_T3]] +) -> _ValidatorType[Union[_T1, _T2, _T3]]: ... +@overload +def instance_of(type: Tuple[type, ...]) -> _ValidatorType[Any]: ... def provides(interface: Any) -> _ValidatorType[Any]: ... def optional( validator: Union[_ValidatorType[_T], List[_ValidatorType[_T]]] ) -> _ValidatorType[Optional[_T]]: ... def in_(options: Container[_T]) -> _ValidatorType[_T]: ... def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def matches_re( + regex: AnyStr, + flags: int = ..., + func: Optional[ + Callable[[AnyStr, AnyStr, int], Optional[Match[AnyStr]]] + ] = ..., +) -> _ValidatorType[AnyStr]: ... +def deep_iterable( + member_validator: _ValidatorType[_T], + iterable_validator: Optional[_ValidatorType[_I]] = ..., +) -> _ValidatorType[_I]: ... +def deep_mapping( + key_validator: _ValidatorType[_K], + value_validator: _ValidatorType[_V], + mapping_validator: Optional[_ValidatorType[_M]] = ..., +) -> _ValidatorType[_M]: ... +def is_callable() -> _ValidatorType[_T]: ... diff --git a/pipenv/vendor/backports/__init__.py b/pipenv/vendor/backports/__init__.py index 0c64b4c1..cf897602 100644 --- a/pipenv/vendor/backports/__init__.py +++ b/pipenv/vendor/backports/__init__.py @@ -1,5 +1,5 @@ __path__ = __import__('pkgutil').extend_path(__path__, __name__) +from . import shutil_get_terminal_size from . import weakref from . import enum -from . import shutil_get_terminal_size from . import functools_lru_cache diff --git a/pipenv/vendor/blindspin/__init__.py b/pipenv/vendor/blindspin/__init__.py deleted file mode 100644 index a1230e83..00000000 --- a/pipenv/vendor/blindspin/__init__.py +++ /dev/null @@ -1,73 +0,0 @@ -# -*- coding: utf-8 -*- - -import sys -import threading -import time -import itertools - - -class Spinner(object): - spinner_cycle = itertools.cycle(u'⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏') - - def __init__(self, beep=False, force=False): - self.beep = beep - self.force = force - self.stop_running = None - self.spin_thread = None - - def start(self): - if sys.stdout.isatty() or self.force: - self.stop_running = threading.Event() - self.spin_thread = threading.Thread(target=self.init_spin) - self.spin_thread.start() - - def stop(self): - if self.spin_thread: - self.stop_running.set() - self.spin_thread.join() - - def init_spin(self): - while not self.stop_running.is_set(): - next_val = next(self.spinner_cycle) - if sys.version_info[0] == 2: - next_val = next_val.encode('utf-8') - sys.stdout.write(next_val) - sys.stdout.flush() - time.sleep(0.07) - sys.stdout.write('\b') - - def __enter__(self): - self.start() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.stop() - if self.beep: - sys.stdout.write('\7') - sys.stdout.flush() - return False - - -def spinner(beep=False, force=False): - """This function creates a context manager that is used to display a - spinner on stdout as long as the context has not exited. - - The spinner is created only if stdout is not redirected, or if the spinner - is forced using the `force` parameter. - - Parameters - ---------- - beep : bool - Beep when spinner finishes. - force : bool - Force creation of spinner even when stdout is redirected. - - Example - ------- - - with spinner(): - do_something() - do_something_else() - - """ - return Spinner(beep, force) diff --git a/pipenv/vendor/cached_property.py b/pipenv/vendor/cached_property.py index a06be97a..125f6195 100644 --- a/pipenv/vendor/cached_property.py +++ b/pipenv/vendor/cached_property.py @@ -2,7 +2,7 @@ __author__ = "Daniel Greenfeld" __email__ = "pydanny@gmail.com" -__version__ = "1.4.3" +__version__ = "1.5.1" __license__ = "BSD" from time import time diff --git a/pipenv/vendor/cerberus/__init__.py b/pipenv/vendor/cerberus/__init__.py index 4b528cff..1e0f0d54 100644 --- a/pipenv/vendor/cerberus/__init__.py +++ b/pipenv/vendor/cerberus/__init__.py @@ -10,20 +10,23 @@ from __future__ import absolute_import +from pkg_resources import get_distribution, DistributionNotFound + from cerberus.validator import DocumentError, Validator -from cerberus.schema import (rules_set_registry, schema_registry, Registry, - SchemaError) +from cerberus.schema import rules_set_registry, schema_registry, SchemaError from cerberus.utils import TypeDefinition -__version__ = "1.2" +try: + __version__ = get_distribution("Cerberus").version +except DistributionNotFound: + __version__ = "unknown" __all__ = [ DocumentError.__name__, - Registry.__name__, SchemaError.__name__, TypeDefinition.__name__, Validator.__name__, - 'schema_registry', - 'rules_set_registry' + "schema_registry", + "rules_set_registry", ] diff --git a/pipenv/vendor/cerberus/errors.py b/pipenv/vendor/cerberus/errors.py index 4c497eeb..41564cab 100644 --- a/pipenv/vendor/cerberus/errors.py +++ b/pipenv/vendor/cerberus/errors.py @@ -3,12 +3,12 @@ from __future__ import absolute_import -from collections import defaultdict, namedtuple, MutableMapping +from collections import defaultdict, namedtuple from copy import copy, deepcopy from functools import wraps from pprint import pformat -from cerberus.platform import PYTHON_VERSION +from cerberus.platform import PYTHON_VERSION, MutableMapping from cerberus.utils import compare_paths_lt, quote_string @@ -54,6 +54,7 @@ UNALLOWED_VALUE = ErrorDefinition(0x44, 'allowed') UNALLOWED_VALUES = ErrorDefinition(0x45, 'allowed') FORBIDDEN_VALUE = ErrorDefinition(0x46, 'forbidden') FORBIDDEN_VALUES = ErrorDefinition(0x47, 'forbidden') +MISSING_MEMBERS = ErrorDefinition(0x48, 'contains') # other NORMALIZATION = ErrorDefinition(0x60, None) @@ -66,9 +67,10 @@ SETTING_DEFAULT_FAILED = ErrorDefinition(0x64, 'default_setter') ERROR_GROUP = ErrorDefinition(0x80, None) MAPPING_SCHEMA = ErrorDefinition(0x81, 'schema') SEQUENCE_SCHEMA = ErrorDefinition(0x82, 'schema') -KEYSCHEMA = ErrorDefinition(0x83, 'keyschema') -VALUESCHEMA = ErrorDefinition(0x84, 'valueschema') -BAD_ITEMS = ErrorDefinition(0x8f, 'items') +# TODO remove KEYSCHEMA AND VALUESCHEMA with next major release +KEYSRULES = KEYSCHEMA = ErrorDefinition(0x83, 'keysrules') +VALUESRULES = VALUESCHEMA = ErrorDefinition(0x84, 'valuesrules') +BAD_ITEMS = ErrorDefinition(0x8F, 'items') LOGICAL = ErrorDefinition(0x90, None) NONEOF = ErrorDefinition(0x91, 'noneof') @@ -79,8 +81,7 @@ ALLOF = ErrorDefinition(0x94, 'allof') """ SchemaError messages """ -SCHEMA_ERROR_DEFINITION_TYPE = \ - "schema definition for field '{0}' must be a dict" +SCHEMA_ERROR_DEFINITION_TYPE = "schema definition for field '{0}' must be a dict" SCHEMA_ERROR_MISSING = "validation schema missing" @@ -89,8 +90,8 @@ SCHEMA_ERROR_MISSING = "validation schema missing" class ValidationError(object): """ A simple class to store and query basic error information. """ - def __init__(self, document_path, schema_path, code, rule, constraint, - value, info): + + def __init__(self, document_path, schema_path, code, rule, constraint, value, info): self.document_path = document_path """ The path to the field within the document that caused the error. Type: :class:`tuple` """ @@ -115,8 +116,7 @@ class ValidationError(object): def __hash__(self): """ Expects that all other properties are transitively determined. """ - return hash(self.document_path) ^ hash(self.schema_path) \ - ^ hash(self.code) + return hash(self.document_path) ^ hash(self.schema_path) ^ hash(self.code) def __lt__(self, other): if self.document_path != other.document_path: @@ -125,20 +125,24 @@ class ValidationError(object): return compare_paths_lt(self.schema_path, other.schema_path) def __repr__(self): - return "{class_name} @ {memptr} ( " \ - "document_path={document_path}," \ - "schema_path={schema_path}," \ - "code={code}," \ - "constraint={constraint}," \ - "value={value}," \ - "info={info} )"\ - .format(class_name=self.__class__.__name__, memptr=hex(id(self)), # noqa: E501 - document_path=self.document_path, - schema_path=self.schema_path, - code=hex(self.code), - constraint=quote_string(self.constraint), - value=quote_string(self.value), - info=self.info) + return ( + "{class_name} @ {memptr} ( " + "document_path={document_path}," + "schema_path={schema_path}," + "code={code}," + "constraint={constraint}," + "value={value}," + "info={info} )".format( + class_name=self.__class__.__name__, + memptr=hex(id(self)), # noqa: E501 + document_path=self.document_path, + schema_path=self.schema_path, + code=hex(self.code), + constraint=quote_string(self.constraint), + value=quote_string(self.value), + info=self.info, + ) + ) @property def child_errors(self): @@ -190,11 +194,13 @@ class ErrorList(list): """ A list for :class:`~cerberus.errors.ValidationError` instances that can be queried with the ``in`` keyword for a particular :class:`~cerberus.errors.ErrorDefinition`. """ + def __contains__(self, error_definition): - for code in (x.code for x in self): - if code == error_definition.code: - return True - return False + if not isinstance(error_definition, ErrorDefinition): + raise TypeError + + wanted_code = error_definition.code + return any(x.code == wanted_code for x in self) class ErrorTreeNode(MutableMapping): @@ -203,14 +209,10 @@ class ErrorTreeNode(MutableMapping): def __init__(self, path, parent_node): self.parent_node = parent_node self.tree_root = self.parent_node.tree_root - self.path = path[:self.parent_node.depth + 1] + self.path = path[: self.parent_node.depth + 1] self.errors = ErrorList() self.descendants = {} - def __add__(self, error): - self.add(error) - return self - def __contains__(self, item): if isinstance(item, ErrorDefinition): return item in self.errors @@ -228,6 +230,7 @@ class ErrorTreeNode(MutableMapping): for error in self.errors: if item.code == error.code: return error + return None else: return self.descendants.get(item) @@ -258,14 +261,16 @@ class ErrorTreeNode(MutableMapping): if key not in self.descendants: self[key] = ErrorTreeNode(error_path, self) + node = self[key] + if len(error_path) == self.depth + 1: - self[key].errors.append(error) - self[key].errors.sort() + node.errors.append(error) + node.errors.sort() if error.is_group_error: for child_error in error.child_errors: - self.tree_root += child_error + self.tree_root.add(child_error) else: - self[key] += error + node.add(error) def _path_of_(self, error): return getattr(error, self.tree_type + '_path') @@ -274,14 +279,15 @@ class ErrorTreeNode(MutableMapping): class ErrorTree(ErrorTreeNode): """ Base class for :class:`~cerberus.errors.DocumentErrorTree` and :class:`~cerberus.errors.SchemaErrorTree`. """ - def __init__(self, errors=[]): + + def __init__(self, errors=()): self.parent_node = None self.tree_root = self self.path = () self.errors = ErrorList() self.descendants = {} for error in errors: - self += error + self.add(error) def add(self, error): """ Add an error to the tree. @@ -323,18 +329,21 @@ class ErrorTree(ErrorTreeNode): class DocumentErrorTree(ErrorTree): """ Implements a dict-like class to query errors by indexes following the structure of a validated document. """ + tree_type = 'document' class SchemaErrorTree(ErrorTree): """ Implements a dict-like class to query errors by indexes following the structure of the used schema. """ + tree_type = 'schema' class BaseErrorHandler(object): """ Base class for all error handlers. Subclasses are identified as error-handlers with an instance-test. """ + def __init__(self, *args, **kwargs): """ Optionally initialize a new instance. """ pass @@ -411,9 +420,9 @@ def encode_unicode(f): This decorator ensures that if legacy Python is used unicode strings are encoded before passing to a function. """ + @wraps(f) def wrapped(obj, error): - def _encode(value): """Helper encoding unicode strings into binary utf-8""" if isinstance(value, unicode): # noqa: F821 @@ -436,56 +445,52 @@ class BasicErrorHandler(BaseErrorHandler): through :class:`str` a pretty-formatted representation of that tree is returned. """ - messages = {0x00: "{0}", - 0x01: "document is missing", - 0x02: "required field", - 0x03: "unknown field", - 0x04: "field '{0}' is required", - 0x05: "depends on these values: {constraint}", - 0x06: "{0} must not be present with '{field}'", - - 0x21: "'{0}' is not a document, must be a dict", - 0x22: "empty values not allowed", - 0x23: "null value not allowed", - 0x24: "must be of {constraint} type", - 0x25: "must be of dict type", - 0x26: "length of list should be {constraint}, it is {0}", - 0x27: "min length is {constraint}", - 0x28: "max length is {constraint}", - - 0x41: "value does not match regex '{constraint}'", - 0x42: "min value is {constraint}", - 0x43: "max value is {constraint}", - 0x44: "unallowed value {value}", - 0x45: "unallowed values {0}", - 0x46: "unallowed value {value}", - 0x47: "unallowed values {0}", - - 0x61: "field '{field}' cannot be coerced: {0}", - 0x62: "field '{field}' cannot be renamed: {0}", - 0x63: "field is read-only", - 0x64: "default value for '{field}' cannot be set: {0}", - - 0x81: "mapping doesn't validate subschema: {0}", - 0x82: "one or more sequence-items don't validate: {0}", - 0x83: "one or more keys of a mapping don't validate: {0}", - 0x84: "one or more values in a mapping don't validate: {0}", - 0x85: "one or more sequence-items don't validate: {0}", - - 0x91: "one or more definitions validate", - 0x92: "none or more than one rule validate", - 0x93: "no definitions validate", - 0x94: "one or more definitions don't validate" - } + messages = { + 0x00: "{0}", + 0x01: "document is missing", + 0x02: "required field", + 0x03: "unknown field", + 0x04: "field '{0}' is required", + 0x05: "depends on these values: {constraint}", + 0x06: "{0} must not be present with '{field}'", + 0x21: "'{0}' is not a document, must be a dict", + 0x22: "empty values not allowed", + 0x23: "null value not allowed", + 0x24: "must be of {constraint} type", + 0x25: "must be of dict type", + 0x26: "length of list should be {0}, it is {1}", + 0x27: "min length is {constraint}", + 0x28: "max length is {constraint}", + 0x41: "value does not match regex '{constraint}'", + 0x42: "min value is {constraint}", + 0x43: "max value is {constraint}", + 0x44: "unallowed value {value}", + 0x45: "unallowed values {0}", + 0x46: "unallowed value {value}", + 0x47: "unallowed values {0}", + 0x48: "missing members {0}", + 0x61: "field '{field}' cannot be coerced: {0}", + 0x62: "field '{field}' cannot be renamed: {0}", + 0x63: "field is read-only", + 0x64: "default value for '{field}' cannot be set: {0}", + 0x81: "mapping doesn't validate subschema: {0}", + 0x82: "one or more sequence-items don't validate: {0}", + 0x83: "one or more keys of a mapping don't validate: {0}", + 0x84: "one or more values in a mapping don't validate: {0}", + 0x85: "one or more sequence-items don't validate: {0}", + 0x91: "one or more definitions validate", + 0x92: "none or more than one rule validate", + 0x93: "no definitions validate", + 0x94: "one or more definitions don't validate", + } def __init__(self, tree=None): self.tree = {} if tree is None else tree - def __call__(self, errors=None): - if errors is not None: - self.clear() - self.extend(errors) + def __call__(self, errors): + self.clear() + self.extend(errors) return self.pretty_tree def __str__(self): @@ -511,8 +516,9 @@ class BasicErrorHandler(BaseErrorHandler): elif error.is_group_error: self._insert_group_error(error) elif error.code in self.messages: - self._insert_error(error.document_path, - self._format_message(error.field, error)) + self._insert_error( + error.document_path, self._format_message(error.field, error) + ) def clear(self): self.tree = {} @@ -522,8 +528,8 @@ class BasicErrorHandler(BaseErrorHandler): def _format_message(self, field, error): return self.messages[error.code].format( - *error.info, constraint=error.constraint, - field=field, value=error.value) + *error.info, constraint=error.constraint, field=field, value=error.value + ) def _insert_error(self, path, node): """ Adds an error or sub-tree to :attr:tree. @@ -559,14 +565,14 @@ class BasicErrorHandler(BaseErrorHandler): elif child_error.is_group_error: self._insert_group_error(child_error) else: - self._insert_error(child_error.document_path, - self._format_message(child_error.field, - child_error)) + self._insert_error( + child_error.document_path, + self._format_message(child_error.field, child_error), + ) def _insert_logic_error(self, error): field = error.field - self._insert_error(error.document_path, - self._format_message(field, error)) + self._insert_error(error.document_path, self._format_message(field, error)) for definition_errors in error.definitions_errors.values(): for child_error in definition_errors: @@ -575,8 +581,10 @@ class BasicErrorHandler(BaseErrorHandler): elif child_error.is_group_error: self._insert_group_error(child_error) else: - self._insert_error(child_error.document_path, - self._format_message(field, child_error)) + self._insert_error( + child_error.document_path, + self._format_message(field, child_error), + ) def _purge_empty_dicts(self, error_list): subtree = error_list[-1] diff --git a/pipenv/vendor/cerberus/platform.py b/pipenv/vendor/cerberus/platform.py index eca9858d..66b1d5f0 100644 --- a/pipenv/vendor/cerberus/platform.py +++ b/pipenv/vendor/cerberus/platform.py @@ -12,3 +12,29 @@ if PYTHON_VERSION < 3: else: _str_type = str _int_types = (int,) + + +if PYTHON_VERSION < 3.3: + from collections import ( # noqa: F401 + Callable, + Container, + Hashable, + Iterable, + Mapping, + MutableMapping, + Sequence, + Set, + Sized, + ) +else: + from collections.abc import ( # noqa: F401 + Callable, + Container, + Hashable, + Iterable, + Mapping, + MutableMapping, + Sequence, + Set, + Sized, + ) diff --git a/pipenv/vendor/cerberus/schema.py b/pipenv/vendor/cerberus/schema.py index 3ddce172..3841384e 100644 --- a/pipenv/vendor/cerberus/schema.py +++ b/pipenv/vendor/cerberus/schema.py @@ -1,13 +1,23 @@ from __future__ import absolute_import -from collections import (Callable, Hashable, Iterable, Mapping, - MutableMapping, Sequence) from copy import copy +from warnings import warn from cerberus import errors -from cerberus.platform import _str_type -from cerberus.utils import (get_Validator_class, validator_factory, - mapping_hash, TypeDefinition) +from cerberus.platform import ( + _str_type, + Callable, + Hashable, + Mapping, + MutableMapping, + Sequence, +) +from cerberus.utils import ( + get_Validator_class, + validator_factory, + mapping_hash, + TypeDefinition, +) class _Abort(Exception): @@ -17,6 +27,7 @@ class _Abort(Exception): class SchemaError(Exception): """ Raised when the validation schema is missing, has the wrong format or contains errors. """ + pass @@ -26,18 +37,19 @@ class DefinitionSchema(MutableMapping): def __new__(cls, *args, **kwargs): if 'SchemaValidator' not in globals(): global SchemaValidator - SchemaValidator = validator_factory('SchemaValidator', - SchemaValidatorMixin) + SchemaValidator = validator_factory('SchemaValidator', SchemaValidatorMixin) types_mapping = SchemaValidator.types_mapping.copy() - types_mapping.update({ - 'callable': TypeDefinition('callable', (Callable,), ()), - 'hashable': TypeDefinition('hashable', (Hashable,), ()) - }) + types_mapping.update( + { + 'callable': TypeDefinition('callable', (Callable,), ()), + 'hashable': TypeDefinition('hashable', (Hashable,), ()), + } + ) SchemaValidator.types_mapping = types_mapping return super(DefinitionSchema, cls).__new__(cls) - def __init__(self, validator, schema={}): + def __init__(self, validator, schema): """ :param validator: An instance of Validator-(sub-)class that uses this schema. @@ -45,8 +57,7 @@ class DefinitionSchema(MutableMapping): one. """ if not isinstance(validator, get_Validator_class()): - raise RuntimeError('validator argument must be a Validator-' - 'instance.') + raise RuntimeError('validator argument must be a Validator-' 'instance.') self.validator = validator if isinstance(schema, _str_type): @@ -56,14 +67,16 @@ class DefinitionSchema(MutableMapping): try: schema = dict(schema) except Exception: - raise SchemaError( - errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) + raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) self.validation_schema = SchemaValidationSchema(validator) self.schema_validator = SchemaValidator( - None, allow_unknown=self.validation_schema, + None, + allow_unknown=self.validation_schema, error_handler=errors.SchemaErrorHandler, - target_schema=schema, target_validator=validator) + target_schema=schema, + target_validator=validator, + ) schema = self.expand(schema) self.validate(schema) @@ -98,7 +111,10 @@ class DefinitionSchema(MutableMapping): self.schema[key] = value def __str__(self): - return str(self.schema) + if hasattr(self, "schema"): + return str(self.schema) + else: + return "No schema data is set yet." def copy(self): return self.__class__(self.validator, self.schema.copy()) @@ -110,6 +126,10 @@ class DefinitionSchema(MutableMapping): schema = cls._expand_subschemas(schema) except Exception: pass + + # TODO remove this with the next major release + schema = cls._rename_deprecated_rulenames(schema) + return schema @classmethod @@ -119,31 +139,33 @@ class DefinitionSchema(MutableMapping): :param schema: The schema-definition to expand. :return: The expanded schema-definition. """ - def is_of_rule(x): - return isinstance(x, _str_type) and \ - x.startswith(('allof_', 'anyof_', 'noneof_', 'oneof_')) - for field in schema: - for of_rule in (x for x in schema[field] if is_of_rule(x)): - operator, rule = of_rule.split('_') - schema[field].update({operator: []}) - for value in schema[field][of_rule]: - schema[field][operator].append({rule: value}) - del schema[field][of_rule] + def is_of_rule(x): + return isinstance(x, _str_type) and x.startswith( + ('allof_', 'anyof_', 'noneof_', 'oneof_') + ) + + for field, rules in schema.items(): + for of_rule in [x for x in rules if is_of_rule(x)]: + operator, rule = of_rule.split('_', 1) + rules.update({operator: []}) + for value in rules[of_rule]: + rules[operator].append({rule: value}) + del rules[of_rule] return schema @classmethod def _expand_subschemas(cls, schema): def has_schema_rule(): - return isinstance(schema[field], Mapping) and \ - 'schema' in schema[field] + return isinstance(schema[field], Mapping) and 'schema' in schema[field] def has_mapping_schema(): """ Tries to determine heuristically if the schema-constraints are aimed to mappings. """ try: - return all(isinstance(x, Mapping) for x - in schema[field]['schema'].values()) + return all( + isinstance(x, Mapping) for x in schema[field]['schema'].values() + ) except TypeError: return False @@ -153,13 +175,12 @@ class DefinitionSchema(MutableMapping): elif has_mapping_schema(): schema[field]['schema'] = cls.expand(schema[field]['schema']) else: # assumes schema-constraints for a sequence - schema[field]['schema'] = \ - cls.expand({0: schema[field]['schema']})[0] + schema[field]['schema'] = cls.expand({0: schema[field]['schema']})[0] - for rule in ('keyschema', 'valueschema'): + # TODO remove the last two values in the tuple with the next major release + for rule in ('keysrules', 'valuesrules', 'keyschema', 'valueschema'): if rule in schema[field]: - schema[field][rule] = \ - cls.expand({0: schema[field][rule]})[0] + schema[field][rule] = cls.expand({0: schema[field][rule]})[0] for rule in ('allof', 'anyof', 'items', 'noneof', 'oneof'): if rule in schema[field]: @@ -171,6 +192,12 @@ class DefinitionSchema(MutableMapping): schema[field][rule] = new_rules_definition return schema + def get(self, item, default=None): + return self.schema.get(item, default) + + def items(self): + return self.schema.items() + def update(self, schema): try: schema = self.expand(schema) @@ -178,31 +205,64 @@ class DefinitionSchema(MutableMapping): _new_schema.update(schema) self.validate(_new_schema) except ValueError: - raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE - .format(schema)) + raise SchemaError(errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema)) except Exception as e: raise e else: self.schema = _new_schema + # TODO remove with next major release + @staticmethod + def _rename_deprecated_rulenames(schema): + for field, rules in schema.items(): + + if isinstance(rules, str): # registry reference + continue + + for old, new in ( + ('keyschema', 'keysrules'), + ('validator', 'check_with'), + ('valueschema', 'valuesrules'), + ): + + if old not in rules: + continue + + if new in rules: + raise RuntimeError( + "The rule '{new}' is also present with its old " + "name '{old}' in the same set of rules." + ) + + warn( + "The rule '{old}' was renamed to '{new}'. The old name will " + "not be available in the next major release of " + "Cerberus.".format(old=old, new=new), + DeprecationWarning, + ) + schema[field][new] = schema[field][old] + schema[field].pop(old) + + return schema + def regenerate_validation_schema(self): self.validation_schema = SchemaValidationSchema(self.validator) def validate(self, schema=None): + """ Validates a schema that defines rules against supported rules. + + :param schema: The schema to be validated as a legal cerberus schema + according to the rules of the assigned Validator object. + Raises a :class:`~cerberus.base.SchemaError` when an invalid + schema is encountered. """ if schema is None: schema = self.schema - _hash = (mapping_hash(schema), - mapping_hash(self.validator.types_mapping)) + _hash = (mapping_hash(schema), mapping_hash(self.validator.types_mapping)) if _hash not in self.validator._valid_schemas: self._validate(schema) self.validator._valid_schemas.add(_hash) def _validate(self, schema): - """ Validates a schema that defines rules against supported rules. - - :param schema: The schema to be validated as a legal cerberus schema - according to the rules of this Validator object. - """ if isinstance(schema, _str_type): schema = self.validator.schema_registry.get(schema, schema) @@ -212,8 +272,7 @@ class DefinitionSchema(MutableMapping): schema = copy(schema) for field in schema: if isinstance(schema[field], _str_type): - schema[field] = rules_set_registry.get(schema[field], - schema[field]) + schema[field] = rules_set_registry.get(schema[field], schema[field]) if not self.schema_validator(schema, normalize=False): raise SchemaError(self.schema_validator.errors) @@ -236,31 +295,31 @@ class UnvalidatedSchema(DefinitionSchema): class SchemaValidationSchema(UnvalidatedSchema): def __init__(self, validator): - self.schema = {'allow_unknown': False, - 'schema': validator.rules, - 'type': 'dict'} + self.schema = { + 'allow_unknown': False, + 'schema': validator.rules, + 'type': 'dict', + } class SchemaValidatorMixin(object): - """ This validator is extended to validate schemas passed to a Cerberus + """ This validator mixin provides mechanics to validate schemas passed to a Cerberus validator. """ + + def __init__(self, *args, **kwargs): + kwargs.setdefault('known_rules_set_refs', set()) + kwargs.setdefault('known_schema_refs', set()) + super(SchemaValidatorMixin, self).__init__(*args, **kwargs) + @property def known_rules_set_refs(self): """ The encountered references to rules set registry items. """ - return self._config.get('known_rules_set_refs', ()) - - @known_rules_set_refs.setter - def known_rules_set_refs(self, value): - self._config['known_rules_set_refs'] = value + return self._config['known_rules_set_refs'] @property def known_schema_refs(self): """ The encountered references to schema registry items. """ - return self._config.get('known_schema_refs', ()) - - @known_schema_refs.setter - def known_schema_refs(self, value): - self._config['known_schema_refs'] = value + return self._config['known_schema_refs'] @property def target_schema(self): @@ -272,35 +331,13 @@ class SchemaValidatorMixin(object): """ The validator whose schema is being validated. """ return self._config['target_validator'] - def _validate_logical(self, rule, field, value): - """ {'allowed': ('allof', 'anyof', 'noneof', 'oneof')} """ - if not isinstance(value, Sequence): - self._error(field, errors.BAD_TYPE) - return - - validator = self._get_child_validator( - document_crumb=rule, allow_unknown=False, - schema=self.target_validator.validation_rules) - - for constraints in value: - _hash = (mapping_hash({'turing': constraints}), - mapping_hash(self.target_validator.types_mapping)) - if _hash in self.target_validator._valid_schemas: - continue - - validator(constraints, normalize=False) - if validator._errors: - self._error(validator._errors) - else: - self.target_validator._valid_schemas.add(_hash) - - def _validator_bulk_schema(self, field, value): + def _check_with_bulk_schema(self, field, value): # resolve schema registry reference if isinstance(value, _str_type): if value in self.known_rules_set_refs: return else: - self.known_rules_set_refs += (value,) + self.known_rules_set_refs.add(value) definition = self.target_validator.rules_set_registry.get(value) if definition is None: self._error(field, 'Rules set definition %s not found.' % value) @@ -308,28 +345,32 @@ class SchemaValidatorMixin(object): else: value = definition - _hash = (mapping_hash({'turing': value}), - mapping_hash(self.target_validator.types_mapping)) + _hash = ( + mapping_hash({'turing': value}), + mapping_hash(self.target_validator.types_mapping), + ) if _hash in self.target_validator._valid_schemas: return validator = self._get_child_validator( - document_crumb=field, allow_unknown=False, - schema=self.target_validator.rules) + document_crumb=field, + allow_unknown=False, + schema=self.target_validator.rules, + ) validator(value, normalize=False) if validator._errors: self._error(validator._errors) else: self.target_validator._valid_schemas.add(_hash) - def _validator_dependencies(self, field, value): + def _check_with_dependencies(self, field, value): if isinstance(value, _str_type): pass elif isinstance(value, Mapping): validator = self._get_child_validator( document_crumb=field, - schema={'valueschema': {'type': 'list'}}, - allow_unknown=True + schema={'valuesrules': {'type': 'list'}}, + allow_unknown=True, ) if not validator(value, normalize=False): self._error(validator._errors) @@ -338,54 +379,36 @@ class SchemaValidatorMixin(object): path = self.document_path + (field,) self._error(path, 'All dependencies must be a hashable type.') - def _validator_handler(self, field, value): - if isinstance(value, Callable): - return - if isinstance(value, _str_type): - if value not in self.target_validator.validators + \ - self.target_validator.coercers: - self._error(field, '%s is no valid coercer' % value) - elif isinstance(value, Iterable): - for handler in value: - self._validator_handler(field, handler) - - def _validator_items(self, field, value): + def _check_with_items(self, field, value): for i, schema in enumerate(value): - self._validator_bulk_schema((field, i), schema) + self._check_with_bulk_schema((field, i), schema) - def _validator_schema(self, field, value): + def _check_with_schema(self, field, value): try: value = self._handle_schema_reference_for_validator(field, value) except _Abort: return - _hash = (mapping_hash(value), - mapping_hash(self.target_validator.types_mapping)) + _hash = (mapping_hash(value), mapping_hash(self.target_validator.types_mapping)) if _hash in self.target_validator._valid_schemas: return validator = self._get_child_validator( - document_crumb=field, - schema=None, allow_unknown=self.root_allow_unknown) + document_crumb=field, schema=None, allow_unknown=self.root_allow_unknown + ) validator(self._expand_rules_set_refs(value), normalize=False) if validator._errors: self._error(validator._errors) else: self.target_validator._valid_schemas.add(_hash) - def _handle_schema_reference_for_validator(self, field, value): - if not isinstance(value, _str_type): - return value - if value in self.known_schema_refs: - raise _Abort - - self.known_schema_refs += (value,) - definition = self.target_validator.schema_registry.get(value) - if definition is None: - path = self.document_path + (field,) - self._error(path, 'Schema definition {} not found.'.format(value)) - raise _Abort - return definition + def _check_with_type(self, field, value): + value = set((value,)) if isinstance(value, _str_type) else set(value) + invalid_constraints = value - set(self.target_validator.types) + if invalid_constraints: + self._error( + field, 'Unsupported types: {}'.format(', '.join(invalid_constraints)) + ) def _expand_rules_set_refs(self, schema): result = {} @@ -396,15 +419,46 @@ class SchemaValidatorMixin(object): result[k] = v return result - def _validator_type(self, field, value): - value = (value,) if isinstance(value, _str_type) else value - invalid_constraints = () - for constraint in value: - if constraint not in self.target_validator.types: - invalid_constraints += (constraint,) - if invalid_constraints: + def _handle_schema_reference_for_validator(self, field, value): + if not isinstance(value, _str_type): + return value + if value in self.known_schema_refs: + raise _Abort + + self.known_schema_refs.add(value) + definition = self.target_validator.schema_registry.get(value) + if definition is None: path = self.document_path + (field,) - self._error(path, 'Unsupported types: %s' % invalid_constraints) + self._error(path, 'Schema definition {} not found.'.format(value)) + raise _Abort + return definition + + def _validate_logical(self, rule, field, value): + """ {'allowed': ('allof', 'anyof', 'noneof', 'oneof')} """ + if not isinstance(value, Sequence): + self._error(field, errors.BAD_TYPE) + return + + validator = self._get_child_validator( + document_crumb=rule, + allow_unknown=False, + schema=self.target_validator.validation_rules, + ) + + for constraints in value: + _hash = ( + mapping_hash({'turing': constraints}), + mapping_hash(self.target_validator.types_mapping), + ) + if _hash in self.target_validator._valid_schemas: + continue + + validator(constraints, normalize=False) + if validator._errors: + self._error(validator._errors) + else: + self.target_validator._valid_schemas.add(_hash) + #### diff --git a/pipenv/vendor/cerberus/tests/__init__.py b/pipenv/vendor/cerberus/tests/__init__.py index cc1c27dc..c014f3b1 100644 --- a/pipenv/vendor/cerberus/tests/__init__.py +++ b/pipenv/vendor/cerberus/tests/__init__.py @@ -1,22 +1,23 @@ # -*- coding: utf-8 -*- +import re + import pytest from cerberus import errors, Validator, SchemaError, DocumentError from cerberus.tests.conftest import sample_schema -def assert_exception(exception, document={}, schema=None, validator=None, - msg=None): +def assert_exception(exception, document={}, schema=None, validator=None, msg=None): """ Tests whether a specific exception is raised. Optionally also tests whether the exception message is as expected. """ if validator is None: validator = Validator() if msg is None: - with pytest.raises(exception) as excinfo: + with pytest.raises(exception): validator(document, schema) else: - with pytest.raises(exception, message=msg) as excinfo: # noqa: F841 + with pytest.raises(exception, match=re.escape(msg)): validator(document, schema) @@ -32,8 +33,15 @@ def assert_document_error(*args): assert_exception(DocumentError, *args) -def assert_fail(document, schema=None, validator=None, update=False, - error=None, errors=None, child_errors=None): +def assert_fail( + document, + schema=None, + validator=None, + update=False, + error=None, + errors=None, + child_errors=None, +): """ Tests whether a validation fails. """ if validator is None: validator = Validator(sample_schema) @@ -45,8 +53,7 @@ def assert_fail(document, schema=None, validator=None, update=False, assert not (error is not None and errors is not None) assert not (errors is not None and child_errors is not None), ( - 'child_errors can only be tested in ' - 'conjunction with the error parameter' + 'child_errors can only be tested in ' 'conjunction with the error parameter' ) assert not (child_errors is not None and error is None) if error is not None: @@ -99,7 +106,8 @@ def assert_has_error(_errors, d_path, s_path, error_def, constraint, info=()): else: break else: - raise AssertionError(""" + raise AssertionError( + """ Error with properties: document_path={doc_path} schema_path={schema_path} @@ -108,9 +116,15 @@ def assert_has_error(_errors, d_path, s_path, error_def, constraint, info=()): info={info} not found in errors: {errors} - """.format(doc_path=d_path, schema_path=s_path, - code=hex(error.code), info=info, - constraint=constraint, errors=_errors)) + """.format( + doc_path=d_path, + schema_path=s_path, + code=hex(error.code), + info=info, + constraint=constraint, + errors=_errors, + ) + ) return i @@ -133,8 +147,9 @@ def assert_not_has_error(_errors, *args, **kwargs): def assert_bad_type(field, data_type, value): - assert_fail({field: value}, - error=(field, (field, 'type'), errors.BAD_TYPE, data_type)) + assert_fail( + {field: value}, error=(field, (field, 'type'), errors.BAD_TYPE, data_type) + ) def assert_normalized(document, expected, schema=None, validator=None): diff --git a/pipenv/vendor/cerberus/tests/conftest.py b/pipenv/vendor/cerberus/tests/conftest.py index 3b4395ea..776c97bc 100644 --- a/pipenv/vendor/cerberus/tests/conftest.py +++ b/pipenv/vendor/cerberus/tests/conftest.py @@ -23,67 +23,27 @@ def validator(): sample_schema = { - 'a_string': { - 'type': 'string', - 'minlength': 2, - 'maxlength': 10 - }, - 'a_binary': { - 'type': 'binary', - 'minlength': 2, - 'maxlength': 10 - }, - 'a_nullable_integer': { - 'type': 'integer', - 'nullable': True - }, - 'an_integer': { - 'type': 'integer', - 'min': 1, - 'max': 100, - }, - 'a_restricted_integer': { - 'type': 'integer', - 'allowed': [-1, 0, 1], - }, - 'a_boolean': { - 'type': 'boolean', - }, - 'a_datetime': { - 'type': 'datetime', - }, - 'a_float': { - 'type': 'float', - 'min': 1, - 'max': 100, - }, - 'a_number': { - 'type': 'number', - 'min': 1, - 'max': 100, - }, - 'a_set': { - 'type': 'set', - }, - 'one_or_more_strings': { - 'type': ['string', 'list'], - 'schema': {'type': 'string'} - }, + 'a_string': {'type': 'string', 'minlength': 2, 'maxlength': 10}, + 'a_binary': {'type': 'binary', 'minlength': 2, 'maxlength': 10}, + 'a_nullable_integer': {'type': 'integer', 'nullable': True}, + 'an_integer': {'type': 'integer', 'min': 1, 'max': 100}, + 'a_restricted_integer': {'type': 'integer', 'allowed': [-1, 0, 1]}, + 'a_boolean': {'type': 'boolean', 'meta': 'can haz two distinct states'}, + 'a_datetime': {'type': 'datetime', 'meta': {'format': '%a, %d. %b %Y'}}, + 'a_float': {'type': 'float', 'min': 1, 'max': 100}, + 'a_number': {'type': 'number', 'min': 1, 'max': 100}, + 'a_set': {'type': 'set'}, + 'one_or_more_strings': {'type': ['string', 'list'], 'schema': {'type': 'string'}}, 'a_regex_email': { 'type': 'string', - 'regex': '^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$' + 'regex': r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', }, - 'a_readonly_string': { - 'type': 'string', - 'readonly': True, - }, - 'a_restricted_string': { - 'type': 'string', - 'allowed': ["agent", "client", "vendor"], - }, - 'an_array': { + 'a_readonly_string': {'type': 'string', 'readonly': True}, + 'a_restricted_string': {'type': 'string', 'allowed': ['agent', 'client', 'vendor']}, + 'an_array': {'type': 'list', 'allowed': ['agent', 'client', 'vendor']}, + 'an_array_from_set': { 'type': 'list', - 'allowed': ["agent", "client", "vendor"], + 'allowed': set(['agent', 'client', 'vendor']), }, 'a_list_of_dicts': { 'type': 'list', @@ -97,38 +57,25 @@ sample_schema = { }, 'a_list_of_values': { 'type': 'list', - 'items': [{'type': 'string'}, {'type': 'integer'}, ] - }, - 'a_list_of_integers': { - 'type': 'list', - 'schema': {'type': 'integer'}, + 'items': [{'type': 'string'}, {'type': 'integer'}], }, + 'a_list_of_integers': {'type': 'list', 'schema': {'type': 'integer'}}, 'a_dict': { 'type': 'dict', 'schema': { 'address': {'type': 'string'}, - 'city': {'type': 'string', 'required': True} + 'city': {'type': 'string', 'required': True}, }, }, - 'a_dict_with_valueschema': { - 'type': 'dict', - 'valueschema': {'type': 'integer'} - }, - 'a_dict_with_keyschema': { - 'type': 'dict', - 'keyschema': {'type': 'string', 'regex': '[a-z]+'} - }, + 'a_dict_with_valuesrules': {'type': 'dict', 'valuesrules': {'type': 'integer'}}, 'a_list_length': { 'type': 'list', 'schema': {'type': 'integer'}, 'minlength': 2, 'maxlength': 5, }, - 'a_nullable_field_without_type': { - 'nullable': True - }, - 'a_not_nullable_field_without_type': { - }, + 'a_nullable_field_without_type': {'nullable': True}, + 'a_not_nullable_field_without_type': {}, } sample_document = {'name': 'john doe'} diff --git a/pipenv/vendor/cerberus/tests/test_assorted.py b/pipenv/vendor/cerberus/tests/test_assorted.py index 641adb7e..b84ef810 100644 --- a/pipenv/vendor/cerberus/tests/test_assorted.py +++ b/pipenv/vendor/cerberus/tests/test_assorted.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- from decimal import Decimal +from pkg_resources import Distribution, DistributionNotFound from pytest import mark @@ -8,6 +9,37 @@ from cerberus import TypeDefinition, Validator from cerberus.tests import assert_fail, assert_success from cerberus.utils import validator_factory from cerberus.validator import BareValidator +from cerberus.platform import PYTHON_VERSION + + +if PYTHON_VERSION > 3 and PYTHON_VERSION < 3.4: + from imp import reload +elif PYTHON_VERSION >= 3.4: + from importlib import reload +else: + pass # Python 2.x + + +def test_pkgresources_version(monkeypatch): + def create_fake_distribution(name): + return Distribution(project_name="cerberus", version="1.2.3") + + with monkeypatch.context() as m: + cerberus = __import__("cerberus") + m.setattr("pkg_resources.get_distribution", create_fake_distribution) + reload(cerberus) + assert cerberus.__version__ == "1.2.3" + + +def test_version_not_found(monkeypatch): + def raise_distribution_not_found(name): + raise DistributionNotFound("pkg_resources cannot get distribution") + + with monkeypatch.context() as m: + cerberus = __import__("cerberus") + m.setattr("pkg_resources.get_distribution", raise_distribution_not_found) + reload(cerberus) + assert cerberus.__version__ == "unknown" def test_clear_cache(validator): @@ -23,8 +55,11 @@ def test_docstring(validator): # Test that testing with the sample schema works as expected # as there might be rules with side-effects in it -@mark.parametrize('test,document', ((assert_fail, {'an_integer': 60}), - (assert_success, {'an_integer': 110}))) + +@mark.parametrize( + "test,document", + ((assert_fail, {"an_integer": 60}), (assert_success, {"an_integer": 110})), +) def test_that_test_fails(test, document): try: test(document) @@ -35,42 +70,42 @@ def test_that_test_fails(test, document): def test_dynamic_types(): - decimal_type = TypeDefinition('decimal', (Decimal,), ()) - document = {'measurement': Decimal(0)} - schema = {'measurement': {'type': 'decimal'}} + decimal_type = TypeDefinition("decimal", (Decimal,), ()) + document = {"measurement": Decimal(0)} + schema = {"measurement": {"type": "decimal"}} validator = Validator() - validator.types_mapping['decimal'] = decimal_type + validator.types_mapping["decimal"] = decimal_type assert_success(document, schema, validator) class MyValidator(Validator): types_mapping = Validator.types_mapping.copy() - types_mapping['decimal'] = decimal_type + types_mapping["decimal"] = decimal_type + validator = MyValidator() assert_success(document, schema, validator) def test_mro(): - assert Validator.__mro__ == (Validator, BareValidator, object), \ - Validator.__mro__ + assert Validator.__mro__ == (Validator, BareValidator, object), Validator.__mro__ def test_mixin_init(): class Mixin(object): def __init__(self, *args, **kwargs): - kwargs['test'] = True + kwargs["test"] = True super(Mixin, self).__init__(*args, **kwargs) - MyValidator = validator_factory('MyValidator', Mixin) + MyValidator = validator_factory("MyValidator", Mixin) validator = MyValidator() - assert validator._config['test'] + assert validator._config["test"] def test_sub_init(): class MyValidator(Validator): def __init__(self, *args, **kwargs): - kwargs['test'] = True + kwargs["test"] = True super(MyValidator, self).__init__(*args, **kwargs) validator = MyValidator() - assert validator._config['test'] + assert validator._config["test"] diff --git a/pipenv/vendor/cerberus/tests/test_customization.py b/pipenv/vendor/cerberus/tests/test_customization.py index 6055894d..8bc3f464 100644 --- a/pipenv/vendor/cerberus/tests/test_customization.py +++ b/pipenv/vendor/cerberus/tests/test_customization.py @@ -1,12 +1,13 @@ # -*- coding: utf-8 -*- +from pytest import mark + import cerberus from cerberus.tests import assert_fail, assert_success from cerberus.tests.conftest import sample_schema def test_contextual_data_preservation(): - class InheritedValidator(cerberus.Validator): def __init__(self, *args, **kwargs): if 'working_dir' in kwargs: @@ -18,9 +19,9 @@ def test_contextual_data_preservation(): return True assert 'test' in InheritedValidator.types - v = InheritedValidator({'test': {'type': 'list', - 'schema': {'type': 'test'}}}, - working_dir='/tmp') + v = InheritedValidator( + {'test': {'type': 'list', 'schema': {'type': 'test'}}}, working_dir='/tmp' + ) assert_success({'test': ['foo']}, validator=v) @@ -42,25 +43,47 @@ def test_docstring_parsing(): assert 'bar' in CustomValidator.validation_rules -def test_issue_265(): +# TODO remove 'validator' as rule parameter with the next major release +@mark.parametrize('rule', ('check_with', 'validator')) +def test_check_with_method(rule): + # https://github.com/pyeve/cerberus/issues/265 + class MyValidator(cerberus.Validator): + def _check_with_oddity(self, field, value): + if not value & 1: + self._error(field, "Must be an odd number") + + v = MyValidator(schema={'amount': {rule: 'oddity'}}) + assert_success(document={'amount': 1}, validator=v) + assert_fail( + document={'amount': 2}, + validator=v, + error=('amount', (), cerberus.errors.CUSTOM, None, ('Must be an odd number',)), + ) + + +# TODO remove test with the next major release +@mark.parametrize('rule', ('check_with', 'validator')) +def test_validator_method(rule): class MyValidator(cerberus.Validator): def _validator_oddity(self, field, value): if not value & 1: self._error(field, "Must be an odd number") - v = MyValidator(schema={'amount': {'validator': 'oddity'}}) + v = MyValidator(schema={'amount': {rule: 'oddity'}}) assert_success(document={'amount': 1}, validator=v) - assert_fail(document={'amount': 2}, validator=v, - error=('amount', (), cerberus.errors.CUSTOM, None, - ('Must be an odd number',))) + assert_fail( + document={'amount': 2}, + validator=v, + error=('amount', (), cerberus.errors.CUSTOM, None, ('Must be an odd number',)), + ) def test_schema_validation_can_be_disabled_in_schema_setter(): - class NonvalidatingValidator(cerberus.Validator): """ Skips schema validation to speed up initialization """ + @cerberus.Validator.schema.setter def schema(self, schema): if schema is None: diff --git a/pipenv/vendor/cerberus/tests/test_errors.py b/pipenv/vendor/cerberus/tests/test_errors.py index df33964f..d64da78e 100644 --- a/pipenv/vendor/cerberus/tests/test_errors.py +++ b/pipenv/vendor/cerberus/tests/test_errors.py @@ -24,14 +24,14 @@ def test__error_1(): def test__error_2(): - v = Validator(schema={'foo': {'keyschema': {'type': 'integer'}}}) + v = Validator(schema={'foo': {'keysrules': {'type': 'integer'}}}) v.document = {'foo': {'0': 'bar'}} - v._error('foo', errors.KEYSCHEMA, ()) + v._error('foo', errors.KEYSRULES, ()) error = v._errors[0] assert error.document_path == ('foo',) - assert error.schema_path == ('foo', 'keyschema') + assert error.schema_path == ('foo', 'keysrules') assert error.code == 0x83 - assert error.rule == 'keyschema' + assert error.rule == 'keysrules' assert error.constraint == {'type': 'integer'} assert error.value == {'0': 'bar'} assert error.info == ((),) @@ -40,8 +40,10 @@ def test__error_2(): def test__error_3(): - valids = [{'type': 'string', 'regex': '0x[0-9a-f]{2}'}, - {'type': 'integer', 'min': 0, 'max': 255}] + valids = [ + {'type': 'string', 'regex': '0x[0-9a-f]{2}'}, + {'type': 'integer', 'min': 0, 'max': 255}, + ] v = Validator(schema={'foo': {'oneof': valids}}) v.document = {'foo': '0x100'} v._error('foo', errors.ONEOF, (), 0, 2) @@ -77,8 +79,9 @@ def test_error_tree_from_subschema(validator): assert 'bar' in s_error_tree['foo']['schema'] assert 'type' in s_error_tree['foo']['schema']['bar'] assert s_error_tree['foo']['schema']['bar']['type'].errors[0].value == 0 - assert s_error_tree.fetch_errors_from( - ('foo', 'schema', 'bar', 'type'))[0].value == 0 + assert ( + s_error_tree.fetch_errors_from(('foo', 'schema', 'bar', 'type'))[0].value == 0 + ) def test_error_tree_from_anyof(validator): @@ -98,12 +101,17 @@ def test_error_tree_from_anyof(validator): def test_nested_error_paths(validator): - schema = {'a_dict': {'keyschema': {'type': 'integer'}, - 'valueschema': {'regex': '[a-z]*'}}, - 'a_list': {'schema': {'type': 'string', - 'oneof_regex': ['[a-z]*$', '[A-Z]*']}}} - document = {'a_dict': {0: 'abc', 'one': 'abc', 2: 'aBc', 'three': 'abC'}, - 'a_list': [0, 'abc', 'abC']} + schema = { + 'a_dict': { + 'keysrules': {'type': 'integer'}, + 'valuesrules': {'regex': '[a-z]*'}, + }, + 'a_list': {'schema': {'type': 'string', 'oneof_regex': ['[a-z]*$', '[A-Z]*']}}, + } + document = { + 'a_dict': {0: 'abc', 'one': 'abc', 2: 'aBc', 'three': 'abC'}, + 'a_list': [0, 'abc', 'abC'], + } assert_fail(document, schema, validator=validator) _det = validator.document_error_tree @@ -120,35 +128,59 @@ def test_nested_error_paths(validator): assert len(_det['a_dict'][2].errors) == 1 assert len(_det['a_dict']['three'].errors) == 2 - assert len(_set['a_dict']['keyschema'].errors) == 1 - assert len(_set['a_dict']['valueschema'].errors) == 1 + assert len(_set['a_dict']['keysrules'].errors) == 1 + assert len(_set['a_dict']['valuesrules'].errors) == 1 - assert len(_set['a_dict']['keyschema']['type'].errors) == 2 - assert len(_set['a_dict']['valueschema']['regex'].errors) == 2 + assert len(_set['a_dict']['keysrules']['type'].errors) == 2 + assert len(_set['a_dict']['valuesrules']['regex'].errors) == 2 _ref_err = ValidationError( - ('a_dict', 'one'), ('a_dict', 'keyschema', 'type'), - errors.BAD_TYPE.code, 'type', 'integer', 'one', ()) + ('a_dict', 'one'), + ('a_dict', 'keysrules', 'type'), + errors.BAD_TYPE.code, + 'type', + 'integer', + 'one', + (), + ) assert _det['a_dict']['one'].errors[0] == _ref_err - assert _set['a_dict']['keyschema']['type'].errors[0] == _ref_err + assert _set['a_dict']['keysrules']['type'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_dict', 2), ('a_dict', 'valueschema', 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'aBc', ()) + ('a_dict', 2), + ('a_dict', 'valuesrules', 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'aBc', + (), + ) assert _det['a_dict'][2].errors[0] == _ref_err - assert _set['a_dict']['valueschema']['regex'].errors[0] == _ref_err + assert _set['a_dict']['valuesrules']['regex'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_dict', 'three'), ('a_dict', 'keyschema', 'type'), - errors.BAD_TYPE.code, 'type', 'integer', 'three', ()) + ('a_dict', 'three'), + ('a_dict', 'keysrules', 'type'), + errors.BAD_TYPE.code, + 'type', + 'integer', + 'three', + (), + ) assert _det['a_dict']['three'].errors[0] == _ref_err - assert _set['a_dict']['keyschema']['type'].errors[1] == _ref_err + assert _set['a_dict']['keysrules']['type'].errors[1] == _ref_err _ref_err = ValidationError( - ('a_dict', 'three'), ('a_dict', 'valueschema', 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_dict', 'three'), + ('a_dict', 'valuesrules', 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_dict']['three'].errors[1] == _ref_err - assert _set['a_dict']['valueschema']['regex'].errors[1] == _ref_err + assert _set['a_dict']['valuesrules']['regex'].errors[1] == _ref_err assert len(_det['a_list'].errors) == 1 assert len(_det['a_list'][0].errors) == 1 @@ -161,34 +193,56 @@ def test_nested_error_paths(validator): assert len(_set['a_list']['schema']['oneof'][1]['regex'].errors) == 1 _ref_err = ValidationError( - ('a_list', 0), ('a_list', 'schema', 'type'), errors.BAD_TYPE.code, - 'type', 'string', 0, ()) + ('a_list', 0), + ('a_list', 'schema', 'type'), + errors.BAD_TYPE.code, + 'type', + 'string', + 0, + (), + ) assert _det['a_list'][0].errors[0] == _ref_err assert _set['a_list']['schema']['type'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof'), errors.ONEOF.code, - 'oneof', 'irrelevant_at_this_point', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof'), + errors.ONEOF.code, + 'oneof', + 'irrelevant_at_this_point', + 'abC', + (), + ) assert _det['a_list'][2].errors[0] == _ref_err assert _set['a_list']['schema']['oneof'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof', 0, 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof', 0, 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_list'][2].errors[1] == _ref_err assert _set['a_list']['schema']['oneof'][0]['regex'].errors[0] == _ref_err _ref_err = ValidationError( - ('a_list', 2), ('a_list', 'schema', 'oneof', 1, 'regex'), - errors.REGEX_MISMATCH.code, 'regex', '[a-z]*$', 'abC', ()) + ('a_list', 2), + ('a_list', 'schema', 'oneof', 1, 'regex'), + errors.REGEX_MISMATCH.code, + 'regex', + '[a-z]*$', + 'abC', + (), + ) assert _det['a_list'][2].errors[2] == _ref_err assert _set['a_list']['schema']['oneof'][1]['regex'].errors[0] == _ref_err def test_queries(): - schema = {'foo': {'type': 'dict', - 'schema': - {'bar': {'type': 'number'}}}} + schema = {'foo': {'type': 'dict', 'schema': {'bar': {'type': 'number'}}}} document = {'foo': {'bar': 'zero'}} validator = Validator(schema) validator(document) @@ -202,59 +256,81 @@ def test_queries(): assert errors.MAPPING_SCHEMA in validator.document_error_tree['foo'] assert errors.BAD_TYPE in validator.document_error_tree['foo']['bar'] assert errors.MAPPING_SCHEMA in validator.schema_error_tree['foo']['schema'] - assert errors.BAD_TYPE in \ - validator.schema_error_tree['foo']['schema']['bar']['type'] + assert ( + errors.BAD_TYPE in validator.schema_error_tree['foo']['schema']['bar']['type'] + ) - assert (validator.document_error_tree['foo'][errors.MAPPING_SCHEMA] - .child_errors[0].code == errors.BAD_TYPE.code) + assert ( + validator.document_error_tree['foo'][errors.MAPPING_SCHEMA].child_errors[0].code + == errors.BAD_TYPE.code + ) def test_basic_error_handler(): handler = errors.BasicErrorHandler() _errors, ref = [], {} - _errors.append(ValidationError( - ['foo'], ['foo'], 0x63, 'readonly', True, None, ())) + _errors.append(ValidationError(['foo'], ['foo'], 0x63, 'readonly', True, None, ())) ref.update({'foo': [handler.messages[0x63]]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['bar'], ['foo'], 0x42, 'min', 1, 2, ())) + _errors.append(ValidationError(['bar'], ['foo'], 0x42, 'min', 1, 2, ())) ref.update({'bar': [handler.messages[0x42].format(constraint=1)]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x24, 'type', 'string', - True, ())) - ref.update({'zap': [{'foo': [handler.messages[0x24].format( - constraint='string')]}]}) + _errors.append( + ValidationError( + ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x24, 'type', 'string', True, () + ) + ) + ref.update({'zap': [{'foo': [handler.messages[0x24].format(constraint='string')]}]}) assert handler(_errors) == ref - _errors.append(ValidationError( - ['zap', 'foo'], ['zap', 'schema', 'foo'], 0x41, 'regex', - '^p[äe]ng$', 'boom', ())) - ref['zap'][0]['foo'].append( - handler.messages[0x41].format(constraint='^p[äe]ng$')) + _errors.append( + ValidationError( + ['zap', 'foo'], + ['zap', 'schema', 'foo'], + 0x41, + 'regex', + '^p[äe]ng$', + 'boom', + (), + ) + ) + ref['zap'][0]['foo'].append(handler.messages[0x41].format(constraint='^p[äe]ng$')) assert handler(_errors) == ref def test_basic_error_of_errors(validator): - schema = {'foo': {'oneof': [ - {'type': 'integer'}, - {'type': 'string'} - ]}} + schema = {'foo': {'oneof': [{'type': 'integer'}, {'type': 'string'}]}} document = {'foo': 23.42} - error = ('foo', ('foo', 'oneof'), errors.ONEOF, - schema['foo']['oneof'], ()) + error = ('foo', ('foo', 'oneof'), errors.ONEOF, schema['foo']['oneof'], ()) child_errors = [ (error[0], error[1] + (0, 'type'), errors.BAD_TYPE, 'integer'), - (error[0], error[1] + (1, 'type'), errors.BAD_TYPE, 'string') + (error[0], error[1] + (1, 'type'), errors.BAD_TYPE, 'string'), ] - assert_fail(document, schema, validator=validator, - error=error, child_errors=child_errors) + assert_fail( + document, schema, validator=validator, error=error, child_errors=child_errors + ) assert validator.errors == { - 'foo': [errors.BasicErrorHandler.messages[0x92], - {'oneof definition 0': ['must be of integer type'], - 'oneof definition 1': ['must be of string type']} - ] + 'foo': [ + errors.BasicErrorHandler.messages[0x92], + { + 'oneof definition 0': ['must be of integer type'], + 'oneof definition 1': ['must be of string type'], + }, + ] } + + +def test_wrong_amount_of_items(validator): + # https://github.com/pyeve/cerberus/issues/505 + validator.schema = { + 'test_list': { + 'type': 'list', + 'required': True, + 'items': [{'type': 'string'}, {'type': 'string'}], + } + } + validator({'test_list': ['test']}) + assert validator.errors == {'test_list': ["length of list should be 2, it is 1"]} diff --git a/pipenv/vendor/cerberus/tests/test_normalization.py b/pipenv/vendor/cerberus/tests/test_normalization.py index 6e06f553..adc281ef 100644 --- a/pipenv/vendor/cerberus/tests/test_normalization.py +++ b/pipenv/vendor/cerberus/tests/test_normalization.py @@ -1,10 +1,21 @@ # -*- coding: utf-8 -*- +from copy import deepcopy from tempfile import NamedTemporaryFile +from pytest import mark + from cerberus import Validator, errors -from cerberus.tests import (assert_fail, assert_has_error, assert_normalized, - assert_success) +from cerberus.tests import ( + assert_fail, + assert_has_error, + assert_normalized, + assert_success, +) + + +def must_not_be_called(*args, **kwargs): + raise RuntimeError('This shall not be called.') def test_coerce(): @@ -15,21 +26,31 @@ def test_coerce(): def test_coerce_in_dictschema(): - schema = {'thing': {'type': 'dict', - 'schema': {'amount': {'coerce': int}}}} + schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} assert_normalized(document, expected, schema) def test_coerce_in_listschema(): - schema = {'things': {'type': 'list', - 'schema': {'coerce': int}}} + schema = {'things': {'type': 'list', 'schema': {'coerce': int}}} document = {'things': ['1', '2', '3']} expected = {'things': [1, 2, 3]} assert_normalized(document, expected, schema) +def test_coerce_in_listitems(): + schema = {'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str}]}} + document = {'things': ['1', 2]} + expected = {'things': [1, '2']} + assert_normalized(document, expected, schema) + + validator = Validator(schema) + document['things'].append(3) + assert not validator(document) + assert validator.document['things'] == document['things'] + + def test_coerce_in_dictschema_in_listschema(): item_schema = {'type': 'dict', 'schema': {'amount': {'coerce': int}}} schema = {'things': {'type': 'list', 'schema': item_schema}} @@ -39,9 +60,7 @@ def test_coerce_in_dictschema_in_listschema(): def test_coerce_not_destructive(): - schema = { - 'amount': {'coerce': int} - } + schema = {'amount': {'coerce': int}} v = Validator(schema) doc = {'amount': '1'} v.validate(doc) @@ -52,16 +71,48 @@ def test_coerce_catches_ValueError(): schema = {'amount': {'coerce': int}} _errors = assert_fail({'amount': 'not_a_number'}, schema) _errors[0].info = () # ignore exception message here - assert_has_error(_errors, 'amount', ('amount', 'coerce'), - errors.COERCION_FAILED, int) + assert_has_error( + _errors, 'amount', ('amount', 'coerce'), errors.COERCION_FAILED, int + ) + + +def test_coerce_in_listitems_catches_ValueError(): + schema = {'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str}]}} + document = {'things': ['not_a_number', 2]} + _errors = assert_fail(document, schema) + _errors[0].info = () # ignore exception message here + assert_has_error( + _errors, + ('things', 0), + ('things', 'items', 'coerce'), + errors.COERCION_FAILED, + int, + ) def test_coerce_catches_TypeError(): schema = {'name': {'coerce': str.lower}} _errors = assert_fail({'name': 1234}, schema) _errors[0].info = () # ignore exception message here - assert_has_error(_errors, 'name', ('name', 'coerce'), - errors.COERCION_FAILED, str.lower) + assert_has_error( + _errors, 'name', ('name', 'coerce'), errors.COERCION_FAILED, str.lower + ) + + +def test_coerce_in_listitems_catches_TypeError(): + schema = { + 'things': {'type': 'list', 'items': [{'coerce': int}, {'coerce': str.lower}]} + } + document = {'things': ['1', 2]} + _errors = assert_fail(document, schema) + _errors[0].info = () # ignore exception message here + assert_has_error( + _errors, + ('things', 1), + ('things', 'items', 'coerce'), + errors.COERCION_FAILED, + str.lower, + ) def test_coerce_unknown(): @@ -88,16 +139,16 @@ def test_custom_coerce_and_rename(): def test_coerce_chain(): - drop_prefix = lambda x: x[2:] - upper = lambda x: x.upper() + drop_prefix = lambda x: x[2:] # noqa: E731 + upper = lambda x: x.upper() # noqa: E731 schema = {'foo': {'coerce': [hex, drop_prefix, upper]}} assert_normalized({'foo': 15}, {'foo': 'F'}, schema) def test_coerce_chain_aborts(validator): def dont_do_me(value): - raise AssertionError('The coercion chain did not abort after an ' - 'error.') + raise AssertionError('The coercion chain did not abort after an ' 'error.') + schema = {'foo': {'coerce': [hex, dont_do_me]}} validator({'foo': '0'}, schema) assert errors.COERCION_FAILED in validator._errors @@ -105,12 +156,12 @@ def test_coerce_chain_aborts(validator): def test_coerce_non_digit_in_sequence(validator): # https://github.com/pyeve/cerberus/issues/211 - schema = {'data': {'type': 'list', - 'schema': {'type': 'integer', 'coerce': int}}} + schema = {'data': {'type': 'list', 'schema': {'type': 'integer', 'coerce': int}}} document = {'data': ['q']} assert validator.validated(document, schema) is None - assert (validator.validated(document, schema, always_return_document=True) - == document) # noqa: W503 + assert ( + validator.validated(document, schema, always_return_document=True) == document + ) # noqa: W503 def test_nullables_dont_fail_coerce(): @@ -119,6 +170,18 @@ def test_nullables_dont_fail_coerce(): assert_normalized(document, document, schema) +def test_nullables_fail_coerce_on_non_null_values(validator): + def failing_coercion(value): + raise Exception("expected to fail") + + schema = {'foo': {'coerce': failing_coercion, 'nullable': True, 'type': 'integer'}} + document = {'foo': None} + assert_normalized(document, document, schema) + + validator({'foo': 2}, schema) + assert errors.COERCION_FAILED in validator._errors + + def test_normalized(): schema = {'amount': {'coerce': int}} document = {'amount': '2'} @@ -154,9 +217,13 @@ def test_purge_unknown(): def test_purge_unknown_in_subschema(): - schema = {'foo': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}}, - 'purge_unknown': True}} + schema = { + 'foo': { + 'type': 'dict', + 'schema': {'foo': {'type': 'string'}}, + 'purge_unknown': True, + } + } document = {'foo': {'bar': ''}} expected = {'foo': {}} assert_normalized(document, expected, schema) @@ -175,8 +242,7 @@ def test_issue_147_complex(): def test_issue_147_nested_dict(): - schema = {'thing': {'type': 'dict', - 'schema': {'amount': {'coerce': int}}}} + schema = {'thing': {'type': 'dict', 'schema': {'amount': {'coerce': int}}}} ref_obj = '2' document = {'thing': {'amount': ref_obj}} normalized = Validator(schema).normalized(document) @@ -186,20 +252,21 @@ def test_issue_147_nested_dict(): assert document['thing']['amount'] is ref_obj -def test_coerce_in_valueschema(): +def test_coerce_in_valuesrules(): # https://github.com/pyeve/cerberus/issues/155 - schema = {'thing': {'type': 'dict', - 'valueschema': {'coerce': int, - 'type': 'integer'}}} + schema = { + 'thing': {'type': 'dict', 'valuesrules': {'coerce': int, 'type': 'integer'}} + } document = {'thing': {'amount': '2'}} expected = {'thing': {'amount': 2}} assert_normalized(document, expected, schema) -def test_coerce_in_keyschema(): +def test_coerce_in_keysrules(): # https://github.com/pyeve/cerberus/issues/155 - schema = {'thing': {'type': 'dict', - 'keyschema': {'coerce': int, 'type': 'integer'}}} + schema = { + 'thing': {'type': 'dict', 'keysrules': {'coerce': int, 'type': 'integer'}} + } document = {'thing': {'5': 'foo'}} expected = {'thing': {5: 'foo'}} assert_normalized(document, expected, schema) @@ -207,8 +274,7 @@ def test_coerce_in_keyschema(): def test_coercion_of_sequence_items(validator): # https://github.com/pyeve/cerberus/issues/161 - schema = {'a_list': {'type': 'list', 'schema': {'type': 'float', - 'coerce': float}}} + schema = {'a_list': {'type': 'list', 'schema': {'type': 'float', 'coerce': float}}} document = {'a_list': [3, 4, 5]} expected = {'a_list': [3.0, 4.0, 5.0]} assert_normalized(document, expected, schema, validator) @@ -216,110 +282,76 @@ def test_coercion_of_sequence_items(validator): assert isinstance(x, float) -def test_default_missing(): - _test_default_missing({'default': 'bar_value'}) - - -def test_default_setter_missing(): - _test_default_missing({'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_missing(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_missing(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value'} expected = {'foo': 'foo_value', 'bar': 'bar_value'} assert_normalized(document, expected, schema) -def test_default_existent(): - _test_default_existent({'default': 'bar_value'}) - - -def test_default_setter_existent(): - def raise_error(doc): - raise RuntimeError('should not be called') - _test_default_existent({'default_setter': raise_error}) - - -def _test_default_existent(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': must_not_be_called}) +) +def test_default_existent(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value', 'bar': 'non_default'} assert_normalized(document, document.copy(), schema) -def test_default_none_nullable(): - _test_default_none_nullable({'default': 'bar_value'}) - - -def test_default_setter_none_nullable(): - def raise_error(doc): - raise RuntimeError('should not be called') - _test_default_none_nullable({'default_setter': raise_error}) - - -def _test_default_none_nullable(default): - bar_schema = {'type': 'string', - 'nullable': True} +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': must_not_be_called}) +) +def test_default_none_nullable(default): + bar_schema = {'type': 'string', 'nullable': True} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} document = {'foo': 'foo_value', 'bar': None} assert_normalized(document, document.copy(), schema) -def test_default_none_nonnullable(): - _test_default_none_nullable({'default': 'bar_value'}) - - -def test_default_setter_none_nonnullable(): - _test_default_none_nullable( - {'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_none_nonnullable(default): - bar_schema = {'type': 'string', - 'nullable': False} +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_none_nonnullable(default): + bar_schema = {'type': 'string', 'nullable': False} bar_schema.update(default) - schema = {'foo': {'type': 'string'}, - 'bar': bar_schema} - document = {'foo': 'foo_value', 'bar': 'bar_value'} - assert_normalized(document, document.copy(), schema) + schema = {'foo': {'type': 'string'}, 'bar': bar_schema} + document = {'foo': 'foo_value', 'bar': None} + expected = {'foo': 'foo_value', 'bar': 'bar_value'} + assert_normalized(document, expected, schema) def test_default_none_default_value(): - schema = {'foo': {'type': 'string'}, - 'bar': {'type': 'string', - 'nullable': True, - 'default': None}} + schema = { + 'foo': {'type': 'string'}, + 'bar': {'type': 'string', 'nullable': True, 'default': None}, + } document = {'foo': 'foo_value'} expected = {'foo': 'foo_value', 'bar': None} assert_normalized(document, expected, schema) -def test_default_missing_in_subschema(): - _test_default_missing_in_subschema({'default': 'bar_value'}) - - -def test_default_setter_missing_in_subschema(): - _test_default_missing_in_subschema( - {'default_setter': lambda doc: 'bar_value'}) - - -def _test_default_missing_in_subschema(default): +@mark.parametrize( + 'default', ({'default': 'bar_value'}, {'default_setter': lambda doc: 'bar_value'}) +) +def test_default_missing_in_subschema(default): bar_schema = {'type': 'string'} bar_schema.update(default) - schema = {'thing': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}, - 'bar': bar_schema}}} + schema = { + 'thing': { + 'type': 'dict', + 'schema': {'foo': {'type': 'string'}, 'bar': bar_schema}, + } + } document = {'thing': {'foo': 'foo_value'}} - expected = {'thing': {'foo': 'foo_value', - 'bar': 'bar_value'}} + expected = {'thing': {'foo': 'foo_value', 'bar': 'bar_value'}} assert_normalized(document, expected, schema) @@ -328,8 +360,7 @@ def test_depending_default_setters(): 'a': {'type': 'integer'}, 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1}, 'c': {'type': 'integer', 'default_setter': lambda d: d['b'] * 2}, - 'd': {'type': 'integer', - 'default_setter': lambda d: d['b'] + d['c']} + 'd': {'type': 'integer', 'default_setter': lambda d: d['b'] + d['c']}, } document = {'a': 1} expected = {'a': 1, 'b': 2, 'c': 4, 'd': 6} @@ -339,7 +370,7 @@ def test_depending_default_setters(): def test_circular_depending_default_setters(validator): schema = { 'a': {'type': 'integer', 'default_setter': lambda d: d['b'] + 1}, - 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1} + 'b': {'type': 'integer', 'default_setter': lambda d: d['a'] + 1}, } validator({}, schema) assert errors.SETTING_DEFAULT_FAILED in validator._errors @@ -353,14 +384,16 @@ def test_issue_250(): 'schema': { 'type': 'dict', 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } + 'schema': {'a': {'type': 'string'}}, + }, } } document = {'list': {'is_a': 'mapping'}} - assert_fail(document, schema, - error=('list', ('list', 'type'), errors.BAD_TYPE, - schema['list']['type'])) + assert_fail( + document, + schema, + error=('list', ('list', 'type'), errors.BAD_TYPE, schema['list']['type']), + ) def test_issue_250_no_type_pass_on_list(): @@ -370,7 +403,7 @@ def test_issue_250_no_type_pass_on_list(): 'schema': { 'allow_unknown': True, 'type': 'dict', - 'schema': {'a': {'type': 'string'}} + 'schema': {'a': {'type': 'string'}}, } } } @@ -381,28 +414,25 @@ def test_issue_250_no_type_pass_on_list(): def test_issue_250_no_type_fail_on_dict(): # https://github.com/pyeve/cerberus/issues/250 schema = { - 'list': { - 'schema': { - 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } - } + 'list': {'schema': {'allow_unknown': True, 'schema': {'a': {'type': 'string'}}}} } document = {'list': {'a': {'a': 'known'}}} - assert_fail(document, schema, - error=('list', ('list', 'schema'), errors.BAD_TYPE_FOR_SCHEMA, - schema['list']['schema'])) + assert_fail( + document, + schema, + error=( + 'list', + ('list', 'schema'), + errors.BAD_TYPE_FOR_SCHEMA, + schema['list']['schema'], + ), + ) def test_issue_250_no_type_fail_pass_on_other(): # https://github.com/pyeve/cerberus/issues/250 schema = { - 'list': { - 'schema': { - 'allow_unknown': True, - 'schema': {'a': {'type': 'string'}} - } - } + 'list': {'schema': {'allow_unknown': True, 'schema': {'a': {'type': 'string'}}}} } document = {'list': 1} assert_normalized(document, document, schema) @@ -416,21 +446,20 @@ def test_allow_unknown_with_of_rules(): { 'type': 'dict', 'allow_unknown': True, - 'schema': {'known': {'type': 'string'}} - }, - { - 'type': 'dict', - 'schema': {'known': {'type': 'string'}} + 'schema': {'known': {'type': 'string'}}, }, + {'type': 'dict', 'schema': {'known': {'type': 'string'}}}, ] } } # check regression and that allow unknown does not cause any different # than expected behaviour for one-of. document = {'test': {'known': 's'}} - assert_fail(document, schema, - error=('test', ('test', 'oneof'), - errors.ONEOF, schema['test']['oneof'])) + assert_fail( + document, + schema, + error=('test', ('test', 'oneof'), errors.ONEOF, schema['test']['oneof']), + ) # check that allow_unknown is actually applied document = {'test': {'known': 's', 'unknown': 'asd'}} assert_success(document, schema) @@ -439,18 +468,20 @@ def test_allow_unknown_with_of_rules(): def test_271_normalising_tuples(): # https://github.com/pyeve/cerberus/issues/271 schema = { - 'my_field': { - 'type': 'list', - 'schema': {'type': ('string', 'number', 'dict')} - } + 'my_field': {'type': 'list', 'schema': {'type': ('string', 'number', 'dict')}} } - document = {'my_field': ('foo', 'bar', 42, 'albert', - 'kandinsky', {'items': 23})} + document = {'my_field': ('foo', 'bar', 42, 'albert', 'kandinsky', {'items': 23})} assert_success(document, schema) normalized = Validator(schema).normalized(document) - assert normalized['my_field'] == ('foo', 'bar', 42, 'albert', - 'kandinsky', {'items': 23}) + assert normalized['my_field'] == ( + 'foo', + 'bar', + 42, + 'albert', + 'kandinsky', + {'items': 23}, + ) def test_allow_unknown_wo_schema(): @@ -472,14 +503,41 @@ def test_allow_unknown_with_purge_unknown_subdocument(): schema = { 'foo': { 'type': 'dict', - 'schema': { - 'bar': { - 'type': 'string' - } - }, - 'allow_unknown': True + 'schema': {'bar': {'type': 'string'}}, + 'allow_unknown': True, } } document = {'foo': {'bar': 'baz', 'corge': False}, 'thud': 'xyzzy'} expected = {'foo': {'bar': 'baz', 'corge': False}} assert_normalized(document, expected, schema, validator) + + +def test_purge_readonly(): + schema = { + 'description': {'type': 'string', 'maxlength': 500}, + 'last_updated': {'readonly': True}, + } + validator = Validator(schema=schema, purge_readonly=True) + document = {'description': 'it is a thing'} + expected = deepcopy(document) + document['last_updated'] = 'future' + assert_normalized(document, expected, validator=validator) + + +def test_defaults_in_allow_unknown_schema(): + schema = {'meta': {'type': 'dict'}, 'version': {'type': 'string'}} + allow_unknown = { + 'type': 'dict', + 'schema': { + 'cfg_path': {'type': 'string', 'default': 'cfg.yaml'}, + 'package': {'type': 'string'}, + }, + } + validator = Validator(schema=schema, allow_unknown=allow_unknown) + + document = {'version': '1.2.3', 'plugin_foo': {'package': 'foo'}} + expected = { + 'version': '1.2.3', + 'plugin_foo': {'package': 'foo', 'cfg_path': 'cfg.yaml'}, + } + assert_normalized(document, expected, schema, validator) diff --git a/pipenv/vendor/cerberus/tests/test_registries.py b/pipenv/vendor/cerberus/tests/test_registries.py index 05f01c52..b628952d 100644 --- a/pipenv/vendor/cerberus/tests/test_registries.py +++ b/pipenv/vendor/cerberus/tests/test_registries.py @@ -1,14 +1,17 @@ # -*- coding: utf-8 -*- from cerberus import schema_registry, rules_set_registry, Validator -from cerberus.tests import (assert_fail, assert_normalized, - assert_schema_error, assert_success) +from cerberus.tests import ( + assert_fail, + assert_normalized, + assert_schema_error, + assert_success, +) def test_schema_registry_simple(): schema_registry.add('foo', {'bar': {'type': 'string'}}) - schema = {'a': {'schema': 'foo'}, - 'b': {'schema': 'foo'}} + schema = {'a': {'schema': 'foo'}, 'b': {'schema': 'foo'}} document = {'a': {'bar': 'a'}, 'b': {'bar': 'b'}} assert_success(document, schema) @@ -33,23 +36,22 @@ def test_allow_unknown_as_reference(): def test_recursion(): - rules_set_registry.add('self', - {'type': 'dict', 'allow_unknown': 'self'}) + rules_set_registry.add('self', {'type': 'dict', 'allow_unknown': 'self'}) v = Validator(allow_unknown='self') assert_success({0: {1: {2: {}}}}, {}, v) def test_references_remain_unresolved(validator): - rules_set_registry.extend((('boolean', {'type': 'boolean'}), - ('booleans', {'valueschema': 'boolean'}))) + rules_set_registry.extend( + (('boolean', {'type': 'boolean'}), ('booleans', {'valuesrules': 'boolean'})) + ) validator.schema = {'foo': 'booleans'} assert 'booleans' == validator.schema['foo'] - assert 'boolean' == rules_set_registry._storage['booleans']['valueschema'] + assert 'boolean' == rules_set_registry._storage['booleans']['valuesrules'] def test_rules_registry_with_anyof_type(): - rules_set_registry.add('string_or_integer', - {'anyof_type': ['string', 'integer']}) + rules_set_registry.add('string_or_integer', {'anyof_type': ['string', 'integer']}) schema = {'soi': 'string_or_integer'} assert_success({'soi': 'hello'}, schema) diff --git a/pipenv/vendor/cerberus/tests/test_schema.py b/pipenv/vendor/cerberus/tests/test_schema.py index 1776cae3..509f446f 100644 --- a/pipenv/vendor/cerberus/tests/test_schema.py +++ b/pipenv/vendor/cerberus/tests/test_schema.py @@ -1,5 +1,7 @@ # -*- coding: utf-8 -*- +import re + import pytest from cerberus import Validator, errors, SchemaError @@ -9,14 +11,14 @@ from cerberus.tests import assert_schema_error def test_empty_schema(): validator = Validator() - with pytest.raises(SchemaError, message=errors.SCHEMA_ERROR_MISSING): + with pytest.raises(SchemaError, match=errors.SCHEMA_ERROR_MISSING): validator({}, schema=None) def test_bad_schema_type(validator): schema = "this string should really be dict" - exp_msg = errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema) - with pytest.raises(SchemaError, message=exp_msg): + msg = errors.SCHEMA_ERROR_DEFINITION_TYPE.format(schema) + with pytest.raises(SchemaError, match=msg): validator.schema = schema @@ -28,23 +30,21 @@ def test_bad_schema_type_field(validator): def test_unknown_rule(validator): - message = "{'foo': [{'unknown': ['unknown rule']}]}" - with pytest.raises(SchemaError, message=message): + msg = "{'foo': [{'unknown': ['unknown rule']}]}" + with pytest.raises(SchemaError, match=re.escape(msg)): validator.schema = {'foo': {'unknown': 'rule'}} def test_unknown_type(validator): - field = 'name' - value = 'catch_me' - message = str({field: [{'type': ['unallowed value %s' % value]}]}) - with pytest.raises(SchemaError, message=message): - validator.schema = {'foo': {'unknown': 'rule'}} + msg = str({'foo': [{'type': ['Unsupported types: unknown']}]}) + with pytest.raises(SchemaError, match=re.escape(msg)): + validator.schema = {'foo': {'type': 'unknown'}} def test_bad_schema_definition(validator): field = 'name' - message = str({field: ['must be of dict type']}) - with pytest.raises(SchemaError, message=message): + msg = str({field: ['must be of dict type']}) + with pytest.raises(SchemaError, match=re.escape(msg)): validator.schema = {field: 'this should really be a dict'} @@ -61,14 +61,14 @@ def test_normalization_rules_are_invalid_in_of_rules(): def test_anyof_allof_schema_validate(): # make sure schema with 'anyof' and 'allof' constraints are checked # correctly - schema = {'doc': {'type': 'dict', - 'anyof': [ - {'schema': [{'param': {'type': 'number'}}]}]}} + schema = { + 'doc': {'type': 'dict', 'anyof': [{'schema': [{'param': {'type': 'number'}}]}]} + } assert_schema_error({'doc': 'this is my document'}, schema) - schema = {'doc': {'type': 'dict', - 'allof': [ - {'schema': [{'param': {'type': 'number'}}]}]}} + schema = { + 'doc': {'type': 'dict', 'allof': [{'schema': [{'param': {'type': 'number'}}]}]} + } assert_schema_error({'doc': 'this is my document'}, schema) @@ -88,24 +88,87 @@ def test_validated_schema_cache(): v = Validator({'foozifix': {'coerce': int}}) assert len(v._valid_schemas) == cache_size - max_cache_size = 147 - assert cache_size <= max_cache_size, \ - "There's an unexpected high amount (%s) of cached valid " \ - "definition schemas. Unless you added further tests, " \ - "there are good chances that something is wrong. " \ - "If you added tests with new schemas, you can try to " \ - "adjust the variable `max_cache_size` according to " \ + max_cache_size = 161 + assert cache_size <= max_cache_size, ( + "There's an unexpected high amount (%s) of cached valid " + "definition schemas. Unless you added further tests, " + "there are good chances that something is wrong. " + "If you added tests with new schemas, you can try to " + "adjust the variable `max_cache_size` according to " "the added schemas." % cache_size + ) def test_expansion_in_nested_schema(): schema = {'detroit': {'schema': {'anyof_regex': ['^Aladdin', 'Sane$']}}} v = Validator(schema) - assert (v.schema['detroit']['schema'] == - {'anyof': [{'regex': '^Aladdin'}, {'regex': 'Sane$'}]}) + assert v.schema['detroit']['schema'] == { + 'anyof': [{'regex': '^Aladdin'}, {'regex': 'Sane$'}] + } def test_unvalidated_schema_can_be_copied(): schema = UnvalidatedSchema() schema_copy = schema.copy() assert schema_copy == schema + + +# TODO remove with next major release +def test_deprecated_rule_names_in_valueschema(): + def check_with(field, value, error): + pass + + schema = { + "field_1": { + "type": "dict", + "valueschema": { + "type": "dict", + "keyschema": {"type": "string"}, + "valueschema": {"type": "string"}, + }, + }, + "field_2": { + "type": "list", + "items": [ + {"keyschema": {}}, + {"validator": check_with}, + {"valueschema": {}}, + ], + }, + } + + validator = Validator(schema) + + assert validator.schema == { + "field_1": { + "type": "dict", + "valuesrules": { + "type": "dict", + "keysrules": {"type": "string"}, + "valuesrules": {"type": "string"}, + }, + }, + "field_2": { + "type": "list", + "items": [ + {"keysrules": {}}, + {"check_with": check_with}, + {"valuesrules": {}}, + ], + }, + } + + +def test_anyof_check_with(): + def foo(field, value, error): + pass + + def bar(field, value, error): + pass + + schema = {'field': {'anyof_check_with': [foo, bar]}} + validator = Validator(schema) + + assert validator.schema == { + 'field': {'anyof': [{'check_with': foo}, {'check_with': bar}]} + } diff --git a/pipenv/vendor/cerberus/tests/test_utils.py b/pipenv/vendor/cerberus/tests/test_utils.py new file mode 100644 index 00000000..6ab38790 --- /dev/null +++ b/pipenv/vendor/cerberus/tests/test_utils.py @@ -0,0 +1,11 @@ +from cerberus.utils import compare_paths_lt + + +def test_compare_paths(): + lesser = ('a_dict', 'keysrules') + greater = ('a_dict', 'valuesrules') + assert compare_paths_lt(lesser, greater) + + lesser += ('type',) + greater += ('regex',) + assert compare_paths_lt(lesser, greater) diff --git a/pipenv/vendor/cerberus/tests/test_validation.py b/pipenv/vendor/cerberus/tests/test_validation.py index 1f828fac..ead79517 100644 --- a/pipenv/vendor/cerberus/tests/test_validation.py +++ b/pipenv/vendor/cerberus/tests/test_validation.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- +import itertools import re import sys from datetime import datetime, date @@ -10,29 +11,34 @@ from pytest import mark from cerberus import errors, Validator from cerberus.tests import ( - assert_bad_type, assert_document_error, assert_fail, assert_has_error, - assert_not_has_error, assert_success + assert_bad_type, + assert_document_error, + assert_fail, + assert_has_error, + assert_not_has_error, + assert_success, ) from cerberus.tests.conftest import sample_schema def test_empty_document(): - assert_document_error(None, sample_schema, None, - errors.DOCUMENT_MISSING) + assert_document_error(None, sample_schema, None, errors.DOCUMENT_MISSING) def test_bad_document_type(): document = "not a dict" assert_document_error( - document, sample_schema, None, - errors.DOCUMENT_FORMAT.format(document) + document, sample_schema, None, errors.DOCUMENT_FORMAT.format(document) ) def test_unknown_field(validator): field = 'surname' - assert_fail({field: 'doe'}, validator=validator, - error=(field, (), errors.UNKNOWN_FIELD, None)) + assert_fail( + {field: 'doe'}, + validator=validator, + error=(field, (), errors.UNKNOWN_FIELD, None), + ) assert validator.errors == {field: ['unknown field']} @@ -45,14 +51,19 @@ def test_empty_field_definition(document): def test_required_field(schema): field = 'a_required_string' required_string_extension = { - 'a_required_string': {'type': 'string', - 'minlength': 2, - 'maxlength': 10, - 'required': True}} + 'a_required_string': { + 'type': 'string', + 'minlength': 2, + 'maxlength': 10, + 'required': True, + } + } schema.update(required_string_extension) - assert_fail({'an_integer': 1}, schema, - error=(field, (field, 'required'), errors.REQUIRED_FIELD, - True)) + assert_fail( + {'an_integer': 1}, + schema, + error=(field, (field, 'required'), errors.REQUIRED_FIELD, True), + ) def test_nullable_field(): @@ -64,22 +75,23 @@ def test_nullable_field(): assert_fail({'a_not_nullable_field_without_type': None}) +def test_nullable_skips_allowed(): + schema = {'role': {'allowed': ['agent', 'client', 'supplier'], 'nullable': True}} + assert_success({'role': None}, schema) + + def test_readonly_field(): field = 'a_readonly_string' - assert_fail({field: 'update me if you can'}, - error=(field, (field, 'readonly'), errors.READONLY_FIELD, True)) + assert_fail( + {field: 'update me if you can'}, + error=(field, (field, 'readonly'), errors.READONLY_FIELD, True), + ) def test_readonly_field_first_rule(): # test that readonly rule is checked before any other rule, and blocks. # See #63. - schema = { - 'a_readonly_number': { - 'type': 'integer', - 'readonly': True, - 'max': 1 - } - } + schema = {'a_readonly_number': {'type': 'integer', 'readonly': True, 'max': 1}} v = Validator(schema) v.validate({'a_readonly_number': 2}) # it would be a list if there's more than one error; we get a dict @@ -89,28 +101,34 @@ def test_readonly_field_first_rule(): def test_readonly_field_with_default_value(): schema = { - 'created': { - 'type': 'string', - 'readonly': True, - 'default': 'today' - }, + 'created': {'type': 'string', 'readonly': True, 'default': 'today'}, 'modified': { 'type': 'string', 'readonly': True, - 'default_setter': lambda d: d['created'] - } + 'default_setter': lambda d: d['created'], + }, } assert_success({}, schema) - expected_errors = [('created', ('created', 'readonly'), - errors.READONLY_FIELD, - schema['created']['readonly']), - ('modified', ('modified', 'readonly'), - errors.READONLY_FIELD, - schema['modified']['readonly'])] - assert_fail({'created': 'tomorrow', 'modified': 'today'}, - schema, errors=expected_errors) - assert_fail({'created': 'today', 'modified': 'today'}, - schema, errors=expected_errors) + expected_errors = [ + ( + 'created', + ('created', 'readonly'), + errors.READONLY_FIELD, + schema['created']['readonly'], + ), + ( + 'modified', + ('modified', 'readonly'), + errors.READONLY_FIELD, + schema['modified']['readonly'], + ), + ] + assert_fail( + {'created': 'tomorrow', 'modified': 'today'}, schema, errors=expected_errors + ) + assert_fail( + {'created': 'today', 'modified': 'today'}, schema, errors=expected_errors + ) def test_nested_readonly_field_with_default_value(): @@ -118,33 +136,40 @@ def test_nested_readonly_field_with_default_value(): 'some_field': { 'type': 'dict', 'schema': { - 'created': { - 'type': 'string', - 'readonly': True, - 'default': 'today' - }, + 'created': {'type': 'string', 'readonly': True, 'default': 'today'}, 'modified': { 'type': 'string', 'readonly': True, - 'default_setter': lambda d: d['created'] - } - } + 'default_setter': lambda d: d['created'], + }, + }, } } assert_success({'some_field': {}}, schema) expected_errors = [ - (('some_field', 'created'), - ('some_field', 'schema', 'created', 'readonly'), - errors.READONLY_FIELD, - schema['some_field']['schema']['created']['readonly']), - (('some_field', 'modified'), - ('some_field', 'schema', 'modified', 'readonly'), - errors.READONLY_FIELD, - schema['some_field']['schema']['modified']['readonly'])] - assert_fail({'some_field': {'created': 'tomorrow', 'modified': 'now'}}, - schema, errors=expected_errors) - assert_fail({'some_field': {'created': 'today', 'modified': 'today'}}, - schema, errors=expected_errors) + ( + ('some_field', 'created'), + ('some_field', 'schema', 'created', 'readonly'), + errors.READONLY_FIELD, + schema['some_field']['schema']['created']['readonly'], + ), + ( + ('some_field', 'modified'), + ('some_field', 'schema', 'modified', 'readonly'), + errors.READONLY_FIELD, + schema['some_field']['schema']['modified']['readonly'], + ), + ] + assert_fail( + {'some_field': {'created': 'tomorrow', 'modified': 'now'}}, + schema, + errors=expected_errors, + ) + assert_fail( + {'some_field': {'created': 'today', 'modified': 'today'}}, + schema, + errors=expected_errors, + ) def test_repeated_readonly(validator): @@ -195,44 +220,73 @@ def test_bad_max_length(schema): field = 'a_string' max_length = schema[field]['maxlength'] value = "".join(choice(ascii_lowercase) for i in range(max_length + 1)) - assert_fail({field: value}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (len(value),), + ), + ) def test_bad_max_length_binary(schema): field = 'a_binary' max_length = schema[field]['maxlength'] value = b'\x00' * (max_length + 1) - assert_fail({field: value}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (len(value),), + ), + ) def test_bad_min_length(schema): field = 'a_string' min_length = schema[field]['minlength'] value = "".join(choice(ascii_lowercase) for i in range(min_length - 1)) - assert_fail({field: value}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (len(value),), + ), + ) def test_bad_min_length_binary(schema): field = 'a_binary' min_length = schema[field]['minlength'] value = b'\x00' * (min_length - 1) - assert_fail({field: value}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (len(value),))) + assert_fail( + {field: value}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (len(value),), + ), + ) def test_bad_max_value(schema): def assert_bad_max_value(field, inc): max_value = schema[field]['max'] value = max_value + inc - assert_fail({field: value}, - error=(field, (field, 'max'), errors.MAX_VALUE, max_value)) + assert_fail( + {field: value}, error=(field, (field, 'max'), errors.MAX_VALUE, max_value) + ) field = 'an_integer' assert_bad_max_value(field, 1) @@ -246,9 +300,9 @@ def test_bad_min_value(schema): def assert_bad_min_value(field, inc): min_value = schema[field]['min'] value = min_value - inc - assert_fail({field: value}, - error=(field, (field, 'min'), - errors.MIN_VALUE, min_value)) + assert_fail( + {field: value}, error=(field, (field, 'min'), errors.MIN_VALUE, min_value) + ) field = 'an_integer' assert_bad_min_value(field, 1) @@ -261,65 +315,112 @@ def test_bad_min_value(schema): def test_bad_schema(): field = 'a_dict' subschema_field = 'address' - schema = {field: {'type': 'dict', - 'schema': {subschema_field: {'type': 'string'}, - 'city': {'type': 'string', 'required': True}} - }} + schema = { + field: { + 'type': 'dict', + 'schema': { + subschema_field: {'type': 'string'}, + 'city': {'type': 'string', 'required': True}, + }, + } + } document = {field: {subschema_field: 34}} validator = Validator(schema) assert_fail( - document, validator=validator, - error=(field, (field, 'schema'), errors.MAPPING_SCHEMA, - validator.schema['a_dict']['schema']), + document, + validator=validator, + error=( + field, + (field, 'schema'), + errors.MAPPING_SCHEMA, + validator.schema['a_dict']['schema'], + ), child_errors=[ - ((field, subschema_field), - (field, 'schema', subschema_field, 'type'), - errors.BAD_TYPE, 'string'), - ((field, 'city'), (field, 'schema', 'city', 'required'), - errors.REQUIRED_FIELD, True)] + ( + (field, subschema_field), + (field, 'schema', subschema_field, 'type'), + errors.BAD_TYPE, + 'string', + ), + ( + (field, 'city'), + (field, 'schema', 'city', 'required'), + errors.REQUIRED_FIELD, + True, + ), + ], ) handler = errors.BasicErrorHandler assert field in validator.errors assert subschema_field in validator.errors[field][-1] - assert handler.messages[errors.BAD_TYPE.code].format(constraint='string') \ + assert ( + handler.messages[errors.BAD_TYPE.code].format(constraint='string') in validator.errors[field][-1][subschema_field] + ) assert 'city' in validator.errors[field][-1] - assert (handler.messages[errors.REQUIRED_FIELD.code] - in validator.errors[field][-1]['city']) + assert ( + handler.messages[errors.REQUIRED_FIELD.code] + in validator.errors[field][-1]['city'] + ) -def test_bad_valueschema(): - field = 'a_dict_with_valueschema' +def test_bad_valuesrules(): + field = 'a_dict_with_valuesrules' schema_field = 'a_string' value = {schema_field: 'not an integer'} exp_child_errors = [ - ((field, schema_field), (field, 'valueschema', 'type'), errors.BAD_TYPE, - 'integer')] - assert_fail({field: value}, - error=(field, (field, 'valueschema'), errors.VALUESCHEMA, - {'type': 'integer'}), child_errors=exp_child_errors) + ( + (field, schema_field), + (field, 'valuesrules', 'type'), + errors.BAD_TYPE, + 'integer', + ) + ] + assert_fail( + {field: value}, + error=(field, (field, 'valuesrules'), errors.VALUESRULES, {'type': 'integer'}), + child_errors=exp_child_errors, + ) def test_bad_list_of_values(validator): field = 'a_list_of_values' value = ['a string', 'not an integer'] - assert_fail({field: value}, validator=validator, - error=(field, (field, 'items'), errors.BAD_ITEMS, - [{'type': 'string'}, {'type': 'integer'}]), - child_errors=[((field, 1), (field, 'items', 1, 'type'), - errors.BAD_TYPE, 'integer')]) + assert_fail( + {field: value}, + validator=validator, + error=( + field, + (field, 'items'), + errors.BAD_ITEMS, + [{'type': 'string'}, {'type': 'integer'}], + ), + child_errors=[ + ((field, 1), (field, 'items', 1, 'type'), errors.BAD_TYPE, 'integer') + ], + ) - assert (errors.BasicErrorHandler.messages[errors.BAD_TYPE.code]. - format(constraint='integer') - in validator.errors[field][-1][1]) + assert ( + errors.BasicErrorHandler.messages[errors.BAD_TYPE.code].format( + constraint='integer' + ) + in validator.errors[field][-1][1] + ) value = ['a string', 10, 'an extra item'] - assert_fail({field: value}, - error=(field, (field, 'items'), errors.ITEMS_LENGTH, - [{'type': 'string'}, {'type': 'integer'}], (2, 3))) + assert_fail( + {field: value}, + error=( + field, + (field, 'items'), + errors.ITEMS_LENGTH, + [{'type': 'string'}, {'type': 'integer'}], + (2, 3), + ), + ) def test_bad_list_of_integers(): @@ -330,58 +431,81 @@ def test_bad_list_of_integers(): def test_bad_list_of_dicts(): field = 'a_list_of_dicts' - map_schema = {'sku': {'type': 'string'}, - 'price': {'type': 'integer', 'required': True}} + map_schema = { + 'sku': {'type': 'string'}, + 'price': {'type': 'integer', 'required': True}, + } seq_schema = {'type': 'dict', 'schema': map_schema} schema = {field: {'type': 'list', 'schema': seq_schema}} validator = Validator(schema) value = [{'sku': 'KT123', 'price': '100'}] document = {field: value} - assert_fail(document, validator=validator, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - seq_schema), - child_errors=[((field, 0), (field, 'schema', 'schema'), - errors.MAPPING_SCHEMA, map_schema)]) + assert_fail( + document, + validator=validator, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, seq_schema), + child_errors=[ + ((field, 0), (field, 'schema', 'schema'), errors.MAPPING_SCHEMA, map_schema) + ], + ) assert field in validator.errors assert 0 in validator.errors[field][-1] assert 'price' in validator.errors[field][-1][0][-1] - exp_msg = errors.BasicErrorHandler.messages[errors.BAD_TYPE.code] \ - .format(constraint='integer') + exp_msg = errors.BasicErrorHandler.messages[errors.BAD_TYPE.code].format( + constraint='integer' + ) assert exp_msg in validator.errors[field][-1][0][-1]['price'] value = ["not a dict"] - exp_child_errors = [((field, 0), (field, 'schema', 'type'), - errors.BAD_TYPE, 'dict', ())] - assert_fail({field: value}, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - seq_schema), - child_errors=exp_child_errors) + exp_child_errors = [ + ((field, 0), (field, 'schema', 'type'), errors.BAD_TYPE, 'dict', ()) + ] + assert_fail( + {field: value}, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, seq_schema), + child_errors=exp_child_errors, + ) def test_array_unallowed(): field = 'an_array' value = ['agent', 'client', 'profit'] - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUES, - ['agent', 'client', 'vendor'], ['profit'])) + assert_fail( + {field: value}, + error=( + field, + (field, 'allowed'), + errors.UNALLOWED_VALUES, + ['agent', 'client', 'vendor'], + ['profit'], + ), + ) def test_string_unallowed(): field = 'a_restricted_string' value = 'profit' - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, - ['agent', 'client', 'vendor'], value)) + assert_fail( + {field: value}, + error=( + field, + (field, 'allowed'), + errors.UNALLOWED_VALUE, + ['agent', 'client', 'vendor'], + value, + ), + ) def test_integer_unallowed(): field = 'a_restricted_integer' value = 2 - assert_fail({field: value}, - error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, - [-1, 0, 1], value)) + assert_fail( + {field: value}, + error=(field, (field, 'allowed'), errors.UNALLOWED_VALUE, [-1, 0, 1], value), + ) def test_integer_allowed(): @@ -389,10 +513,14 @@ def test_integer_allowed(): def test_validate_update(): - assert_success({'an_integer': 100, - 'a_dict': {'address': 'adr'}, - 'a_list_of_dicts': [{'sku': 'let'}] - }, update=True) + assert_success( + { + 'an_integer': 100, + 'a_dict': {'address': 'adr'}, + 'a_list_of_dicts': [{'sku': 'let'}], + }, + update=True, + ) def test_string(): @@ -437,24 +565,35 @@ def test_one_of_two_types(validator): field = 'one_or_more_strings' assert_success({field: 'foo'}) assert_success({field: ['foo', 'bar']}) - exp_child_errors = [((field, 1), (field, 'schema', 'type'), - errors.BAD_TYPE, 'string')] - assert_fail({field: ['foo', 23]}, validator=validator, - error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, - {'type': 'string'}), - child_errors=exp_child_errors) - assert_fail({field: 23}, - error=((field,), (field, 'type'), errors.BAD_TYPE, - ['string', 'list'])) + exp_child_errors = [ + ((field, 1), (field, 'schema', 'type'), errors.BAD_TYPE, 'string') + ] + assert_fail( + {field: ['foo', 23]}, + validator=validator, + error=(field, (field, 'schema'), errors.SEQUENCE_SCHEMA, {'type': 'string'}), + child_errors=exp_child_errors, + ) + assert_fail( + {field: 23}, + error=((field,), (field, 'type'), errors.BAD_TYPE, ['string', 'list']), + ) assert validator.errors == {field: [{1: ['must be of string type']}]} def test_regex(validator): field = 'a_regex_email' assert_success({field: 'valid.email@gmail.com'}, validator=validator) - assert_fail({field: 'invalid'}, update=True, - error=(field, (field, 'regex'), errors.REGEX_MISMATCH, - '^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$')) + assert_fail( + {field: 'invalid'}, + update=True, + error=( + field, + (field, 'regex'), + errors.REGEX_MISMATCH, + r'^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$', + ), + ) def test_a_list_of_dicts(): @@ -462,7 +601,7 @@ def test_a_list_of_dicts(): { 'a_list_of_dicts': [ {'sku': 'AK345', 'price': 100}, - {'sku': 'YZ069', 'price': 25} + {'sku': 'YZ069', 'price': 25}, ] } ) @@ -472,50 +611,84 @@ def test_a_list_of_values(): assert_success({'a_list_of_values': ['hello', 100]}) +def test_an_array_from_set(): + assert_success({'an_array_from_set': ['agent', 'client']}) + + def test_a_list_of_integers(): assert_success({'a_list_of_integers': [99, 100]}) def test_a_dict(schema): - assert_success({'a_dict': {'address': 'i live here', - 'city': 'in my own town'}}) + assert_success({'a_dict': {'address': 'i live here', 'city': 'in my own town'}}) assert_fail( {'a_dict': {'address': 8545}}, - error=('a_dict', ('a_dict', 'schema'), errors.MAPPING_SCHEMA, - schema['a_dict']['schema']), - child_errors=[(('a_dict', 'address'), - ('a_dict', 'schema', 'address', 'type'), - errors.BAD_TYPE, 'string'), - (('a_dict', 'city'), - ('a_dict', 'schema', 'city', 'required'), - errors.REQUIRED_FIELD, True)] + error=( + 'a_dict', + ('a_dict', 'schema'), + errors.MAPPING_SCHEMA, + schema['a_dict']['schema'], + ), + child_errors=[ + ( + ('a_dict', 'address'), + ('a_dict', 'schema', 'address', 'type'), + errors.BAD_TYPE, + 'string', + ), + ( + ('a_dict', 'city'), + ('a_dict', 'schema', 'city', 'required'), + errors.REQUIRED_FIELD, + True, + ), + ], ) -def test_a_dict_with_valueschema(validator): - assert_success({'a_dict_with_valueschema': - {'an integer': 99, 'another integer': 100}}) +def test_a_dict_with_valuesrules(validator): + assert_success( + {'a_dict_with_valuesrules': {'an integer': 99, 'another integer': 100}} + ) error = ( - 'a_dict_with_valueschema', ('a_dict_with_valueschema', 'valueschema'), - errors.VALUESCHEMA, {'type': 'integer'}) + 'a_dict_with_valuesrules', + ('a_dict_with_valuesrules', 'valuesrules'), + errors.VALUESRULES, + {'type': 'integer'}, + ) child_errors = [ - (('a_dict_with_valueschema', 'a string'), - ('a_dict_with_valueschema', 'valueschema', 'type'), - errors.BAD_TYPE, 'integer')] + ( + ('a_dict_with_valuesrules', 'a string'), + ('a_dict_with_valuesrules', 'valuesrules', 'type'), + errors.BAD_TYPE, + 'integer', + ) + ] - assert_fail({'a_dict_with_valueschema': {'a string': '99'}}, - validator=validator, error=error, child_errors=child_errors) + assert_fail( + {'a_dict_with_valuesrules': {'a string': '99'}}, + validator=validator, + error=error, + child_errors=child_errors, + ) - assert 'valueschema' in \ - validator.schema_error_tree['a_dict_with_valueschema'] + assert 'valuesrules' in validator.schema_error_tree['a_dict_with_valuesrules'] v = validator.schema_error_tree - assert len(v['a_dict_with_valueschema']['valueschema'].descendants) == 1 + assert len(v['a_dict_with_valuesrules']['valuesrules'].descendants) == 1 -def test_a_dict_with_keyschema(): - assert_success({'a_dict_with_keyschema': {'key': 'value'}}) - assert_fail({'a_dict_with_keyschema': {'KEY': 'value'}}) +# TODO remove 'keyschema' as rule with the next major release +@mark.parametrize('rule', ('keysrules', 'keyschema')) +def test_keysrules(rule): + schema = { + 'a_dict_with_keysrules': { + 'type': 'dict', + rule: {'type': 'string', 'regex': '[a-z]+'}, + } + } + assert_success({'a_dict_with_keysrules': {'key': 'value'}}, schema=schema) + assert_fail({'a_dict_with_keysrules': {'KEY': 'value'}}, schema=schema) def test_a_list_length(schema): @@ -523,17 +696,31 @@ def test_a_list_length(schema): min_length = schema[field]['minlength'] max_length = schema[field]['maxlength'] - assert_fail({field: [1] * (min_length - 1)}, - error=(field, (field, 'minlength'), errors.MIN_LENGTH, - min_length, (min_length - 1,))) + assert_fail( + {field: [1] * (min_length - 1)}, + error=( + field, + (field, 'minlength'), + errors.MIN_LENGTH, + min_length, + (min_length - 1,), + ), + ) for i in range(min_length, max_length): value = [1] * i assert_success({field: value}) - assert_fail({field: [1] * (max_length + 1)}, - error=(field, (field, 'maxlength'), errors.MAX_LENGTH, - max_length, (max_length + 1,))) + assert_fail( + {field: [1] * (max_length + 1)}, + error=( + field, + (field, 'maxlength'), + errors.MAX_LENGTH, + max_length, + (max_length + 1,), + ), + ) def test_custom_datatype(): @@ -544,11 +731,12 @@ def test_custom_datatype(): schema = {'test_field': {'type': 'objectid'}} validator = MyValidator(schema) - assert_success({'test_field': '50ad188438345b1049c88a28'}, - validator=validator) - assert_fail({'test_field': 'hello'}, validator=validator, - error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, - 'objectid')) + assert_success({'test_field': '50ad188438345b1049c88a28'}, validator=validator) + assert_fail( + {'test_field': 'hello'}, + validator=validator, + error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, 'objectid'), + ) def test_custom_datatype_rule(): @@ -565,12 +753,16 @@ def test_custom_datatype_rule(): schema = {'test_field': {'min_number': 1, 'type': 'number'}} validator = MyValidator(schema) - assert_fail({'test_field': '0'}, validator=validator, - error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, - 'number')) - assert_fail({'test_field': 0}, validator=validator, - error=('test_field', (), errors.CUSTOM, None, - ('Below the min',))) + assert_fail( + {'test_field': '0'}, + validator=validator, + error=('test_field', ('test_field', 'type'), errors.BAD_TYPE, 'number'), + ) + assert_fail( + {'test_field': 0}, + validator=validator, + error=('test_field', (), errors.CUSTOM, None, ('Below the min',)), + ) assert validator.errors == {'test_field': ['Below the min']} @@ -584,14 +776,17 @@ def test_custom_validator(): schema = {'test_field': {'isodd': True}} validator = MyValidator(schema) assert_success({'test_field': 7}, validator=validator) - assert_fail({'test_field': 6}, validator=validator, - error=('test_field', (), errors.CUSTOM, None, - ('Not an odd number',))) + assert_fail( + {'test_field': 6}, + validator=validator, + error=('test_field', (), errors.CUSTOM, None, ('Not an odd number',)), + ) assert validator.errors == {'test_field': ['Not an odd number']} -@mark.parametrize('value, _type', - (('', 'string'), ((), 'list'), ({}, 'dict'), ([], 'list'))) +@mark.parametrize( + 'value, _type', (('', 'string'), ((), 'list'), ({}, 'dict'), ([], 'list')) +) def test_empty_values(value, _type): field = 'test' schema = {field: {'type': _type}} @@ -600,17 +795,18 @@ def test_empty_values(value, _type): assert_success(document, schema) schema[field]['empty'] = False - assert_fail(document, schema, - error=(field, (field, 'empty'), - errors.EMPTY_NOT_ALLOWED, False)) + assert_fail( + document, + schema, + error=(field, (field, 'empty'), errors.EMPTY_NOT_ALLOWED, False), + ) schema[field]['empty'] = True assert_success(document, schema) def test_empty_skips_regex(validator): - schema = {'foo': {'empty': True, 'regex': r'\d?\d\.\d\d', - 'type': 'string'}} + schema = {'foo': {'empty': True, 'regex': r'\d?\d\.\d\d', 'type': 'string'}} assert validator({'foo': ''}, schema) @@ -625,8 +821,9 @@ def test_ignore_none_values(): validator.schema[field]['required'] = True validator.schema.validate() _errors = assert_fail(document, validator=validator) - assert_not_has_error(_errors, field, (field, 'required'), - errors.REQUIRED_FIELD, True) + assert_not_has_error( + _errors, field, (field, 'required'), errors.REQUIRED_FIELD, True + ) # Test ignore None behaviour validator = Validator(schema, ignore_none_values=True) @@ -635,10 +832,8 @@ def test_ignore_none_values(): assert_success(document, validator=validator) validator.schema[field]['required'] = True _errors = assert_fail(schema=schema, document=document, validator=validator) - assert_has_error(_errors, field, (field, 'required'), errors.REQUIRED_FIELD, - True) - assert_not_has_error(_errors, field, (field, 'type'), errors.BAD_TYPE, - 'string') + assert_has_error(_errors, field, (field, 'required'), errors.REQUIRED_FIELD, True) + assert_not_has_error(_errors, field, (field, 'type'), errors.BAD_TYPE, 'string') def test_unknown_keys(): @@ -677,8 +872,7 @@ def test_unknown_keys_list_of_dicts(validator): # test that allow_unknown is honored even for subdicts in lists. # https://github.com/pyeve/cerberus/issues/67. validator.allow_unknown = True - document = {'a_list_of_dicts': [{'sku': 'YZ069', 'price': 25, - 'extra': True}]} + document = {'a_list_of_dicts': [{'sku': 'YZ069', 'price': 25, 'extra': True}]} assert_success(document, validator=validator) @@ -692,8 +886,7 @@ def test_unknown_keys_retain_custom_rules(): validator = CustomValidator({}) validator.allow_unknown = {"type": "foo"} - assert_success(document={"fred": "foo", "barney": "foo"}, - validator=validator) + assert_success(document={"fred": "foo", "barney": "foo"}, validator=validator) def test_nested_unknown_keys(): @@ -701,16 +894,10 @@ def test_nested_unknown_keys(): 'field1': { 'type': 'dict', 'allow_unknown': True, - 'schema': {'nested1': {'type': 'string'}} - } - } - document = { - 'field1': { - 'nested1': 'foo', - 'arb1': 'bar', - 'arb2': 42 + 'schema': {'nested1': {'type': 'string'}}, } } + document = {'field1': {'nested1': 'foo', 'arb1': 'bar', 'arb2': 42}} assert_success(document=document, schema=schema) schema['field1']['allow_unknown'] = {'type': 'string'} @@ -739,8 +926,7 @@ def test_callable_validator(): def test_dependencies_field(): - schema = {'test_field': {'dependencies': 'foo'}, - 'foo': {'type': 'string'}} + schema = {'test_field': {'dependencies': 'foo'}, 'foo': {'type': 'string'}} assert_success({'test_field': 'foobar', 'foo': 'bar'}, schema) assert_fail({'test_field': 'foobar'}, schema) @@ -749,10 +935,9 @@ def test_dependencies_list(): schema = { 'test_field': {'dependencies': ['foo', 'bar']}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } - assert_success({'test_field': 'foobar', 'foo': 'bar', 'bar': 'foo'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'bar', 'bar': 'foo'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'bar'}, schema) @@ -760,7 +945,7 @@ def test_dependencies_list_with_required_field(): schema = { 'test_field': {'required': True, 'dependencies': ['foo', 'bar']}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } # False: all dependencies missing assert_fail({'test_field': 'foobar'}, schema) @@ -784,27 +969,23 @@ def test_dependencies_list_with_subodcuments_fields(): 'test_field': {'dependencies': ['a_dict.foo', 'a_dict.bar']}, 'a_dict': { 'type': 'dict', - 'schema': { - 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} - } - } + 'schema': {'foo': {'type': 'string'}, 'bar': {'type': 'string'}}, + }, } - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema + ) assert_fail({'test_field': 'foobar', 'a_dict': {}}, schema) - assert_fail({'test_field': 'foobar', - 'a_dict': {'foo': 'foo'}}, schema) + assert_fail({'test_field': 'foobar', 'a_dict': {'foo': 'foo'}}, schema) def test_dependencies_dict(): schema = { 'test_field': {'dependencies': {'foo': 'foo', 'bar': 'bar'}}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } - assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'foo'}, schema) assert_fail({'test_field': 'foobar', 'foo': 'bar'}, schema) assert_fail({'test_field': 'foobar', 'bar': 'bar'}, schema) @@ -814,12 +995,9 @@ def test_dependencies_dict(): def test_dependencies_dict_with_required_field(): schema = { - 'test_field': { - 'required': True, - 'dependencies': {'foo': 'foo', 'bar': 'bar'} - }, + 'test_field': {'required': True, 'dependencies': {'foo': 'foo', 'bar': 'bar'}}, 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} + 'bar': {'type': 'string'}, } # False: all dependencies missing assert_fail({'test_field': 'foobar'}, schema) @@ -833,8 +1011,7 @@ def test_dependencies_dict_with_required_field(): # False: dependency missing assert_fail({'foo': 'bar'}, schema) - assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, - schema) + assert_success({'test_field': 'foobar', 'foo': 'foo', 'bar': 'bar'}, schema) # True: dependencies are validated but field is not required schema['test_field']['required'] = False @@ -843,10 +1020,7 @@ def test_dependencies_dict_with_required_field(): def test_dependencies_field_satisfy_nullable_field(): # https://github.com/pyeve/cerberus/issues/305 - schema = { - 'foo': {'nullable': True}, - 'bar': {'dependencies': 'foo'} - } + schema = {'foo': {'nullable': True}, 'bar': {'dependencies': 'foo'}} assert_success({'foo': None, 'bar': 1}, schema) assert_success({'foo': None}, schema) @@ -857,7 +1031,7 @@ def test_dependencies_field_with_mutually_dependent_nullable_fields(): # https://github.com/pyeve/cerberus/pull/306 schema = { 'foo': {'dependencies': 'bar', 'nullable': True}, - 'bar': {'dependencies': 'foo', 'nullable': True} + 'bar': {'dependencies': 'foo', 'nullable': True}, } assert_success({'foo': None, 'bar': None}, schema) assert_success({'foo': 1, 'bar': 1}, schema) @@ -868,63 +1042,75 @@ def test_dependencies_field_with_mutually_dependent_nullable_fields(): def test_dependencies_dict_with_subdocuments_fields(): schema = { - 'test_field': {'dependencies': {'a_dict.foo': ['foo', 'bar'], - 'a_dict.bar': 'bar'}}, + 'test_field': { + 'dependencies': {'a_dict.foo': ['foo', 'bar'], 'a_dict.bar': 'bar'} + }, 'a_dict': { 'type': 'dict', - 'schema': { - 'foo': {'type': 'string'}, - 'bar': {'type': 'string'} - } - } + 'schema': {'foo': {'type': 'string'}, 'bar': {'type': 'string'}}, + }, } - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema) - assert_success({'test_field': 'foobar', - 'a_dict': {'foo': 'bar', 'bar': 'bar'}}, schema) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'bar'}}, schema + ) + assert_success( + {'test_field': 'foobar', 'a_dict': {'foo': 'bar', 'bar': 'bar'}}, schema + ) assert_fail({'test_field': 'foobar', 'a_dict': {}}, schema) - assert_fail({'test_field': 'foobar', - 'a_dict': {'foo': 'foo', 'bar': 'foo'}}, schema) - assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'foo'}}, - schema) - assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'bar'}}, - schema) + assert_fail( + {'test_field': 'foobar', 'a_dict': {'foo': 'foo', 'bar': 'foo'}}, schema + ) + assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'foo'}}, schema) + assert_fail({'test_field': 'foobar', 'a_dict': {'bar': 'bar'}}, schema) def test_root_relative_dependencies(): # https://github.com/pyeve/cerberus/issues/288 subschema = {'version': {'dependencies': '^repo'}} - schema = {'package': {'allow_unknown': True, 'schema': subschema}, - 'repo': {}} + schema = {'package': {'allow_unknown': True, 'schema': subschema}, 'repo': {}} assert_fail( - {'package': {'repo': 'somewhere', 'version': 0}}, schema, - error=('package', ('package', 'schema'), - errors.MAPPING_SCHEMA, subschema), - child_errors=[( - ('package', 'version'), - ('package', 'schema', 'version', 'dependencies'), - errors.DEPENDENCIES_FIELD, '^repo', ('^repo',) - )] + {'package': {'repo': 'somewhere', 'version': 0}}, + schema, + error=('package', ('package', 'schema'), errors.MAPPING_SCHEMA, subschema), + child_errors=[ + ( + ('package', 'version'), + ('package', 'schema', 'version', 'dependencies'), + errors.DEPENDENCIES_FIELD, + '^repo', + ('^repo',), + ) + ], ) assert_success({'repo': 'somewhere', 'package': {'version': 1}}, schema) def test_dependencies_errors(): - v = Validator({'field1': {'required': False}, - 'field2': {'required': True, - 'dependencies': {'field1': ['one', 'two']}}}) - assert_fail({'field1': 'three', 'field2': 7}, validator=v, - error=('field2', ('field2', 'dependencies'), - errors.DEPENDENCIES_FIELD_VALUE, - {'field1': ['one', 'two']}, ({'field1': 'three'},))) + v = Validator( + { + 'field1': {'required': False}, + 'field2': {'required': True, 'dependencies': {'field1': ['one', 'two']}}, + } + ) + assert_fail( + {'field1': 'three', 'field2': 7}, + validator=v, + error=( + 'field2', + ('field2', 'dependencies'), + errors.DEPENDENCIES_FIELD_VALUE, + {'field1': ['one', 'two']}, + ({'field1': 'three'},), + ), + ) def test_options_passed_to_nested_validators(validator): - validator.schema = {'sub_dict': {'type': 'dict', - 'schema': {'foo': {'type': 'string'}}}} + validator.schema = { + 'sub_dict': {'type': 'dict', 'schema': {'foo': {'type': 'string'}}} + } validator.allow_unknown = True - assert_success({'sub_dict': {'foo': 'bar', 'unknown': True}}, - validator=validator) + assert_success({'sub_dict': {'foo': 'bar', 'unknown': True}}, validator=validator) def test_self_root_document(): @@ -937,8 +1123,7 @@ def test_self_root_document(): class MyValidator(Validator): def _validate_root_doc(self, root_doc, field, value): """ {'type': 'boolean'} """ - if ('sub' not in self.root_document or - len(self.root_document['sub']) != 2): + if 'sub' not in self.root_document or len(self.root_document['sub']) != 2: self._error(field, 'self.context is not the root doc!') schema = { @@ -947,17 +1132,13 @@ def test_self_root_document(): 'root_doc': True, 'schema': { 'type': 'dict', - 'schema': { - 'foo': { - 'type': 'string', - 'root_doc': True - } - } - } + 'schema': {'foo': {'type': 'string', 'root_doc': True}}, + }, } } - assert_success({'sub': [{'foo': 'bar'}, {'foo': 'baz'}]}, - validator=MyValidator(schema)) + assert_success( + {'sub': [{'foo': 'bar'}, {'foo': 'baz'}]}, validator=MyValidator(schema) + ) def test_validator_rule(validator): @@ -967,11 +1148,14 @@ def test_validator_rule(validator): validator.schema = { 'name': {'validator': validate_name}, - 'age': {'type': 'integer'} + 'age': {'type': 'integer'}, } - assert_fail({'name': 'ItsMe', 'age': 2}, validator=validator, - error=('name', (), errors.CUSTOM, None, ('must be lowercase',))) + assert_fail( + {'name': 'ItsMe', 'age': 2}, + validator=validator, + error=('name', (), errors.CUSTOM, None, ('must be lowercase',)), + ) assert validator.errors == {'name': ['must be lowercase']} assert_success({'name': 'itsme', 'age': 2}, validator=validator) @@ -992,23 +1176,20 @@ def test_anyof(): assert_success(doc, schema) # prop1 must be either a number between 0 and 10 or 100 and 110 - schema = {'prop1': {'anyof': - [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} + schema = {'prop1': {'anyof': [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} doc = {'prop1': 105} assert_success(doc, schema) # prop1 must be either a number between 0 and 10 or 100 and 110 - schema = {'prop1': {'anyof': - [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} + schema = {'prop1': {'anyof': [{'min': 0, 'max': 10}, {'min': 100, 'max': 110}]}} doc = {'prop1': 50} assert_fail(doc, schema) # prop1 must be an integer that is either be # greater than or equal to 0, or greater than or equal to 10 - schema = {'prop1': {'type': 'integer', - 'anyof': [{'min': 0}, {'min': 10}]}} + schema = {'prop1': {'type': 'integer', 'anyof': [{'min': 0}, {'min': 10}]}} assert_success({'prop1': 10}, schema) # test that intermediate schemas do not sustain assert 'type' not in schema['prop1']['anyof'][0] @@ -1019,12 +1200,14 @@ def test_anyof(): exp_child_errors = [ (('prop1',), ('prop1', 'anyof', 0, 'min'), errors.MIN_VALUE, 0), - (('prop1',), ('prop1', 'anyof', 1, 'min'), errors.MIN_VALUE, 10) + (('prop1',), ('prop1', 'anyof', 1, 'min'), errors.MIN_VALUE, 10), ] - assert_fail({'prop1': -1}, schema, - error=(('prop1',), ('prop1', 'anyof'), errors.ANYOF, - [{'min': 0}, {'min': 10}]), - child_errors=exp_child_errors) + assert_fail( + {'prop1': -1}, + schema, + error=(('prop1',), ('prop1', 'anyof'), errors.ANYOF, [{'min': 0}, {'min': 10}]), + child_errors=exp_child_errors, + ) doc = {'prop1': 5.5} assert_fail(doc, schema) doc = {'prop1': '5.5'} @@ -1033,8 +1216,7 @@ def test_anyof(): def test_allof(): # prop1 has to be a float between 0 and 10 - schema = {'prop1': {'allof': [ - {'type': 'float'}, {'min': 0}, {'max': 10}]}} + schema = {'prop1': {'allof': [{'type': 'float'}, {'min': 0}, {'max': 10}]}} doc = {'prop1': -1} assert_fail(doc, schema) doc = {'prop1': 5} @@ -1067,8 +1249,7 @@ def test_unicode_allowed(): assert_success(doc, schema) -@mark.skipif(sys.version_info[0] < 3, - reason='requires python 3.x') +@mark.skipif(sys.version_info[0] < 3, reason='requires python 3.x') def test_unicode_allowed_py3(): """ All strings are unicode in Python 3.x. Input doc and schema have equal strings and validation yield success.""" @@ -1079,8 +1260,7 @@ def test_unicode_allowed_py3(): assert_success(doc, schema) -@mark.skipif(sys.version_info[0] > 2, - reason='requires python 2.x') +@mark.skipif(sys.version_info[0] > 2, reason='requires python 2.x') def test_unicode_allowed_py2(): """ Python 2.x encodes value of allowed using default encoding if the string includes characters outside ASCII range. Produced string @@ -1098,10 +1278,12 @@ def test_oneof(): # - greater than 0 # - equal to -5, 5, or 15 - schema = {'prop1': {'type': 'integer', 'oneof': [ - {'min': 0}, - {'min': 10}, - {'allowed': [-5, 5, 15]}]}} + schema = { + 'prop1': { + 'type': 'integer', + 'oneof': [{'min': 0}, {'min': 10}, {'allowed': [-5, 5, 15]}], + } + } # document is not valid # prop1 not greater than 0, 10 or equal to -5 @@ -1144,10 +1326,12 @@ def test_noneof(): # - greater than 0 # - equal to -5, 5, or 15 - schema = {'prop1': {'type': 'integer', 'noneof': [ - {'min': 0}, - {'min': 10}, - {'allowed': [-5, 5, 15]}]}} + schema = { + 'prop1': { + 'type': 'integer', + 'noneof': [{'min': 0}, {'min': 10}, {'allowed': [-5, 5, 15]}], + } + } # document is valid doc = {'prop1': -1} @@ -1179,11 +1363,14 @@ def test_noneof(): def test_anyof_allof(): # prop1 can be any number outside of [0-10] - schema = {'prop1': {'allof': [{'anyof': [{'type': 'float'}, - {'type': 'integer'}]}, - {'anyof': [{'min': 10}, - {'max': 0}]} - ]}} + schema = { + 'prop1': { + 'allof': [ + {'anyof': [{'type': 'float'}, {'type': 'integer'}]}, + {'anyof': [{'min': 10}, {'max': 0}]}, + ] + } + } doc = {'prop1': 11} assert_success(doc, schema) @@ -1206,15 +1393,19 @@ def test_anyof_allof(): def test_anyof_schema(validator): # test that a list of schemas can be specified. - valid_parts = [{'schema': {'model number': {'type': 'string'}, - 'count': {'type': 'integer'}}}, - {'schema': {'serial number': {'type': 'string'}, - 'count': {'type': 'integer'}}}] + valid_parts = [ + {'schema': {'model number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + {'schema': {'serial number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + ] valid_item = {'type': ['dict', 'string'], 'anyof': valid_parts} schema = {'parts': {'type': 'list', 'schema': valid_item}} - document = {'parts': [{'model number': 'MX-009', 'count': 100}, - {'serial number': '898-001'}, - 'misc']} + document = { + 'parts': [ + {'model number': 'MX-009', 'count': 100}, + {'serial number': '898-001'}, + 'misc', + ] + } # document is valid. each entry in 'parts' matches a type or schema assert_success(document, schema, validator=validator) @@ -1232,18 +1423,25 @@ def test_anyof_schema(validator): # and invalid. numbers are not allowed. exp_child_errors = [ - (('parts', 3), ('parts', 'schema', 'anyof'), errors.ANYOF, - valid_parts), - (('parts', 4), ('parts', 'schema', 'type'), errors.BAD_TYPE, - ['dict', 'string']) + (('parts', 3), ('parts', 'schema', 'anyof'), errors.ANYOF, valid_parts), + ( + ('parts', 4), + ('parts', 'schema', 'type'), + errors.BAD_TYPE, + ['dict', 'string'], + ), ] - _errors = assert_fail(document, schema, validator=validator, - error=('parts', ('parts', 'schema'), - errors.SEQUENCE_SCHEMA, valid_item), - child_errors=exp_child_errors) - assert_not_has_error(_errors, ('parts', 4), ('parts', 'schema', 'anyof'), - errors.ANYOF, valid_parts) + _errors = assert_fail( + document, + schema, + validator=validator, + error=('parts', ('parts', 'schema'), errors.SEQUENCE_SCHEMA, valid_item), + child_errors=exp_child_errors, + ) + assert_not_has_error( + _errors, ('parts', 4), ('parts', 'schema', 'anyof'), errors.ANYOF, valid_parts + ) # tests errors.BasicErrorHandler's tree representation v_errors = validator.errors @@ -1260,15 +1458,23 @@ def test_anyof_schema(validator): def test_anyof_2(): # these two schema should be the same - schema1 = {'prop': {'anyof': [{'type': 'dict', - 'schema': { - 'val': {'type': 'integer'}}}, - {'type': 'dict', - 'schema': { - 'val': {'type': 'string'}}}]}} - schema2 = {'prop': {'type': 'dict', 'anyof': [ - {'schema': {'val': {'type': 'integer'}}}, - {'schema': {'val': {'type': 'string'}}}]}} + schema1 = { + 'prop': { + 'anyof': [ + {'type': 'dict', 'schema': {'val': {'type': 'integer'}}}, + {'type': 'dict', 'schema': {'val': {'type': 'string'}}}, + ] + } + } + schema2 = { + 'prop': { + 'type': 'dict', + 'anyof': [ + {'schema': {'val': {'type': 'integer'}}}, + {'schema': {'val': {'type': 'string'}}}, + ], + } + } doc = {'prop': {'val': 0}} assert_success(doc, schema1) @@ -1290,47 +1496,69 @@ def test_anyof_type(): def test_oneof_schema(): - schema = {'oneof_schema': {'type': 'dict', - 'oneof_schema': - [{'digits': {'type': 'integer', - 'min': 0, 'max': 99}}, - {'text': {'type': 'string', - 'regex': '^[0-9]{2}$'}}]}} + schema = { + 'oneof_schema': { + 'type': 'dict', + 'oneof_schema': [ + {'digits': {'type': 'integer', 'min': 0, 'max': 99}}, + {'text': {'type': 'string', 'regex': '^[0-9]{2}$'}}, + ], + } + } assert_success({'oneof_schema': {'digits': 19}}, schema) assert_success({'oneof_schema': {'text': '84'}}, schema) assert_fail({'oneof_schema': {'digits': 19, 'text': '84'}}, schema) def test_nested_oneof_type(): - schema = {'nested_oneof_type': - {'valueschema': {'oneof_type': ['string', 'integer']}}} + schema = { + 'nested_oneof_type': {'valuesrules': {'oneof_type': ['string', 'integer']}} + } assert_success({'nested_oneof_type': {'foo': 'a'}}, schema) assert_success({'nested_oneof_type': {'bar': 3}}, schema) def test_nested_oneofs(validator): - validator.schema = {'abc': { - 'type': 'dict', - 'oneof_schema': [ - {'foo': { - 'type': 'dict', - 'schema': {'bar': {'oneof_type': ['integer', 'float']}} - }}, - {'baz': {'type': 'string'}} - ]}} + validator.schema = { + 'abc': { + 'type': 'dict', + 'oneof_schema': [ + { + 'foo': { + 'type': 'dict', + 'schema': {'bar': {'oneof_type': ['integer', 'float']}}, + } + }, + {'baz': {'type': 'string'}}, + ], + } + } document = {'abc': {'foo': {'bar': 'bad'}}} expected_errors = { 'abc': [ 'none or more than one rule validate', - {'oneof definition 0': [ - {'foo': [{'bar': [ - 'none or more than one rule validate', - {'oneof definition 0': ['must be of integer type'], - 'oneof definition 1': ['must be of float type']} - ]}]}], - 'oneof definition 1': [{'foo': ['unknown field']}]} + { + 'oneof definition 0': [ + { + 'foo': [ + { + 'bar': [ + 'none or more than one rule validate', + { + 'oneof definition 0': [ + 'must be of integer type' + ], + 'oneof definition 1': ['must be of float type'], + }, + ] + } + ] + } + ], + 'oneof definition 1': [{'foo': ['unknown field']}], + }, ] } @@ -1339,21 +1567,23 @@ def test_nested_oneofs(validator): def test_no_of_validation_if_type_fails(validator): - valid_parts = [{'schema': {'model number': {'type': 'string'}, - 'count': {'type': 'integer'}}}, - {'schema': {'serial number': {'type': 'string'}, - 'count': {'type': 'integer'}}}] - validator.schema = {'part': {'type': ['dict', 'string'], - 'anyof': valid_parts}} + valid_parts = [ + {'schema': {'model number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + {'schema': {'serial number': {'type': 'string'}, 'count': {'type': 'integer'}}}, + ] + validator.schema = {'part': {'type': ['dict', 'string'], 'anyof': valid_parts}} document = {'part': 10} _errors = assert_fail(document, validator=validator) assert len(_errors) == 1 def test_issue_107(validator): - schema = {'info': {'type': 'dict', - 'schema': {'name': {'type': 'string', - 'required': True}}}} + schema = { + 'info': { + 'type': 'dict', + 'schema': {'name': {'type': 'string', 'required': True}}, + } + } document = {'info': {'name': 'my name'}} assert_success(document, schema, validator=validator) @@ -1369,20 +1599,23 @@ def test_dont_type_validate_nulled_values(validator): def test_dependencies_error(validator): - schema = {'field1': {'required': False}, - 'field2': {'required': True, - 'dependencies': {'field1': ['one', 'two']}}} + schema = { + 'field1': {'required': False}, + 'field2': {'required': True, 'dependencies': {'field1': ['one', 'two']}}, + } validator.validate({'field2': 7}, schema) - exp_msg = errors.BasicErrorHandler \ - .messages[errors.DEPENDENCIES_FIELD_VALUE.code] \ - .format(field='field2', constraint={'field1': ['one', 'two']}) + exp_msg = errors.BasicErrorHandler.messages[ + errors.DEPENDENCIES_FIELD_VALUE.code + ].format(field='field2', constraint={'field1': ['one', 'two']}) assert validator.errors == {'field2': [exp_msg]} def test_dependencies_on_boolean_field_with_one_value(): # https://github.com/pyeve/cerberus/issues/138 - schema = {'deleted': {'type': 'boolean'}, - 'text': {'dependencies': {'deleted': False}}} + schema = { + 'deleted': {'type': 'boolean'}, + 'text': {'dependencies': {'deleted': False}}, + } try: assert_success({'text': 'foo', 'deleted': False}, schema) assert_fail({'text': 'foo', 'deleted': True}, schema) @@ -1392,15 +1625,18 @@ def test_dependencies_on_boolean_field_with_one_value(): raise AssertionError( "Bug #138 still exists, couldn't use boolean in dependency " "without putting it in a list.\n" - "'some_field': True vs 'some_field: [True]") + "'some_field': True vs 'some_field: [True]" + ) else: raise def test_dependencies_on_boolean_field_with_value_in_list(): # https://github.com/pyeve/cerberus/issues/138 - schema = {'deleted': {'type': 'boolean'}, - 'text': {'dependencies': {'deleted': [False]}}} + schema = { + 'deleted': {'type': 'boolean'}, + 'text': {'dependencies': {'deleted': [False]}}, + } assert_success({'text': 'foo', 'deleted': False}, schema) assert_fail({'text': 'foo', 'deleted': True}, schema) @@ -1423,9 +1659,10 @@ def test_document_path(): def test_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field'}, - 'that_field': {'type': 'dict'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field'}, + 'that_field': {'type': 'dict'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_success({}, schema) @@ -1433,10 +1670,10 @@ def test_excludes(): def test_mutual_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field'}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field'}, + 'that_field': {'type': 'dict', 'excludes': 'this_field'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_success({}, schema) @@ -1444,12 +1681,10 @@ def test_mutual_excludes(): def test_required_excludes(): - schema = {'this_field': {'type': 'dict', - 'excludes': 'that_field', - 'required': True}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field', - 'required': True}} + schema = { + 'this_field': {'type': 'dict', 'excludes': 'that_field', 'required': True}, + 'that_field': {'type': 'dict', 'excludes': 'this_field', 'required': True}, + } assert_success({'this_field': {}}, schema, update=False) assert_success({'that_field': {}}, schema, update=False) assert_fail({}, schema) @@ -1457,11 +1692,11 @@ def test_required_excludes(): def test_multiples_exclusions(): - schema = {'this_field': {'type': 'dict', - 'excludes': ['that_field', 'bazo_field']}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field'}, - 'bazo_field': {'type': 'dict'}} + schema = { + 'this_field': {'type': 'dict', 'excludes': ['that_field', 'bazo_field']}, + 'that_field': {'type': 'dict', 'excludes': 'this_field'}, + 'bazo_field': {'type': 'dict'}, + } assert_success({'this_field': {}}, schema) assert_success({'that_field': {}}, schema) assert_fail({'this_field': {}, 'that_field': {}}, schema) @@ -1471,21 +1706,28 @@ def test_multiples_exclusions(): def test_bad_excludes_fields(validator): - validator.schema = {'this_field': {'type': 'dict', - 'excludes': ['that_field', 'bazo_field'], - 'required': True}, - 'that_field': {'type': 'dict', - 'excludes': 'this_field', - 'required': True}} + validator.schema = { + 'this_field': { + 'type': 'dict', + 'excludes': ['that_field', 'bazo_field'], + 'required': True, + }, + 'that_field': {'type': 'dict', 'excludes': 'this_field', 'required': True}, + } assert_fail({'that_field': {}, 'this_field': {}}, validator=validator) handler = errors.BasicErrorHandler - assert (validator.errors == - {'that_field': - [handler.messages[errors.EXCLUDES_FIELD.code].format( - "'this_field'", field="that_field")], - 'this_field': - [handler.messages[errors.EXCLUDES_FIELD.code].format( - "'that_field', 'bazo_field'", field="this_field")]}) + assert validator.errors == { + 'that_field': [ + handler.messages[errors.EXCLUDES_FIELD.code].format( + "'this_field'", field="that_field" + ) + ], + 'this_field': [ + handler.messages[errors.EXCLUDES_FIELD.code].format( + "'that_field', 'bazo_field'", field="this_field" + ) + ], + } def test_boolean_is_not_a_number(): @@ -1511,17 +1753,29 @@ def test_forbidden(): assert_success({'user': 'alice'}, schema) +def test_forbidden_number(): + schema = {'amount': {'forbidden': (0, 0.0)}} + assert_fail({'amount': 0}, schema) + assert_fail({'amount': 0.0}, schema) + + def test_mapping_with_sequence_schema(): schema = {'list': {'schema': {'allowed': ['a', 'b', 'c']}}} document = {'list': {'is_a': 'mapping'}} - assert_fail(document, schema, - error=('list', ('list', 'schema'), errors.BAD_TYPE_FOR_SCHEMA, - schema['list']['schema'])) + assert_fail( + document, + schema, + error=( + 'list', + ('list', 'schema'), + errors.BAD_TYPE_FOR_SCHEMA, + schema['list']['schema'], + ), + ) def test_sequence_with_mapping_schema(): - schema = {'list': {'schema': {'foo': {'allowed': ['a', 'b', 'c']}}, - 'type': 'dict'}} + schema = {'list': {'schema': {'foo': {'allowed': ['a', 'b', 'c']}}, 'type': 'dict'}} document = {'list': ['a', 'b', 'c']} assert_fail(document, schema) @@ -1529,19 +1783,24 @@ def test_sequence_with_mapping_schema(): def test_type_error_aborts_validation(): schema = {'foo': {'type': 'string', 'allowed': ['a']}} document = {'foo': 0} - assert_fail(document, schema, - error=('foo', ('foo', 'type'), errors.BAD_TYPE, 'string')) + assert_fail( + document, schema, error=('foo', ('foo', 'type'), errors.BAD_TYPE, 'string') + ) def test_dependencies_in_oneof(): # https://github.com/pyeve/cerberus/issues/241 - schema = {'a': {'type': 'integer', - 'oneof': [ - {'allowed': [1], 'dependencies': 'b'}, - {'allowed': [2], 'dependencies': 'c'} - ]}, - 'b': {}, - 'c': {}} + schema = { + 'a': { + 'type': 'integer', + 'oneof': [ + {'allowed': [1], 'dependencies': 'b'}, + {'allowed': [2], 'dependencies': 'c'}, + ], + }, + 'b': {}, + 'c': {}, + } assert_success({'a': 1, 'b': 'foo'}, schema) assert_success({'a': 2, 'c': 'bar'}, schema) assert_fail({'a': 1, 'c': 'foo'}, schema) @@ -1556,12 +1815,9 @@ def test_allow_unknown_with_oneof_rules(validator): { 'type': 'dict', 'allow_unknown': True, - 'schema': {'known': {'type': 'string'}} - }, - { - 'type': 'dict', - 'schema': {'known': {'type': 'string'}} + 'schema': {'known': {'type': 'string'}}, }, + {'type': 'dict', 'schema': {'known': {'type': 'string'}}}, ] } } @@ -1571,9 +1827,122 @@ def test_allow_unknown_with_oneof_rules(validator): validator(document, schema) _errors = validator._errors assert len(_errors) == 1 - assert_has_error(_errors, 'test', ('test', 'oneof'), - errors.ONEOF, schema['test']['oneof']) + assert_has_error( + _errors, 'test', ('test', 'oneof'), errors.ONEOF, schema['test']['oneof'] + ) assert len(_errors[0].child_errors) == 0 # check that allow_unknown is actually applied document = {'test': {'known': 's', 'unknown': 'asd'}} assert_success(document, validator=validator) + + +@mark.parametrize('constraint', (('Graham Chapman', 'Eric Idle'), 'Terry Gilliam')) +def test_contains(constraint): + validator = Validator({'actors': {'contains': constraint}}) + + document = {'actors': ('Graham Chapman', 'Eric Idle', 'Terry Gilliam')} + assert validator(document) + + document = {'actors': ('Eric idle', 'Terry Jones', 'John Cleese', 'Michael Palin')} + assert not validator(document) + assert errors.MISSING_MEMBERS in validator.document_error_tree['actors'] + missing_actors = validator.document_error_tree['actors'][ + errors.MISSING_MEMBERS + ].info[0] + assert any(x in missing_actors for x in ('Eric Idle', 'Terry Gilliam')) + + +def test_require_all_simple(): + schema = {'foo': {'type': 'string'}} + validator = Validator(require_all=True) + assert_fail( + {}, + schema, + validator, + error=('foo', '__require_all__', errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + + +def test_require_all_override_by_required(): + schema = {'foo': {'type': 'string', 'required': False}} + validator = Validator(require_all=True) + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'bar'}, schema, validator) + + schema = {'foo': {'type': 'string', 'required': True}} + validator.require_all = True + assert_fail( + {}, + schema, + validator, + error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + validator.require_all = False + assert_fail( + {}, + schema, + validator, + error=('foo', ('foo', 'required'), errors.REQUIRED_FIELD, True), + ) + assert_success({'foo': 'bar'}, schema, validator) + + +@mark.parametrize( + "validator_require_all, sub_doc_require_all", + list(itertools.product([True, False], repeat=2)), +) +def test_require_all_override_by_subdoc_require_all( + validator_require_all, sub_doc_require_all +): + sub_schema = {"bar": {"type": "string"}} + schema = { + "foo": { + "type": "dict", + "require_all": sub_doc_require_all, + "schema": sub_schema, + } + } + validator = Validator(require_all=validator_require_all) + + assert_success({"foo": {"bar": "baz"}}, schema, validator) + if validator_require_all: + assert_fail({}, schema, validator) + else: + assert_success({}, schema, validator) + if sub_doc_require_all: + assert_fail({"foo": {}}, schema, validator) + else: + assert_success({"foo": {}}, schema, validator) + + +def test_require_all_and_exclude(): + schema = { + 'foo': {'type': 'string', 'excludes': 'bar'}, + 'bar': {'type': 'string', 'excludes': 'foo'}, + } + validator = Validator(require_all=True) + assert_fail( + {}, + schema, + validator, + errors=[ + ('foo', '__require_all__', errors.REQUIRED_FIELD, True), + ('bar', '__require_all__', errors.REQUIRED_FIELD, True), + ], + ) + assert_success({'foo': 'value'}, schema, validator) + assert_success({'bar': 'value'}, schema, validator) + assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator) + validator.require_all = False + assert_success({}, schema, validator) + assert_success({'foo': 'value'}, schema, validator) + assert_success({'bar': 'value'}, schema, validator) + assert_fail({'foo': 'value', 'bar': 'value'}, schema, validator) diff --git a/pipenv/vendor/cerberus/utils.py b/pipenv/vendor/cerberus/utils.py index f10d3976..5a015d64 100644 --- a/pipenv/vendor/cerberus/utils.py +++ b/pipenv/vendor/cerberus/utils.py @@ -1,12 +1,11 @@ from __future__ import absolute_import -from collections import Mapping, namedtuple, Sequence +from collections import namedtuple -from cerberus.platform import _int_types, _str_type +from cerberus.platform import _int_types, _str_type, Mapping, Sequence, Set -TypeDefinition = namedtuple('TypeDefinition', - 'name,included_types,excluded_types') +TypeDefinition = namedtuple('TypeDefinition', 'name,included_types,excluded_types') """ This class is used to define types that can be used as value in the :attr:`~cerberus.Validator.types_mapping` property. @@ -19,19 +18,33 @@ contained in ``excluded_types``. def compare_paths_lt(x, y): - for i in range(min(len(x), len(y))): - if isinstance(x[i], type(y[i])): - if x[i] != y[i]: - return x[i] < y[i] - elif isinstance(x[i], _int_types): + min_length = min(len(x), len(y)) + + if x[:min_length] == y[:min_length]: + return len(x) == min_length + + for i in range(min_length): + a, b = x[i], y[i] + + for _type in (_int_types, _str_type, tuple): + if isinstance(a, _type): + if isinstance(b, _type): + break + else: + return True + + if a == b: + continue + elif a < b: return True - elif isinstance(y[i], _int_types): + else: return False - return len(x) < len(y) + + raise RuntimeError def drop_item_from_tuple(t, i): - return t[:i] + t[i + 1:] + return t[:i] + t[i + 1 :] def get_Validator_class(): @@ -50,26 +63,24 @@ def mapping_to_frozenset(mapping): equal. As it is used to identify equality of schemas, this can be considered okay as definitions are semantically equal regardless the container type. """ - mapping = mapping.copy() + + aggregation = {} + for key, value in mapping.items(): if isinstance(value, Mapping): - mapping[key] = mapping_to_frozenset(value) + aggregation[key] = mapping_to_frozenset(value) elif isinstance(value, Sequence): value = list(value) for i, item in enumerate(value): if isinstance(item, Mapping): value[i] = mapping_to_frozenset(item) - mapping[key] = tuple(value) - return frozenset(mapping.items()) + aggregation[key] = tuple(value) + elif isinstance(value, Set): + aggregation[key] = frozenset(value) + else: + aggregation[key] = value - -def isclass(obj): - try: - issubclass(obj, object) - except TypeError: - return False - else: - return True + return frozenset(aggregation.items()) def quote_string(value): diff --git a/pipenv/vendor/cerberus/validator.py b/pipenv/vendor/cerberus/validator.py index 27a29053..ed1c1536 100644 --- a/pipenv/vendor/cerberus/validator.py +++ b/pipenv/vendor/cerberus/validator.py @@ -11,28 +11,41 @@ from __future__ import absolute_import from ast import literal_eval -from collections import Hashable, Iterable, Mapping, Sequence from copy import copy from datetime import date, datetime import re from warnings import warn from cerberus import errors -from cerberus.platform import _int_types, _str_type -from cerberus.schema import (schema_registry, rules_set_registry, - DefinitionSchema, SchemaError) -from cerberus.utils import (drop_item_from_tuple, isclass, - readonly_classproperty, TypeDefinition) - +from cerberus.platform import ( + _int_types, + _str_type, + Container, + Hashable, + Iterable, + Mapping, + Sequence, + Sized, +) +from cerberus.schema import ( + schema_registry, + rules_set_registry, + DefinitionSchema, + SchemaError, +) +from cerberus.utils import drop_item_from_tuple, readonly_classproperty, TypeDefinition toy_error_handler = errors.ToyErrorHandler() def dummy_for_rule_validation(rule_constraints): def dummy(self, constraint, field, value): - raise RuntimeError('Dummy method called. Its purpose is to hold just' - 'validation constraints for a rule in its ' - 'docstring.') + raise RuntimeError( + 'Dummy method called. Its purpose is to hold just' + 'validation constraints for a rule in its ' + 'docstring.' + ) + f = dummy f.__doc__ = rule_constraints return f @@ -40,12 +53,14 @@ def dummy_for_rule_validation(rule_constraints): class DocumentError(Exception): """ Raised when the target document is missing or has the wrong format """ + pass class _SchemaRuleTypeError(Exception): """ Raised when a schema (list) validation encounters a mapping. Not supposed to be used outside this module. """ + pass @@ -76,9 +91,15 @@ class BareValidator(object): :param allow_unknown: See :attr:`~cerberus.Validator.allow_unknown`. Defaults to ``False``. :type allow_unknown: :class:`bool` or any :term:`mapping` + :param require_all: See :attr:`~cerberus.Validator.require_all`. + Defaults to ``False``. + :type require_all: :class:`bool` :param purge_unknown: See :attr:`~cerberus.Validator.purge_unknown`. Defaults to to ``False``. :type purge_unknown: :class:`bool` + :param purge_readonly: Removes all fields that are defined as ``readonly`` in the + normalization phase. + :type purge_readonly: :class:`bool` :param error_handler: The error handler that formats the result of :attr:`~cerberus.Validator.errors`. When given as two-value tuple with an error-handler @@ -98,28 +119,18 @@ class BareValidator(object): """ Rules that will be processed in that order before any other. Type: :class:`tuple` """ types_mapping = { - 'binary': - TypeDefinition('binary', (bytes, bytearray), ()), - 'boolean': - TypeDefinition('boolean', (bool,), ()), - 'date': - TypeDefinition('date', (date,), ()), - 'datetime': - TypeDefinition('datetime', (datetime,), ()), - 'dict': - TypeDefinition('dict', (Mapping,), ()), - 'float': - TypeDefinition('float', (float, _int_types), ()), - 'integer': - TypeDefinition('integer', (_int_types,), ()), - 'list': - TypeDefinition('list', (Sequence,), (_str_type,)), - 'number': - TypeDefinition('number', (_int_types, float), (bool,)), - 'set': - TypeDefinition('set', (set,), ()), - 'string': - TypeDefinition('string', (_str_type), ()) + 'binary': TypeDefinition('binary', (bytes, bytearray), ()), + 'boolean': TypeDefinition('boolean', (bool,), ()), + 'container': TypeDefinition('container', (Container,), (_str_type,)), + 'date': TypeDefinition('date', (date,), ()), + 'datetime': TypeDefinition('datetime', (datetime,), ()), + 'dict': TypeDefinition('dict', (Mapping,), ()), + 'float': TypeDefinition('float', (float, _int_types), ()), + 'integer': TypeDefinition('integer', (_int_types,), ()), + 'list': TypeDefinition('list', (Sequence,), (_str_type,)), + 'number': TypeDefinition('number', (_int_types, float), (bool,)), + 'set': TypeDefinition('set', (set,), ()), + 'string': TypeDefinition('string', (_str_type), ()), } """ This mapping holds all available constraints for the type rule and their assigned :class:`~cerberus.TypeDefinition`. """ @@ -131,7 +142,8 @@ class BareValidator(object): """ The arguments will be treated as with this signature: __init__(self, schema=None, ignore_none_values=False, - allow_unknown=False, purge_unknown=False, + allow_unknown=False, require_all=False, + purge_unknown=False, purge_readonly=False, error_handler=errors.BasicErrorHandler) """ @@ -168,6 +180,7 @@ class BareValidator(object): self.__store_config(args, kwargs) self.schema = kwargs.get('schema', None) self.allow_unknown = kwargs.get('allow_unknown', False) + self.require_all = kwargs.get('require_all', False) self._remaining_rules = [] """ Keeps track of the rules that are next in line to be evaluated during the validation of a field. @@ -182,8 +195,9 @@ class BareValidator(object): error_handler, eh_config = error_handler else: eh_config = {} - if isclass(error_handler) and \ - issubclass(error_handler, errors.BaseErrorHandler): + if isinstance(error_handler, type) and issubclass( + error_handler, errors.BaseErrorHandler + ): return error_handler(**eh_config) elif isinstance(error_handler, errors.BaseErrorHandler): return error_handler @@ -192,12 +206,17 @@ class BareValidator(object): def __store_config(self, args, kwargs): """ Assign args to kwargs and store configuration. """ - signature = ('schema', 'ignore_none_values', 'allow_unknown', - 'purge_unknown') - for i, p in enumerate(signature[:len(args)]): + signature = ( + 'schema', + 'ignore_none_values', + 'allow_unknown', + 'require_all', + 'purge_unknown', + 'purge_readonly', + ) + for i, p in enumerate(signature[: len(args)]): if p in kwargs: - raise TypeError("__init__ got multiple values for argument " - "'%s'" % p) + raise TypeError("__init__ got multiple values for argument " "'%s'" % p) else: kwargs[p] = args[i] self._config = kwargs @@ -251,8 +270,8 @@ class BareValidator(object): self._errors.extend(args[0]) self._errors.sort() for error in args[0]: - self.document_error_tree += error - self.schema_error_tree += error + self.document_error_tree.add(error) + self.schema_error_tree.add(error) self.error_handler.emit(error) elif len(args) == 2 and isinstance(args[1], _str_type): self._error(args[0], errors.CUSTOM, args[1]) @@ -262,7 +281,7 @@ class BareValidator(object): rule = args[1].rule info = args[2:] - document_path = self.document_path + (field, ) + document_path = self.document_path + (field,) schema_path = self.schema_path if code != errors.UNKNOWN_FIELD.code and rule is not None: @@ -274,6 +293,10 @@ class BareValidator(object): field_definitions = self._resolve_rules_set(self.schema[field]) if rule == 'nullable': constraint = field_definitions.get(rule, False) + elif rule == 'required': + constraint = field_definitions.get(rule, self.require_all) + if rule not in field_definitions: + schema_path = "__require_all__" else: constraint = field_definitions[rule] @@ -284,8 +307,7 @@ class BareValidator(object): ) self._error([self.recent_error]) - def _get_child_validator(self, document_crumb=None, schema_crumb=None, - **kwargs): + def _get_child_validator(self, document_crumb=None, schema_crumb=None, **kwargs): """ Creates a new instance of Validator-(sub-)class. All initial parameters of the parent are passed to the initialization, unless a parameter is given as an explicit *keyword*-parameter. @@ -309,6 +331,7 @@ class BareValidator(object): child_config['is_child'] = True child_config['error_handler'] = toy_error_handler child_config['root_allow_unknown'] = self.allow_unknown + child_config['root_require_all'] = self.require_all child_config['root_document'] = self.document child_config['root_schema'] = self.schema @@ -318,14 +341,14 @@ class BareValidator(object): child_validator.document_path = self.document_path else: if not isinstance(document_crumb, tuple): - document_crumb = (document_crumb, ) + document_crumb = (document_crumb,) child_validator.document_path = self.document_path + document_crumb if schema_crumb is None: child_validator.schema_path = self.schema_path else: if not isinstance(schema_crumb, tuple): - schema_crumb = (schema_crumb, ) + schema_crumb = (schema_crumb,) child_validator.schema_path = self.schema_path + schema_crumb return child_validator @@ -334,8 +357,10 @@ class BareValidator(object): methodname = '_{0}_{1}'.format(domain, rule.replace(' ', '_')) result = getattr(self, methodname, None) if result is None: - raise RuntimeError("There's no handler for '{}' in the '{}' " - "domain.".format(rule, domain)) + raise RuntimeError( + "There's no handler for '{}' in the '{}' " + "domain.".format(rule, domain) + ) return result def _drop_nodes_from_errorpaths(self, _errors, dp_items, sp_items): @@ -351,14 +376,15 @@ class BareValidator(object): sp_basedepth = len(self.schema_path) for error in _errors: for i in sorted(dp_items, reverse=True): - error.document_path = \ - drop_item_from_tuple(error.document_path, dp_basedepth + i) + error.document_path = drop_item_from_tuple( + error.document_path, dp_basedepth + i + ) for i in sorted(sp_items, reverse=True): - error.schema_path = \ - drop_item_from_tuple(error.schema_path, sp_basedepth + i) + error.schema_path = drop_item_from_tuple( + error.schema_path, sp_basedepth + i + ) if error.child_errors: - self._drop_nodes_from_errorpaths(error.child_errors, - dp_items, sp_items) + self._drop_nodes_from_errorpaths(error.child_errors, dp_items, sp_items) def _lookup_field(self, path): """ Searches for a field as defined by path. This method is used by the @@ -377,8 +403,7 @@ class BareValidator(object): """ if path.startswith('^'): path = path[1:] - context = self.document if path.startswith('^') \ - else self.root_document + context = self.document if path.startswith('^') else self.root_document else: context = self.document @@ -386,7 +411,7 @@ class BareValidator(object): for part in parts: if part not in context: return None, None - context = context.get(part) + context = context.get(part, {}) return parts[-1], context @@ -421,6 +446,17 @@ class BareValidator(object): DefinitionSchema(self, {'allow_unknown': value}) self._config['allow_unknown'] = value + @property + def require_all(self): + """ If ``True`` known fields that are defined in the schema will + be required. + Type: :class:`bool` """ + return self._config.get('require_all', False) + + @require_all.setter + def require_all(self, value): + self._config['require_all'] = value + @property def errors(self): """ The errors of the last processing formatted by the handler that is @@ -455,7 +491,7 @@ class BareValidator(object): @property def purge_unknown(self): - """ If ``True`` unknown fields will be deleted from the document + """ If ``True``, unknown fields will be deleted from the document unless a validation is called with disabled normalization. Also see :ref:`purging-unknown-fields`. Type: :class:`bool` """ return self._config.get('purge_unknown', False) @@ -464,12 +500,29 @@ class BareValidator(object): def purge_unknown(self, value): self._config['purge_unknown'] = value + @property + def purge_readonly(self): + """ If ``True``, fields declared as readonly will be deleted from the + document unless a validation is called with disabled normalization. + Type: :class:`bool` """ + return self._config.get('purge_readonly', False) + + @purge_readonly.setter + def purge_readonly(self, value): + self._config['purge_readonly'] = value + @property def root_allow_unknown(self): """ The :attr:`~cerberus.Validator.allow_unknown` attribute of the first level ancestor of a child validator. """ return self._config.get('root_allow_unknown', self.allow_unknown) + @property + def root_require_all(self): + """ The :attr:`~cerberus.Validator.require_all` attribute of + the first level ancestor of a child validator. """ + return self._config.get('root_require_all', self.require_all) + @property def root_document(self): """ The :attr:`~cerberus.Validator.document` attribute of the @@ -524,12 +577,12 @@ class BareValidator(object): def types(cls): """ The constraints that can be used for the 'type' rule. Type: A tuple of strings. """ - redundant_types = \ - set(cls.types_mapping) & set(cls._types_from_methods) + redundant_types = set(cls.types_mapping) & set(cls._types_from_methods) if redundant_types: - warn("These types are defined both with a method and in the" - "'types_mapping' property of this validator: %s" - % redundant_types) + warn( + "These types are defined both with a method and in the" + "'types_mapping' property of this validator: %s" % redundant_types + ) return tuple(cls.types_mapping) + cls._types_from_methods @@ -554,8 +607,7 @@ class BareValidator(object): if document is None: raise DocumentError(errors.DOCUMENT_MISSING) if not isinstance(document, Mapping): - raise DocumentError( - errors.DOCUMENT_FORMAT.format(document)) + raise DocumentError(errors.DOCUMENT_FORMAT.format(document)) self.error_handler.start(self) def _drop_remaining_rules(self, *rules): @@ -608,6 +660,8 @@ class BareValidator(object): self.__normalize_rename_fields(mapping, schema) if self.purge_unknown and not self.allow_unknown: self._normalize_purge_unknown(mapping, schema) + if self.purge_readonly: + self.__normalize_purge_readonly(mapping, schema) # Check `readonly` fields before applying default values because # a field's schema definition might contain both `readonly` and # `default`. @@ -631,13 +685,23 @@ class BareValidator(object): for field in mapping: if field in schema and 'coerce' in schema[field]: mapping[field] = self.__normalize_coerce( - schema[field]['coerce'], field, mapping[field], - schema[field].get('nullable', False), error) - elif isinstance(self.allow_unknown, Mapping) and \ - 'coerce' in self.allow_unknown: + schema[field]['coerce'], + field, + mapping[field], + schema[field].get('nullable', False), + error, + ) + elif ( + isinstance(self.allow_unknown, Mapping) + and 'coerce' in self.allow_unknown + ): mapping[field] = self.__normalize_coerce( - self.allow_unknown['coerce'], field, mapping[field], - self.allow_unknown.get('nullable', False), error) + self.allow_unknown['coerce'], + field, + mapping[field], + self.allow_unknown.get('nullable', False), + error, + ) def __normalize_coerce(self, processor, field, value, nullable, error): if isinstance(processor, _str_type): @@ -646,52 +710,60 @@ class BareValidator(object): elif isinstance(processor, Iterable): result = value for p in processor: - result = self.__normalize_coerce(p, field, result, - nullable, error) - if errors.COERCION_FAILED in \ - self.document_error_tree.fetch_errors_from( - self.document_path + (field,)): + result = self.__normalize_coerce(p, field, result, nullable, error) + if ( + errors.COERCION_FAILED + in self.document_error_tree.fetch_errors_from( + self.document_path + (field,) + ) + ): break return result try: return processor(value) except Exception as e: - if not nullable and e is not TypeError: + if not (nullable and value is None): self._error(field, error, str(e)) return value def __normalize_containers(self, mapping, schema): for field in mapping: - if field not in schema: - continue + rules = set(schema.get(field, ())) + # TODO: This check conflates validation and normalization if isinstance(mapping[field], Mapping): - if 'keyschema' in schema[field]: - self.__normalize_mapping_per_keyschema( - field, mapping, schema[field]['keyschema']) - if 'valueschema' in schema[field]: - self.__normalize_mapping_per_valueschema( - field, mapping, schema[field]['valueschema']) - if set(schema[field]) & set(('allow_unknown', 'purge_unknown', - 'schema')): + if 'keysrules' in rules: + self.__normalize_mapping_per_keysrules( + field, mapping, schema[field]['keysrules'] + ) + if 'valuesrules' in rules: + self.__normalize_mapping_per_valuesrules( + field, mapping, schema[field]['valuesrules'] + ) + if rules & set( + ('allow_unknown', 'purge_unknown', 'schema') + ) or isinstance(self.allow_unknown, Mapping): try: - self.__normalize_mapping_per_schema( - field, mapping, schema) + self.__normalize_mapping_per_schema(field, mapping, schema) except _SchemaRuleTypeError: pass + elif isinstance(mapping[field], _str_type): continue - elif isinstance(mapping[field], Sequence) and \ - 'schema' in schema[field]: - self.__normalize_sequence(field, mapping, schema) - def __normalize_mapping_per_keyschema(self, field, mapping, property_rules): + elif isinstance(mapping[field], Sequence): + if 'schema' in rules: + self.__normalize_sequence_per_schema(field, mapping, schema) + elif 'items' in rules: + self.__normalize_sequence_per_items(field, mapping, schema) + + def __normalize_mapping_per_keysrules(self, field, mapping, property_rules): schema = dict(((k, property_rules) for k in mapping[field])) document = dict(((k, k) for k in mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'keyschema'), - schema=schema) + document_crumb=field, schema_crumb=(field, 'keysrules'), schema=schema + ) result = validator.normalized(document, always_return_document=True) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2, 4]) @@ -700,46 +772,72 @@ class BareValidator(object): if k == result[k]: continue if result[k] in mapping[field]: - warn("Normalizing keys of {path}: {key} already exists, " - "its value is replaced." - .format(path='.'.join(self.document_path + (field,)), - key=k)) + warn( + "Normalizing keys of {path}: {key} already exists, " + "its value is replaced.".format( + path='.'.join(str(x) for x in self.document_path + (field,)), + key=k, + ) + ) mapping[field][result[k]] = mapping[field][k] else: mapping[field][result[k]] = mapping[field][k] del mapping[field][k] - def __normalize_mapping_per_valueschema(self, field, mapping, value_rules): + def __normalize_mapping_per_valuesrules(self, field, mapping, value_rules): schema = dict(((k, value_rules) for k in mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'valueschema'), - schema=schema) - mapping[field] = validator.normalized(mapping[field], - always_return_document=True) + document_crumb=field, schema_crumb=(field, 'valuesrules'), schema=schema + ) + mapping[field] = validator.normalized( + mapping[field], always_return_document=True + ) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) self._error(validator._errors) def __normalize_mapping_per_schema(self, field, mapping, schema): + rules = schema.get(field, {}) + if not rules and isinstance(self.allow_unknown, Mapping): + rules = self.allow_unknown validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema[field].get('schema', {}), - allow_unknown=schema[field].get('allow_unknown', self.allow_unknown), # noqa: E501 - purge_unknown=schema[field].get('purge_unknown', self.purge_unknown)) # noqa: E501 + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=rules.get('schema', {}), + allow_unknown=rules.get('allow_unknown', self.allow_unknown), # noqa: E501 + purge_unknown=rules.get('purge_unknown', self.purge_unknown), + require_all=rules.get('require_all', self.require_all), + ) # noqa: E501 value_type = type(mapping[field]) - result_value = validator.normalized(mapping[field], - always_return_document=True) + result_value = validator.normalized(mapping[field], always_return_document=True) mapping[field] = value_type(result_value) if validator._errors: self._error(validator._errors) - def __normalize_sequence(self, field, mapping, schema): - schema = dict(((k, schema[field]['schema']) - for k in range(len(mapping[field])))) + def __normalize_sequence_per_schema(self, field, mapping, schema): + schema = dict( + ((k, schema[field]['schema']) for k in range(len(mapping[field]))) + ) document = dict((k, v) for k, v in enumerate(mapping[field])) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema) + document_crumb=field, schema_crumb=(field, 'schema'), schema=schema + ) + value_type = type(mapping[field]) + result = validator.normalized(document, always_return_document=True) + mapping[field] = value_type(result.values()) + if validator._errors: + self._drop_nodes_from_errorpaths(validator._errors, [], [2]) + self._error(validator._errors) + + def __normalize_sequence_per_items(self, field, mapping, schema): + rules, values = schema[field]['items'], mapping[field] + if len(rules) != len(values): + return + schema = dict(((k, v) for k, v in enumerate(rules))) + document = dict((k, v) for k, v in enumerate(values)) + validator = self._get_child_validator( + document_crumb=field, schema_crumb=(field, 'items'), schema=schema + ) value_type = type(mapping[field]) result = validator.normalized(document, always_return_document=True) mapping[field] = value_type(result.values()) @@ -747,12 +845,17 @@ class BareValidator(object): self._drop_nodes_from_errorpaths(validator._errors, [], [2]) self._error(validator._errors) + @staticmethod + def __normalize_purge_readonly(mapping, schema): + for field in [x for x in mapping if schema.get(x, {}).get('readonly', False)]: + mapping.pop(field) + return mapping + @staticmethod def _normalize_purge_unknown(mapping, schema): """ {'type': 'boolean'} """ - for field in tuple(mapping): - if field not in schema: - del mapping[field] + for field in [x for x in mapping if x not in schema]: + mapping.pop(field) return mapping def __normalize_rename_fields(self, mapping, schema): @@ -760,10 +863,13 @@ class BareValidator(object): if field in schema: self._normalize_rename(mapping, schema, field) self._normalize_rename_handler(mapping, schema, field) - elif isinstance(self.allow_unknown, Mapping) and \ - 'rename_handler' in self.allow_unknown: + elif ( + isinstance(self.allow_unknown, Mapping) + and 'rename_handler' in self.allow_unknown + ): self._normalize_rename_handler( - mapping, {field: self.allow_unknown}, field) + mapping, {field: self.allow_unknown}, field + ) return mapping def _normalize_rename(self, mapping, schema, field): @@ -783,47 +889,62 @@ class BareValidator(object): if 'rename_handler' not in schema[field]: return new_name = self.__normalize_coerce( - schema[field]['rename_handler'], field, field, - False, errors.RENAMING_FAILED) + schema[field]['rename_handler'], field, field, False, errors.RENAMING_FAILED + ) if new_name != field: mapping[new_name] = mapping[field] del mapping[field] def __validate_readonly_fields(self, mapping, schema): - for field in (x for x in schema if x in mapping and - self._resolve_rules_set(schema[x]).get('readonly')): - self._validate_readonly(schema[field]['readonly'], field, - mapping[field]) + for field in ( + x + for x in schema + if x in mapping and self._resolve_rules_set(schema[x]).get('readonly') + ): + self._validate_readonly(schema[field]['readonly'], field, mapping[field]) def __normalize_default_fields(self, mapping, schema): - fields = [x for x in schema if x not in mapping or - mapping[x] is None and not schema[x].get('nullable', False)] + empty_fields = [ + x + for x in schema + if x not in mapping + or ( + mapping[x] is None # noqa: W503 + and not schema[x].get('nullable', False) + ) # noqa: W503 + ] + try: - fields_with_default = [x for x in fields if 'default' in schema[x]] + fields_with_default = [x for x in empty_fields if 'default' in schema[x]] except TypeError: raise _SchemaRuleTypeError for field in fields_with_default: self._normalize_default(mapping, schema, field) known_fields_states = set() - fields = [x for x in fields if 'default_setter' in schema[x]] - while fields: - field = fields.pop(0) + fields_with_default_setter = [ + x for x in empty_fields if 'default_setter' in schema[x] + ] + while fields_with_default_setter: + field = fields_with_default_setter.pop(0) try: self._normalize_default_setter(mapping, schema, field) except KeyError: - fields.append(field) + fields_with_default_setter.append(field) except Exception as e: self._error(field, errors.SETTING_DEFAULT_FAILED, str(e)) - fields_state = tuple(fields) - if fields_state in known_fields_states: - for field in fields: - self._error(field, errors.SETTING_DEFAULT_FAILED, - 'Circular dependencies of default setters.') + fields_processing_state = hash(tuple(fields_with_default_setter)) + if fields_processing_state in known_fields_states: + for field in fields_with_default_setter: + self._error( + field, + errors.SETTING_DEFAULT_FAILED, + 'Circular dependencies of default setters.', + ) break else: - known_fields_states.add(fields_state) + known_fields_states.add(fields_processing_state) def _normalize_default(self, mapping, schema, field): """ {'nullable': True} """ @@ -837,8 +958,7 @@ class BareValidator(object): if 'default_setter' in schema[field]: setter = schema[field]['default_setter'] if isinstance(setter, _str_type): - setter = self.__get_rule_handler('normalize_default_setter', - setter) + setter = self.__get_rule_handler('normalize_default_setter', setter) mapping[field] = setter(mapping) # # Validating @@ -904,11 +1024,10 @@ class BareValidator(object): if isinstance(self.allow_unknown, (Mapping, _str_type)): # validate that unknown fields matches the schema # for unknown_fields - schema_crumb = 'allow_unknown' if self.is_child \ - else '__allow_unknown__' + schema_crumb = 'allow_unknown' if self.is_child else '__allow_unknown__' validator = self._get_child_validator( - schema_crumb=schema_crumb, - schema={field: self.allow_unknown}) + schema_crumb=schema_crumb, schema={field: self.allow_unknown} + ) if not validator({field: value}, normalize=False): self._error(validator._errors) else: @@ -924,14 +1043,21 @@ class BareValidator(object): definitions = self._resolve_rules_set(definitions) value = self.document[field] - rules_queue = [x for x in self.priority_validations - if x in definitions or x in self.mandatory_validations] - rules_queue.extend(x for x in self.mandatory_validations - if x not in rules_queue) - rules_queue.extend(x for x in definitions - if x not in rules_queue and - x not in self.normalization_rules and - x not in ('allow_unknown', 'required')) + rules_queue = [ + x + for x in self.priority_validations + if x in definitions or x in self.mandatory_validations + ] + rules_queue.extend( + x for x in self.mandatory_validations if x not in rules_queue + ) + rules_queue.extend( + x + for x in definitions + if x not in rules_queue + and x not in self.normalization_rules + and x not in ('allow_unknown', 'require_all', 'meta', 'required') + ) self._remaining_rules = rules_queue while self._remaining_rules: @@ -952,10 +1078,11 @@ class BareValidator(object): _validate_allow_unknown = dummy_for_rule_validation( """ {'oneof': [{'type': 'boolean'}, {'type': ['dict', 'string'], - 'validator': 'bulk_schema'}]} """) + 'check_with': 'bulk_schema'}]} """ + ) def _validate_allowed(self, allowed_values, field, value): - """ {'type': 'list'} """ + """ {'type': 'container'} """ if isinstance(value, Iterable) and not isinstance(value, _str_type): unallowed = set(value) - set(allowed_values) if unallowed: @@ -964,10 +1091,54 @@ class BareValidator(object): if value not in allowed_values: self._error(field, errors.UNALLOWED_VALUE, value) + def _validate_check_with(self, checks, field, value): + """ {'oneof': [ + {'type': 'callable'}, + {'type': 'list', + 'schema': {'oneof': [{'type': 'callable'}, + {'type': 'string'}]}}, + {'type': 'string'} + ]} """ + if isinstance(checks, _str_type): + try: + value_checker = self.__get_rule_handler('check_with', checks) + # TODO remove on next major release + except RuntimeError: + value_checker = self.__get_rule_handler('validator', checks) + warn( + "The 'validator' rule was renamed to 'check_with'. Please update " + "your schema and method names accordingly.", + DeprecationWarning, + ) + value_checker(field, value) + elif isinstance(checks, Iterable): + for v in checks: + self._validate_check_with(v, field, value) + else: + checks(field, value, self._error) + + def _validate_contains(self, expected_values, field, value): + """ {'empty': False } """ + if not isinstance(value, Iterable): + return + + if not isinstance(expected_values, Iterable) or isinstance( + expected_values, _str_type + ): + expected_values = set((expected_values,)) + else: + expected_values = set(expected_values) + + missing_values = expected_values - set(value) + if missing_values: + self._error(field, errors.MISSING_MEMBERS, missing_values) + def _validate_dependencies(self, dependencies, field, value): """ {'type': ('dict', 'hashable', 'list'), - 'validator': 'dependencies'} """ - if isinstance(dependencies, _str_type): + 'check_with': 'dependencies'} """ + if isinstance(dependencies, _str_type) or not isinstance( + dependencies, (Iterable, Mapping) + ): dependencies = (dependencies,) if isinstance(dependencies, Sequence): @@ -975,20 +1146,24 @@ class BareValidator(object): elif isinstance(dependencies, Mapping): self.__validate_dependencies_mapping(dependencies, field) - if self.document_error_tree.fetch_node_from( - self.schema_path + (field, 'dependencies')) is not None: + if ( + self.document_error_tree.fetch_node_from( + self.schema_path + (field, 'dependencies') + ) + is not None + ): return True def __validate_dependencies_mapping(self, dependencies, field): validated_dependencies_counter = 0 error_info = {} for dependency_name, dependency_values in dependencies.items(): - if (not isinstance(dependency_values, Sequence) or - isinstance(dependency_values, _str_type)): + if not isinstance(dependency_values, Sequence) or isinstance( + dependency_values, _str_type + ): dependency_values = [dependency_values] - wanted_field, wanted_field_value = \ - self._lookup_field(dependency_name) + wanted_field, wanted_field_value = self._lookup_field(dependency_name) if wanted_field_value in dependency_values: validated_dependencies_counter += 1 else: @@ -1004,59 +1179,71 @@ class BareValidator(object): def _validate_empty(self, empty, field, value): """ {'type': 'boolean'} """ - if isinstance(value, Iterable) and len(value) == 0: + if isinstance(value, Sized) and len(value) == 0: self._drop_remaining_rules( - 'allowed', 'forbidden', 'items', 'minlength', 'maxlength', - 'regex', 'validator') + 'allowed', + 'forbidden', + 'items', + 'minlength', + 'maxlength', + 'regex', + 'check_with', + ) if not empty: self._error(field, errors.EMPTY_NOT_ALLOWED) - def _validate_excludes(self, excludes, field, value): + def _validate_excludes(self, excluded_fields, field, value): """ {'type': ('hashable', 'list'), 'schema': {'type': 'hashable'}} """ - if isinstance(excludes, Hashable): - excludes = [excludes] + if isinstance(excluded_fields, Hashable): + excluded_fields = [excluded_fields] - # Save required field to be checked latter - if 'required' in self.schema[field] and self.schema[field]['required']: + # Mark the currently evaluated field as not required for now if it actually is. + # One of the so marked will be needed to pass when required fields are checked. + if self.schema[field].get('required', self.require_all): self._unrequired_by_excludes.add(field) - for exclude in excludes: - if (exclude in self.schema and - 'required' in self.schema[exclude] and - self.schema[exclude]['required']): - self._unrequired_by_excludes.add(exclude) + for excluded_field in excluded_fields: + if excluded_field in self.schema and self.schema[field].get( + 'required', self.require_all + ): - if [True for key in excludes if key in self.document]: - # Wrap each field in `excludes` list between quotes - exclusion_str = ', '.join("'{0}'" - .format(word) for word in excludes) + self._unrequired_by_excludes.add(excluded_field) + + if any(excluded_field in self.document for excluded_field in excluded_fields): + exclusion_str = ', '.join( + "'{0}'".format(field) for field in excluded_fields + ) self._error(field, errors.EXCLUDES_FIELD, exclusion_str) def _validate_forbidden(self, forbidden_values, field, value): """ {'type': 'list'} """ - if isinstance(value, _str_type): - if value in forbidden_values: - self._error(field, errors.FORBIDDEN_VALUE, value) - elif isinstance(value, Sequence): + if isinstance(value, Sequence) and not isinstance(value, _str_type): forbidden = set(value) & set(forbidden_values) if forbidden: self._error(field, errors.FORBIDDEN_VALUES, list(forbidden)) - elif isinstance(value, int): + else: if value in forbidden_values: self._error(field, errors.FORBIDDEN_VALUE, value) def _validate_items(self, items, field, values): - """ {'type': 'list', 'validator': 'items'} """ + """ {'type': 'list', 'check_with': 'items'} """ if len(items) != len(values): self._error(field, errors.ITEMS_LENGTH, len(items), len(values)) else: - schema = dict((i, definition) for i, definition in enumerate(items)) # noqa: E501 - validator = self._get_child_validator(document_crumb=field, - schema_crumb=(field, 'items'), # noqa: E501 - schema=schema) - if not validator(dict((i, value) for i, value in enumerate(values)), - update=self.update, normalize=False): + schema = dict( + (i, definition) for i, definition in enumerate(items) + ) # noqa: E501 + validator = self._get_child_validator( + document_crumb=field, + schema_crumb=(field, 'items'), # noqa: E501 + schema=schema, + ) + if not validator( + dict((i, value) for i, value in enumerate(values)), + update=self.update, + normalize=False, + ): self._error(field, errors.BAD_ITEMS, validator._errors) def __validate_logical(self, operator, definitions, field, value): @@ -1074,8 +1261,8 @@ class BareValidator(object): schema[field]['allow_unknown'] = self.allow_unknown validator = self._get_child_validator( - schema_crumb=(field, operator, i), - schema=schema, allow_unknown=True) + schema_crumb=(field, operator, i), schema=schema, allow_unknown=True + ) if validator(self.document, update=self.update, normalize=False): valid_counter += 1 else: @@ -1086,35 +1273,27 @@ class BareValidator(object): def _validate_anyof(self, definitions, field, value): """ {'type': 'list', 'logical': 'anyof'} """ - valids, _errors = \ - self.__validate_logical('anyof', definitions, field, value) + valids, _errors = self.__validate_logical('anyof', definitions, field, value) if valids < 1: - self._error(field, errors.ANYOF, _errors, - valids, len(definitions)) + self._error(field, errors.ANYOF, _errors, valids, len(definitions)) def _validate_allof(self, definitions, field, value): """ {'type': 'list', 'logical': 'allof'} """ - valids, _errors = \ - self.__validate_logical('allof', definitions, field, value) + valids, _errors = self.__validate_logical('allof', definitions, field, value) if valids < len(definitions): - self._error(field, errors.ALLOF, _errors, - valids, len(definitions)) + self._error(field, errors.ALLOF, _errors, valids, len(definitions)) def _validate_noneof(self, definitions, field, value): """ {'type': 'list', 'logical': 'noneof'} """ - valids, _errors = \ - self.__validate_logical('noneof', definitions, field, value) + valids, _errors = self.__validate_logical('noneof', definitions, field, value) if valids > 0: - self._error(field, errors.NONEOF, _errors, - valids, len(definitions)) + self._error(field, errors.NONEOF, _errors, valids, len(definitions)) def _validate_oneof(self, definitions, field, value): """ {'type': 'list', 'logical': 'oneof'} """ - valids, _errors = \ - self.__validate_logical('oneof', definitions, field, value) + valids, _errors = self.__validate_logical('oneof', definitions, field, value) if valids != 1: - self._error(field, errors.ONEOF, _errors, - valids, len(definitions)) + self._error(field, errors.ONEOF, _errors, valids, len(definitions)) def _validate_max(self, max_value, field, value): """ {'nullable': False } """ @@ -1137,6 +1316,8 @@ class BareValidator(object): if isinstance(value, Iterable) and len(value) > max_length: self._error(field, errors.MAX_LENGTH, len(value)) + _validate_meta = dummy_for_rule_validation('') + def _validate_minlength(self, min_length, field, value): """ {'type': 'integer'} """ if isinstance(value, Iterable) and len(value) < min_length: @@ -1148,23 +1329,33 @@ class BareValidator(object): if not nullable: self._error(field, errors.NOT_NULLABLE) self._drop_remaining_rules( - 'empty', 'forbidden', 'items', 'keyschema', 'min', 'max', - 'minlength', 'maxlength', 'regex', 'schema', 'type', - 'valueschema') + 'allowed', + 'empty', + 'forbidden', + 'items', + 'keysrules', + 'min', + 'max', + 'minlength', + 'maxlength', + 'regex', + 'schema', + 'type', + 'valuesrules', + ) - def _validate_keyschema(self, schema, field, value): - """ {'type': ['dict', 'string'], 'validator': 'bulk_schema', + def _validate_keysrules(self, schema, field, value): + """ {'type': ['dict', 'string'], 'check_with': 'bulk_schema', 'forbidden': ['rename', 'rename_handler']} """ if isinstance(value, Mapping): validator = self._get_child_validator( document_crumb=field, - schema_crumb=(field, 'keyschema'), - schema=dict(((k, schema) for k in value.keys()))) - if not validator(dict(((k, k) for k in value.keys())), - normalize=False): - self._drop_nodes_from_errorpaths(validator._errors, - [], [2, 4]) - self._error(field, errors.KEYSCHEMA, validator._errors) + schema_crumb=(field, 'keysrules'), + schema=dict(((k, schema) for k in value.keys())), + ) + if not validator(dict(((k, k) for k in value.keys())), normalize=False): + self._drop_nodes_from_errorpaths(validator._errors, [], [2, 4]) + self._error(field, errors.KEYSRULES, validator._errors) def _validate_readonly(self, readonly, field, value): """ {'type': 'boolean'} """ @@ -1174,9 +1365,12 @@ class BareValidator(object): # If the document was normalized (and therefore already been # checked for readonly fields), we still have to return True # if an error was filed. - has_error = errors.READONLY_FIELD in \ - self.document_error_tree.fetch_errors_from( - self.document_path + (field,)) + has_error = ( + errors.READONLY_FIELD + in self.document_error_tree.fetch_errors_from( + self.document_path + (field,) + ) + ) if self._is_normalized and has_error: self._drop_remaining_rules() @@ -1192,41 +1386,47 @@ class BareValidator(object): _validate_required = dummy_for_rule_validation(""" {'type': 'boolean'} """) + _validate_require_all = dummy_for_rule_validation(""" {'type': 'boolean'} """) + def __validate_required_fields(self, document): """ Validates that required fields are not missing. :param document: The document being validated. """ try: - required = set(field for field, definition in self.schema.items() - if self._resolve_rules_set(definition). - get('required') is True) + required = set( + field + for field, definition in self.schema.items() + if self._resolve_rules_set(definition).get('required', self.require_all) + is True + ) except AttributeError: if self.is_child and self.schema_path[-1] == 'schema': raise _SchemaRuleTypeError else: raise required -= self._unrequired_by_excludes - missing = required - set(field for field in document - if document.get(field) is not None or - not self.ignore_none_values) + missing = required - set( + field + for field in document + if document.get(field) is not None or not self.ignore_none_values + ) for field in missing: self._error(field, errors.REQUIRED_FIELD) - # At least on field from self._unrequired_by_excludes should be - # present in document + # At least one field from self._unrequired_by_excludes should be present in + # document. if self._unrequired_by_excludes: - fields = set(field for field in document - if document.get(field) is not None) + fields = set(field for field in document if document.get(field) is not None) if self._unrequired_by_excludes.isdisjoint(fields): for field in self._unrequired_by_excludes - fields: self._error(field, errors.REQUIRED_FIELD) def _validate_schema(self, schema, field, value): """ {'type': ['dict', 'string'], - 'anyof': [{'validator': 'schema'}, - {'validator': 'bulk_schema'}]} """ + 'anyof': [{'check_with': 'schema'}, + {'check_with': 'bulk_schema'}]} """ if schema is None: return @@ -1237,12 +1437,15 @@ class BareValidator(object): def __validate_schema_mapping(self, field, schema, value): schema = self._resolve_schema(schema) - allow_unknown = self.schema[field].get('allow_unknown', - self.allow_unknown) - validator = self._get_child_validator(document_crumb=field, - schema_crumb=(field, 'schema'), - schema=schema, - allow_unknown=allow_unknown) + allow_unknown = self.schema[field].get('allow_unknown', self.allow_unknown) + require_all = self.schema[field].get('require_all', self.require_all) + validator = self._get_child_validator( + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=schema, + allow_unknown=allow_unknown, + require_all=require_all, + ) try: if not validator(value, update=self.update, normalize=False): self._error(field, errors.MAPPING_SCHEMA, validator._errors) @@ -1253,10 +1456,16 @@ class BareValidator(object): def __validate_schema_sequence(self, field, schema, value): schema = dict(((i, schema) for i in range(len(value)))) validator = self._get_child_validator( - document_crumb=field, schema_crumb=(field, 'schema'), - schema=schema, allow_unknown=self.allow_unknown) - validator(dict(((i, v) for i, v in enumerate(value))), - update=self.update, normalize=False) + document_crumb=field, + schema_crumb=(field, 'schema'), + schema=schema, + allow_unknown=self.allow_unknown, + ) + validator( + dict(((i, v) for i, v in enumerate(value))), + update=self.update, + normalize=False, + ) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) @@ -1264,7 +1473,7 @@ class BareValidator(object): def _validate_type(self, data_type, field, value): """ {'type': ['string', 'list'], - 'validator': 'type'} """ + 'check_with': 'type'} """ if not data_type: return @@ -1275,8 +1484,9 @@ class BareValidator(object): # this implementation still supports custom type validation methods type_definition = self.types_mapping.get(_type) if type_definition is not None: - matched = isinstance(value, type_definition.included_types) \ - and not isinstance(value, type_definition.excluded_types) + matched = isinstance( + value, type_definition.included_types + ) and not isinstance(value, type_definition.excluded_types) else: type_handler = self.__get_rule_handler('validate_type', _type) matched = type_handler(value) @@ -1293,43 +1503,28 @@ class BareValidator(object): self._error(field, errors.BAD_TYPE) self._drop_remaining_rules() - def _validate_validator(self, validator, field, value): - """ {'oneof': [ - {'type': 'callable'}, - {'type': 'list', - 'schema': {'oneof': [{'type': 'callable'}, - {'type': 'string'}]}}, - {'type': 'string'} - ]} """ - if isinstance(validator, _str_type): - validator = self.__get_rule_handler('validator', validator) - validator(field, value) - elif isinstance(validator, Iterable): - for v in validator: - self._validate_validator(v, field, value) - else: - validator(field, value, self._error) - - def _validate_valueschema(self, schema, field, value): - """ {'type': ['dict', 'string'], 'validator': 'bulk_schema', + def _validate_valuesrules(self, schema, field, value): + """ {'type': ['dict', 'string'], 'check_with': 'bulk_schema', 'forbidden': ['rename', 'rename_handler']} """ - schema_crumb = (field, 'valueschema') + schema_crumb = (field, 'valuesrules') if isinstance(value, Mapping): validator = self._get_child_validator( - document_crumb=field, schema_crumb=schema_crumb, - schema=dict((k, schema) for k in value)) + document_crumb=field, + schema_crumb=schema_crumb, + schema=dict((k, schema) for k in value), + ) validator(value, update=self.update, normalize=False) if validator._errors: self._drop_nodes_from_errorpaths(validator._errors, [], [2]) - self._error(field, errors.VALUESCHEMA, validator._errors) + self._error(field, errors.VALUESRULES, validator._errors) -RULE_SCHEMA_SEPARATOR = \ - "The rule's arguments are validated against this schema:" +RULE_SCHEMA_SEPARATOR = "The rule's arguments are validated against this schema:" class InspectedValidator(type): """ Metaclass for all validators """ + def __new__(cls, *args): if '__doc__' not in args[2]: args[2].update({'__doc__': args[1][0].__doc__}) @@ -1337,8 +1532,11 @@ class InspectedValidator(type): def __init__(cls, *args): def attributes_with_prefix(prefix): - return tuple(x.split('_', 2)[-1] for x in dir(cls) - if x.startswith('_' + prefix)) + return tuple( + x[len(prefix) + 2 :] + for x in dir(cls) + if x.startswith('_' + prefix + '_') + ) super(InspectedValidator, cls).__init__(*args) @@ -1346,20 +1544,27 @@ class InspectedValidator(type): for attribute in attributes_with_prefix('validate'): # TODO remove inspection of type test methods in next major release if attribute.startswith('type_'): - cls._types_from_methods += (attribute[len('type_'):],) + cls._types_from_methods += (attribute[len('type_') :],) else: - cls.validation_rules[attribute] = \ - cls.__get_rule_schema('_validate_' + attribute) + cls.validation_rules[attribute] = cls.__get_rule_schema( + '_validate_' + attribute + ) # TODO remove on next major release if cls._types_from_methods: - warn("Methods for type testing are deprecated, use TypeDefinition " - "and the 'types_mapping'-property of a Validator-instance " - "instead.", DeprecationWarning) + warn( + "Methods for type testing are deprecated, use TypeDefinition " + "and the 'types_mapping'-property of a Validator-instance " + "instead.", + DeprecationWarning, + ) - cls.validators = tuple(x for x in attributes_with_prefix('validator')) - x = cls.validation_rules['validator']['oneof'] - x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.validators + # TODO remove second summand on next major release + cls.checkers = tuple(x for x in attributes_with_prefix('check_with')) + tuple( + x for x in attributes_with_prefix('validator') + ) + x = cls.validation_rules['check_with']['oneof'] + x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.checkers for rule in (x for x in cls.mandatory_validations if x != 'nullable'): cls.validation_rules[rule]['required'] = True @@ -1367,19 +1572,20 @@ class InspectedValidator(type): cls.coercers, cls.default_setters, cls.normalization_rules = (), (), {} for attribute in attributes_with_prefix('normalize'): if attribute.startswith('coerce_'): - cls.coercers += (attribute[len('coerce_'):],) + cls.coercers += (attribute[len('coerce_') :],) elif attribute.startswith('default_setter_'): - cls.default_setters += (attribute[len('default_setter_'):],) + cls.default_setters += (attribute[len('default_setter_') :],) else: - cls.normalization_rules[attribute] = \ - cls.__get_rule_schema('_normalize_' + attribute) + cls.normalization_rules[attribute] = cls.__get_rule_schema( + '_normalize_' + attribute + ) for rule in ('coerce', 'rename_handler'): x = cls.normalization_rules[rule]['oneof'] - x[1]['schema']['oneof'][1]['allowed'] = \ - x[2]['allowed'] = cls.coercers - cls.normalization_rules['default_setter']['oneof'][1]['allowed'] = \ - cls.default_setters + x[1]['schema']['oneof'][1]['allowed'] = x[2]['allowed'] = cls.coercers + cls.normalization_rules['default_setter']['oneof'][1][ + 'allowed' + ] = cls.default_setters cls.rules = {} cls.rules.update(cls.validation_rules) @@ -1397,9 +1603,11 @@ class InspectedValidator(type): except Exception: result = {} - if not result: - warn("No validation schema is defined for the arguments of rule " - "'%s'" % method_name.split('_', 2)[-1]) + if not result and method_name != '_validate_meta': + warn( + "No validation schema is defined for the arguments of rule " + "'%s'" % method_name.split('_', 2)[-1] + ) return result diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index 50f2e130..1e2dfac7 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import contents, where -__version__ = "2018.10.15" +__version__ = "2020.04.05.1" diff --git a/pipenv/vendor/certifi/__main__.py b/pipenv/vendor/certifi/__main__.py index 5f1da0dd..8945b5da 100644 --- a/pipenv/vendor/certifi/__main__.py +++ b/pipenv/vendor/certifi/__main__.py @@ -1,2 +1,12 @@ -from certifi import where -print(where()) +import argparse + +from certifi import contents, where + +parser = argparse.ArgumentParser() +parser.add_argument("-c", "--contents", action="store_true") +args = parser.parse_args() + +if args.contents: + print(contents()) +else: + print(where()) diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index e75d85b3..ece147c9 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -771,36 +771,6 @@ vEsXCS+0yx5DaMkHJ8HSXPfqIbloEpw8nL+e/IBcm2PN7EeqJSdnoDfzAIJ9VNep +OkuE6N36B9K -----END CERTIFICATE----- -# Issuer: CN=Class 2 Primary CA O=Certplus -# Subject: CN=Class 2 Primary CA O=Certplus -# Label: "Certplus Class 2 Primary CA" -# Serial: 177770208045934040241468760488327595043 -# MD5 Fingerprint: 88:2c:8c:52:b8:a2:3c:f3:f7:bb:03:ea:ae:ac:42:0b -# SHA1 Fingerprint: 74:20:74:41:72:9c:dd:92:ec:79:31:d8:23:10:8d:c2:81:92:e2:bb -# SHA256 Fingerprint: 0f:99:3c:8a:ef:97:ba:af:56:87:14:0e:d5:9a:d1:82:1b:b4:af:ac:f0:aa:9a:58:b5:d5:7a:33:8a:3a:fb:cb ------BEGIN CERTIFICATE----- -MIIDkjCCAnqgAwIBAgIRAIW9S/PY2uNp9pTXX8OlRCMwDQYJKoZIhvcNAQEFBQAw -PTELMAkGA1UEBhMCRlIxETAPBgNVBAoTCENlcnRwbHVzMRswGQYDVQQDExJDbGFz -cyAyIFByaW1hcnkgQ0EwHhcNOTkwNzA3MTcwNTAwWhcNMTkwNzA2MjM1OTU5WjA9 -MQswCQYDVQQGEwJGUjERMA8GA1UEChMIQ2VydHBsdXMxGzAZBgNVBAMTEkNsYXNz -IDIgUHJpbWFyeSBDQTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANxQ -ltAS+DXSCHh6tlJw/W/uz7kRy1134ezpfgSN1sxvc0NXYKwzCkTsA18cgCSR5aiR -VhKC9+Ar9NuuYS6JEI1rbLqzAr3VNsVINyPi8Fo3UjMXEuLRYE2+L0ER4/YXJQyL -kcAbmXuZVg2v7tK8R1fjeUl7NIknJITesezpWE7+Tt9avkGtrAjFGA7v0lPubNCd -EgETjdyAYveVqUSISnFOYFWe2yMZeVYHDD9jC1yw4r5+FfyUM1hBOHTE4Y+L3yas -H7WLO7dDWWuwJKZtkIvEcupdM5i3y95ee++U8Rs+yskhwcWYAqqi9lt3m/V+llU0 -HGdpwPFC40es/CgcZlUCAwEAAaOBjDCBiTAPBgNVHRMECDAGAQH/AgEKMAsGA1Ud -DwQEAwIBBjAdBgNVHQ4EFgQU43Mt38sOKAze3bOkynm4jrvoMIkwEQYJYIZIAYb4 -QgEBBAQDAgEGMDcGA1UdHwQwMC4wLKAqoCiGJmh0dHA6Ly93d3cuY2VydHBsdXMu -Y29tL0NSTC9jbGFzczIuY3JsMA0GCSqGSIb3DQEBBQUAA4IBAQCnVM+IRBnL39R/ -AN9WM2K191EBkOvDP9GIROkkXe/nFL0gt5o8AP5tn9uQ3Nf0YtaLcF3n5QRIqWh8 -yfFC82x/xXp8HVGIutIKPidd3i1RTtMTZGnkLuPT55sJmabglZvOGtd/vjzOUrMR -FcEPF80Du5wlFbqidon8BvEY0JNLDnyCt6X09l/+7UCmnYR0ObncHoUW2ikbhiMA -ybuJfm6AiB4vFLQDJKgybwOaRywwvlbGp0ICcBvqQNi6BQNwB6SW//1IMwrh3KWB -kJtN3X3n57LNXMhqlfil9o3EXXgIvnsG1knPGTZQIy4I5p4FTUcY1Rbpsda2ENW7 -l7+ijrRU ------END CERTIFICATE----- - # Issuer: CN=DST Root CA X3 O=Digital Signature Trust Co. # Subject: CN=DST Root CA X3 O=Digital Signature Trust Co. # Label: "DST Root CA X3" @@ -1219,36 +1189,6 @@ t0QmwCbAr1UwnjvVNioZBPRcHv/PLLf/0P2HQBHVESO7SMAhqaQoLf0V+LBOK/Qw WyH8EZE0vkHve52Xdf+XlcCWWC/qu0bXu+TZLg== -----END CERTIFICATE----- -# Issuer: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center -# Subject: CN=Deutsche Telekom Root CA 2 O=Deutsche Telekom AG OU=T-TeleSec Trust Center -# Label: "Deutsche Telekom Root CA 2" -# Serial: 38 -# MD5 Fingerprint: 74:01:4a:91:b1:08:c4:58:ce:47:cd:f0:dd:11:53:08 -# SHA1 Fingerprint: 85:a4:08:c0:9c:19:3e:5d:51:58:7d:cd:d6:13:30:fd:8c:de:37:bf -# SHA256 Fingerprint: b6:19:1a:50:d0:c3:97:7f:7d:a9:9b:cd:aa:c8:6a:22:7d:ae:b9:67:9e:c7:0b:a3:b0:c9:d9:22:71:c1:70:d3 ------BEGIN CERTIFICATE----- -MIIDnzCCAoegAwIBAgIBJjANBgkqhkiG9w0BAQUFADBxMQswCQYDVQQGEwJERTEc -MBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxlU2Vj -IFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290IENB -IDIwHhcNOTkwNzA5MTIxMTAwWhcNMTkwNzA5MjM1OTAwWjBxMQswCQYDVQQGEwJE -RTEcMBoGA1UEChMTRGV1dHNjaGUgVGVsZWtvbSBBRzEfMB0GA1UECxMWVC1UZWxl -U2VjIFRydXN0IENlbnRlcjEjMCEGA1UEAxMaRGV1dHNjaGUgVGVsZWtvbSBSb290 -IENBIDIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCrC6M14IspFLEU -ha88EOQ5bzVdSq7d6mGNlUn0b2SjGmBmpKlAIoTZ1KXleJMOaAGtuU1cOs7TuKhC -QN/Po7qCWWqSG6wcmtoIKyUn+WkjR/Hg6yx6m/UTAtB+NHzCnjwAWav12gz1Mjwr -rFDa1sPeg5TKqAyZMg4ISFZbavva4VhYAUlfckE8FQYBjl2tqriTtM2e66foai1S -NNs671x1Udrb8zH57nGYMsRUFUQM+ZtV7a3fGAigo4aKSe5TBY8ZTNXeWHmb0moc -QqvF1afPaA+W5OFhmHZhyJF81j4A4pFQh+GdCuatl9Idxjp9y7zaAzTVjlsB9WoH -txa2bkp/AgMBAAGjQjBAMB0GA1UdDgQWBBQxw3kbuvVT1xfgiXotF2wKsyudMzAP -BgNVHRMECDAGAQH/AgEFMA4GA1UdDwEB/wQEAwIBBjANBgkqhkiG9w0BAQUFAAOC -AQEAlGRZrTlk5ynrE/5aw4sTV8gEJPB0d8Bg42f76Ymmg7+Wgnxu1MM9756Abrsp -tJh6sTtU6zkXR34ajgv8HzFZMQSyzhfzLMdiNlXiItiJVbSYSKpk+tYcNthEeFpa -IzpXl/V6ME+un2pMSyuOoAPjPuCp1NJ70rOo4nI8rZ7/gFnkm0W09juwzTkZmDLl -6iFhkOQxIY40sfcvNUqFENrnijchvllj4PKFiDFT1FQUhXB59C4Gdyd1Lx+4ivn+ -xbrYNuSD7Odlt79jWvNGr4GUN9RBjNYj1h7P9WgbRGOiWrqnNVmh5XAFmw4jV5mU -Cm26OWMohpLzGITY+9HPBVZkVw== ------END CERTIFICATE----- - # Issuer: CN=Cybertrust Global Root O=Cybertrust, Inc # Subject: CN=Cybertrust Global Root O=Cybertrust, Inc # Label: "Cybertrust Global Root" @@ -2200,6 +2140,45 @@ t/2jioSgrGK+KwmHNPBqAbubKVY8/gA3zyNs8U6qtnRGEmyR7jTV7JqR50S+kDFy SjnRBUkLp7Y3gaVdjKozXoEofKd9J+sAro03 -----END CERTIFICATE----- +# Issuer: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes +# Subject: CN=EC-ACC O=Agencia Catalana de Certificacio (NIF Q-0801176-I) OU=Serveis Publics de Certificacio/Vegeu https://www.catcert.net/verarrel (c)03/Jerarquia Entitats de Certificacio Catalanes +# Label: "EC-ACC" +# Serial: -23701579247955709139626555126524820479 +# MD5 Fingerprint: eb:f5:9d:29:0d:61:f9:42:1f:7c:c2:ba:6d:e3:15:09 +# SHA1 Fingerprint: 28:90:3a:63:5b:52:80:fa:e6:77:4c:0b:6d:a7:d6:ba:a6:4a:f2:e8 +# SHA256 Fingerprint: 88:49:7f:01:60:2f:31:54:24:6a:e2:8c:4d:5a:ef:10:f1:d8:7e:bb:76:62:6f:4a:e0:b7:f9:5b:a7:96:87:99 +-----BEGIN CERTIFICATE----- +MIIFVjCCBD6gAwIBAgIQ7is969Qh3hSoYqwE893EATANBgkqhkiG9w0BAQUFADCB +8zELMAkGA1UEBhMCRVMxOzA5BgNVBAoTMkFnZW5jaWEgQ2F0YWxhbmEgZGUgQ2Vy +dGlmaWNhY2lvIChOSUYgUS0wODAxMTc2LUkpMSgwJgYDVQQLEx9TZXJ2ZWlzIFB1 +YmxpY3MgZGUgQ2VydGlmaWNhY2lvMTUwMwYDVQQLEyxWZWdldSBodHRwczovL3d3 +dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAoYykwMzE1MDMGA1UECxMsSmVyYXJxdWlh +IEVudGl0YXRzIGRlIENlcnRpZmljYWNpbyBDYXRhbGFuZXMxDzANBgNVBAMTBkVD +LUFDQzAeFw0wMzAxMDcyMzAwMDBaFw0zMTAxMDcyMjU5NTlaMIHzMQswCQYDVQQG +EwJFUzE7MDkGA1UEChMyQWdlbmNpYSBDYXRhbGFuYSBkZSBDZXJ0aWZpY2FjaW8g +KE5JRiBRLTA4MDExNzYtSSkxKDAmBgNVBAsTH1NlcnZlaXMgUHVibGljcyBkZSBD +ZXJ0aWZpY2FjaW8xNTAzBgNVBAsTLFZlZ2V1IGh0dHBzOi8vd3d3LmNhdGNlcnQu +bmV0L3ZlcmFycmVsIChjKTAzMTUwMwYDVQQLEyxKZXJhcnF1aWEgRW50aXRhdHMg +ZGUgQ2VydGlmaWNhY2lvIENhdGFsYW5lczEPMA0GA1UEAxMGRUMtQUNDMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAsyLHT+KXQpWIR4NA9h0X84NzJB5R +85iKw5K4/0CQBXCHYMkAqbWUZRkiFRfCQ2xmRJoNBD45b6VLeqpjt4pEndljkYRm +4CgPukLjbo73FCeTae6RDqNfDrHrZqJyTxIThmV6PttPB/SnCWDaOkKZx7J/sxaV +HMf5NLWUhdWZXqBIoH7nF2W4onW4HvPlQn2v7fOKSGRdghST2MDk/7NQcvJ29rNd +QlB50JQ+awwAvthrDk4q7D7SzIKiGGUzE3eeml0aE9jD2z3Il3rucO2n5nzbcc8t +lGLfbdb1OL4/pYUKGbio2Al1QnDE6u/LDsg0qBIimAy4E5S2S+zw0JDnJwIDAQAB +o4HjMIHgMB0GA1UdEQQWMBSBEmVjX2FjY0BjYXRjZXJ0Lm5ldDAPBgNVHRMBAf8E +BTADAQH/MA4GA1UdDwEB/wQEAwIBBjAdBgNVHQ4EFgQUoMOLRKo3pUW/l4Ba0fF4 +opvpXY0wfwYDVR0gBHgwdjB0BgsrBgEEAfV4AQMBCjBlMCwGCCsGAQUFBwIBFiBo +dHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbDA1BggrBgEFBQcCAjApGidW +ZWdldSBodHRwczovL3d3dy5jYXRjZXJ0Lm5ldC92ZXJhcnJlbCAwDQYJKoZIhvcN +AQEFBQADggEBAKBIW4IB9k1IuDlVNZyAelOZ1Vr/sXE7zDkJlF7W2u++AVtd0x7Y +/X1PzaBB4DSTv8vihpw3kpBWHNzrKQXlxJ7HNd+KDM3FIUPpqojlNcAZQmNaAl6k +SBg6hW/cnbw/nZzBh7h6YQjpdwt/cKt63dmXLGQehb+8dJahw3oS7AwaboMMPOhy +Rp/7SNVel+axofjk70YllJyJ22k4vuxcDlbHZVHlUIiIv0LVKz3l+bqeLrPK9HOS +Agu+TGbrIP65y7WZf+a2E/rKS03Z7lNGBjvGTq2TWoF+bCpLagVFjPIhpDGQh2xl +nJ2lYJU6Un/10asIbvPuW/mIPX64b24D5EI= +-----END CERTIFICATE----- + # Issuer: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority # Subject: CN=Hellenic Academic and Research Institutions RootCA 2011 O=Hellenic Academic and Research Institutions Cert. Authority # Label: "Hellenic Academic and Research Institutions RootCA 2011" @@ -3453,46 +3432,6 @@ AAoACxGV2lZFA4gKn2fQ1XmxqI1AbQ3CekD6819kR5LLU7m7Wc5P/dAVUwHY3+vZ 5nbv0CO7O6l5s9UCKc2Jo5YPSjXnTkLAdc0Hz+Ys63su -----END CERTIFICATE----- -# Issuer: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 -# Subject: CN=Certinomis - Root CA O=Certinomis OU=0002 433998903 -# Label: "Certinomis - Root CA" -# Serial: 1 -# MD5 Fingerprint: 14:0a:fd:8d:a8:28:b5:38:69:db:56:7e:61:22:03:3f -# SHA1 Fingerprint: 9d:70:bb:01:a5:a4:a0:18:11:2e:f7:1c:01:b9:32:c5:34:e7:88:a8 -# SHA256 Fingerprint: 2a:99:f5:bc:11:74:b7:3c:bb:1d:62:08:84:e0:1c:34:e5:1c:cb:39:78:da:12:5f:0e:33:26:88:83:bf:41:58 ------BEGIN CERTIFICATE----- -MIIFkjCCA3qgAwIBAgIBATANBgkqhkiG9w0BAQsFADBaMQswCQYDVQQGEwJGUjET -MBEGA1UEChMKQ2VydGlub21pczEXMBUGA1UECxMOMDAwMiA0MzM5OTg5MDMxHTAb -BgNVBAMTFENlcnRpbm9taXMgLSBSb290IENBMB4XDTEzMTAyMTA5MTcxOFoXDTMz -MTAyMTA5MTcxOFowWjELMAkGA1UEBhMCRlIxEzARBgNVBAoTCkNlcnRpbm9taXMx -FzAVBgNVBAsTDjAwMDIgNDMzOTk4OTAzMR0wGwYDVQQDExRDZXJ0aW5vbWlzIC0g -Um9vdCBDQTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANTMCQosP5L2 -fxSeC5yaah1AMGT9qt8OHgZbn1CF6s2Nq0Nn3rD6foCWnoR4kkjW4znuzuRZWJfl -LieY6pOod5tK8O90gC3rMB+12ceAnGInkYjwSond3IjmFPnVAy//ldu9n+ws+hQV -WZUKxkd8aRi5pwP5ynapz8dvtF4F/u7BUrJ1Mofs7SlmO/NKFoL21prbcpjp3vDF -TKWrteoB4owuZH9kb/2jJZOLyKIOSY008B/sWEUuNKqEUL3nskoTuLAPrjhdsKkb -5nPJWqHZZkCqqU2mNAKthH6yI8H7KsZn9DS2sJVqM09xRLWtwHkziOC/7aOgFLSc -CbAK42C++PhmiM1b8XcF4LVzbsF9Ri6OSyemzTUK/eVNfaoqoynHWmgE6OXWk6Ri -wsXm9E/G+Z8ajYJJGYrKWUM66A0ywfRMEwNvbqY/kXPLynNvEiCL7sCCeN5LLsJJ -wx3tFvYk9CcbXFcx3FXuqB5vbKziRcxXV4p1VxngtViZSTYxPDMBbRZKzbgqg4SG -m/lg0h9tkQPTYKbVPZrdd5A9NaSfD171UkRpucC63M9933zZxKyGIjK8e2uR73r4 -F2iw4lNVYC2vPsKD2NkJK/DAZNuHi5HMkesE/Xa0lZrmFAYb1TQdvtj/dBxThZng -WVJKYe2InmtJiUZ+IFrZ50rlau7SZRFDAgMBAAGjYzBhMA4GA1UdDwEB/wQEAwIB -BjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBTvkUz1pcMw6C8I6tNxIqSSaHh0 -2TAfBgNVHSMEGDAWgBTvkUz1pcMw6C8I6tNxIqSSaHh02TANBgkqhkiG9w0BAQsF -AAOCAgEAfj1U2iJdGlg+O1QnurrMyOMaauo++RLrVl89UM7g6kgmJs95Vn6RHJk/ -0KGRHCwPT5iVWVO90CLYiF2cN/z7ZMF4jIuaYAnq1fohX9B0ZedQxb8uuQsLrbWw -F6YSjNRieOpWauwK0kDDPAUwPk2Ut59KA9N9J0u2/kTO+hkzGm2kQtHdzMjI1xZS -g081lLMSVX3l4kLr5JyTCcBMWwerx20RoFAXlCOotQqSD7J6wWAsOMwaplv/8gzj -qh8c3LigkyfeY+N/IZ865Z764BNqdeuWXGKRlI5nU7aJ+BIJy29SWwNyhlCVCNSN -h4YVH5Uk2KRvms6knZtt0rJ2BobGVgjF6wnaNsIbW0G+YSrjcOa4pvi2WsS9Iff/ -ql+hbHY5ZtbqTFXhADObE5hjyW/QASAJN1LnDE8+zbz1X5YnpyACleAu6AdBBR8V -btaw5BngDwKTACdyxYvRVB9dSsNAl35VpnzBMwQUAR1JIGkLGZOdblgi90AMRgwj -Y/M50n92Uaf0yKHxDHYiI0ZSKS3io0EHVmmY0gUJvGnHWmHNj4FgFU2A3ZDifcRQ -8ow7bkrHxuaAKzyBvBGAFhAn1/DNP3nMcyrDflOR1m749fPH0FFNjkulW+YZFzvW -gQncItzujrnEj1PhZ7szuIgVRs/taTX/dQ1G885x4cVrhkIGuUE= ------END CERTIFICATE----- - # Issuer: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed # Subject: CN=OISTE WISeKey Global Root GB CA O=WISeKey OU=OISTE Foundation Endorsed # Label: "OISTE WISeKey Global Root GB CA" @@ -4268,3 +4207,435 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign Root CA - G1 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign Root CA - G1" +# Serial: 235931866688319308814040 +# MD5 Fingerprint: 9c:42:84:57:dd:cb:0b:a7:2e:95:ad:b6:f3:da:bc:ac +# SHA1 Fingerprint: 8a:c7:ad:8f:73:ac:4e:c1:b5:75:4d:a5:40:f4:fc:cf:7c:b5:8e:8c +# SHA256 Fingerprint: 40:f6:af:03:46:a9:9a:a1:cd:1d:55:5a:4e:9c:ce:62:c7:f9:63:46:03:ee:40:66:15:83:3d:c8:c8:d0:03:67 +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Subject: CN=emSign ECC Root CA - G3 O=eMudhra Technologies Limited OU=emSign PKI +# Label: "emSign ECC Root CA - G3" +# Serial: 287880440101571086945156 +# MD5 Fingerprint: ce:0b:72:d1:9f:88:8e:d0:50:03:e8:e3:b8:8b:67:40 +# SHA1 Fingerprint: 30:43:fa:4f:f2:57:dc:a0:c3:80:ee:2e:58:ea:78:b2:3f:e6:bb:c1 +# SHA256 Fingerprint: 86:a1:ec:ba:08:9c:4a:8d:3b:be:27:34:c6:12:ba:34:1d:81:3e:04:3c:f9:e8:a8:62:cd:5c:57:a3:6b:be:6b +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +# Issuer: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign Root CA - C1 O=eMudhra Inc OU=emSign PKI +# Label: "emSign Root CA - C1" +# Serial: 825510296613316004955058 +# MD5 Fingerprint: d8:e3:5d:01:21:fa:78:5a:b0:df:ba:d2:ee:2a:5f:68 +# SHA1 Fingerprint: e7:2e:f1:df:fc:b2:09:28:cf:5d:d4:d5:67:37:b1:51:cb:86:4f:01 +# SHA256 Fingerprint: 12:56:09:aa:30:1d:a0:a2:49:b9:7a:82:39:cb:6a:34:21:6f:44:dc:ac:9f:39:54:b1:42:92:f2:e8:c8:60:8f +-----BEGIN CERTIFICATE----- +MIIDczCCAlugAwIBAgILAK7PALrEzzL4Q7IwDQYJKoZIhvcNAQELBQAwVjELMAkG +A1UEBhMCVVMxEzARBgNVBAsTCmVtU2lnbiBQS0kxFDASBgNVBAoTC2VNdWRocmEg +SW5jMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEMxMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowVjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMRwwGgYDVQQDExNlbVNpZ24gUm9v +dCBDQSAtIEMxMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAz+upufGZ +BczYKCFK83M0UYRWEPWgTywS4/oTmifQz/l5GnRfHXk5/Fv4cI7gklL35CX5VIPZ +HdPIWoU/Xse2B+4+wM6ar6xWQio5JXDWv7V7Nq2s9nPczdcdioOl+yuQFTdrHCZH +3DspVpNqs8FqOp099cGXOFgFixwR4+S0uF2FHYP+eF8LRWgYSKVGczQ7/g/IdrvH +GPMF0Ybzhe3nudkyrVWIzqa2kbBPrH4VI5b2P/AgNBbeCsbEBEV5f6f9vtKppa+c +xSMq9zwhbL2vj07FOrLzNBL834AaSaTUqZX3noleoomslMuoaJuvimUnzYnu3Yy1 +aylwQ6BpC+S5DwIDAQABo0IwQDAdBgNVHQ4EFgQU/qHgcB4qAzlSWkK+XJGFehiq +TbUwDgYDVR0PAQH/BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQEL +BQADggEBAMJKVvoVIXsoounlHfv4LcQ5lkFMOycsxGwYFYDGrK9HWS8mC+M2sO87 +/kOXSTKZEhVb3xEp/6tT+LvBeA+snFOvV71ojD1pM/CjoCNjO2RnIkSt1XHLVip4 +kqNPEjE2NuLe/gDEo2APJ62gsIq1NnpSob0n9CAnYuhNlCQT5AoE6TyrLshDCUrG +YQTlSTR+08TI9Q/Aqum6VF7zYytPT1DU/rl7mYw9wC68AivTxEDkigcxHpvOJpkT ++xHqmiIMERnHXhuBUDDIlhJu58tBf5E7oke3VIAb3ADMmpDqw8NQBmIMMMAVSKeo +WXzhriKi4gp6D/piq1JM4fHfyr6DDUI= +-----END CERTIFICATE----- + +# Issuer: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Subject: CN=emSign ECC Root CA - C3 O=eMudhra Inc OU=emSign PKI +# Label: "emSign ECC Root CA - C3" +# Serial: 582948710642506000014504 +# MD5 Fingerprint: 3e:53:b3:a3:81:ee:d7:10:f8:d3:b0:1d:17:92:f5:d5 +# SHA1 Fingerprint: b6:af:43:c2:9b:81:53:7d:f6:ef:6b:c3:1f:1f:60:15:0c:ee:48:66 +# SHA256 Fingerprint: bc:4d:80:9b:15:18:9d:78:db:3e:1d:8c:f4:f9:72:6a:79:5d:a1:64:3c:a5:f1:35:8e:1d:db:0e:dc:0d:7e:b3 +-----BEGIN CERTIFICATE----- +MIICKzCCAbGgAwIBAgIKe3G2gla4EnycqDAKBggqhkjOPQQDAzBaMQswCQYDVQQG +EwJVUzETMBEGA1UECxMKZW1TaWduIFBLSTEUMBIGA1UEChMLZU11ZGhyYSBJbmMx +IDAeBgNVBAMTF2VtU2lnbiBFQ0MgUm9vdCBDQSAtIEMzMB4XDTE4MDIxODE4MzAw +MFoXDTQzMDIxODE4MzAwMFowWjELMAkGA1UEBhMCVVMxEzARBgNVBAsTCmVtU2ln +biBQS0kxFDASBgNVBAoTC2VNdWRocmEgSW5jMSAwHgYDVQQDExdlbVNpZ24gRUND +IFJvb3QgQ0EgLSBDMzB2MBAGByqGSM49AgEGBSuBBAAiA2IABP2lYa57JhAd6bci +MK4G9IGzsUJxlTm801Ljr6/58pc1kjZGDoeVjbk5Wum739D+yAdBPLtVb4Ojavti +sIGJAnB9SMVK4+kiVCJNk7tCDK93nCOmfddhEc5lx/h//vXyqaNCMEAwHQYDVR0O +BBYEFPtaSNCAIEDyqOkAB2kZd6fmw/TPMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMB +Af8EBTADAQH/MAoGCCqGSM49BAMDA2gAMGUCMQC02C8Cif22TGK6Q04ThHK1rt0c +3ta13FaPWEBaLd4gTCKDypOofu4SQMfWh0/434UCMBwUZOR8loMRnLDRWmFLpg9J +0wD8ofzkpf9/rdcw0Md3f76BB1UwUCAU9Vc4CqgxUQ== +-----END CERTIFICATE----- + +# Issuer: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Subject: CN=Hongkong Post Root CA 3 O=Hongkong Post +# Label: "Hongkong Post Root CA 3" +# Serial: 46170865288971385588281144162979347873371282084 +# MD5 Fingerprint: 11:fc:9f:bd:73:30:02:8a:fd:3f:f3:58:b9:cb:20:f0 +# SHA1 Fingerprint: 58:a2:d0:ec:20:52:81:5b:c1:f3:f8:64:02:24:4e:c2:8e:02:4b:02 +# SHA256 Fingerprint: 5a:2f:c0:3f:0c:83:b0:90:bb:fa:40:60:4b:09:88:44:6c:76:36:18:3d:f9:84:6e:17:10:1a:44:7f:b8:ef:d6 +-----BEGIN CERTIFICATE----- +MIIFzzCCA7egAwIBAgIUCBZfikyl7ADJk0DfxMauI7gcWqQwDQYJKoZIhvcNAQEL +BQAwbzELMAkGA1UEBhMCSEsxEjAQBgNVBAgTCUhvbmcgS29uZzESMBAGA1UEBxMJ +SG9uZyBLb25nMRYwFAYDVQQKEw1Ib25na29uZyBQb3N0MSAwHgYDVQQDExdIb25n +a29uZyBQb3N0IFJvb3QgQ0EgMzAeFw0xNzA2MDMwMjI5NDZaFw00MjA2MDMwMjI5 +NDZaMG8xCzAJBgNVBAYTAkhLMRIwEAYDVQQIEwlIb25nIEtvbmcxEjAQBgNVBAcT +CUhvbmcgS29uZzEWMBQGA1UEChMNSG9uZ2tvbmcgUG9zdDEgMB4GA1UEAxMXSG9u +Z2tvbmcgUG9zdCBSb290IENBIDMwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIK +AoICAQCziNfqzg8gTr7m1gNt7ln8wlffKWihgw4+aMdoWJwcYEuJQwy51BWy7sFO +dem1p+/l6TWZ5Mwc50tfjTMwIDNT2aa71T4Tjukfh0mtUC1Qyhi+AViiE3CWu4mI +VoBc+L0sPOFMV4i707mV78vH9toxdCim5lSJ9UExyuUmGs2C4HDaOym71QP1mbpV +9WTRYA6ziUm4ii8F0oRFKHyPaFASePwLtVPLwpgchKOesL4jpNrcyCse2m5FHomY +2vkALgbpDDtw1VAliJnLzXNg99X/NWfFobxeq81KuEXryGgeDQ0URhLj0mRiikKY +vLTGCAj4/ahMZJx2Ab0vqWwzD9g/KLg8aQFChn5pwckGyuV6RmXpwtZQQS4/t+Tt +bNe/JgERohYpSms0BpDsE9K2+2p20jzt8NYt3eEV7KObLyzJPivkaTv/ciWxNoZb +x39ri1UbSsUgYT2uy1DhCDq+sI9jQVMwCFk8mB13umOResoQUGC/8Ne8lYePl8X+ +l2oBlKN8W4UdKjk60FSh0Tlxnf0h+bV78OLgAo9uliQlLKAeLKjEiafv7ZkGL7YK +TE/bosw3Gq9HhS2KX8Q0NEwA/RiTZxPRN+ZItIsGxVd7GYYKecsAyVKvQv83j+Gj +Hno9UKtjBucVtT+2RTeUN7F+8kjDf8V1/peNRY8apxpyKBpADwIDAQABo2MwYTAP +BgNVHRMBAf8EBTADAQH/MA4GA1UdDwEB/wQEAwIBBjAfBgNVHSMEGDAWgBQXnc0e +i9Y5K3DTXNSguB+wAPzFYTAdBgNVHQ4EFgQUF53NHovWOStw01zUoLgfsAD8xWEw +DQYJKoZIhvcNAQELBQADggIBAFbVe27mIgHSQpsY1Q7XZiNc4/6gx5LS6ZStS6LG +7BJ8dNVI0lkUmcDrudHr9EgwW62nV3OZqdPlt9EuWSRY3GguLmLYauRwCy0gUCCk +MpXRAJi70/33MvJJrsZ64Ee+bs7Lo3I6LWldy8joRTnU+kLBEUx3XZL7av9YROXr +gZ6voJmtvqkBZss4HTzfQx/0TW60uhdG/H39h4F5ag0zD/ov+BS5gLNdTaqX4fnk +GMX41TiMJjz98iji7lpJiCzfeT2OnpA8vUFKOt1b9pq0zj8lMH8yfaIDlNDceqFS +3m6TjRgm/VWsvY+b0s+v54Ysyx8Jb6NvqYTUc79NoXQbTiNg8swOqn+knEwlqLJm +Ozj/2ZQw9nKEvmhVEA/GcywWaZMH/rFF7buiVWqw2rVKAiUnhde3t4ZEFolsgCs+ +l6mc1X5VTMbeRRAc6uk7nwNT7u56AQIWeNTowr5GdogTPyK7SBIdUgC0An4hGh6c +JfTzPV4e0hz5sy229zdcxsshTrD3mUcYhcErulWuBurQB7Lcq9CClnXO0lD+mefP +L5/ndtFhKvshuzHQqp9HpLIiyhY6UFfEW0NnxWViA0kB60PZ2Pierc+xYw5F9KBa +LJstxabArahH9CdMOA0uG0k7UvToiIMrVCjU8jVStDKDYmlkDJGcn5fqdBb9HxEG +mpv0 +-----END CERTIFICATE----- + +# Issuer: CN=Entrust Root Certification Authority - G4 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2015 Entrust, Inc. - for authorized use only +# Subject: CN=Entrust Root Certification Authority - G4 O=Entrust, Inc. OU=See www.entrust.net/legal-terms/(c) 2015 Entrust, Inc. - for authorized use only +# Label: "Entrust Root Certification Authority - G4" +# Serial: 289383649854506086828220374796556676440 +# MD5 Fingerprint: 89:53:f1:83:23:b7:7c:8e:05:f1:8c:71:38:4e:1f:88 +# SHA1 Fingerprint: 14:88:4e:86:26:37:b0:26:af:59:62:5c:40:77:ec:35:29:ba:96:01 +# SHA256 Fingerprint: db:35:17:d1:f6:73:2a:2d:5a:b9:7c:53:3e:c7:07:79:ee:32:70:a6:2f:b4:ac:42:38:37:24:60:e6:f0:1e:88 +-----BEGIN CERTIFICATE----- +MIIGSzCCBDOgAwIBAgIRANm1Q3+vqTkPAAAAAFVlrVgwDQYJKoZIhvcNAQELBQAw +gb4xCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQL +Ex9TZWUgd3d3LmVudHJ1c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykg +MjAxNSBFbnRydXN0LCBJbmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAw +BgNVBAMTKUVudHJ1c3QgUm9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0 +MB4XDTE1MDUyNzExMTExNloXDTM3MTIyNzExNDExNlowgb4xCzAJBgNVBAYTAlVT +MRYwFAYDVQQKEw1FbnRydXN0LCBJbmMuMSgwJgYDVQQLEx9TZWUgd3d3LmVudHJ1 +c3QubmV0L2xlZ2FsLXRlcm1zMTkwNwYDVQQLEzAoYykgMjAxNSBFbnRydXN0LCBJ +bmMuIC0gZm9yIGF1dGhvcml6ZWQgdXNlIG9ubHkxMjAwBgNVBAMTKUVudHJ1c3Qg +Um9vdCBDZXJ0aWZpY2F0aW9uIEF1dGhvcml0eSAtIEc0MIICIjANBgkqhkiG9w0B +AQEFAAOCAg8AMIICCgKCAgEAsewsQu7i0TD/pZJH4i3DumSXbcr3DbVZwbPLqGgZ +2K+EbTBwXX7zLtJTmeH+H17ZSK9dE43b/2MzTdMAArzE+NEGCJR5WIoV3imz/f3E +T+iq4qA7ec2/a0My3dl0ELn39GjUu9CH1apLiipvKgS1sqbHoHrmSKvS0VnM1n4j +5pds8ELl3FFLFUHtSUrJ3hCX1nbB76W1NhSXNdh4IjVS70O92yfbYVaCNNzLiGAM +C1rlLAHGVK/XqsEQe9IFWrhAnoanw5CGAlZSCXqc0ieCU0plUmr1POeo8pyvi73T +DtTUXm6Hnmo9RR3RXRv06QqsYJn7ibT/mCzPfB3pAqoEmh643IhuJbNsZvc8kPNX +wbMv9W3y+8qh+CmdRouzavbmZwe+LGcKKh9asj5XxNMhIWNlUpEbsZmOeX7m640A +2Vqq6nPopIICR5b+W45UYaPrL0swsIsjdXJ8ITzI9vF01Bx7owVV7rtNOzK+mndm +nqxpkCIHH2E6lr7lmk/MBTwoWdPBDFSoWWG9yHJM6Nyfh3+9nEg2XpWjDrk4JFX8 +dWbrAuMINClKxuMrLzOg2qOGpRKX/YAr2hRC45K9PvJdXmd0LhyIRyk0X+IyqJwl +N4y6mACXi0mWHv0liqzc2thddG5msP9E36EYxr5ILzeUePiVSj9/E15dWf10hkNj +c0kCAwEAAaNCMEAwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYD +VR0OBBYEFJ84xFYjwznooHFs6FRM5Og6sb9nMA0GCSqGSIb3DQEBCwUAA4ICAQAS +5UKme4sPDORGpbZgQIeMJX6tuGguW8ZAdjwD+MlZ9POrYs4QjbRaZIxowLByQzTS +Gwv2LFPSypBLhmb8qoMi9IsabyZIrHZ3CL/FmFz0Jomee8O5ZDIBf9PD3Vht7LGr +hFV0d4QEJ1JrhkzO3bll/9bGXp+aEJlLdWr+aumXIOTkdnrG0CSqkM0gkLpHZPt/ +B7NTeLUKYvJzQ85BK4FqLoUWlFPUa19yIqtRLULVAJyZv967lDtX/Zr1hstWO1uI +AeV8KEsD+UmDfLJ/fOPtjqF/YFOOVZ1QNBIPt5d7bIdKROf1beyAN/BYGW5KaHbw +H5Lk6rWS02FREAutp9lfx1/cH6NcjKF+m7ee01ZvZl4HliDtC3T7Zk6LERXpgUl+ +b7DUUH8i119lAg2m9IUe2K4GS0qn0jFmwvjO5QimpAKWRGhXxNUzzxkvFMSUHHuk +2fCfDrGA4tGeEWSpiBE6doLlYsKA2KSD7ZPvfC+QsDJMlhVoSFLUmQjAJOgc47Ol +IQ6SwJAfzyBfyjs4x7dtOvPmRLgOMWuIjnDrnBdSqEGULoe256YSxXXfW8AKbnuk +5F6G+TaU33fD6Q3AOfF5u0aOq0NZJ7cguyPpVkAh7DE9ZapD8j3fcEThuk0mEDuY +n/PIjhs4ViFqUZPTkcpG2om3PVODLAgfi49T3f+sHw== +-----END CERTIFICATE----- diff --git a/pipenv/vendor/certifi/core.py b/pipenv/vendor/certifi/core.py index eab9d1d1..56b52a3c 100644 --- a/pipenv/vendor/certifi/core.py +++ b/pipenv/vendor/certifi/core.py @@ -1,37 +1,30 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- """ certifi.py ~~~~~~~~~~ -This module returns the installation location of cacert.pem. +This module returns the installation location of cacert.pem or its contents. """ import os -import warnings - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ +try: + from importlib.resources import read_text +except ImportError: + # This fallback will work for Python versions prior to 3.7 that lack the + # importlib.resources module but relies on the existing `where` function + # so won't address issues with environments like PyOxidizer that don't set + # __file__ on modules. + def read_text(_module, _path, encoding="ascii"): + with open(where(), "r", encoding=encoding) as data: + return data.read() def where(): f = os.path.dirname(__file__) - return os.path.join(f, 'cacert.pem') + return os.path.join(f, "cacert.pem") -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - -if __name__ == '__main__': - print(where()) +def contents(): + return read_text("certifi", "cacert.pem", encoding="ascii") diff --git a/pipenv/vendor/click/LICENSE.rst b/pipenv/vendor/click/LICENSE.rst index 87ce152a..d12a8491 100644 --- a/pipenv/vendor/click/LICENSE.rst +++ b/pipenv/vendor/click/LICENSE.rst @@ -1,39 +1,28 @@ -Copyright © 2014 by the Pallets team. +Copyright 2014 Pallets -Some rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: -Redistribution and use in source and binary forms of the software as -well as documentation, with or without modification, are permitted -provided that the following conditions are met: - -- Redistributions of source code must retain the above copyright +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -- Redistributions in binary form must reproduce the above copyright +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. -- Neither the name of the copyright holder nor the names of its +3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, -BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE -COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, -INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT -NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF -THIS SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. - ----- - -Click uses parts of optparse written by Gregory P. Ward and maintained -by the Python Software Foundation. This is limited to code in parser.py. - -Copyright © 2001-2006 Gregory P. Ward. All rights reserved. -Copyright © 2002-2006 Python Software Foundation. All rights reserved. +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/click/__init__.py b/pipenv/vendor/click/__init__.py index d3c33660..3910b803 100644 --- a/pipenv/vendor/click/__init__.py +++ b/pipenv/vendor/click/__init__.py @@ -1,97 +1,79 @@ -# -*- coding: utf-8 -*- """ -click -~~~~~ - Click is a simple Python module inspired by the stdlib optparse to make writing command line scripts fun. Unlike other modules, it's based around a simple API that does not come with too much magic and is composable. - -:copyright: © 2014 by the Pallets team. -:license: BSD, see LICENSE.rst for more details. """ - -# Core classes -from .core import Context, BaseCommand, Command, MultiCommand, Group, \ - CommandCollection, Parameter, Option, Argument - -# Globals +from .core import Argument +from .core import BaseCommand +from .core import Command +from .core import CommandCollection +from .core import Context +from .core import Group +from .core import MultiCommand +from .core import Option +from .core import Parameter +from .decorators import argument +from .decorators import command +from .decorators import confirmation_option +from .decorators import group +from .decorators import help_option +from .decorators import make_pass_decorator +from .decorators import option +from .decorators import pass_context +from .decorators import pass_obj +from .decorators import password_option +from .decorators import version_option +from .exceptions import Abort +from .exceptions import BadArgumentUsage +from .exceptions import BadOptionUsage +from .exceptions import BadParameter +from .exceptions import ClickException +from .exceptions import FileError +from .exceptions import MissingParameter +from .exceptions import NoSuchOption +from .exceptions import UsageError +from .formatting import HelpFormatter +from .formatting import wrap_text from .globals import get_current_context - -# Decorators -from .decorators import pass_context, pass_obj, make_pass_decorator, \ - command, group, argument, option, confirmation_option, \ - password_option, version_option, help_option - -# Types -from .types import ParamType, File, Path, Choice, IntRange, Tuple, \ - DateTime, STRING, INT, FLOAT, BOOL, UUID, UNPROCESSED, FloatRange - -# Utilities -from .utils import echo, get_binary_stream, get_text_stream, open_file, \ - format_filename, get_app_dir, get_os_args - -# Terminal functions -from .termui import prompt, confirm, get_terminal_size, echo_via_pager, \ - progressbar, clear, style, unstyle, secho, edit, launch, getchar, \ - pause - -# Exceptions -from .exceptions import ClickException, UsageError, BadParameter, \ - FileError, Abort, NoSuchOption, BadOptionUsage, BadArgumentUsage, \ - MissingParameter - -# Formatting -from .formatting import HelpFormatter, wrap_text - -# Parsing from .parser import OptionParser - - -__all__ = [ - # Core classes - 'Context', 'BaseCommand', 'Command', 'MultiCommand', 'Group', - 'CommandCollection', 'Parameter', 'Option', 'Argument', - - # Globals - 'get_current_context', - - # Decorators - 'pass_context', 'pass_obj', 'make_pass_decorator', 'command', 'group', - 'argument', 'option', 'confirmation_option', 'password_option', - 'version_option', 'help_option', - - # Types - 'ParamType', 'File', 'Path', 'Choice', 'IntRange', 'Tuple', - 'DateTime', 'STRING', 'INT', 'FLOAT', 'BOOL', 'UUID', 'UNPROCESSED', - 'FloatRange', - - # Utilities - 'echo', 'get_binary_stream', 'get_text_stream', 'open_file', - 'format_filename', 'get_app_dir', 'get_os_args', - - # Terminal functions - 'prompt', 'confirm', 'get_terminal_size', 'echo_via_pager', - 'progressbar', 'clear', 'style', 'unstyle', 'secho', 'edit', 'launch', - 'getchar', 'pause', - - # Exceptions - 'ClickException', 'UsageError', 'BadParameter', 'FileError', - 'Abort', 'NoSuchOption', 'BadOptionUsage', 'BadArgumentUsage', - 'MissingParameter', - - # Formatting - 'HelpFormatter', 'wrap_text', - - # Parsing - 'OptionParser', -] - +from .termui import clear +from .termui import confirm +from .termui import echo_via_pager +from .termui import edit +from .termui import get_terminal_size +from .termui import getchar +from .termui import launch +from .termui import pause +from .termui import progressbar +from .termui import prompt +from .termui import secho +from .termui import style +from .termui import unstyle +from .types import BOOL +from .types import Choice +from .types import DateTime +from .types import File +from .types import FLOAT +from .types import FloatRange +from .types import INT +from .types import IntRange +from .types import ParamType +from .types import Path +from .types import STRING +from .types import Tuple +from .types import UNPROCESSED +from .types import UUID +from .utils import echo +from .utils import format_filename +from .utils import get_app_dir +from .utils import get_binary_stream +from .utils import get_os_args +from .utils import get_text_stream +from .utils import open_file # Controls if click should emit the warning about the use of unicode # literals. disable_unicode_literals_warning = False - -__version__ = '7.0' +__version__ = "7.1.1" diff --git a/pipenv/vendor/click/_bashcomplete.py b/pipenv/vendor/click/_bashcomplete.py index a5f1084c..8bca2448 100644 --- a/pipenv/vendor/click/_bashcomplete.py +++ b/pipenv/vendor/click/_bashcomplete.py @@ -2,20 +2,22 @@ import copy import os import re -from .utils import echo +from .core import Argument +from .core import MultiCommand +from .core import Option from .parser import split_arg_string -from .core import MultiCommand, Option, Argument from .types import Choice +from .utils import echo try: from collections import abc except ImportError: import collections as abc -WORDBREAK = '=' +WORDBREAK = "=" # Note, only BASH version 4.4 and later have the nosort option. -COMPLETION_SCRIPT_BASH = ''' +COMPLETION_SCRIPT_BASH = """ %(complete_func)s() { local IFS=$'\n' COMPREPLY=( $( env COMP_WORDS="${COMP_WORDS[*]}" \\ @@ -28,7 +30,8 @@ COMPLETION_SCRIPT_BASH = ''' local COMPLETION_OPTIONS="" local BASH_VERSION_ARR=(${BASH_VERSION//./ }) # Only BASH version 4.4 and later have the nosort option. - if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] && [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then + if [ ${BASH_VERSION_ARR[0]} -gt 4 ] || ([ ${BASH_VERSION_ARR[0]} -eq 4 ] \ +&& [ ${BASH_VERSION_ARR[1]} -ge 4 ]); then COMPLETION_OPTIONS="-o nosort" fi @@ -36,13 +39,17 @@ COMPLETION_SCRIPT_BASH = ''' } %(complete_func)setup -''' +""" + +COMPLETION_SCRIPT_ZSH = """ +#compdef %(script_names)s -COMPLETION_SCRIPT_ZSH = ''' %(complete_func)s() { local -a completions local -a completions_with_descriptions local -a response + (( ! $+commands[%(script_names)s] )) && return 1 + response=("${(@f)$( env COMP_WORDS=\"${words[*]}\" \\ COMP_CWORD=$((CURRENT-1)) \\ %(autocomplete_var)s=\"complete_zsh\" \\ @@ -57,34 +64,51 @@ COMPLETION_SCRIPT_ZSH = ''' done if [ -n "$completions_with_descriptions" ]; then - _describe -V unsorted completions_with_descriptions -U -Q + _describe -V unsorted completions_with_descriptions -U fi if [ -n "$completions" ]; then - compadd -U -V unsorted -Q -a completions + compadd -U -V unsorted -a completions fi compstate[insert]="automenu" } compdef %(complete_func)s %(script_names)s -''' +""" -_invalid_ident_char_re = re.compile(r'[^a-zA-Z0-9_]') +COMPLETION_SCRIPT_FISH = ( + "complete --no-files --command %(script_names)s --arguments" + ' "(env %(autocomplete_var)s=complete_fish' + " COMP_WORDS=(commandline -cp) COMP_CWORD=(commandline -t)" + ' %(script_names)s)"' +) + +_completion_scripts = { + "bash": COMPLETION_SCRIPT_BASH, + "zsh": COMPLETION_SCRIPT_ZSH, + "fish": COMPLETION_SCRIPT_FISH, +} + +_invalid_ident_char_re = re.compile(r"[^a-zA-Z0-9_]") def get_completion_script(prog_name, complete_var, shell): - cf_name = _invalid_ident_char_re.sub('', prog_name.replace('-', '_')) - script = COMPLETION_SCRIPT_ZSH if shell == 'zsh' else COMPLETION_SCRIPT_BASH - return (script % { - 'complete_func': '_%s_completion' % cf_name, - 'script_names': prog_name, - 'autocomplete_var': complete_var, - }).strip() + ';' + cf_name = _invalid_ident_char_re.sub("", prog_name.replace("-", "_")) + script = _completion_scripts.get(shell, COMPLETION_SCRIPT_BASH) + return ( + script + % { + "complete_func": "_{}_completion".format(cf_name), + "script_names": prog_name, + "autocomplete_var": complete_var, + } + ).strip() + ";" def resolve_ctx(cli, prog_name, args): - """ - Parse into a hierarchy of contexts. Contexts are connected through the parent variable. + """Parse into a hierarchy of contexts. Contexts are connected + through the parent variable. + :param cli: command definition :param prog_name: the program that is running :param args: full list of args @@ -98,8 +122,9 @@ def resolve_ctx(cli, prog_name, args): cmd_name, cmd, args = ctx.command.resolve_command(ctx, args) if cmd is None: return ctx - ctx = cmd.make_context(cmd_name, args, parent=ctx, - resilient_parsing=True) + ctx = cmd.make_context( + cmd_name, args, parent=ctx, resilient_parsing=True + ) args = ctx.protected_args + ctx.args else: # Walk chained subcommand contexts saving the last one. @@ -107,10 +132,14 @@ def resolve_ctx(cli, prog_name, args): cmd_name, cmd, args = ctx.command.resolve_command(ctx, args) if cmd is None: return ctx - sub_ctx = cmd.make_context(cmd_name, args, parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - resilient_parsing=True) + sub_ctx = cmd.make_context( + cmd_name, + args, + parent=ctx, + allow_extra_args=True, + allow_interspersed_args=False, + resilient_parsing=True, + ) args = sub_ctx.args ctx = sub_ctx args = sub_ctx.protected_args + sub_ctx.args @@ -122,25 +151,29 @@ def resolve_ctx(cli, prog_name, args): def start_of_option(param_str): """ :param param_str: param_str to check - :return: whether or not this is the start of an option declaration (i.e. starts "-" or "--") + :return: whether or not this is the start of an option declaration + (i.e. starts "-" or "--") """ - return param_str and param_str[:1] == '-' + return param_str and param_str[:1] == "-" def is_incomplete_option(all_args, cmd_param): """ :param all_args: the full original list of args supplied :param cmd_param: the current command paramter - :return: whether or not the last option declaration (i.e. starts "-" or "--") is incomplete and - corresponds to this cmd_param. In other words whether this cmd_param option can still accept - values + :return: whether or not the last option declaration (i.e. starts + "-" or "--") is incomplete and corresponds to this cmd_param. In + other words whether this cmd_param option can still accept + values """ if not isinstance(cmd_param, Option): return False if cmd_param.is_flag: return False last_option = None - for index, arg_str in enumerate(reversed([arg for arg in all_args if arg != WORDBREAK])): + for index, arg_str in enumerate( + reversed([arg for arg in all_args if arg != WORDBREAK]) + ): if index + 1 > cmd_param.nargs: break if start_of_option(arg_str): @@ -151,10 +184,12 @@ def is_incomplete_option(all_args, cmd_param): def is_incomplete_argument(current_params, cmd_param): """ - :param current_params: the current params and values for this argument as already entered + :param current_params: the current params and values for this + argument as already entered :param cmd_param: the current command parameter - :return: whether or not the last argument is incomplete and corresponds to this cmd_param. In - other words whether or not the this cmd_param argument can still accept values + :return: whether or not the last argument is incomplete and + corresponds to this cmd_param. In other words whether or not the + this cmd_param argument can still accept values """ if not isinstance(cmd_param, Argument): return False @@ -163,8 +198,11 @@ def is_incomplete_argument(current_params, cmd_param): return True if cmd_param.nargs == -1: return True - if isinstance(current_param_values, abc.Iterable) \ - and cmd_param.nargs > 1 and len(current_param_values) < cmd_param.nargs: + if ( + isinstance(current_param_values, abc.Iterable) + and cmd_param.nargs > 1 + and len(current_param_values) < cmd_param.nargs + ): return True return False @@ -180,14 +218,16 @@ def get_user_autocompletions(ctx, args, incomplete, cmd_param): results = [] if isinstance(cmd_param.type, Choice): # Choices don't support descriptions. - results = [(c, None) - for c in cmd_param.type.choices if str(c).startswith(incomplete)] + results = [ + (c, None) for c in cmd_param.type.choices if str(c).startswith(incomplete) + ] elif cmd_param.autocompletion is not None: - dynamic_completions = cmd_param.autocompletion(ctx=ctx, - args=args, - incomplete=incomplete) - results = [c if isinstance(c, tuple) else (c, None) - for c in dynamic_completions] + dynamic_completions = cmd_param.autocompletion( + ctx=ctx, args=args, incomplete=incomplete + ) + results = [ + c if isinstance(c, tuple) else (c, None) for c in dynamic_completions + ] return results @@ -208,15 +248,25 @@ def add_subcommand_completions(ctx, incomplete, completions_out): # Add subcommand completions. if isinstance(ctx.command, MultiCommand): completions_out.extend( - [(c.name, c.get_short_help_str()) for c in get_visible_commands_starting_with(ctx, incomplete)]) + [ + (c.name, c.get_short_help_str()) + for c in get_visible_commands_starting_with(ctx, incomplete) + ] + ) - # Walk up the context list and add any other completion possibilities from chained commands + # Walk up the context list and add any other completion + # possibilities from chained commands while ctx.parent is not None: ctx = ctx.parent if isinstance(ctx.command, MultiCommand) and ctx.command.chain: - remaining_commands = [c for c in get_visible_commands_starting_with(ctx, incomplete) - if c.name not in ctx.protected_args] - completions_out.extend([(c.name, c.get_short_help_str()) for c in remaining_commands]) + remaining_commands = [ + c + for c in get_visible_commands_starting_with(ctx, incomplete) + if c.name not in ctx.protected_args + ] + completions_out.extend( + [(c.name, c.get_short_help_str()) for c in remaining_commands] + ) def get_choices(cli, prog_name, args, incomplete): @@ -233,23 +283,30 @@ def get_choices(cli, prog_name, args, incomplete): if ctx is None: return [] - # In newer versions of bash long opts with '='s are partitioned, but it's easier to parse - # without the '=' + has_double_dash = "--" in all_args + + # In newer versions of bash long opts with '='s are partitioned, but + # it's easier to parse without the '=' if start_of_option(incomplete) and WORDBREAK in incomplete: partition_incomplete = incomplete.partition(WORDBREAK) all_args.append(partition_incomplete[0]) incomplete = partition_incomplete[2] elif incomplete == WORDBREAK: - incomplete = '' + incomplete = "" completions = [] - if start_of_option(incomplete): + if not has_double_dash and start_of_option(incomplete): # completions for partial options for param in ctx.command.params: if isinstance(param, Option) and not param.hidden: - param_opts = [param_opt for param_opt in param.opts + - param.secondary_opts if param_opt not in all_args or param.multiple] - completions.extend([(o, param.help) for o in param_opts if o.startswith(incomplete)]) + param_opts = [ + param_opt + for param_opt in param.opts + param.secondary_opts + if param_opt not in all_args or param.multiple + ] + completions.extend( + [(o, param.help) for o in param_opts if o.startswith(incomplete)] + ) return completions # completion for option values from user supplied values for param in ctx.command.params: @@ -266,28 +323,53 @@ def get_choices(cli, prog_name, args, incomplete): def do_complete(cli, prog_name, include_descriptions): - cwords = split_arg_string(os.environ['COMP_WORDS']) - cword = int(os.environ['COMP_CWORD']) + cwords = split_arg_string(os.environ["COMP_WORDS"]) + cword = int(os.environ["COMP_CWORD"]) args = cwords[1:cword] try: incomplete = cwords[cword] except IndexError: - incomplete = '' + incomplete = "" for item in get_choices(cli, prog_name, args, incomplete): echo(item[0]) if include_descriptions: - # ZSH has trouble dealing with empty array parameters when returned from commands, so use a well defined character '_' to indicate no description is present. - echo(item[1] if item[1] else '_') + # ZSH has trouble dealing with empty array parameters when + # returned from commands, use '_' to indicate no description + # is present. + echo(item[1] if item[1] else "_") + + return True + + +def do_complete_fish(cli, prog_name): + cwords = split_arg_string(os.environ["COMP_WORDS"]) + incomplete = os.environ["COMP_CWORD"] + args = cwords[1:] + + for item in get_choices(cli, prog_name, args, incomplete): + if item[1]: + echo("{arg}\t{desc}".format(arg=item[0], desc=item[1])) + else: + echo(item[0]) return True def bashcomplete(cli, prog_name, complete_var, complete_instr): - if complete_instr.startswith('source'): - shell = 'zsh' if complete_instr == 'source_zsh' else 'bash' + if "_" in complete_instr: + command, shell = complete_instr.split("_", 1) + else: + command = complete_instr + shell = "bash" + + if command == "source": echo(get_completion_script(prog_name, complete_var, shell)) return True - elif complete_instr == 'complete' or complete_instr == 'complete_zsh': - return do_complete(cli, prog_name, complete_instr == 'complete_zsh') + elif command == "complete": + if shell == "fish": + return do_complete_fish(cli, prog_name) + elif shell in {"bash", "zsh"}: + return do_complete(cli, prog_name, shell == "zsh") + return False diff --git a/pipenv/vendor/click/_compat.py b/pipenv/vendor/click/_compat.py index 937e2301..ed57a18f 100644 --- a/pipenv/vendor/click/_compat.py +++ b/pipenv/vendor/click/_compat.py @@ -1,67 +1,80 @@ -import re +# flake8: noqa +import codecs import io import os +import re import sys -import codecs from weakref import WeakKeyDictionary - PY2 = sys.version_info[0] == 2 -CYGWIN = sys.platform.startswith('cygwin') +CYGWIN = sys.platform.startswith("cygwin") +MSYS2 = sys.platform.startswith("win") and ("GCC" in sys.version) # Determine local App Engine environment, per Google's own suggestion -APP_ENGINE = ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) -WIN = sys.platform.startswith('win') and not APP_ENGINE +APP_ENGINE = "APPENGINE_RUNTIME" in os.environ and "Development/" in os.environ.get( + "SERVER_SOFTWARE", "" +) +WIN = sys.platform.startswith("win") and not APP_ENGINE and not MSYS2 DEFAULT_COLUMNS = 80 -_ansi_re = re.compile(r'\033\[((?:\d|;)*)([a-zA-Z])') +_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") def get_filesystem_encoding(): return sys.getfilesystemencoding() or sys.getdefaultencoding() -def _make_text_stream(stream, encoding, errors, - force_readable=False, force_writable=False): +def _make_text_stream( + stream, encoding, errors, force_readable=False, force_writable=False +): if encoding is None: encoding = get_best_encoding(stream) if errors is None: - errors = 'replace' - return _NonClosingTextIOWrapper(stream, encoding, errors, - line_buffering=True, - force_readable=force_readable, - force_writable=force_writable) + errors = "replace" + return _NonClosingTextIOWrapper( + stream, + encoding, + errors, + line_buffering=True, + force_readable=force_readable, + force_writable=force_writable, + ) def is_ascii_encoding(encoding): """Checks if a given encoding is ascii.""" try: - return codecs.lookup(encoding).name == 'ascii' + return codecs.lookup(encoding).name == "ascii" except LookupError: return False def get_best_encoding(stream): """Returns the default stream encoding if not found.""" - rv = getattr(stream, 'encoding', None) or sys.getdefaultencoding() + rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() if is_ascii_encoding(rv): - return 'utf-8' + return "utf-8" return rv class _NonClosingTextIOWrapper(io.TextIOWrapper): - - def __init__(self, stream, encoding, errors, - force_readable=False, force_writable=False, **extra): - self._stream = stream = _FixupStream(stream, force_readable, - force_writable) + def __init__( + self, + stream, + encoding, + errors, + force_readable=False, + force_writable=False, + **extra + ): + self._stream = stream = _FixupStream(stream, force_readable, force_writable) io.TextIOWrapper.__init__(self, stream, encoding, errors, **extra) # The io module is a place where the Python 3 text behavior # was forced upon Python 2, so we need to unbreak # it to look like Python 2. if PY2: + def write(self, x): if isinstance(x, str) or is_bytes(x): try: @@ -105,7 +118,7 @@ class _FixupStream(object): return getattr(self._stream, name) def read1(self, size): - f = getattr(self._stream, 'read1', None) + f = getattr(self._stream, "read1", None) if f is not None: return f(size) # We only dispatch to readline instead of read in Python 2 as we @@ -118,7 +131,7 @@ class _FixupStream(object): def readable(self): if self._force_readable: return True - x = getattr(self._stream, 'readable', None) + x = getattr(self._stream, "readable", None) if x is not None: return x() try: @@ -130,20 +143,20 @@ class _FixupStream(object): def writable(self): if self._force_writable: return True - x = getattr(self._stream, 'writable', None) + x = getattr(self._stream, "writable", None) if x is not None: return x() try: - self._stream.write('') + self._stream.write("") except Exception: try: - self._stream.write(b'') + self._stream.write(b"") except Exception: return False return True def seekable(self): - x = getattr(self._stream, 'seekable', None) + x = getattr(self._stream, "seekable", None) if x is not None: return x() try: @@ -155,17 +168,18 @@ class _FixupStream(object): if PY2: text_type = unicode - bytes = str raw_input = raw_input string_types = (str, unicode) int_types = (int, long) iteritems = lambda x: x.iteritems() range_type = xrange + from pipes import quote as shlex_quote + def is_bytes(x): return isinstance(x, (buffer, bytearray)) - _identifier_re = re.compile(r'^[a-zA-Z_][a-zA-Z0-9_]*$') + _identifier_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") # For Windows, we need to force stdout/stdin/stderr to binary if it's # fetched for that. This obviously is not the most correct way to do @@ -193,6 +207,7 @@ if PY2: except ImportError: pass else: + def set_binary_mode(f): try: fileno = f.fileno() @@ -207,6 +222,7 @@ if PY2: except ImportError: pass else: + def set_binary_mode(f): try: fileno = f.fileno() @@ -224,42 +240,42 @@ if PY2: return set_binary_mode(sys.stdin) def get_binary_stdout(): - _wrap_std_stream('stdout') + _wrap_std_stream("stdout") return set_binary_mode(sys.stdout) def get_binary_stderr(): - _wrap_std_stream('stderr') + _wrap_std_stream("stderr") return set_binary_mode(sys.stderr) def get_text_stdin(encoding=None, errors=None): rv = _get_windows_console_stream(sys.stdin, encoding, errors) if rv is not None: return rv - return _make_text_stream(sys.stdin, encoding, errors, - force_readable=True) + return _make_text_stream(sys.stdin, encoding, errors, force_readable=True) def get_text_stdout(encoding=None, errors=None): - _wrap_std_stream('stdout') + _wrap_std_stream("stdout") rv = _get_windows_console_stream(sys.stdout, encoding, errors) if rv is not None: return rv - return _make_text_stream(sys.stdout, encoding, errors, - force_writable=True) + return _make_text_stream(sys.stdout, encoding, errors, force_writable=True) def get_text_stderr(encoding=None, errors=None): - _wrap_std_stream('stderr') + _wrap_std_stream("stderr") rv = _get_windows_console_stream(sys.stderr, encoding, errors) if rv is not None: return rv - return _make_text_stream(sys.stderr, encoding, errors, - force_writable=True) + return _make_text_stream(sys.stderr, encoding, errors, force_writable=True) def filename_to_ui(value): if isinstance(value, bytes): - value = value.decode(get_filesystem_encoding(), 'replace') + value = value.decode(get_filesystem_encoding(), "replace") return value + + else: import io + text_type = str raw_input = input string_types = (str,) @@ -268,6 +284,8 @@ else: isidentifier = lambda x: x.isidentifier() iteritems = lambda x: iter(x.items()) + from shlex import quote as shlex_quote + def is_bytes(x): return isinstance(x, (bytes, memoryview, bytearray)) @@ -281,10 +299,10 @@ else: def _is_binary_writer(stream, default=False): try: - stream.write(b'') + stream.write(b"") except Exception: try: - stream.write('') + stream.write("") return False except Exception: pass @@ -299,7 +317,7 @@ else: if _is_binary_reader(stream, False): return stream - buf = getattr(stream, 'buffer', None) + buf = getattr(stream, "buffer", None) # Same situation here; this time we assume that the buffer is # actually binary in case it's closed. @@ -314,7 +332,7 @@ else: if _is_binary_writer(stream, False): return stream - buf = getattr(stream, 'buffer', None) + buf = getattr(stream, "buffer", None) # Same situation here; this time we assume that the buffer is # actually binary in case it's closed. @@ -327,136 +345,142 @@ else: # to ASCII. This appears to happen in certain unittest # environments. It's not quite clear what the correct behavior is # but this at least will force Click to recover somehow. - return is_ascii_encoding(getattr(stream, 'encoding', None) or 'ascii') + return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") + + def _is_compat_stream_attr(stream, attr, value): + """A stream attribute is compatible if it is equal to the + desired value or the desired value is unset and the attribute + has a value. + """ + stream_value = getattr(stream, attr, None) + return stream_value == value or (value is None and stream_value is not None) def _is_compatible_text_stream(stream, encoding, errors): - stream_encoding = getattr(stream, 'encoding', None) - stream_errors = getattr(stream, 'errors', None) + """Check if a stream's encoding and errors attributes are + compatible with the desired values. + """ + return _is_compat_stream_attr( + stream, "encoding", encoding + ) and _is_compat_stream_attr(stream, "errors", errors) - # Perfect match. - if stream_encoding == encoding and stream_errors == errors: - return True - - # Otherwise, it's only a compatible stream if we did not ask for - # an encoding. - if encoding is None: - return stream_encoding is not None - - return False - - def _force_correct_text_reader(text_reader, encoding, errors, - force_readable=False): - if _is_binary_reader(text_reader, False): - binary_reader = text_reader + def _force_correct_text_stream( + text_stream, + encoding, + errors, + is_binary, + find_binary, + force_readable=False, + force_writable=False, + ): + if is_binary(text_stream, False): + binary_reader = text_stream else: - # If there is no target encoding set, we need to verify that the - # reader is not actually misconfigured. - if encoding is None and not _stream_is_misconfigured(text_reader): - return text_reader + # If the stream looks compatible, and won't default to a + # misconfigured ascii encoding, return it as-is. + if _is_compatible_text_stream(text_stream, encoding, errors) and not ( + encoding is None and _stream_is_misconfigured(text_stream) + ): + return text_stream - if _is_compatible_text_stream(text_reader, encoding, errors): - return text_reader + # Otherwise, get the underlying binary reader. + binary_reader = find_binary(text_stream) - # If the reader has no encoding, we try to find the underlying - # binary reader for it. If that fails because the environment is - # misconfigured, we silently go with the same reader because this - # is too common to happen. In that case, mojibake is better than - # exceptions. - binary_reader = _find_binary_reader(text_reader) + # If that's not possible, silently use the original reader + # and get mojibake instead of exceptions. if binary_reader is None: - return text_reader + return text_stream - # At this point, we default the errors to replace instead of strict - # because nobody handles those errors anyways and at this point - # we're so fundamentally fucked that nothing can repair it. + # Default errors to replace instead of strict in order to get + # something that works. if errors is None: - errors = 'replace' - return _make_text_stream(binary_reader, encoding, errors, - force_readable=force_readable) + errors = "replace" - def _force_correct_text_writer(text_writer, encoding, errors, - force_writable=False): - if _is_binary_writer(text_writer, False): - binary_writer = text_writer - else: - # If there is no target encoding set, we need to verify that the - # writer is not actually misconfigured. - if encoding is None and not _stream_is_misconfigured(text_writer): - return text_writer + # Wrap the binary stream in a text stream with the correct + # encoding parameters. + return _make_text_stream( + binary_reader, + encoding, + errors, + force_readable=force_readable, + force_writable=force_writable, + ) - if _is_compatible_text_stream(text_writer, encoding, errors): - return text_writer + def _force_correct_text_reader(text_reader, encoding, errors, force_readable=False): + return _force_correct_text_stream( + text_reader, + encoding, + errors, + _is_binary_reader, + _find_binary_reader, + force_readable=force_readable, + ) - # If the writer has no encoding, we try to find the underlying - # binary writer for it. If that fails because the environment is - # misconfigured, we silently go with the same writer because this - # is too common to happen. In that case, mojibake is better than - # exceptions. - binary_writer = _find_binary_writer(text_writer) - if binary_writer is None: - return text_writer - - # At this point, we default the errors to replace instead of strict - # because nobody handles those errors anyways and at this point - # we're so fundamentally fucked that nothing can repair it. - if errors is None: - errors = 'replace' - return _make_text_stream(binary_writer, encoding, errors, - force_writable=force_writable) + def _force_correct_text_writer(text_writer, encoding, errors, force_writable=False): + return _force_correct_text_stream( + text_writer, + encoding, + errors, + _is_binary_writer, + _find_binary_writer, + force_writable=force_writable, + ) def get_binary_stdin(): reader = _find_binary_reader(sys.stdin) if reader is None: - raise RuntimeError('Was not able to determine binary ' - 'stream for sys.stdin.') + raise RuntimeError("Was not able to determine binary stream for sys.stdin.") return reader def get_binary_stdout(): writer = _find_binary_writer(sys.stdout) if writer is None: - raise RuntimeError('Was not able to determine binary ' - 'stream for sys.stdout.') + raise RuntimeError( + "Was not able to determine binary stream for sys.stdout." + ) return writer def get_binary_stderr(): writer = _find_binary_writer(sys.stderr) if writer is None: - raise RuntimeError('Was not able to determine binary ' - 'stream for sys.stderr.') + raise RuntimeError( + "Was not able to determine binary stream for sys.stderr." + ) return writer def get_text_stdin(encoding=None, errors=None): rv = _get_windows_console_stream(sys.stdin, encoding, errors) if rv is not None: return rv - return _force_correct_text_reader(sys.stdin, encoding, errors, - force_readable=True) + return _force_correct_text_reader( + sys.stdin, encoding, errors, force_readable=True + ) def get_text_stdout(encoding=None, errors=None): rv = _get_windows_console_stream(sys.stdout, encoding, errors) if rv is not None: return rv - return _force_correct_text_writer(sys.stdout, encoding, errors, - force_writable=True) + return _force_correct_text_writer( + sys.stdout, encoding, errors, force_writable=True + ) def get_text_stderr(encoding=None, errors=None): rv = _get_windows_console_stream(sys.stderr, encoding, errors) if rv is not None: return rv - return _force_correct_text_writer(sys.stderr, encoding, errors, - force_writable=True) + return _force_correct_text_writer( + sys.stderr, encoding, errors, force_writable=True + ) def filename_to_ui(value): if isinstance(value, bytes): - value = value.decode(get_filesystem_encoding(), 'replace') + value = value.decode(get_filesystem_encoding(), "replace") else: - value = value.encode('utf-8', 'surrogateescape') \ - .decode('utf-8', 'replace') + value = value.encode("utf-8", "surrogateescape").decode("utf-8", "replace") return value def get_streerror(e, default=None): - if hasattr(e, 'strerror'): + if hasattr(e, "strerror"): msg = e.strerror else: if default is not None: @@ -464,60 +488,107 @@ def get_streerror(e, default=None): else: msg = str(e) if isinstance(msg, bytes): - msg = msg.decode('utf-8', 'replace') + msg = msg.decode("utf-8", "replace") return msg -def open_stream(filename, mode='r', encoding=None, errors='strict', - atomic=False): +def _wrap_io_open(file, mode, encoding, errors): + """On Python 2, :func:`io.open` returns a text file wrapper that + requires passing ``unicode`` to ``write``. Need to open the file in + binary mode then wrap it in a subclass that can write ``str`` and + ``unicode``. + + Also handles not passing ``encoding`` and ``errors`` in binary mode. + """ + binary = "b" in mode + + if binary: + kwargs = {} + else: + kwargs = {"encoding": encoding, "errors": errors} + + if not PY2 or binary: + return io.open(file, mode, **kwargs) + + f = io.open(file, "{}b".format(mode.replace("t", ""))) + return _make_text_stream(f, **kwargs) + + +def open_stream(filename, mode="r", encoding=None, errors="strict", atomic=False): + binary = "b" in mode + # Standard streams first. These are simple because they don't need # special handling for the atomic flag. It's entirely ignored. - if filename == '-': - if any(m in mode for m in ['w', 'a', 'x']): - if 'b' in mode: + if filename == "-": + if any(m in mode for m in ["w", "a", "x"]): + if binary: return get_binary_stdout(), False return get_text_stdout(encoding=encoding, errors=errors), False - if 'b' in mode: + if binary: return get_binary_stdin(), False return get_text_stdin(encoding=encoding, errors=errors), False # Non-atomic writes directly go out through the regular open functions. if not atomic: - if encoding is None: - return open(filename, mode), True - return io.open(filename, mode, encoding=encoding, errors=errors), True + return _wrap_io_open(filename, mode, encoding, errors), True # Some usability stuff for atomic writes - if 'a' in mode: + if "a" in mode: raise ValueError( - 'Appending to an existing file is not supported, because that ' - 'would involve an expensive `copy`-operation to a temporary ' - 'file. Open the file in normal `w`-mode and copy explicitly ' - 'if that\'s what you\'re after.' + "Appending to an existing file is not supported, because that" + " would involve an expensive `copy`-operation to a temporary" + " file. Open the file in normal `w`-mode and copy explicitly" + " if that's what you're after." ) - if 'x' in mode: - raise ValueError('Use the `overwrite`-parameter instead.') - if 'w' not in mode: - raise ValueError('Atomic writes only make sense with `w`-mode.') + if "x" in mode: + raise ValueError("Use the `overwrite`-parameter instead.") + if "w" not in mode: + raise ValueError("Atomic writes only make sense with `w`-mode.") # Atomic writes are more complicated. They work by opening a file # as a proxy in the same folder and then using the fdopen # functionality to wrap it in a Python file. Then we wrap it in an # atomic file that moves the file over on close. - import tempfile - fd, tmp_filename = tempfile.mkstemp(dir=os.path.dirname(filename), - prefix='.__atomic-write') + import errno + import random - if encoding is not None: - f = io.open(fd, mode, encoding=encoding, errors=errors) - else: - f = os.fdopen(fd, mode) + try: + perm = os.stat(filename).st_mode + except OSError: + perm = None + flags = os.O_RDWR | os.O_CREAT | os.O_EXCL + + if binary: + flags |= getattr(os, "O_BINARY", 0) + + while True: + tmp_filename = os.path.join( + os.path.dirname(filename), + ".__atomic-write{:08x}".format(random.randrange(1 << 32)), + ) + try: + fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) + break + except OSError as e: + if e.errno == errno.EEXIST or ( + os.name == "nt" + and e.errno == errno.EACCES + and os.path.isdir(e.filename) + and os.access(e.filename, os.W_OK) + ): + continue + raise + + if perm is not None: + os.chmod(tmp_filename, perm) # in case perm includes bits in umask + + f = _wrap_io_open(fd, mode, encoding, errors) return _AtomicFile(f, tmp_filename, os.path.realpath(filename)), True # Used in a destructor call, needs extra protection from interpreter cleanup. -if hasattr(os, 'replace'): +if hasattr(os, "replace"): _replace = os.replace _can_replace = True else: @@ -526,7 +597,6 @@ else: class _AtomicFile(object): - def __init__(self, f, tmp_filename, real_filename): self._f = f self._tmp_filename = tmp_filename @@ -568,14 +638,26 @@ get_winterm_size = None def strip_ansi(value): - return _ansi_re.sub('', value) + return _ansi_re.sub("", value) + + +def _is_jupyter_kernel_output(stream): + if WIN: + # TODO: Couldn't test on Windows, should't try to support until + # someone tests the details wrt colorama. + return + + while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): + stream = stream._stream + + return stream.__class__.__module__.startswith("ipykernel.") def should_strip_ansi(stream=None, color=None): if color is None: if stream is None: stream = sys.stdin - return not isatty(stream) + return not isatty(stream) and not _is_jupyter_kernel_output(stream) return not color @@ -590,16 +672,18 @@ if WIN: def _get_argv_encoding(): import locale + return locale.getpreferredencoding() if PY2: - def raw_input(prompt=''): + + def raw_input(prompt=""): sys.stderr.flush() if prompt: stdout = _default_text_stdout() stdout.write(prompt) stdin = _default_text_stdin() - return stdin.readline().rstrip('\r\n') + return stdin.readline().rstrip("\r\n") try: import colorama @@ -641,11 +725,15 @@ if WIN: def get_winterm_size(): win = colorama.win32.GetConsoleScreenBufferInfo( - colorama.win32.STDOUT).srWindow + colorama.win32.STDOUT + ).srWindow return win.Right - win.Left, win.Bottom - win.Top + + else: + def _get_argv_encoding(): - return getattr(sys.stdin, 'encoding', None) or get_filesystem_encoding() + return getattr(sys.stdin, "encoding", None) or get_filesystem_encoding() _get_windows_console_stream = lambda *x: None _wrap_std_stream = lambda *x: None @@ -664,6 +752,7 @@ def isatty(stream): def _make_cached_stream_func(src_func, wrapper_func): cache = WeakKeyDictionary() + def func(): stream = src_func() try: @@ -679,25 +768,23 @@ def _make_cached_stream_func(src_func, wrapper_func): except Exception: pass return rv + return func -_default_text_stdin = _make_cached_stream_func( - lambda: sys.stdin, get_text_stdin) -_default_text_stdout = _make_cached_stream_func( - lambda: sys.stdout, get_text_stdout) -_default_text_stderr = _make_cached_stream_func( - lambda: sys.stderr, get_text_stderr) +_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) +_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) +_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) binary_streams = { - 'stdin': get_binary_stdin, - 'stdout': get_binary_stdout, - 'stderr': get_binary_stderr, + "stdin": get_binary_stdin, + "stdout": get_binary_stdout, + "stderr": get_binary_stderr, } text_streams = { - 'stdin': get_text_stdin, - 'stdout': get_text_stdout, - 'stderr': get_text_stderr, + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, } diff --git a/pipenv/vendor/click/_termui_impl.py b/pipenv/vendor/click/_termui_impl.py index 00a8e5ef..c6e86cc0 100644 --- a/pipenv/vendor/click/_termui_impl.py +++ b/pipenv/vendor/click/_termui_impl.py @@ -1,34 +1,35 @@ # -*- coding: utf-8 -*- """ -click._termui_impl -~~~~~~~~~~~~~~~~~~ - This module contains implementations for the termui module. To keep the import time of Click down, some infrequently used functionality is placed in this module and only imported as needed. - -:copyright: © 2014 by the Pallets team. -:license: BSD, see LICENSE.rst for more details. """ - +import contextlib +import math import os import sys import time -import math -import contextlib -from ._compat import _default_text_stdout, range_type, PY2, isatty, \ - open_stream, strip_ansi, term_len, get_best_encoding, WIN, int_types, \ - CYGWIN -from .utils import echo + +from ._compat import _default_text_stdout +from ._compat import CYGWIN +from ._compat import get_best_encoding +from ._compat import int_types +from ._compat import isatty +from ._compat import open_stream +from ._compat import range_type +from ._compat import shlex_quote +from ._compat import strip_ansi +from ._compat import term_len +from ._compat import WIN from .exceptions import ClickException +from .utils import echo - -if os.name == 'nt': - BEFORE_BAR = '\r' - AFTER_BAR = '\n' +if os.name == "nt": + BEFORE_BAR = "\r" + AFTER_BAR = "\n" else: - BEFORE_BAR = '\r\033[?25l' - AFTER_BAR = '\033[?25h\n' + BEFORE_BAR = "\r\033[?25l" + AFTER_BAR = "\033[?25h\n" def _length_hint(obj): @@ -44,19 +45,29 @@ def _length_hint(obj): hint = get_hint(obj) except TypeError: return None - if hint is NotImplemented or \ - not isinstance(hint, int_types) or \ - hint < 0: + if hint is NotImplemented or not isinstance(hint, int_types) or hint < 0: return None return hint class ProgressBar(object): - - def __init__(self, iterable, length=None, fill_char='#', empty_char=' ', - bar_template='%(bar)s', info_sep=' ', show_eta=True, - show_percent=None, show_pos=False, item_show_func=None, - label=None, file=None, color=None, width=30): + def __init__( + self, + iterable, + length=None, + fill_char="#", + empty_char=" ", + bar_template="%(bar)s", + info_sep=" ", + show_eta=True, + show_percent=None, + show_pos=False, + item_show_func=None, + label=None, + file=None, + color=None, + width=30, + ): self.fill_char = fill_char self.empty_char = empty_char self.bar_template = bar_template @@ -65,7 +76,7 @@ class ProgressBar(object): self.show_percent = show_percent self.show_pos = show_pos self.item_show_func = item_show_func - self.label = label or '' + self.label = label or "" if file is None: file = _default_text_stdout() self.file = file @@ -77,7 +88,7 @@ class ProgressBar(object): length = _length_hint(iterable) if iterable is None: if length is None: - raise TypeError('iterable or length is required') + raise TypeError("iterable or length is required") iterable = range_type(length) self.iter = iter(iterable) self.length = length @@ -104,10 +115,21 @@ class ProgressBar(object): def __iter__(self): if not self.entered: - raise RuntimeError('You need to use progress bars in a with block.') + raise RuntimeError("You need to use progress bars in a with block.") self.render_progress() return self.generator() + def __next__(self): + # Iteration is defined in terms of a generator function, + # returned by iter(self); use that to define next(). This works + # because `self.iter` is an iterable consumed by that generator, + # so it is re-entry safe. Calling `next(self.generator())` + # twice works and does "what you want". + return next(iter(self)) + + # Python 2 compat + next = __next__ + def is_fast(self): return time.time() - self.start <= self.short_limit @@ -145,20 +167,19 @@ class ProgressBar(object): hours = t % 24 t //= 24 if t > 0: - days = t - return '%dd %02d:%02d:%02d' % (days, hours, minutes, seconds) + return "{}d {:02}:{:02}:{:02}".format(t, hours, minutes, seconds) else: - return '%02d:%02d:%02d' % (hours, minutes, seconds) - return '' + return "{:02}:{:02}:{:02}".format(hours, minutes, seconds) + return "" def format_pos(self): pos = str(self.pos) if self.length_known: - pos += '/%s' % self.length + pos += "/{}".format(self.length) return pos def format_pct(self): - return ('% 4d%%' % int(self.pct * 100))[1:] + return "{: 4}%".format(int(self.pct * 100))[1:] def format_bar(self): if self.length_known: @@ -170,9 +191,13 @@ class ProgressBar(object): else: bar = list(self.empty_char * (self.width or 1)) if self.time_per_iteration != 0: - bar[int((math.cos(self.pos * self.time_per_iteration) - / 2.0 + 0.5) * self.width)] = self.fill_char - bar = ''.join(bar) + bar[ + int( + (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) + * self.width + ) + ] = self.fill_char + bar = "".join(bar) return bar def format_progress_line(self): @@ -193,11 +218,14 @@ class ProgressBar(object): if item_info is not None: info_bits.append(item_info) - return (self.bar_template % { - 'label': self.label, - 'bar': self.format_bar(), - 'info': self.info_sep.join(info_bits) - }).rstrip() + return ( + self.bar_template + % { + "label": self.label, + "bar": self.format_bar(), + "info": self.info_sep.join(info_bits), + } + ).rstrip() def render_progress(self): from .termui import get_terminal_size @@ -214,7 +242,7 @@ class ProgressBar(object): new_width = max(0, get_terminal_size()[0] - clutter_length) if new_width < old_width: buf.append(BEFORE_BAR) - buf.append(' ' * self.max_width) + buf.append(" " * self.max_width) self.max_width = new_width self.width = new_width @@ -229,8 +257,8 @@ class ProgressBar(object): self.max_width = line_len buf.append(line) - buf.append(' ' * (clear_width - line_len)) - line = ''.join(buf) + buf.append(" " * (clear_width - line_len)) + line = "".join(buf) # Render the line only if it changed. if line != self._last_line and not self.is_fast(): @@ -270,13 +298,19 @@ class ProgressBar(object): self.finished = True def generator(self): + """Return a generator which yields the items added to the bar + during construction, and updates the progress bar *after* the + yielded block returns. """ - Returns a generator which yields the items added to the bar during - construction, and updates the progress bar *after* the yielded block - returns. - """ + # WARNING: the iterator interface for `ProgressBar` relies on + # this and only works because this is a simple generator which + # doesn't create or manage additional state. If this function + # changes, the impact should be evaluated both against + # `iter(bar)` and `next(bar)`. `next()` in particular may call + # `self.generator()` repeatedly, and this must remain safe in + # order for that interface to work. if not self.entered: - raise RuntimeError('You need to use progress bars in a with block.') + raise RuntimeError("You need to use progress bars in a with block.") if self.is_hidden: for rv in self.iter: @@ -295,24 +329,28 @@ def pager(generator, color=None): stdout = _default_text_stdout() if not isatty(sys.stdin) or not isatty(stdout): return _nullpager(stdout, generator, color) - pager_cmd = (os.environ.get('PAGER', None) or '').strip() + pager_cmd = (os.environ.get("PAGER", None) or "").strip() if pager_cmd: if WIN: return _tempfilepager(generator, pager_cmd, color) return _pipepager(generator, pager_cmd, color) - if os.environ.get('TERM') in ('dumb', 'emacs'): + if os.environ.get("TERM") in ("dumb", "emacs"): return _nullpager(stdout, generator, color) - if WIN or sys.platform.startswith('os2'): - return _tempfilepager(generator, 'more <', color) - if hasattr(os, 'system') and os.system('(less) 2>/dev/null') == 0: - return _pipepager(generator, 'less', color) + if WIN or sys.platform.startswith("os2"): + return _tempfilepager(generator, "more <", color) + if hasattr(os, "system") and os.system("(less) 2>/dev/null") == 0: + return _pipepager(generator, "less", color) import tempfile + fd, filename = tempfile.mkstemp() os.close(fd) try: - if hasattr(os, 'system') and os.system('more "%s"' % filename) == 0: - return _pipepager(generator, 'more', color) + if ( + hasattr(os, "system") + and os.system("more {}".format(shlex_quote(filename))) == 0 + ): + return _pipepager(generator, "more", color) return _nullpager(stdout, generator, color) finally: os.unlink(filename) @@ -323,28 +361,28 @@ def _pipepager(generator, cmd, color): pager through this might support colors. """ import subprocess + env = dict(os.environ) # If we're piping to less we might support colors under the # condition that - cmd_detail = cmd.rsplit('/', 1)[-1].split() - if color is None and cmd_detail[0] == 'less': - less_flags = os.environ.get('LESS', '') + ' '.join(cmd_detail[1:]) + cmd_detail = cmd.rsplit("/", 1)[-1].split() + if color is None and cmd_detail[0] == "less": + less_flags = "{}{}".format(os.environ.get("LESS", ""), " ".join(cmd_detail[1:])) if not less_flags: - env['LESS'] = '-R' + env["LESS"] = "-R" color = True - elif 'r' in less_flags or 'R' in less_flags: + elif "r" in less_flags or "R" in less_flags: color = True - c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - env=env) + c = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, env=env) encoding = get_best_encoding(c.stdin) try: for text in generator: if not color: text = strip_ansi(text) - c.stdin.write(text.encode(encoding, 'replace')) + c.stdin.write(text.encode(encoding, "replace")) except (IOError, KeyboardInterrupt): pass else: @@ -370,16 +408,17 @@ def _pipepager(generator, cmd, color): def _tempfilepager(generator, cmd, color): """Page through text by invoking a program on a temporary file.""" import tempfile + filename = tempfile.mktemp() # TODO: This never terminates if the passed generator never terminates. text = "".join(generator) if not color: text = strip_ansi(text) encoding = get_best_encoding(sys.stdout) - with open_stream(filename, 'wb')[0] as f: + with open_stream(filename, "wb")[0] as f: f.write(text.encode(encoding)) try: - os.system(cmd + ' "' + filename + '"') + os.system("{} {}".format(shlex_quote(cmd), shlex_quote(filename))) finally: os.unlink(filename) @@ -393,9 +432,7 @@ def _nullpager(stream, generator, color): class Editor(object): - - def __init__(self, editor=None, env=None, require_save=True, - extension='.txt'): + def __init__(self, editor=None, env=None, require_save=True, extension=".txt"): self.editor = editor self.env = env self.require_save = require_save @@ -404,19 +441,20 @@ class Editor(object): def get_editor(self): if self.editor is not None: return self.editor - for key in 'VISUAL', 'EDITOR': + for key in "VISUAL", "EDITOR": rv = os.environ.get(key) if rv: return rv if WIN: - return 'notepad' - for editor in 'vim', 'nano': - if os.system('which %s >/dev/null 2>&1' % editor) == 0: + return "notepad" + for editor in "sensible-editor", "vim", "nano": + if os.system("which {} >/dev/null 2>&1".format(editor)) == 0: return editor - return 'vi' + return "vi" def edit_file(self, filename): import subprocess + editor = self.get_editor() if self.env: environ = os.environ.copy() @@ -424,47 +462,49 @@ class Editor(object): else: environ = None try: - c = subprocess.Popen('%s "%s"' % (editor, filename), - env=environ, shell=True) + c = subprocess.Popen( + "{} {}".format(shlex_quote(editor), shlex_quote(filename)), + env=environ, + shell=True, + ) exit_code = c.wait() if exit_code != 0: - raise ClickException('%s: Editing failed!' % editor) + raise ClickException("{}: Editing failed!".format(editor)) except OSError as e: - raise ClickException('%s: Editing failed: %s' % (editor, e)) + raise ClickException("{}: Editing failed: {}".format(editor, e)) def edit(self, text): import tempfile - text = text or '' - if text and not text.endswith('\n'): - text += '\n' + text = text or "" + if text and not text.endswith("\n"): + text += "\n" - fd, name = tempfile.mkstemp(prefix='editor-', suffix=self.extension) + fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) try: if WIN: - encoding = 'utf-8-sig' - text = text.replace('\n', '\r\n') + encoding = "utf-8-sig" + text = text.replace("\n", "\r\n") else: - encoding = 'utf-8' + encoding = "utf-8" text = text.encode(encoding) - f = os.fdopen(fd, 'wb') + f = os.fdopen(fd, "wb") f.write(text) f.close() timestamp = os.path.getmtime(name) self.edit_file(name) - if self.require_save \ - and os.path.getmtime(name) == timestamp: + if self.require_save and os.path.getmtime(name) == timestamp: return None - f = open(name, 'rb') + f = open(name, "rb") try: rv = f.read() finally: f.close() - return rv.decode('utf-8-sig').replace('\r\n', '\n') + return rv.decode("utf-8-sig").replace("\r\n", "\n") finally: os.unlink(name) @@ -477,18 +517,18 @@ def open_url(url, wait=False, locate=False): import urllib except ImportError: import urllib - if url.startswith('file://'): + if url.startswith("file://"): url = urllib.unquote(url[7:]) return url - if sys.platform == 'darwin': - args = ['open'] + if sys.platform == "darwin": + args = ["open"] if wait: - args.append('-W') + args.append("-W") if locate: - args.append('-R') + args.append("-R") args.append(_unquote_file(url)) - null = open('/dev/null', 'w') + null = open("/dev/null", "w") try: return subprocess.Popen(args, stderr=null).wait() finally: @@ -496,44 +536,42 @@ def open_url(url, wait=False, locate=False): elif WIN: if locate: url = _unquote_file(url) - args = 'explorer /select,"%s"' % _unquote_file( - url.replace('"', '')) + args = "explorer /select,{}".format(shlex_quote(url)) else: - args = 'start %s "" "%s"' % ( - wait and '/WAIT' or '', url.replace('"', '')) + args = 'start {} "" {}'.format("/WAIT" if wait else "", shlex_quote(url)) return os.system(args) elif CYGWIN: if locate: url = _unquote_file(url) - args = 'cygstart "%s"' % (os.path.dirname(url).replace('"', '')) + args = "cygstart {}".format(shlex_quote(os.path.dirname(url))) else: - args = 'cygstart %s "%s"' % ( - wait and '-w' or '', url.replace('"', '')) + args = "cygstart {} {}".format("-w" if wait else "", shlex_quote(url)) return os.system(args) try: if locate: - url = os.path.dirname(_unquote_file(url)) or '.' + url = os.path.dirname(_unquote_file(url)) or "." else: url = _unquote_file(url) - c = subprocess.Popen(['xdg-open', url]) + c = subprocess.Popen(["xdg-open", url]) if wait: return c.wait() return 0 except OSError: - if url.startswith(('http://', 'https://')) and not locate and not wait: + if url.startswith(("http://", "https://")) and not locate and not wait: import webbrowser + webbrowser.open(url) return 0 return 1 def _translate_ch_to_exc(ch): - if ch == u'\x03': + if ch == u"\x03": raise KeyboardInterrupt() - if ch == u'\x04' and not WIN: # Unix-like, Ctrl+D + if ch == u"\x04" and not WIN: # Unix-like, Ctrl+D raise EOFError() - if ch == u'\x1a' and WIN: # Windows, Ctrl+Z + if ch == u"\x1a" and WIN: # Windows, Ctrl+Z raise EOFError() @@ -580,12 +618,14 @@ if WIN: func = msvcrt.getwch rv = func() - if rv in (u'\x00', u'\xe0'): + if rv in (u"\x00", u"\xe0"): # \x00 and \xe0 are control characters that indicate special key, # see above. rv += func() _translate_ch_to_exc(rv) return rv + + else: import tty import termios @@ -593,7 +633,7 @@ else: @contextlib.contextmanager def raw_terminal(): if not isatty(sys.stdin): - f = open('/dev/tty') + f = open("/dev/tty") fd = f.fileno() else: fd = sys.stdin.fileno() @@ -614,7 +654,7 @@ else: def getchar(echo): with raw_terminal() as fd: ch = os.read(fd, 32) - ch = ch.decode(get_best_encoding(sys.stdin), 'replace') + ch = ch.decode(get_best_encoding(sys.stdin), "replace") if echo and isatty(sys.stdout): sys.stdout.write(ch) _translate_ch_to_exc(ch) diff --git a/pipenv/vendor/click/_textwrap.py b/pipenv/vendor/click/_textwrap.py index 7e776031..6959087b 100644 --- a/pipenv/vendor/click/_textwrap.py +++ b/pipenv/vendor/click/_textwrap.py @@ -3,7 +3,6 @@ from contextlib import contextmanager class TextWrapper(textwrap.TextWrapper): - def _handle_long_word(self, reversed_chunks, cur_line, cur_len, width): space_left = max(width - cur_len, 1) @@ -35,4 +34,4 @@ class TextWrapper(textwrap.TextWrapper): if idx > 0: indent = self.subsequent_indent rv.append(indent + line) - return '\n'.join(rv) + return "\n".join(rv) diff --git a/pipenv/vendor/click/_unicodefun.py b/pipenv/vendor/click/_unicodefun.py index 620edff3..781c3652 100644 --- a/pipenv/vendor/click/_unicodefun.py +++ b/pipenv/vendor/click/_unicodefun.py @@ -1,25 +1,19 @@ +import codecs import os import sys -import codecs from ._compat import PY2 -# If someone wants to vendor click, we want to ensure the -# correct package is discovered. Ideally we could use a -# relative import here but unfortunately Python does not -# support that. -click = sys.modules[__name__.rsplit('.', 1)[0]] - - def _find_unicode_literals_frame(): import __future__ - if not hasattr(sys, '_getframe'): # not all Python implementations have it + + if not hasattr(sys, "_getframe"): # not all Python implementations have it return 0 frm = sys._getframe(1) idx = 1 while frm is not None: - if frm.f_globals.get('__name__', '').startswith('click.'): + if frm.f_globals.get("__name__", "").startswith("click."): frm = frm.f_back idx += 1 elif frm.f_code.co_flags & __future__.unicode_literals.compiler_flag: @@ -32,19 +26,27 @@ def _find_unicode_literals_frame(): def _check_for_unicode_literals(): if not __debug__: return - if not PY2 or click.disable_unicode_literals_warning: + + from . import disable_unicode_literals_warning + + if not PY2 or disable_unicode_literals_warning: return bad_frame = _find_unicode_literals_frame() if bad_frame <= 0: return from warnings import warn - warn(Warning('Click detected the use of the unicode_literals ' - '__future__ import. This is heavily discouraged ' - 'because it can introduce subtle bugs in your ' - 'code. You should instead use explicit u"" literals ' - 'for your unicode strings. For more information see ' - 'https://click.palletsprojects.com/python3/'), - stacklevel=bad_frame) + + warn( + Warning( + "Click detected the use of the unicode_literals __future__" + " import. This is heavily discouraged because it can" + " introduce subtle bugs in your code. You should instead" + ' use explicit u"" literals for your unicode strings. For' + " more information see" + " https://click.palletsprojects.com/python3/" + ), + stacklevel=bad_frame, + ) def _verify_python3_env(): @@ -53,73 +55,77 @@ def _verify_python3_env(): return try: import locale + fs_enc = codecs.lookup(locale.getpreferredencoding()).name except Exception: - fs_enc = 'ascii' - if fs_enc != 'ascii': + fs_enc = "ascii" + if fs_enc != "ascii": return - extra = '' - if os.name == 'posix': + extra = "" + if os.name == "posix": import subprocess + try: - rv = subprocess.Popen(['locale', '-a'], stdout=subprocess.PIPE, - stderr=subprocess.PIPE).communicate()[0] + rv = subprocess.Popen( + ["locale", "-a"], stdout=subprocess.PIPE, stderr=subprocess.PIPE + ).communicate()[0] except OSError: - rv = b'' + rv = b"" good_locales = set() has_c_utf8 = False # Make sure we're operating on text here. if isinstance(rv, bytes): - rv = rv.decode('ascii', 'replace') + rv = rv.decode("ascii", "replace") for line in rv.splitlines(): locale = line.strip() - if locale.lower().endswith(('.utf-8', '.utf8')): + if locale.lower().endswith((".utf-8", ".utf8")): good_locales.add(locale) - if locale.lower() in ('c.utf8', 'c.utf-8'): + if locale.lower() in ("c.utf8", "c.utf-8"): has_c_utf8 = True - extra += '\n\n' + extra += "\n\n" if not good_locales: extra += ( - 'Additional information: on this system no suitable UTF-8\n' - 'locales were discovered. This most likely requires resolving\n' - 'by reconfiguring the locale system.' + "Additional information: on this system no suitable" + " UTF-8 locales were discovered. This most likely" + " requires resolving by reconfiguring the locale" + " system." ) elif has_c_utf8: extra += ( - 'This system supports the C.UTF-8 locale which is recommended.\n' - 'You might be able to resolve your issue by exporting the\n' - 'following environment variables:\n\n' - ' export LC_ALL=C.UTF-8\n' - ' export LANG=C.UTF-8' + "This system supports the C.UTF-8 locale which is" + " recommended. You might be able to resolve your issue" + " by exporting the following environment variables:\n\n" + " export LC_ALL=C.UTF-8\n" + " export LANG=C.UTF-8" ) else: extra += ( - 'This system lists a couple of UTF-8 supporting locales that\n' - 'you can pick from. The following suitable locales were\n' - 'discovered: %s' - ) % ', '.join(sorted(good_locales)) + "This system lists a couple of UTF-8 supporting locales" + " that you can pick from. The following suitable" + " locales were discovered: {}".format(", ".join(sorted(good_locales))) + ) bad_locale = None - for locale in os.environ.get('LC_ALL'), os.environ.get('LANG'): - if locale and locale.lower().endswith(('.utf-8', '.utf8')): + for locale in os.environ.get("LC_ALL"), os.environ.get("LANG"): + if locale and locale.lower().endswith((".utf-8", ".utf8")): bad_locale = locale if locale is not None: break if bad_locale is not None: extra += ( - '\n\nClick discovered that you exported a UTF-8 locale\n' - 'but the locale system could not pick up from it because\n' - 'it does not exist. The exported locale is "%s" but it\n' - 'is not supported' - ) % bad_locale + "\n\nClick discovered that you exported a UTF-8 locale" + " but the locale system could not pick up from it" + " because it does not exist. The exported locale is" + " '{}' but it is not supported".format(bad_locale) + ) raise RuntimeError( - 'Click will abort further execution because Python 3 was' - ' configured to use ASCII as encoding for the environment.' - ' Consult https://click.palletsprojects.com/en/7.x/python3/ for' - ' mitigation steps.' + extra + "Click will abort further execution because Python 3 was" + " configured to use ASCII as encoding for the environment." + " Consult https://click.palletsprojects.com/python3/ for" + " mitigation steps.{}".format(extra) ) diff --git a/pipenv/vendor/click/_winconsole.py b/pipenv/vendor/click/_winconsole.py index bbb080dd..b6c4274a 100644 --- a/pipenv/vendor/click/_winconsole.py +++ b/pipenv/vendor/click/_winconsole.py @@ -7,24 +7,42 @@ # compared to the original patches as we do not need to patch # the entire interpreter but just work in our little world of # echo and prmopt. - +import ctypes import io import os import sys -import zlib import time -import ctypes +import zlib +from ctypes import byref +from ctypes import c_char +from ctypes import c_char_p +from ctypes import c_int +from ctypes import c_ssize_t +from ctypes import c_ulong +from ctypes import c_void_p +from ctypes import POINTER +from ctypes import py_object +from ctypes import windll +from ctypes import WinError +from ctypes import WINFUNCTYPE +from ctypes.wintypes import DWORD +from ctypes.wintypes import HANDLE +from ctypes.wintypes import LPCWSTR +from ctypes.wintypes import LPWSTR + import msvcrt -from ._compat import _NonClosingTextIOWrapper, text_type, PY2 -from ctypes import byref, POINTER, c_int, c_char, c_char_p, \ - c_void_p, py_object, c_ssize_t, c_ulong, windll, WINFUNCTYPE + +from ._compat import _NonClosingTextIOWrapper +from ._compat import PY2 +from ._compat import text_type + try: from ctypes import pythonapi + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer PyBuffer_Release = pythonapi.PyBuffer_Release except ImportError: pythonapi = None -from ctypes.wintypes import LPWSTR, LPCWSTR c_ssize_p = POINTER(c_ssize_t) @@ -33,12 +51,15 @@ kernel32 = windll.kernel32 GetStdHandle = kernel32.GetStdHandle ReadConsoleW = kernel32.ReadConsoleW WriteConsoleW = kernel32.WriteConsoleW +GetConsoleMode = kernel32.GetConsoleMode GetLastError = kernel32.GetLastError -GetCommandLineW = WINFUNCTYPE(LPWSTR)( - ('GetCommandLineW', windll.kernel32)) -CommandLineToArgvW = WINFUNCTYPE( - POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( - ('CommandLineToArgvW', windll.shell32)) +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) +LocalFree = WINFUNCTYPE(ctypes.c_void_p, ctypes.c_void_p)( + ("LocalFree", windll.kernel32) +) STDIN_HANDLE = GetStdHandle(-10) @@ -57,27 +78,27 @@ STDIN_FILENO = 0 STDOUT_FILENO = 1 STDERR_FILENO = 2 -EOF = b'\x1a' +EOF = b"\x1a" MAX_BYTES_WRITTEN = 32767 class Py_buffer(ctypes.Structure): _fields_ = [ - ('buf', c_void_p), - ('obj', py_object), - ('len', c_ssize_t), - ('itemsize', c_ssize_t), - ('readonly', c_int), - ('ndim', c_int), - ('format', c_char_p), - ('shape', c_ssize_p), - ('strides', c_ssize_p), - ('suboffsets', c_ssize_p), - ('internal', c_void_p) + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), ] if PY2: - _fields_.insert(-1, ('smalltable', c_ssize_t * 2)) + _fields_.insert(-1, ("smalltable", c_ssize_t * 2)) # On PyPy we cannot get buffers so our ability to operate here is @@ -85,6 +106,7 @@ class Py_buffer(ctypes.Structure): if pythonapi is None: get_buffer = None else: + def get_buffer(obj, writable=False): buf = Py_buffer() flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE @@ -97,7 +119,6 @@ else: class _WindowsConsoleRawIOBase(io.RawIOBase): - def __init__(self, handle): self.handle = handle @@ -107,7 +128,6 @@ class _WindowsConsoleRawIOBase(io.RawIOBase): class _WindowsConsoleReader(_WindowsConsoleRawIOBase): - def readable(self): return True @@ -116,20 +136,26 @@ class _WindowsConsoleReader(_WindowsConsoleRawIOBase): if not bytes_to_be_read: return 0 elif bytes_to_be_read % 2: - raise ValueError('cannot read odd number of bytes from ' - 'UTF-16-LE encoded console') + raise ValueError( + "cannot read odd number of bytes from UTF-16-LE encoded console" + ) buffer = get_buffer(b, writable=True) code_units_to_be_read = bytes_to_be_read // 2 code_units_read = c_ulong() - rv = ReadConsoleW(self.handle, buffer, code_units_to_be_read, - byref(code_units_read), None) + rv = ReadConsoleW( + HANDLE(self.handle), + buffer, + code_units_to_be_read, + byref(code_units_read), + None, + ) if GetLastError() == ERROR_OPERATION_ABORTED: # wait for KeyboardInterrupt time.sleep(0.1) if not rv: - raise OSError('Windows error: %s' % GetLastError()) + raise OSError("Windows error: {}".format(GetLastError())) if buffer[0] == EOF: return 0 @@ -137,27 +163,30 @@ class _WindowsConsoleReader(_WindowsConsoleRawIOBase): class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): - def writable(self): return True @staticmethod def _get_error_message(errno): if errno == ERROR_SUCCESS: - return 'ERROR_SUCCESS' + return "ERROR_SUCCESS" elif errno == ERROR_NOT_ENOUGH_MEMORY: - return 'ERROR_NOT_ENOUGH_MEMORY' - return 'Windows error %s' % errno + return "ERROR_NOT_ENOUGH_MEMORY" + return "Windows error {}".format(errno) def write(self, b): bytes_to_be_written = len(b) buf = get_buffer(b) - code_units_to_be_written = min(bytes_to_be_written, - MAX_BYTES_WRITTEN) // 2 + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 code_units_written = c_ulong() - WriteConsoleW(self.handle, buf, code_units_to_be_written, - byref(code_units_written), None) + WriteConsoleW( + HANDLE(self.handle), + buf, + code_units_to_be_written, + byref(code_units_written), + None, + ) bytes_written = 2 * code_units_written.value if bytes_written == 0 and bytes_to_be_written > 0: @@ -166,7 +195,6 @@ class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): class ConsoleStream(object): - def __init__(self, text_stream, byte_stream): self._text_stream = text_stream self.buffer = byte_stream @@ -195,9 +223,8 @@ class ConsoleStream(object): return self.buffer.isatty() def __repr__(self): - return '<ConsoleStream name=%r encoding=%r>' % ( - self.name, - self.encoding, + return "<ConsoleStream name={!r} encoding={!r}>".format( + self.name, self.encoding ) @@ -207,6 +234,7 @@ class WindowsChunkedWriter(object): attribute access apart from method 'write()' which we wrap to write in limited chunks due to a Windows limitation on binary console streams. """ + def __init__(self, wrapped): # double-underscore everything to prevent clashes with names of # attributes on the wrapped stream object. @@ -221,7 +249,7 @@ class WindowsChunkedWriter(object): while written < total_to_write: to_write = min(total_to_write - written, MAX_BYTES_WRITTEN) - self.__wrapped.write(text[written:written+to_write]) + self.__wrapped.write(text[written : written + to_write]) written += to_write @@ -230,7 +258,11 @@ _wrapped_std_streams = set() def _wrap_std_stream(name): # Python 2 & Windows 7 and below - if PY2 and sys.getwindowsversion()[:2] <= (6, 1) and name not in _wrapped_std_streams: + if ( + PY2 + and sys.getwindowsversion()[:2] <= (6, 1) + and name not in _wrapped_std_streams + ): setattr(sys, name, WindowsChunkedWriter(getattr(sys, name))) _wrapped_std_streams.add(name) @@ -238,43 +270,59 @@ def _wrap_std_stream(name): def _get_text_stdin(buffer_stream): text_stream = _NonClosingTextIOWrapper( io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), - 'utf-16-le', 'strict', line_buffering=True) + "utf-16-le", + "strict", + line_buffering=True, + ) return ConsoleStream(text_stream, buffer_stream) def _get_text_stdout(buffer_stream): text_stream = _NonClosingTextIOWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), - 'utf-16-le', 'strict', line_buffering=True) + "utf-16-le", + "strict", + line_buffering=True, + ) return ConsoleStream(text_stream, buffer_stream) def _get_text_stderr(buffer_stream): text_stream = _NonClosingTextIOWrapper( io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), - 'utf-16-le', 'strict', line_buffering=True) + "utf-16-le", + "strict", + line_buffering=True, + ) return ConsoleStream(text_stream, buffer_stream) if PY2: + def _hash_py_argv(): - return zlib.crc32('\x00'.join(sys.argv[1:])) + return zlib.crc32("\x00".join(sys.argv[1:])) _initial_argv_hash = _hash_py_argv() def _get_windows_argv(): argc = c_int(0) argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) - argv = [argv_unicode[i] for i in range(0, argc.value)] + if not argv_unicode: + raise WinError() + try: + argv = [argv_unicode[i] for i in range(0, argc.value)] + finally: + LocalFree(argv_unicode) + del argv_unicode - if not hasattr(sys, 'frozen'): + if not hasattr(sys, "frozen"): argv = argv[1:] while len(argv) > 0: arg = argv[0] - if not arg.startswith('-') or arg == '-': + if not arg.startswith("-") or arg == "-": break argv = argv[1:] - if arg.startswith(('-c', '-m')): + if arg.startswith(("-c", "-m")): break return argv[1:] @@ -287,15 +335,30 @@ _stream_factories = { } +def _is_console(f): + if not hasattr(f, "fileno"): + return False + + try: + fileno = f.fileno() + except OSError: + return False + + handle = msvcrt.get_osfhandle(fileno) + return bool(GetConsoleMode(handle, byref(DWORD()))) + + def _get_windows_console_stream(f, encoding, errors): - if get_buffer is not None and \ - encoding in ('utf-16-le', None) \ - and errors in ('strict', None) and \ - hasattr(f, 'isatty') and f.isatty(): + if ( + get_buffer is not None + and encoding in ("utf-16-le", None) + and errors in ("strict", None) + and _is_console(f) + ): func = _stream_factories.get(f.fileno()) if func is not None: if not PY2: - f = getattr(f, 'buffer', None) + f = getattr(f, "buffer", None) if f is None: return None else: diff --git a/pipenv/vendor/click/core.py b/pipenv/vendor/click/core.py index 7a1e3422..f58bf26d 100644 --- a/pipenv/vendor/click/core.py +++ b/pipenv/vendor/click/core.py @@ -3,37 +3,51 @@ import inspect import os import sys from contextlib import contextmanager -from itertools import repeat from functools import update_wrapper +from itertools import repeat -from .types import convert_type, IntRange, BOOL -from .utils import PacifyFlushWrapper, make_str, make_default_short_help, \ - echo, get_os_args -from .exceptions import ClickException, UsageError, BadParameter, Abort, \ - MissingParameter, Exit -from .termui import prompt, confirm, style -from .formatting import HelpFormatter, join_options -from .parser import OptionParser, split_opt -from .globals import push_context, pop_context - -from ._compat import PY2, isidentifier, iteritems, string_types -from ._unicodefun import _check_for_unicode_literals, _verify_python3_env - +from ._compat import isidentifier +from ._compat import iteritems +from ._compat import PY2 +from ._compat import string_types +from ._unicodefun import _check_for_unicode_literals +from ._unicodefun import _verify_python3_env +from .exceptions import Abort +from .exceptions import BadParameter +from .exceptions import ClickException +from .exceptions import Exit +from .exceptions import MissingParameter +from .exceptions import UsageError +from .formatting import HelpFormatter +from .formatting import join_options +from .globals import pop_context +from .globals import push_context +from .parser import OptionParser +from .parser import split_opt +from .termui import confirm +from .termui import prompt +from .termui import style +from .types import BOOL +from .types import convert_type +from .types import IntRange +from .utils import echo +from .utils import get_os_args +from .utils import make_default_short_help +from .utils import make_str +from .utils import PacifyFlushWrapper _missing = object() +SUBCOMMAND_METAVAR = "COMMAND [ARGS]..." +SUBCOMMANDS_METAVAR = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." -SUBCOMMAND_METAVAR = 'COMMAND [ARGS]...' -SUBCOMMANDS_METAVAR = 'COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]...' - -DEPRECATED_HELP_NOTICE = ' (DEPRECATED)' -DEPRECATED_INVOKE_NOTICE = 'DeprecationWarning: ' + \ - 'The command %(name)s is deprecated.' +DEPRECATED_HELP_NOTICE = " (DEPRECATED)" +DEPRECATED_INVOKE_NOTICE = "DeprecationWarning: The command %(name)s is deprecated." def _maybe_show_deprecated_notice(cmd): if cmd.deprecated: - echo(style(DEPRECATED_INVOKE_NOTICE % {'name': cmd.name}, fg='red'), err=True) + echo(style(DEPRECATED_INVOKE_NOTICE % {"name": cmd.name}, fg="red"), err=True) def fast_exit(code): @@ -48,12 +62,13 @@ def fast_exit(code): def _bashcomplete(cmd, prog_name, complete_var=None): """Internal handler for the bash completion support.""" if complete_var is None: - complete_var = '_%s_COMPLETE' % (prog_name.replace('-', '_')).upper() + complete_var = "_{}_COMPLETE".format(prog_name.replace("-", "_").upper()) complete_instr = os.environ.get(complete_var) if not complete_instr: return from ._bashcomplete import bashcomplete + if bashcomplete(cmd, prog_name, complete_var, complete_instr): fast_exit(1) @@ -62,19 +77,28 @@ def _check_multicommand(base_command, cmd_name, cmd, register=False): if not base_command.chain or not isinstance(cmd, MultiCommand): return if register: - hint = 'It is not possible to add multi commands as children to ' \ - 'another multi command that is in chain mode' + hint = ( + "It is not possible to add multi commands as children to" + " another multi command that is in chain mode." + ) else: - hint = 'Found a multi command as subcommand to a multi command ' \ - 'that is in chain mode. This is not supported' - raise RuntimeError('%s. Command "%s" is set to chain and "%s" was ' - 'added as subcommand but it in itself is a ' - 'multi command. ("%s" is a %s within a chained ' - '%s named "%s").' % ( - hint, base_command.name, cmd_name, - cmd_name, cmd.__class__.__name__, - base_command.__class__.__name__, - base_command.name)) + hint = ( + "Found a multi command as subcommand to a multi command" + " that is in chain mode. This is not supported." + ) + raise RuntimeError( + "{}. Command '{}' is set to chain and '{}' was added as" + " subcommand but it in itself is a multi command. ('{}' is a {}" + " within a chained {} named '{}').".format( + hint, + base_command.name, + cmd_name, + cmd_name, + cmd.__class__.__name__, + base_command.__class__.__name__, + base_command.name, + ) + ) def batch(iterable, batch_size): @@ -82,25 +106,26 @@ def batch(iterable, batch_size): def invoke_param_callback(callback, ctx, param, value): - code = getattr(callback, '__code__', None) - args = getattr(code, 'co_argcount', 3) + code = getattr(callback, "__code__", None) + args = getattr(code, "co_argcount", 3) if args < 3: - # This will become a warning in Click 3.0: from warnings import warn - warn(Warning('Invoked legacy parameter callback "%s". The new ' - 'signature for such callbacks starting with ' - 'click 2.0 is (ctx, param, value).' - % callback), stacklevel=3) + + warn( + "Parameter callbacks take 3 args, (ctx, param, value). The" + " 2-arg style is deprecated and will be removed in 8.0.".format(callback), + DeprecationWarning, + stacklevel=3, + ) return callback(ctx, value) + return callback(ctx, param, value) @contextmanager def augment_usage_errors(ctx, param=None): - """Context manager that attaches extra information to exceptions that - fly. - """ + """Context manager that attaches extra information to exceptions.""" try: yield except BadParameter as e: @@ -120,11 +145,12 @@ def iter_params_for_processing(invocation_order, declaration_order): for processing and an iterable of parameters that exist, this returns a list in the correct order as they should be processed. """ + def sort_key(item): try: idx = invocation_order.index(item) except ValueError: - idx = float('inf') + idx = float("inf") return (not item.is_eager, idx) return sorted(declaration_order, key=sort_key) @@ -154,6 +180,9 @@ class Context(object): Added the `color`, `ignore_unknown_options`, and `max_content_width` parameters. + .. versionadded:: 7.1 + Added the `show_default` parameter. + :param command: the command class for this context. :param parent: the parent context. :param info_name: the info name for this invocation. Generally this @@ -208,15 +237,30 @@ class Context(object): codes are used in texts that Click prints which is by default not the case. This for instance would affect help output. + :param show_default: if True, shows defaults for all options. + Even if an option is later created with show_default=False, + this command-level setting overrides it. """ - def __init__(self, command, parent=None, info_name=None, obj=None, - auto_envvar_prefix=None, default_map=None, - terminal_width=None, max_content_width=None, - resilient_parsing=False, allow_extra_args=None, - allow_interspersed_args=None, - ignore_unknown_options=None, help_option_names=None, - token_normalize_func=None, color=None): + def __init__( + self, + command, + parent=None, + info_name=None, + obj=None, + auto_envvar_prefix=None, + default_map=None, + terminal_width=None, + max_content_width=None, + resilient_parsing=False, + allow_extra_args=None, + allow_interspersed_args=None, + ignore_unknown_options=None, + help_option_names=None, + token_normalize_func=None, + color=None, + show_default=None, + ): #: the parent context or `None` if none exists. self.parent = parent #: the :class:`Command` for this context. @@ -237,12 +281,14 @@ class Context(object): obj = parent.obj #: the user object stored. self.obj = obj - self._meta = getattr(parent, 'meta', {}) + self._meta = getattr(parent, "meta", {}) #: A dictionary (-like object) with defaults for parameters. - if default_map is None \ - and parent is not None \ - and parent.default_map is not None: + if ( + default_map is None + and parent is not None + and parent.default_map is not None + ): default_map = parent.default_map.get(info_name) self.default_map = default_map @@ -301,7 +347,7 @@ class Context(object): if parent is not None: help_option_names = parent.help_option_names else: - help_option_names = ['--help'] + help_option_names = ["--help"] #: The names for the help options. self.help_option_names = help_option_names @@ -322,13 +368,18 @@ class Context(object): # the command on this level has a name, we can expand the envvar # prefix automatically. if auto_envvar_prefix is None: - if parent is not None \ - and parent.auto_envvar_prefix is not None and \ - self.info_name is not None: - auto_envvar_prefix = '%s_%s' % (parent.auto_envvar_prefix, - self.info_name.upper()) + if ( + parent is not None + and parent.auto_envvar_prefix is not None + and self.info_name is not None + ): + auto_envvar_prefix = "{}_{}".format( + parent.auto_envvar_prefix, self.info_name.upper() + ) else: auto_envvar_prefix = auto_envvar_prefix.upper() + if auto_envvar_prefix is not None: + auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") self.auto_envvar_prefix = auto_envvar_prefix if color is None and parent is not None: @@ -337,6 +388,8 @@ class Context(object): #: Controls if styling output is wanted or not. self.color = color + self.show_default = show_default + self._close_callbacks = [] self._depth = 0 @@ -404,7 +457,7 @@ class Context(object): Example usage:: - LANG_KEY = __name__ + '.lang' + LANG_KEY = f'{__name__}.lang' def set_language(value): ctx = get_current_context() @@ -419,8 +472,9 @@ class Context(object): def make_formatter(self): """Creates the formatter for the help and usage output.""" - return HelpFormatter(width=self.terminal_width, - max_width=self.max_content_width) + return HelpFormatter( + width=self.terminal_width, max_width=self.max_content_width + ) def call_on_close(self, f): """This decorator remembers a function as callback that should be @@ -446,11 +500,11 @@ class Context(object): information on the help page. It's automatically created by combining the info names of the chain of contexts to the root. """ - rv = '' + rv = "" if self.info_name is not None: rv = self.info_name if self.parent is not None: - rv = self.parent.command_path + ' ' + rv + rv = "{} {}".format(self.parent.command_path, rv) return rv.lstrip() def find_root(self): @@ -515,7 +569,7 @@ class Context(object): """ return self.command.get_help(self) - def invoke(*args, **kwargs): + def invoke(*args, **kwargs): # noqa: B902 """Invokes a command callback in exactly the way it expects. There are two ways to invoke this method: @@ -542,8 +596,9 @@ class Context(object): callback = other_cmd.callback ctx = Context(other_cmd, info_name=other_cmd.name, parent=self) if callback is None: - raise TypeError('The given command does not have a ' - 'callback that can be invoked.') + raise TypeError( + "The given command does not have a callback that can be invoked." + ) for param in other_cmd.params: if param.name not in kwargs and param.expose_value: @@ -554,7 +609,7 @@ class Context(object): with ctx: return callback(*args, **kwargs) - def forward(*args, **kwargs): + def forward(*args, **kwargs): # noqa: B902 """Similar to :meth:`invoke` but fills in default keyword arguments from the current context if the other command expects it. This cannot invoke callbacks directly, only other commands. @@ -564,7 +619,7 @@ class Context(object): # It's also possible to invoke another command which might or # might not have a callback. if not isinstance(cmd, Command): - raise TypeError('Callback is not a command.') + raise TypeError("Callback is not a command.") for param in self.params: if param not in kwargs: @@ -594,6 +649,7 @@ class BaseCommand(object): :param context_settings: an optional dictionary with defaults that are passed to the context object. """ + #: the default for the :attr:`Context.allow_extra_args` flag. allow_extra_args = False #: the default for the :attr:`Context.allow_interspersed_args` flag. @@ -612,11 +668,14 @@ class BaseCommand(object): #: an optional dictionary with defaults passed to the context. self.context_settings = context_settings + def __repr__(self): + return "<{} {}>".format(self.__class__.__name__, self.name) + def get_usage(self, ctx): - raise NotImplementedError('Base commands cannot get usage') + raise NotImplementedError("Base commands cannot get usage") def get_help(self, ctx): - raise NotImplementedError('Base commands cannot get help') + raise NotImplementedError("Base commands cannot get help") def make_context(self, info_name, args, parent=None, **extra): """This function when given an info name and arguments will kick @@ -646,17 +705,22 @@ class BaseCommand(object): and parses the arguments, then modifies the context as necessary. This is automatically invoked by :meth:`make_context`. """ - raise NotImplementedError('Base commands do not know how to parse ' - 'arguments.') + raise NotImplementedError("Base commands do not know how to parse arguments.") def invoke(self, ctx): """Given a context, this invokes the command. The default implementation is raising a not implemented error. """ - raise NotImplementedError('Base commands are not invokable by default') + raise NotImplementedError("Base commands are not invokable by default") - def main(self, args=None, prog_name=None, complete_var=None, - standalone_mode=True, **extra): + def main( + self, + args=None, + prog_name=None, + complete_var=None, + standalone_mode=True, + **extra + ): """This is the way to invoke a script with all the bells and whistles as a command line application. This will always terminate the application after a call. If this is not wanted, ``SystemExit`` @@ -703,8 +767,9 @@ class BaseCommand(object): args = list(args) if prog_name is None: - prog_name = make_str(os.path.basename( - sys.argv and sys.argv[0] or __file__)) + prog_name = make_str( + os.path.basename(sys.argv[0] if sys.argv else __file__) + ) # Hook for the Bash completion. This only activates if the Bash # completion is actually enabled, otherwise this is quite a fast @@ -756,7 +821,7 @@ class BaseCommand(object): except Abort: if not standalone_mode: raise - echo('Aborted!', file=sys.stderr) + echo("Aborted!", file=sys.stderr) sys.exit(1) def __call__(self, *args, **kwargs): @@ -771,6 +836,8 @@ class Command(BaseCommand): .. versionchanged:: 2.0 Added the `context_settings` parameter. + .. versionchanged:: 7.1 + Added the `no_args_is_help` parameter. :param name: the name of the command to use unless a group overrides it. :param context_settings: an optional dictionary with defaults that are @@ -785,16 +852,31 @@ class Command(BaseCommand): shown on the command listing of the parent command. :param add_help_option: by default each command registers a ``--help`` option. This can be disabled by this parameter. + :param no_args_is_help: this controls what happens if no arguments are + provided. This option is disabled by default. + If enabled this will add ``--help`` as argument + if no arguments are passed :param hidden: hide this command from help outputs. :param deprecated: issues a message indicating that the command is deprecated. """ - def __init__(self, name, context_settings=None, callback=None, - params=None, help=None, epilog=None, short_help=None, - options_metavar='[OPTIONS]', add_help_option=True, - hidden=False, deprecated=False): + def __init__( + self, + name, + context_settings=None, + callback=None, + params=None, + help=None, + epilog=None, + short_help=None, + options_metavar="[OPTIONS]", + add_help_option=True, + no_args_is_help=False, + hidden=False, + deprecated=False, + ): BaseCommand.__init__(self, name, context_settings) #: the callback to execute when the command fires. This might be #: `None` in which case nothing happens. @@ -805,20 +887,25 @@ class Command(BaseCommand): self.params = params or [] # if a form feed (page break) is found in the help text, truncate help # text to the content preceding the first form feed - if help and '\f' in help: - help = help.split('\f', 1)[0] + if help and "\f" in help: + help = help.split("\f", 1)[0] self.help = help self.epilog = epilog self.options_metavar = options_metavar self.short_help = short_help self.add_help_option = add_help_option + self.no_args_is_help = no_args_is_help self.hidden = hidden self.deprecated = deprecated def get_usage(self, ctx): + """Formats the usage line into a string and returns it. + + Calls :meth:`format_usage` internally. + """ formatter = ctx.make_formatter() self.format_usage(ctx, formatter) - return formatter.getvalue().rstrip('\n') + return formatter.getvalue().rstrip("\n") def get_params(self, ctx): rv = self.params @@ -828,9 +915,12 @@ class Command(BaseCommand): return rv def format_usage(self, ctx, formatter): - """Writes the usage line into the formatter.""" + """Writes the usage line into the formatter. + + This is a low-level method called by :meth:`get_usage`. + """ pieces = self.collect_usage_pieces(ctx) - formatter.write_usage(ctx.command_path, ' '.join(pieces)) + formatter.write_usage(ctx.command_path, " ".join(pieces)) def collect_usage_pieces(self, ctx): """Returns all the pieces that go into the usage line and returns @@ -859,10 +949,15 @@ class Command(BaseCommand): if value and not ctx.resilient_parsing: echo(ctx.get_help(), color=ctx.color) ctx.exit() - return Option(help_options, is_flag=True, - is_eager=True, expose_value=False, - callback=show_help, - help='Show this message and exit.') + + return Option( + help_options, + is_flag=True, + is_eager=True, + expose_value=False, + callback=show_help, + help="Show this message and exit.", + ) def make_parser(self, ctx): """Creates the underlying option parser for this command.""" @@ -872,21 +967,31 @@ class Command(BaseCommand): return parser def get_help(self, ctx): - """Formats the help into a string and returns it. This creates a - formatter and will call into the following formatting methods: + """Formats the help into a string and returns it. + + Calls :meth:`format_help` internally. """ formatter = ctx.make_formatter() self.format_help(ctx, formatter) - return formatter.getvalue().rstrip('\n') + return formatter.getvalue().rstrip("\n") def get_short_help_str(self, limit=45): - """Gets short help for the command or makes it by shortening the long help string.""" - return self.short_help or self.help and make_default_short_help(self.help, limit) or '' + """Gets short help for the command or makes it by shortening the + long help string. + """ + return ( + self.short_help + or self.help + and make_default_short_help(self.help, limit) + or "" + ) def format_help(self, ctx, formatter): """Writes the help into the formatter if it exists. - This calls into the following methods: + This is a low-level method called by :meth:`get_help`. + + This calls the following methods: - :meth:`format_usage` - :meth:`format_help_text` @@ -921,7 +1026,7 @@ class Command(BaseCommand): opts.append(rv) if opts: - with formatter.section('Options'): + with formatter.section("Options"): formatter.write_dl(opts) def format_epilog(self, ctx, formatter): @@ -932,17 +1037,22 @@ class Command(BaseCommand): formatter.write_text(self.epilog) def parse_args(self, ctx, args): + if not args and self.no_args_is_help and not ctx.resilient_parsing: + echo(ctx.get_help(), color=ctx.color) + ctx.exit() + parser = self.make_parser(ctx) opts, args, param_order = parser.parse_args(args=args) - for param in iter_params_for_processing( - param_order, self.get_params(ctx)): + for param in iter_params_for_processing(param_order, self.get_params(ctx)): value, args = param.handle_parse_result(ctx, opts, args) if args and not ctx.allow_extra_args and not ctx.resilient_parsing: - ctx.fail('Got unexpected extra argument%s (%s)' - % (len(args) != 1 and 's' or '', - ' '.join(map(make_str, args)))) + ctx.fail( + "Got unexpected extra argument{} ({})".format( + "s" if len(args) != 1 else "", " ".join(map(make_str, args)) + ) + ) ctx.args = args return args @@ -979,12 +1089,20 @@ class MultiCommand(Command): :param result_callback: the result callback to attach to this multi command. """ + allow_extra_args = True allow_interspersed_args = False - def __init__(self, name=None, invoke_without_command=False, - no_args_is_help=None, subcommand_metavar=None, - chain=False, result_callback=None, **attrs): + def __init__( + self, + name=None, + invoke_without_command=False, + no_args_is_help=None, + subcommand_metavar=None, + chain=False, + result_callback=None, + **attrs + ): Command.__init__(self, name, **attrs) if no_args_is_help is None: no_args_is_help = not invoke_without_command @@ -1004,8 +1122,10 @@ class MultiCommand(Command): if self.chain: for param in self.params: if isinstance(param, Argument) and not param.required: - raise RuntimeError('Multi commands in chain mode cannot ' - 'have optional arguments.') + raise RuntimeError( + "Multi commands in chain mode cannot have" + " optional arguments." + ) def collect_usage_pieces(self, ctx): rv = Command.collect_usage_pieces(self, ctx) @@ -1041,16 +1161,19 @@ class MultiCommand(Command): :param replace: if set to `True` an already existing result callback will be removed. """ + def decorator(f): old_callback = self.result_callback if old_callback is None or replace: self.result_callback = f return f + def function(__value, *args, **kwargs): - return f(old_callback(__value, *args, **kwargs), - *args, **kwargs) + return f(old_callback(__value, *args, **kwargs), *args, **kwargs) + self.result_callback = rv = update_wrapper(function, f) return rv + return decorator def format_commands(self, ctx, formatter): @@ -1078,7 +1201,7 @@ class MultiCommand(Command): rows.append((subcommand, help)) if rows: - with formatter.section('Commands'): + with formatter.section("Commands"): formatter.write_dl(rows) def parse_args(self, ctx, args): @@ -1098,8 +1221,7 @@ class MultiCommand(Command): def invoke(self, ctx): def _process_result(value): if self.result_callback is not None: - value = ctx.invoke(self.result_callback, value, - **ctx.params) + value = ctx.invoke(self.result_callback, value, **ctx.params) return value if not ctx.protected_args: @@ -1115,7 +1237,7 @@ class MultiCommand(Command): with ctx: Command.invoke(self, ctx) return _process_result([]) - ctx.fail('Missing command.') + ctx.fail("Missing command.") # Fetch args back out args = ctx.protected_args + ctx.args @@ -1142,7 +1264,7 @@ class MultiCommand(Command): # set to ``*`` to inform the command that subcommands are executed # but nothing else. with ctx: - ctx.invoked_subcommand = args and '*' or None + ctx.invoked_subcommand = "*" if args else None Command.invoke(self, ctx) # Otherwise we make every single context and invoke them in a @@ -1151,9 +1273,13 @@ class MultiCommand(Command): contexts = [] while args: cmd_name, cmd, args = self.resolve_command(ctx, args) - sub_ctx = cmd.make_context(cmd_name, args, parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False) + sub_ctx = cmd.make_context( + cmd_name, + args, + parent=ctx, + allow_extra_args=True, + allow_interspersed_args=False, + ) contexts.append(sub_ctx) args, sub_ctx.args = sub_ctx.args, [] @@ -1185,7 +1311,7 @@ class MultiCommand(Command): if cmd is None and not ctx.resilient_parsing: if split_opt(cmd_name)[0]: self.parse_args(ctx, ctx.args) - ctx.fail('No such command "%s".' % original_cmd_name) + ctx.fail("No such command '{}'.".format(original_cmd_name)) return cmd_name, cmd, args[1:] @@ -1220,7 +1346,7 @@ class Group(MultiCommand): """ name = name or cmd.name if name is None: - raise TypeError('Command has no name.') + raise TypeError("Command has no name.") _check_multicommand(self, name, cmd, register=True) self.commands[name] = cmd @@ -1230,10 +1356,13 @@ class Group(MultiCommand): immediately registers the created command with this instance by calling into :meth:`add_command`. """ + from .decorators import command + def decorator(f): cmd = command(*args, **kwargs)(f) self.add_command(cmd) return cmd + return decorator def group(self, *args, **kwargs): @@ -1242,10 +1371,13 @@ class Group(MultiCommand): immediately registers the created command with this instance by calling into :meth:`add_command`. """ + from .decorators import group + def decorator(f): cmd = group(*args, **kwargs)(f) self.add_command(cmd) return cmd + return decorator def get_command(self, ctx, cmd_name): @@ -1294,12 +1426,6 @@ class Parameter(object): Some settings are supported by both options and arguments. - .. versionchanged:: 2.0 - Changed signature for parameter callback to also be passed the - parameter. In Click 2.0, the old callback format will still work, - but it will raise a warning to give you change to migrate the - code easier. - :param param_decls: the parameter declarations for this option or argument. This is a list of flags or argument names. @@ -1312,8 +1438,7 @@ class Parameter(object): without any arguments. :param callback: a callback that should be executed after the parameter was matched. This is called as ``fn(ctx, param, - value)`` and needs to return the value. Before Click - 2.0, the signature was ``(ctx, value)``. + value)`` and needs to return the value. :param nargs: the number of arguments to match. If not ``1`` the return value is a tuple instead of single value. The default for nargs is ``1`` (except if the type is a tuple, then it's @@ -1327,15 +1452,36 @@ class Parameter(object): order of processing. :param envvar: a string or list of strings that are environment variables that should be checked. - """ - param_type_name = 'parameter' - def __init__(self, param_decls=None, type=None, required=False, - default=None, callback=None, nargs=None, metavar=None, - expose_value=True, is_eager=False, envvar=None, - autocompletion=None): - self.name, self.opts, self.secondary_opts = \ - self._parse_decls(param_decls or (), expose_value) + .. versionchanged:: 7.1 + Empty environment variables are ignored rather than taking the + empty string value. This makes it possible for scripts to clear + variables if they can't unset them. + + .. versionchanged:: 2.0 + Changed signature for parameter callback to also be passed the + parameter. The old callback format will still work, but it will + raise a warning to give you a chance to migrate the code easier. + """ + param_type_name = "parameter" + + def __init__( + self, + param_decls=None, + type=None, + required=False, + default=None, + callback=None, + nargs=None, + metavar=None, + expose_value=True, + is_eager=False, + envvar=None, + autocompletion=None, + ): + self.name, self.opts, self.secondary_opts = self._parse_decls( + param_decls or (), expose_value + ) self.type = convert_type(type, default) @@ -1358,6 +1504,9 @@ class Parameter(object): self.envvar = envvar self.autocompletion = autocompletion + def __repr__(self): + return "<{} {}>".format(self.__class__.__name__, self.name) + @property def human_readable_name(self): """Returns the human readable name of this parameter. This is the @@ -1372,7 +1521,7 @@ class Parameter(object): if metavar is None: metavar = self.type.name.upper() if self.nargs != 1: - metavar += '...' + metavar += "..." return metavar def get_default(self, ctx): @@ -1402,10 +1551,11 @@ class Parameter(object): """ if self.type.is_composite: if self.nargs <= 1: - raise TypeError('Attempted to invoke composite type ' - 'but nargs has been set to %s. This is ' - 'not supported; nargs needs to be set to ' - 'a fixed value > 1.' % self.nargs) + raise TypeError( + "Attempted to invoke composite type but nargs has" + " been set to {}. This is not supported; nargs" + " needs to be set to a fixed value > 1.".format(self.nargs) + ) if self.multiple: return tuple(self.type(x or (), self, ctx) for x in value or ()) return self.type(value or (), self, ctx) @@ -1414,6 +1564,7 @@ class Parameter(object): if level == 0: return self.type(value, self, ctx) return tuple(_convert(x, level - 1) for x in value or ()) + return _convert(value, (self.nargs != 1) + bool(self.multiple)) def process_value(self, ctx, value): @@ -1454,7 +1605,10 @@ class Parameter(object): if rv is not None: return rv else: - return os.environ.get(self.envvar) + rv = os.environ.get(self.envvar) + + if rv != "": + return rv def value_from_envvar(self, ctx): rv = self.resolve_envvar_value(ctx) @@ -1473,8 +1627,7 @@ class Parameter(object): value = None if self.callback is not None: try: - value = invoke_param_callback( - self.callback, ctx, self, value) + value = invoke_param_callback(self.callback, ctx, self, value) except Exception: if not ctx.resilient_parsing: raise @@ -1494,7 +1647,7 @@ class Parameter(object): indicate which param caused the error. """ hint_list = self.opts or [self.human_readable_name] - return ' / '.join('"%s"' % x for x in hint_list) + return " / ".join(repr(x) for x in hint_list) class Option(Parameter): @@ -1535,19 +1688,33 @@ class Option(Parameter): :param help: the help string. :param hidden: hide this option from help outputs. """ - param_type_name = 'option' - def __init__(self, param_decls=None, show_default=False, - prompt=False, confirmation_prompt=False, - hide_input=False, is_flag=None, flag_value=None, - multiple=False, count=False, allow_from_autoenv=True, - type=None, help=None, hidden=False, show_choices=True, - show_envvar=False, **attrs): - default_is_missing = attrs.get('default', _missing) is _missing + param_type_name = "option" + + def __init__( + self, + param_decls=None, + show_default=False, + prompt=False, + confirmation_prompt=False, + hide_input=False, + is_flag=None, + flag_value=None, + multiple=False, + count=False, + allow_from_autoenv=True, + type=None, + help=None, + hidden=False, + show_choices=True, + show_envvar=False, + **attrs + ): + default_is_missing = attrs.get("default", _missing) is _missing Parameter.__init__(self, param_decls, type=type, **attrs) if prompt is True: - prompt_text = self.name.replace('_', ' ').capitalize() + prompt_text = self.name.replace("_", " ").capitalize() elif prompt is False: prompt_text = None else: @@ -1569,8 +1736,7 @@ class Option(Parameter): flag_value = not self.default self.is_flag = is_flag self.flag_value = flag_value - if self.is_flag and isinstance(self.flag_value, bool) \ - and type is None: + if self.is_flag and isinstance(self.flag_value, bool) and type in [None, bool]: self.type = BOOL self.is_bool_flag = True else: @@ -1594,22 +1760,22 @@ class Option(Parameter): # Sanity check for stuff we don't support if __debug__: if self.nargs < 0: - raise TypeError('Options cannot have nargs < 0') + raise TypeError("Options cannot have nargs < 0") if self.prompt and self.is_flag and not self.is_bool_flag: - raise TypeError('Cannot prompt for flags that are not bools.') + raise TypeError("Cannot prompt for flags that are not bools.") if not self.is_bool_flag and self.secondary_opts: - raise TypeError('Got secondary option for non boolean flag.') - if self.is_bool_flag and self.hide_input \ - and self.prompt is not None: - raise TypeError('Hidden input does not work with boolean ' - 'flag prompts.') + raise TypeError("Got secondary option for non boolean flag.") + if self.is_bool_flag and self.hide_input and self.prompt is not None: + raise TypeError("Hidden input does not work with boolean flag prompts.") if self.count: if self.multiple: - raise TypeError('Options cannot be multiple and count ' - 'at the same time.') + raise TypeError( + "Options cannot be multiple and count at the same time." + ) elif self.is_flag: - raise TypeError('Options cannot be count and flags at ' - 'the same time.') + raise TypeError( + "Options cannot be count and flags at the same time." + ) def _parse_decls(self, decls, expose_value): opts = [] @@ -1620,10 +1786,10 @@ class Option(Parameter): for decl in decls: if isidentifier(decl): if name is not None: - raise TypeError('Name defined twice') + raise TypeError("Name defined twice") name = decl else: - split_char = decl[:1] == '/' and ';' or '/' + split_char = ";" if decl[:1] == "/" else "/" if split_char in decl: first, second = decl.split(split_char, 1) first = first.rstrip() @@ -1639,49 +1805,51 @@ class Option(Parameter): if name is None and possible_names: possible_names.sort(key=lambda x: -len(x[0])) # group long options first - name = possible_names[0][1].replace('-', '_').lower() + name = possible_names[0][1].replace("-", "_").lower() if not isidentifier(name): name = None if name is None: if not expose_value: return None, opts, secondary_opts - raise TypeError('Could not determine name for option') + raise TypeError("Could not determine name for option") if not opts and not secondary_opts: - raise TypeError('No options defined but a name was passed (%s). ' - 'Did you mean to declare an argument instead ' - 'of an option?' % name) + raise TypeError( + "No options defined but a name was passed ({}). Did you" + " mean to declare an argument instead of an option?".format(name) + ) return name, opts, secondary_opts def add_to_parser(self, parser, ctx): kwargs = { - 'dest': self.name, - 'nargs': self.nargs, - 'obj': self, + "dest": self.name, + "nargs": self.nargs, + "obj": self, } if self.multiple: - action = 'append' + action = "append" elif self.count: - action = 'count' + action = "count" else: - action = 'store' + action = "store" if self.is_flag: - kwargs.pop('nargs', None) + kwargs.pop("nargs", None) + action_const = "{}_const".format(action) if self.is_bool_flag and self.secondary_opts: - parser.add_option(self.opts, action=action + '_const', - const=True, **kwargs) - parser.add_option(self.secondary_opts, action=action + - '_const', const=False, **kwargs) + parser.add_option(self.opts, action=action_const, const=True, **kwargs) + parser.add_option( + self.secondary_opts, action=action_const, const=False, **kwargs + ) else: - parser.add_option(self.opts, action=action + '_const', - const=self.flag_value, - **kwargs) + parser.add_option( + self.opts, action=action_const, const=self.flag_value, **kwargs + ) else: - kwargs['action'] = action + kwargs["action"] = action parser.add_option(self.opts, **kwargs) def get_help_record(self, ctx): @@ -1694,46 +1862,50 @@ class Option(Parameter): if any_slashes: any_prefix_is_slash[:] = [True] if not self.is_flag and not self.count: - rv += ' ' + self.make_metavar() + rv += " {}".format(self.make_metavar()) return rv rv = [_write_opts(self.opts)] if self.secondary_opts: rv.append(_write_opts(self.secondary_opts)) - help = self.help or '' + help = self.help or "" extra = [] if self.show_envvar: envvar = self.envvar if envvar is None: - if self.allow_from_autoenv and \ - ctx.auto_envvar_prefix is not None: - envvar = '%s_%s' % (ctx.auto_envvar_prefix, self.name.upper()) + if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None: + envvar = "{}_{}".format(ctx.auto_envvar_prefix, self.name.upper()) if envvar is not None: - extra.append('env var: %s' % ( - ', '.join('%s' % d for d in envvar) - if isinstance(envvar, (list, tuple)) - else envvar, )) - if self.default is not None and self.show_default: + extra.append( + "env var: {}".format( + ", ".join(str(d) for d in envvar) + if isinstance(envvar, (list, tuple)) + else envvar + ) + ) + if self.default is not None and (self.show_default or ctx.show_default): if isinstance(self.show_default, string_types): - default_string = '({})'.format(self.show_default) + default_string = "({})".format(self.show_default) elif isinstance(self.default, (list, tuple)): - default_string = ', '.join('%s' % d for d in self.default) + default_string = ", ".join(str(d) for d in self.default) elif inspect.isfunction(self.default): default_string = "(dynamic)" else: default_string = self.default - extra.append('default: {}'.format(default_string)) + extra.append("default: {}".format(default_string)) if self.required: - extra.append('required') + extra.append("required") if extra: - help = '%s[%s]' % (help and help + ' ' or '', '; '.join(extra)) + help = "{}[{}]".format( + "{} ".format(help) if help else "", "; ".join(extra) + ) - return ((any_prefix_is_slash and '; ' or ' / ').join(rv), help) + return ("; " if any_prefix_is_slash else " / ").join(rv), help def get_default(self, ctx): - # If we're a non boolean flag out default is more complex because + # If we're a non boolean flag our default is more complex because # we need to look at all flags in the same group to figure out # if we're the the default one in which case we return the flag # value as default. @@ -1758,18 +1930,22 @@ class Option(Parameter): if self.is_bool_flag: return confirm(self.prompt, default) - return prompt(self.prompt, default=default, type=self.type, - hide_input=self.hide_input, show_choices=self.show_choices, - confirmation_prompt=self.confirmation_prompt, - value_proc=lambda x: self.process_value(ctx, x)) + return prompt( + self.prompt, + default=default, + type=self.type, + hide_input=self.hide_input, + show_choices=self.show_choices, + confirmation_prompt=self.confirmation_prompt, + value_proc=lambda x: self.process_value(ctx, x), + ) def resolve_envvar_value(self, ctx): rv = Parameter.resolve_envvar_value(self, ctx) if rv is not None: return rv - if self.allow_from_autoenv and \ - ctx.auto_envvar_prefix is not None: - envvar = '%s_%s' % (ctx.auto_envvar_prefix, self.name.upper()) + if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None: + envvar = "{}_{}".format(ctx.auto_envvar_prefix, self.name.upper()) return os.environ.get(envvar) def value_from_envvar(self, ctx): @@ -1784,8 +1960,7 @@ class Option(Parameter): return rv def full_process_value(self, ctx, value): - if value is None and self.prompt is not None \ - and not ctx.resilient_parsing: + if value is None and self.prompt is not None and not ctx.resilient_parsing: return self.prompt_for_value(ctx) return Parameter.full_process_value(self, ctx, value) @@ -1797,18 +1972,20 @@ class Argument(Parameter): All parameters are passed onwards to the parameter constructor. """ - param_type_name = 'argument' + + param_type_name = "argument" def __init__(self, param_decls, required=None, **attrs): if required is None: - if attrs.get('default') is not None: + if attrs.get("default") is not None: required = False else: - required = attrs.get('nargs', 1) > 0 + required = attrs.get("nargs", 1) > 0 Parameter.__init__(self, param_decls, required=required, **attrs) if self.default is not None and self.nargs < 0: - raise TypeError('nargs=-1 in combination with a default value ' - 'is not supported.') + raise TypeError( + "nargs=-1 in combination with a default value is not supported." + ) @property def human_readable_name(self): @@ -1823,34 +2000,31 @@ class Argument(Parameter): if not var: var = self.name.upper() if not self.required: - var = '[%s]' % var + var = "[{}]".format(var) if self.nargs != 1: - var += '...' + var += "..." return var def _parse_decls(self, decls, expose_value): if not decls: if not expose_value: return None, [], [] - raise TypeError('Could not determine name for argument') + raise TypeError("Could not determine name for argument") if len(decls) == 1: name = arg = decls[0] - name = name.replace('-', '_').lower() + name = name.replace("-", "_").lower() else: - raise TypeError('Arguments take exactly one ' - 'parameter declaration, got %d' % len(decls)) + raise TypeError( + "Arguments take exactly one parameter declaration, got" + " {}".format(len(decls)) + ) return name, [arg], [] def get_usage_pieces(self, ctx): return [self.make_metavar()] def get_error_hint(self, ctx): - return '"%s"' % self.make_metavar() + return repr(self.make_metavar()) def add_to_parser(self, parser, ctx): - parser.add_argument(dest=self.name, nargs=self.nargs, - obj=self) - - -# Circular dependency between decorators and core -from .decorators import command, group + parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) diff --git a/pipenv/vendor/click/decorators.py b/pipenv/vendor/click/decorators.py index c57c5308..c7b5af6c 100644 --- a/pipenv/vendor/click/decorators.py +++ b/pipenv/vendor/click/decorators.py @@ -1,20 +1,25 @@ -import sys import inspect - +import sys from functools import update_wrapper from ._compat import iteritems from ._unicodefun import _check_for_unicode_literals -from .utils import echo +from .core import Argument +from .core import Command +from .core import Group +from .core import Option from .globals import get_current_context +from .utils import echo def pass_context(f): """Marks a callback as wanting to receive the current context object as first argument. """ + def new_func(*args, **kwargs): return f(get_current_context(), *args, **kwargs) + return update_wrapper(new_func, f) @@ -23,8 +28,10 @@ def pass_obj(f): context onwards (:attr:`Context.obj`). This is useful if that object represents the state of a nested system. """ + def new_func(*args, **kwargs): return f(get_current_context().obj, *args, **kwargs) + return update_wrapper(new_func, f) @@ -50,6 +57,7 @@ def make_pass_decorator(object_type, ensure=False): :param ensure: if set to `True`, a new object will be created and remembered on the context if it's not there yet. """ + def decorator(f): def new_func(*args, **kwargs): ctx = get_current_context() @@ -58,35 +66,41 @@ def make_pass_decorator(object_type, ensure=False): else: obj = ctx.find_object(object_type) if obj is None: - raise RuntimeError('Managed to invoke callback without a ' - 'context object of type %r existing' - % object_type.__name__) + raise RuntimeError( + "Managed to invoke callback without a context" + " object of type '{}' existing".format(object_type.__name__) + ) return ctx.invoke(f, obj, *args, **kwargs) + return update_wrapper(new_func, f) + return decorator def _make_command(f, name, attrs, cls): if isinstance(f, Command): - raise TypeError('Attempted to convert a callback into a ' - 'command twice.') + raise TypeError("Attempted to convert a callback into a command twice.") try: params = f.__click_params__ params.reverse() del f.__click_params__ except AttributeError: params = [] - help = attrs.get('help') + help = attrs.get("help") if help is None: help = inspect.getdoc(f) if isinstance(help, bytes): - help = help.decode('utf-8') + help = help.decode("utf-8") else: help = inspect.cleandoc(help) - attrs['help'] = help + attrs["help"] = help _check_for_unicode_literals() - return cls(name=name or f.__name__.lower().replace('_', '-'), - callback=f, params=params, **attrs) + return cls( + name=name or f.__name__.lower().replace("_", "-"), + callback=f, + params=params, + **attrs + ) def command(name=None, cls=None, **attrs): @@ -94,9 +108,9 @@ def command(name=None, cls=None, **attrs): callback. This will also automatically attach all decorated :func:`option`\s and :func:`argument`\s as parameters to the command. - The name of the command defaults to the name of the function. If you - want to change that, you can pass the intended name as the first - argument. + The name of the command defaults to the name of the function with + underscores replaced by dashes. If you want to change that, you can + pass the intended name as the first argument. All keyword arguments are forwarded to the underlying command class. @@ -111,10 +125,12 @@ def command(name=None, cls=None, **attrs): """ if cls is None: cls = Command + def decorator(f): cmd = _make_command(f, name, attrs, cls) cmd.__doc__ = f.__doc__ return cmd + return decorator @@ -123,7 +139,7 @@ def group(name=None, **attrs): works otherwise the same as :func:`command` just that the `cls` parameter is set to :class:`Group`. """ - attrs.setdefault('cls', Group) + attrs.setdefault("cls", Group) return command(name, **attrs) @@ -131,7 +147,7 @@ def _param_memo(f, param): if isinstance(f, Command): f.params.append(param) else: - if not hasattr(f, '__click_params__'): + if not hasattr(f, "__click_params__"): f.__click_params__ = [] f.__click_params__.append(param) @@ -146,10 +162,12 @@ def argument(*param_decls, **attrs): :param cls: the argument class to instantiate. This defaults to :class:`Argument`. """ + def decorator(f): - ArgumentClass = attrs.pop('cls', Argument) + ArgumentClass = attrs.pop("cls", Argument) _param_memo(f, ArgumentClass(param_decls, **attrs)) return f + return decorator @@ -163,15 +181,17 @@ def option(*param_decls, **attrs): :param cls: the option class to instantiate. This defaults to :class:`Option`. """ + def decorator(f): # Issue 926, copy attrs, so pre-defined options can re-use the same cls= option_attrs = attrs.copy() - if 'help' in option_attrs: - option_attrs['help'] = inspect.cleandoc(option_attrs['help']) - OptionClass = option_attrs.pop('cls', Option) + if "help" in option_attrs: + option_attrs["help"] = inspect.cleandoc(option_attrs["help"]) + OptionClass = option_attrs.pop("cls", Option) _param_memo(f, OptionClass(param_decls, **option_attrs)) return f + return decorator @@ -192,16 +212,19 @@ def confirmation_option(*param_decls, **attrs): def dropdb(): pass """ + def decorator(f): def callback(ctx, param, value): if not value: ctx.abort() - attrs.setdefault('is_flag', True) - attrs.setdefault('callback', callback) - attrs.setdefault('expose_value', False) - attrs.setdefault('prompt', 'Do you want to continue?') - attrs.setdefault('help', 'Confirm the action without prompting.') - return option(*(param_decls or ('--yes',)), **attrs)(f) + + attrs.setdefault("is_flag", True) + attrs.setdefault("callback", callback) + attrs.setdefault("expose_value", False) + attrs.setdefault("prompt", "Do you want to continue?") + attrs.setdefault("help", "Confirm the action without prompting.") + return option(*(param_decls or ("--yes",)), **attrs)(f) + return decorator @@ -217,11 +240,13 @@ def password_option(*param_decls, **attrs): def changeadmin(password): pass """ + def decorator(f): - attrs.setdefault('prompt', True) - attrs.setdefault('confirmation_prompt', True) - attrs.setdefault('hide_input', True) - return option(*(param_decls or ('--password',)), **attrs)(f) + attrs.setdefault("prompt", True) + attrs.setdefault("confirmation_prompt", True) + attrs.setdefault("hide_input", True) + return option(*(param_decls or ("--password",)), **attrs)(f) + return decorator @@ -238,14 +263,14 @@ def version_option(version=None, *param_decls, **attrs): :param others: everything else is forwarded to :func:`option`. """ if version is None: - if hasattr(sys, '_getframe'): - module = sys._getframe(1).f_globals.get('__name__') + if hasattr(sys, "_getframe"): + module = sys._getframe(1).f_globals.get("__name__") else: - module = '' + module = "" def decorator(f): - prog_name = attrs.pop('prog_name', None) - message = attrs.pop('message', '%(prog)s, version %(version)s') + prog_name = attrs.pop("prog_name", None) + message = attrs.pop("message", "%(prog)s, version %(version)s") def callback(ctx, param, value): if not value or ctx.resilient_parsing: @@ -261,25 +286,23 @@ def version_option(version=None, *param_decls, **attrs): pass else: for dist in pkg_resources.working_set: - scripts = dist.get_entry_map().get('console_scripts') or {} - for script_name, entry_point in iteritems(scripts): + scripts = dist.get_entry_map().get("console_scripts") or {} + for _, entry_point in iteritems(scripts): if entry_point.module_name == module: ver = dist.version break if ver is None: - raise RuntimeError('Could not determine version') - echo(message % { - 'prog': prog, - 'version': ver, - }, color=ctx.color) + raise RuntimeError("Could not determine version") + echo(message % {"prog": prog, "version": ver}, color=ctx.color) ctx.exit() - attrs.setdefault('is_flag', True) - attrs.setdefault('expose_value', False) - attrs.setdefault('is_eager', True) - attrs.setdefault('help', 'Show the version and exit.') - attrs['callback'] = callback - return option(*(param_decls or ('--version',)), **attrs)(f) + attrs.setdefault("is_flag", True) + attrs.setdefault("expose_value", False) + attrs.setdefault("is_eager", True) + attrs.setdefault("help", "Show the version and exit.") + attrs["callback"] = callback + return option(*(param_decls or ("--version",)), **attrs)(f) + return decorator @@ -293,19 +316,18 @@ def help_option(*param_decls, **attrs): All arguments are forwarded to :func:`option`. """ + def decorator(f): def callback(ctx, param, value): if value and not ctx.resilient_parsing: echo(ctx.get_help(), color=ctx.color) ctx.exit() - attrs.setdefault('is_flag', True) - attrs.setdefault('expose_value', False) - attrs.setdefault('help', 'Show this message and exit.') - attrs.setdefault('is_eager', True) - attrs['callback'] = callback - return option(*(param_decls or ('--help',)), **attrs)(f) + + attrs.setdefault("is_flag", True) + attrs.setdefault("expose_value", False) + attrs.setdefault("help", "Show this message and exit.") + attrs.setdefault("is_eager", True) + attrs["callback"] = callback + return option(*(param_decls or ("--help",)), **attrs)(f) + return decorator - - -# Circular dependencies between core and decorators -from .core import Command, Group, Argument, Option diff --git a/pipenv/vendor/click/exceptions.py b/pipenv/vendor/click/exceptions.py index 6fa17658..592ee38f 100644 --- a/pipenv/vendor/click/exceptions.py +++ b/pipenv/vendor/click/exceptions.py @@ -1,10 +1,12 @@ -from ._compat import PY2, filename_to_ui, get_text_stderr +from ._compat import filename_to_ui +from ._compat import get_text_stderr +from ._compat import PY2 from .utils import echo def _join_param_hints(param_hint): if isinstance(param_hint, (tuple, list)): - return ' / '.join('"%s"' % x for x in param_hint) + return " / ".join(repr(x) for x in param_hint) return param_hint @@ -18,7 +20,7 @@ class ClickException(Exception): ctor_msg = message if PY2: if ctor_msg is not None: - ctor_msg = ctor_msg.encode('utf-8') + ctor_msg = ctor_msg.encode("utf-8") Exception.__init__(self, ctor_msg) self.message = message @@ -32,12 +34,12 @@ class ClickException(Exception): __unicode__ = __str__ def __str__(self): - return self.message.encode('utf-8') + return self.message.encode("utf-8") def show(self, file=None): if file is None: file = get_text_stderr() - echo('Error: %s' % self.format_message(), file=file) + echo("Error: {}".format(self.format_message()), file=file) class UsageError(ClickException): @@ -48,26 +50,27 @@ class UsageError(ClickException): :param ctx: optionally the context that caused this error. Click will fill in the context automatically in some situations. """ + exit_code = 2 def __init__(self, message, ctx=None): ClickException.__init__(self, message) self.ctx = ctx - self.cmd = self.ctx and self.ctx.command or None + self.cmd = self.ctx.command if self.ctx else None def show(self, file=None): if file is None: file = get_text_stderr() color = None - hint = '' - if (self.cmd is not None and - self.cmd.get_help_option(self.ctx) is not None): - hint = ('Try "%s %s" for help.\n' - % (self.ctx.command_path, self.ctx.help_option_names[0])) + hint = "" + if self.cmd is not None and self.cmd.get_help_option(self.ctx) is not None: + hint = "Try '{} {}' for help.\n".format( + self.ctx.command_path, self.ctx.help_option_names[0] + ) if self.ctx is not None: color = self.ctx.color - echo(self.ctx.get_usage() + '\n%s' % hint, file=file, color=color) - echo('Error: %s' % self.format_message(), file=file, color=color) + echo("{}\n{}".format(self.ctx.get_usage(), hint), file=file, color=color) + echo("Error: {}".format(self.format_message()), file=file, color=color) class BadParameter(UsageError): @@ -88,8 +91,7 @@ class BadParameter(UsageError): each item is quoted and separated. """ - def __init__(self, message, ctx=None, param=None, - param_hint=None): + def __init__(self, message, ctx=None, param=None, param_hint=None): UsageError.__init__(self, message, ctx) self.param = param self.param_hint = param_hint @@ -100,10 +102,10 @@ class BadParameter(UsageError): elif self.param is not None: param_hint = self.param.get_error_hint(self.ctx) else: - return 'Invalid value: %s' % self.message + return "Invalid value: {}".format(self.message) param_hint = _join_param_hints(param_hint) - return 'Invalid value for %s: %s' % (param_hint, self.message) + return "Invalid value for {}: {}".format(param_hint, self.message) class MissingParameter(BadParameter): @@ -118,8 +120,9 @@ class MissingParameter(BadParameter): ``'option'`` or ``'argument'``. """ - def __init__(self, message=None, ctx=None, param=None, - param_hint=None, param_type=None): + def __init__( + self, message=None, ctx=None, param=None, param_hint=None, param_type=None + ): BadParameter.__init__(self, message, ctx, param, param_hint) self.param_type = param_type @@ -141,17 +144,30 @@ class MissingParameter(BadParameter): msg_extra = self.param.type.get_missing_message(self.param) if msg_extra: if msg: - msg += '. ' + msg_extra + msg += ". {}".format(msg_extra) else: msg = msg_extra - return 'Missing %s%s%s%s' % ( + return "Missing {}{}{}{}".format( param_type, - param_hint and ' %s' % param_hint or '', - msg and '. ' or '.', - msg or '', + " {}".format(param_hint) if param_hint else "", + ". " if msg else ".", + msg or "", ) + def __str__(self): + if self.message is None: + param_name = self.param.name if self.param else None + return "missing parameter: {}".format(param_name) + else: + return self.message + + if PY2: + __unicode__ = __str__ + + def __str__(self): + return self.__unicode__().encode("utf-8") + class NoSuchOption(UsageError): """Raised if click attempted to handle an option that does not @@ -160,10 +176,9 @@ class NoSuchOption(UsageError): .. versionadded:: 4.0 """ - def __init__(self, option_name, message=None, possibilities=None, - ctx=None): + def __init__(self, option_name, message=None, possibilities=None, ctx=None): if message is None: - message = 'no such option: %s' % option_name + message = "no such option: {}".format(option_name) UsageError.__init__(self, message, ctx) self.option_name = option_name self.possibilities = possibilities @@ -172,11 +187,11 @@ class NoSuchOption(UsageError): bits = [self.message] if self.possibilities: if len(self.possibilities) == 1: - bits.append('Did you mean %s?' % self.possibilities[0]) + bits.append("Did you mean {}?".format(self.possibilities[0])) else: possibilities = sorted(self.possibilities) - bits.append('(Possible options: %s)' % ', '.join(possibilities)) - return ' '.join(bits) + bits.append("(Possible options: {})".format(", ".join(possibilities))) + return " ".join(bits) class BadOptionUsage(UsageError): @@ -212,13 +227,13 @@ class FileError(ClickException): def __init__(self, filename, hint=None): ui_filename = filename_to_ui(filename) if hint is None: - hint = 'unknown error' + hint = "unknown error" ClickException.__init__(self, hint) self.ui_filename = ui_filename self.filename = filename def format_message(self): - return 'Could not open file %s: %s' % (self.ui_filename, self.message) + return "Could not open file {}: {}".format(self.ui_filename, self.message) class Abort(RuntimeError): @@ -231,5 +246,8 @@ class Exit(RuntimeError): :param code: the status code to exit with. """ + + __slots__ = ("exit_code",) + def __init__(self, code=0): self.exit_code = code diff --git a/pipenv/vendor/click/formatting.py b/pipenv/vendor/click/formatting.py index a3d6a4d3..319c7f61 100644 --- a/pipenv/vendor/click/formatting.py +++ b/pipenv/vendor/click/formatting.py @@ -1,8 +1,8 @@ from contextlib import contextmanager -from .termui import get_terminal_size -from .parser import split_opt -from ._compat import term_len +from ._compat import term_len +from .parser import split_opt +from .termui import get_terminal_size # Can force a width. This is used by the test system FORCED_WIDTH = None @@ -19,11 +19,12 @@ def measure_table(rows): def iter_rows(rows, col_count): for row in rows: row = tuple(row) - yield row + ('',) * (col_count - len(row)) + yield row + ("",) * (col_count - len(row)) -def wrap_text(text, width=78, initial_indent='', subsequent_indent='', - preserve_paragraphs=False): +def wrap_text( + text, width=78, initial_indent="", subsequent_indent="", preserve_paragraphs=False +): """A helper function that intelligently wraps text. By default, it assumes that it operates on a single paragraph of text but if the `preserve_paragraphs` parameter is provided it will intelligently @@ -43,10 +44,14 @@ def wrap_text(text, width=78, initial_indent='', subsequent_indent='', intelligently handle paragraphs. """ from ._textwrap import TextWrapper + text = text.expandtabs() - wrapper = TextWrapper(width, initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - replace_whitespace=False) + wrapper = TextWrapper( + width, + initial_indent=initial_indent, + subsequent_indent=subsequent_indent, + replace_whitespace=False, + ) if not preserve_paragraphs: return wrapper.fill(text) @@ -57,10 +62,10 @@ def wrap_text(text, width=78, initial_indent='', subsequent_indent='', def _flush_par(): if not buf: return - if buf[0].strip() == '\b': - p.append((indent or 0, True, '\n'.join(buf[1:]))) + if buf[0].strip() == "\b": + p.append((indent or 0, True, "\n".join(buf[1:]))) else: - p.append((indent or 0, False, ' '.join(buf))) + p.append((indent or 0, False, " ".join(buf))) del buf[:] for line in text.splitlines(): @@ -77,13 +82,13 @@ def wrap_text(text, width=78, initial_indent='', subsequent_indent='', rv = [] for indent, raw, text in p: - with wrapper.extra_indent(' ' * indent): + with wrapper.extra_indent(" " * indent): if raw: rv.append(wrapper.indent_only(text)) else: rv.append(wrapper.fill(text)) - return '\n\n'.join(rv) + return "\n\n".join(rv) class HelpFormatter(object): @@ -122,53 +127,65 @@ class HelpFormatter(object): """Decreases the indentation.""" self.current_indent -= self.indent_increment - def write_usage(self, prog, args='', prefix='Usage: '): + def write_usage(self, prog, args="", prefix="Usage: "): """Writes a usage line into the buffer. :param prog: the program name. :param args: whitespace separated list of arguments. :param prefix: the prefix for the first line. """ - usage_prefix = '%*s%s ' % (self.current_indent, prefix, prog) + usage_prefix = "{:>{w}}{} ".format(prefix, prog, w=self.current_indent) text_width = self.width - self.current_indent if text_width >= (term_len(usage_prefix) + 20): # The arguments will fit to the right of the prefix. - indent = ' ' * term_len(usage_prefix) - self.write(wrap_text(args, text_width, - initial_indent=usage_prefix, - subsequent_indent=indent)) + indent = " " * term_len(usage_prefix) + self.write( + wrap_text( + args, + text_width, + initial_indent=usage_prefix, + subsequent_indent=indent, + ) + ) else: # The prefix is too long, put the arguments on the next line. self.write(usage_prefix) - self.write('\n') - indent = ' ' * (max(self.current_indent, term_len(prefix)) + 4) - self.write(wrap_text(args, text_width, - initial_indent=indent, - subsequent_indent=indent)) + self.write("\n") + indent = " " * (max(self.current_indent, term_len(prefix)) + 4) + self.write( + wrap_text( + args, text_width, initial_indent=indent, subsequent_indent=indent + ) + ) - self.write('\n') + self.write("\n") def write_heading(self, heading): """Writes a heading into the buffer.""" - self.write('%*s%s:\n' % (self.current_indent, '', heading)) + self.write("{:>{w}}{}:\n".format("", heading, w=self.current_indent)) def write_paragraph(self): """Writes a paragraph into the buffer.""" if self.buffer: - self.write('\n') + self.write("\n") def write_text(self, text): """Writes re-indented text into the buffer. This rewraps and preserves paragraphs. """ text_width = max(self.width - self.current_indent, 11) - indent = ' ' * self.current_indent - self.write(wrap_text(text, text_width, - initial_indent=indent, - subsequent_indent=indent, - preserve_paragraphs=True)) - self.write('\n') + indent = " " * self.current_indent + self.write( + wrap_text( + text, + text_width, + initial_indent=indent, + subsequent_indent=indent, + preserve_paragraphs=True, + ) + ) + self.write("\n") def write_dl(self, rows, col_max=30, col_spacing=2): """Writes a definition list into the buffer. This is how options @@ -182,30 +199,40 @@ class HelpFormatter(object): rows = list(rows) widths = measure_table(rows) if len(widths) != 2: - raise TypeError('Expected two columns for definition list') + raise TypeError("Expected two columns for definition list") first_col = min(widths[0], col_max) + col_spacing for first, second in iter_rows(rows, len(widths)): - self.write('%*s%s' % (self.current_indent, '', first)) + self.write("{:>{w}}{}".format("", first, w=self.current_indent)) if not second: - self.write('\n') + self.write("\n") continue if term_len(first) <= first_col - col_spacing: - self.write(' ' * (first_col - term_len(first))) + self.write(" " * (first_col - term_len(first))) else: - self.write('\n') - self.write(' ' * (first_col + self.current_indent)) + self.write("\n") + self.write(" " * (first_col + self.current_indent)) text_width = max(self.width - first_col - 2, 10) - lines = iter(wrap_text(second, text_width).splitlines()) + wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True) + lines = wrapped_text.splitlines() + if lines: - self.write(next(lines) + '\n') - for line in lines: - self.write('%*s%s\n' % ( - first_col + self.current_indent, '', line)) + self.write("{}\n".format(lines[0])) + + for line in lines[1:]: + self.write( + "{:>{w}}{}\n".format( + "", line, w=first_col + self.current_indent + ) + ) + + if len(lines) > 1: + # separate long help from next option + self.write("\n") else: - self.write('\n') + self.write("\n") @contextmanager def section(self, name): @@ -233,7 +260,7 @@ class HelpFormatter(object): def getvalue(self): """Returns the buffer contents.""" - return ''.join(self.buffer) + return "".join(self.buffer) def join_options(options): @@ -246,11 +273,11 @@ def join_options(options): any_prefix_is_slash = False for opt in options: prefix = split_opt(opt)[0] - if prefix == '/': + if prefix == "/": any_prefix_is_slash = True rv.append((len(prefix), opt)) rv.sort(key=lambda x: x[0]) - rv = ', '.join(x[1] for x in rv) + rv = ", ".join(x[1] for x in rv) return rv, any_prefix_is_slash diff --git a/pipenv/vendor/click/globals.py b/pipenv/vendor/click/globals.py index 843b594a..1649f9a0 100644 --- a/pipenv/vendor/click/globals.py +++ b/pipenv/vendor/click/globals.py @@ -1,6 +1,5 @@ from threading import local - _local = local() @@ -15,20 +14,20 @@ def get_current_context(silent=False): .. versionadded:: 5.0 - :param silent: is set to `True` the return value is `None` if no context + :param silent: if set to `True` the return value is `None` if no context is available. The default behavior is to raise a :exc:`RuntimeError`. """ try: - return getattr(_local, 'stack')[-1] + return _local.stack[-1] except (AttributeError, IndexError): if not silent: - raise RuntimeError('There is no active click context.') + raise RuntimeError("There is no active click context.") def push_context(ctx): """Pushes a new context to the current stack.""" - _local.__dict__.setdefault('stack', []).append(ctx) + _local.__dict__.setdefault("stack", []).append(ctx) def pop_context(): diff --git a/pipenv/vendor/click/parser.py b/pipenv/vendor/click/parser.py index 1c3ae9c8..f43ebfe9 100644 --- a/pipenv/vendor/click/parser.py +++ b/pipenv/vendor/click/parser.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- """ -click.parser -~~~~~~~~~~~~ - This module started out as largely a copy paste from the stdlib's optparse module with the features removed that we do not need from optparse because we implement them in Click on a higher level (for @@ -14,12 +11,20 @@ The reason this is a different module and not optparse from the stdlib is that there are differences in 2.x and 3.x about the error messages generated and optparse in the stdlib uses gettext for no good reason and might cause us issues. -""" +Click uses parts of optparse written by Gregory P. Ward and maintained +by the Python Software Foundation. This is limited to code in parser.py. + +Copyright 2001-2006 Gregory P. Ward. All rights reserved. +Copyright 2002-2006 Python Software Foundation. All rights reserved. +""" import re from collections import deque -from .exceptions import UsageError, NoSuchOption, BadOptionUsage, \ - BadArgumentUsage + +from .exceptions import BadArgumentUsage +from .exceptions import BadOptionUsage +from .exceptions import NoSuchOption +from .exceptions import UsageError def _unpack_args(args, nargs_spec): @@ -59,7 +64,7 @@ def _unpack_args(args, nargs_spec): rv.append(tuple(x)) elif nargs < 0: if spos is not None: - raise TypeError('Cannot have two nargs < 0') + raise TypeError("Cannot have two nargs < 0") spos = len(rv) rv.append(None) @@ -68,21 +73,21 @@ def _unpack_args(args, nargs_spec): if spos is not None: rv[spos] = tuple(args) args = [] - rv[spos + 1:] = reversed(rv[spos + 1:]) + rv[spos + 1 :] = reversed(rv[spos + 1 :]) return tuple(rv), list(args) def _error_opt_args(nargs, opt): if nargs == 1: - raise BadOptionUsage(opt, '%s option requires an argument' % opt) - raise BadOptionUsage(opt, '%s option requires %d arguments' % (opt, nargs)) + raise BadOptionUsage(opt, "{} option requires an argument".format(opt)) + raise BadOptionUsage(opt, "{} option requires {} arguments".format(opt, nargs)) def split_opt(opt): first = opt[:1] if first.isalnum(): - return '', opt + return "", opt if opt[1:2] == first: return opt[:2], opt[2:] return first, opt[1:] @@ -98,13 +103,14 @@ def normalize_opt(opt, ctx): def split_arg_string(string): """Given an argument string this attempts to split it into small parts.""" rv = [] - for match in re.finditer(r"('([^'\\]*(?:\\.[^'\\]*)*)'" - r'|"([^"\\]*(?:\\.[^"\\]*)*)"' - r'|\S+)\s*', string, re.S): + for match in re.finditer( + r"('([^'\\]*(?:\\.[^'\\]*)*)'|\"([^\"\\]*(?:\\.[^\"\\]*)*)\"|\S+)\s*", + string, + re.S, + ): arg = match.group().strip() - if arg[:1] == arg[-1:] and arg[:1] in '"\'': - arg = arg[1:-1].encode('ascii', 'backslashreplace') \ - .decode('unicode-escape') + if arg[:1] == arg[-1:] and arg[:1] in "\"'": + arg = arg[1:-1].encode("ascii", "backslashreplace").decode("unicode-escape") try: arg = type(string)(arg) except UnicodeError: @@ -114,7 +120,6 @@ def split_arg_string(string): class Option(object): - def __init__(self, opts, dest, action=None, nargs=1, const=None, obj=None): self._short_opts = [] self._long_opts = [] @@ -123,8 +128,7 @@ class Option(object): for opt in opts: prefix, value = split_opt(opt) if not prefix: - raise ValueError('Invalid start character for option (%s)' - % opt) + raise ValueError("Invalid start character for option ({})".format(opt)) self.prefixes.add(prefix[0]) if len(prefix) == 1 and len(value) == 1: self._short_opts.append(opt) @@ -133,7 +137,7 @@ class Option(object): self.prefixes.add(prefix) if action is None: - action = 'store' + action = "store" self.dest = dest self.action = action @@ -143,26 +147,25 @@ class Option(object): @property def takes_value(self): - return self.action in ('store', 'append') + return self.action in ("store", "append") def process(self, value, state): - if self.action == 'store': + if self.action == "store": state.opts[self.dest] = value - elif self.action == 'store_const': + elif self.action == "store_const": state.opts[self.dest] = self.const - elif self.action == 'append': + elif self.action == "append": state.opts.setdefault(self.dest, []).append(value) - elif self.action == 'append_const': + elif self.action == "append_const": state.opts.setdefault(self.dest, []).append(self.const) - elif self.action == 'count': + elif self.action == "count": state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 else: - raise ValueError('unknown action %r' % self.action) + raise ValueError("unknown action '{}'".format(self.action)) state.order.append(self.obj) class Argument(object): - def __init__(self, dest, nargs=1, obj=None): self.dest = dest self.nargs = nargs @@ -174,14 +177,14 @@ class Argument(object): if holes == len(value): value = None elif holes != 0: - raise BadArgumentUsage('argument %s takes %d values' - % (self.dest, self.nargs)) + raise BadArgumentUsage( + "argument {} takes {} values".format(self.dest, self.nargs) + ) state.opts[self.dest] = value state.order.append(self.obj) class ParsingState(object): - def __init__(self, rargs): self.opts = {} self.largs = [] @@ -222,11 +225,10 @@ class OptionParser(object): self.ignore_unknown_options = ctx.ignore_unknown_options self._short_opt = {} self._long_opt = {} - self._opt_prefixes = set(['-', '--']) + self._opt_prefixes = {"-", "--"} self._args = [] - def add_option(self, opts, dest, action=None, nargs=1, const=None, - obj=None): + def add_option(self, opts, dest, action=None, nargs=1, const=None, obj=None): """Adds a new option named `dest` to the parser. The destination is not inferred (unlike with optparse) and needs to be explicitly provided. Action can be any of ``store``, ``store_const``, @@ -238,8 +240,7 @@ class OptionParser(object): if obj is None: obj = dest opts = [normalize_opt(opt, self.ctx) for opt in opts] - option = Option(opts, dest, action=action, nargs=nargs, - const=const, obj=obj) + option = Option(opts, dest, action=action, nargs=nargs, const=const, obj=obj) self._opt_prefixes.update(option.prefixes) for opt in option._short_opts: self._short_opt[opt] = option @@ -273,8 +274,9 @@ class OptionParser(object): return state.opts, state.largs, state.order def _process_args_for_args(self, state): - pargs, args = _unpack_args(state.largs + state.rargs, - [x.nargs for x in self._args]) + pargs, args = _unpack_args( + state.largs + state.rargs, [x.nargs for x in self._args] + ) for idx, arg in enumerate(self._args): arg.process(pargs[idx], state) @@ -288,7 +290,7 @@ class OptionParser(object): arglen = len(arg) # Double dashes always handled explicitly regardless of what # prefixes are valid. - if arg == '--': + if arg == "--": return elif arg[:1] in self._opt_prefixes and arglen > 1: self._process_opts(arg, state) @@ -320,8 +322,7 @@ class OptionParser(object): def _match_long_opt(self, opt, explicit_value, state): if opt not in self._long_opt: - possibilities = [word for word in self._long_opt - if word.startswith(opt)] + possibilities = [word for word in self._long_opt if word.startswith(opt)] raise NoSuchOption(opt, possibilities=possibilities, ctx=self.ctx) option = self._long_opt[opt] @@ -343,7 +344,7 @@ class OptionParser(object): del state.rargs[:nargs] elif explicit_value is not None: - raise BadOptionUsage(opt, '%s option does not take a value' % opt) + raise BadOptionUsage(opt, "{} option does not take a value".format(opt)) else: value = None @@ -395,15 +396,15 @@ class OptionParser(object): # to the state as new larg. This way there is basic combinatorics # that can be achieved while still ignoring unknown arguments. if self.ignore_unknown_options and unknown_options: - state.largs.append(prefix + ''.join(unknown_options)) + state.largs.append("{}{}".format(prefix, "".join(unknown_options))) def _process_opts(self, arg, state): explicit_value = None # Long option handling happens in two parts. The first part is # supporting explicitly attached values. In any case, we will try # to long match the option first. - if '=' in arg: - long_opt, explicit_value = arg.split('=', 1) + if "=" in arg: + long_opt, explicit_value = arg.split("=", 1) else: long_opt = arg norm_long_opt = normalize_opt(long_opt, self.ctx) diff --git a/pipenv/vendor/click/termui.py b/pipenv/vendor/click/termui.py index bf9a3aa1..02ef9e9f 100644 --- a/pipenv/vendor/click/termui.py +++ b/pipenv/vendor/click/termui.py @@ -1,60 +1,89 @@ -import os -import sys -import struct import inspect +import io import itertools +import os +import struct +import sys -from ._compat import raw_input, text_type, string_types, \ - isatty, strip_ansi, get_winterm_size, DEFAULT_COLUMNS, WIN -from .utils import echo -from .exceptions import Abort, UsageError -from .types import convert_type, Choice, Path +from ._compat import DEFAULT_COLUMNS +from ._compat import get_winterm_size +from ._compat import isatty +from ._compat import raw_input +from ._compat import string_types +from ._compat import strip_ansi +from ._compat import text_type +from ._compat import WIN +from .exceptions import Abort +from .exceptions import UsageError from .globals import resolve_color_default - +from .types import Choice +from .types import convert_type +from .types import Path +from .utils import echo +from .utils import LazyFile # The prompt functions to use. The doc tools currently override these # functions to customize how they work. visible_prompt_func = raw_input _ansi_colors = { - 'black': 30, - 'red': 31, - 'green': 32, - 'yellow': 33, - 'blue': 34, - 'magenta': 35, - 'cyan': 36, - 'white': 37, - 'reset': 39, - 'bright_black': 90, - 'bright_red': 91, - 'bright_green': 92, - 'bright_yellow': 93, - 'bright_blue': 94, - 'bright_magenta': 95, - 'bright_cyan': 96, - 'bright_white': 97, + "black": 30, + "red": 31, + "green": 32, + "yellow": 33, + "blue": 34, + "magenta": 35, + "cyan": 36, + "white": 37, + "reset": 39, + "bright_black": 90, + "bright_red": 91, + "bright_green": 92, + "bright_yellow": 93, + "bright_blue": 94, + "bright_magenta": 95, + "bright_cyan": 96, + "bright_white": 97, } -_ansi_reset_all = '\033[0m' +_ansi_reset_all = "\033[0m" def hidden_prompt_func(prompt): import getpass + return getpass.getpass(prompt) -def _build_prompt(text, suffix, show_default=False, default=None, show_choices=True, type=None): +def _build_prompt( + text, suffix, show_default=False, default=None, show_choices=True, type=None +): prompt = text if type is not None and show_choices and isinstance(type, Choice): - prompt += ' (' + ", ".join(map(str, type.choices)) + ')' + prompt += " ({})".format(", ".join(map(str, type.choices))) if default is not None and show_default: - prompt = '%s [%s]' % (prompt, default) + prompt = "{} [{}]".format(prompt, _format_default(default)) return prompt + suffix -def prompt(text, default=None, hide_input=False, confirmation_prompt=False, - type=None, value_proc=None, prompt_suffix=': ', show_default=True, - err=False, show_choices=True): +def _format_default(default): + if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"): + return default.name + + return default + + +def prompt( + text, + default=None, + hide_input=False, + confirmation_prompt=False, + type=None, + value_proc=None, + prompt_suffix=": ", + show_default=True, + err=False, + show_choices=True, +): """Prompts a user for input. This is a convenience function that can be used to prompt a user for input later. @@ -92,12 +121,12 @@ def prompt(text, default=None, hide_input=False, confirmation_prompt=False, result = None def prompt_func(text): - f = hide_input and hidden_prompt_func or visible_prompt_func + f = hidden_prompt_func if hide_input else visible_prompt_func try: # Write the prompt separately so that we get nice # coloring through colorama on Windows echo(text, nl=False, err=err) - return f('') + return f("") except (KeyboardInterrupt, EOFError): # getpass doesn't print a newline if the user aborts input with ^C. # Allegedly this behavior is inherited from getpass(3). @@ -109,7 +138,9 @@ def prompt(text, default=None, hide_input=False, confirmation_prompt=False, if value_proc is None: value_proc = convert_type(type, default) - prompt = _build_prompt(text, prompt_suffix, show_default, default, show_choices, type) + prompt = _build_prompt( + text, prompt_suffix, show_default, default, show_choices, type + ) while 1: while 1: @@ -125,21 +156,22 @@ def prompt(text, default=None, hide_input=False, confirmation_prompt=False, try: result = value_proc(value) except UsageError as e: - echo('Error: %s' % e.message, err=err) + echo("Error: {}".format(e.message), err=err) # noqa: B306 continue if not confirmation_prompt: return result while 1: - value2 = prompt_func('Repeat for confirmation: ') + value2 = prompt_func("Repeat for confirmation: ") if value2: break if value == value2: return result - echo('Error: the two entered values do not match', err=err) + echo("Error: the two entered values do not match", err=err) -def confirm(text, default=False, abort=False, prompt_suffix=': ', - show_default=True, err=False): +def confirm( + text, default=False, abort=False, prompt_suffix=": ", show_default=True, err=False +): """Prompts for confirmation (yes/no question). If the user aborts the input by sending a interrupt signal this @@ -157,24 +189,25 @@ def confirm(text, default=False, abort=False, prompt_suffix=': ', :param err: if set to true the file defaults to ``stderr`` instead of ``stdout``, the same as with echo. """ - prompt = _build_prompt(text, prompt_suffix, show_default, - default and 'Y/n' or 'y/N') + prompt = _build_prompt( + text, prompt_suffix, show_default, "Y/n" if default else "y/N" + ) while 1: try: # Write the prompt separately so that we get nice # coloring through colorama on Windows echo(prompt, nl=False, err=err) - value = visible_prompt_func('').lower().strip() + value = visible_prompt_func("").lower().strip() except (KeyboardInterrupt, EOFError): raise Abort() - if value in ('y', 'yes'): + if value in ("y", "yes"): rv = True - elif value in ('n', 'no'): + elif value in ("n", "no"): rv = False - elif value == '': + elif value == "": rv = default else: - echo('Error: invalid input', err=err) + echo("Error: invalid input", err=err) continue break if abort and not rv: @@ -189,7 +222,8 @@ def get_terminal_size(): # If shutil has get_terminal_size() (Python 3.3 and later) use that if sys.version_info >= (3, 3): import shutil - shutil_get_terminal_size = getattr(shutil, 'get_terminal_size', None) + + shutil_get_terminal_size = getattr(shutil, "get_terminal_size", None) if shutil_get_terminal_size: sz = shutil_get_terminal_size() return sz.columns, sz.lines @@ -207,8 +241,8 @@ def get_terminal_size(): try: import fcntl import termios - cr = struct.unpack( - 'hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, '1234')) + + cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) except Exception: return return cr @@ -224,8 +258,7 @@ def get_terminal_size(): except Exception: pass if not cr or not cr[0] or not cr[1]: - cr = (os.environ.get('LINES', 25), - os.environ.get('COLUMNS', DEFAULT_COLUMNS)) + cr = (os.environ.get("LINES", 25), os.environ.get("COLUMNS", DEFAULT_COLUMNS)) return int(cr[1]), int(cr[0]) @@ -251,18 +284,29 @@ def echo_via_pager(text_or_generator, color=None): i = iter(text_or_generator) # convert every element of i to a text type if necessary - text_generator = (el if isinstance(el, string_types) else text_type(el) - for el in i) + text_generator = (el if isinstance(el, string_types) else text_type(el) for el in i) from ._termui_impl import pager + return pager(itertools.chain(text_generator, "\n"), color) -def progressbar(iterable=None, length=None, label=None, show_eta=True, - show_percent=None, show_pos=False, - item_show_func=None, fill_char='#', empty_char='-', - bar_template='%(label)s [%(bar)s] %(info)s', - info_sep=' ', width=36, file=None, color=None): +def progressbar( + iterable=None, + length=None, + label=None, + show_eta=True, + show_percent=None, + show_pos=False, + item_show_func=None, + fill_char="#", + empty_char="-", + bar_template="%(label)s [%(bar)s] %(info)s", + info_sep=" ", + width=36, + file=None, + color=None, +): """This function creates an iterable context manager that can be used to iterate over something while showing a progress bar. It will either iterate over the `iterable` or `length` items (that are counted @@ -272,11 +316,17 @@ def progressbar(iterable=None, length=None, label=None, show_eta=True, will not be rendered if the file is not a terminal. The context manager creates the progress bar. When the context - manager is entered the progress bar is already displayed. With every + manager is entered the progress bar is already created. With every iteration over the progress bar, the iterable passed to the bar is advanced and the bar is updated. When the context manager exits, a newline is printed and the progress bar is finalized on screen. + Note: The progress bar is currently designed for use cases where the + total progress can be expected to take at least several seconds. + Because of this, the ProgressBar class object won't display + progress that is considered too fast, and progress where the time + between steps is less than a second. + No printing must happen or the progress bar will be unintentionally destroyed. @@ -342,13 +392,24 @@ def progressbar(iterable=None, length=None, label=None, show_eta=True, which is not the case by default. """ from ._termui_impl import ProgressBar + color = resolve_color_default(color) - return ProgressBar(iterable=iterable, length=length, show_eta=show_eta, - show_percent=show_percent, show_pos=show_pos, - item_show_func=item_show_func, fill_char=fill_char, - empty_char=empty_char, bar_template=bar_template, - info_sep=info_sep, file=file, label=label, - width=width, color=color) + return ProgressBar( + iterable=iterable, + length=length, + show_eta=show_eta, + show_percent=show_percent, + show_pos=show_pos, + item_show_func=item_show_func, + fill_char=fill_char, + empty_char=empty_char, + bar_template=bar_template, + info_sep=info_sep, + file=file, + label=label, + width=width, + color=color, + ) def clear(): @@ -364,13 +425,22 @@ def clear(): # clear the screen by shelling out. Otherwise we can use an escape # sequence. if WIN: - os.system('cls') + os.system("cls") else: - sys.stdout.write('\033[2J\033[1;1H') + sys.stdout.write("\033[2J\033[1;1H") -def style(text, fg=None, bg=None, bold=None, dim=None, underline=None, - blink=None, reverse=None, reset=True): +def style( + text, + fg=None, + bg=None, + bold=None, + dim=None, + underline=None, + blink=None, + reverse=None, + reset=True, +): """Styles a text with ANSI styles and returns the new string. By default the styling is self contained which means that at the end of the string a reset code is issued. This can be prevented by @@ -425,28 +495,28 @@ def style(text, fg=None, bg=None, bold=None, dim=None, underline=None, bits = [] if fg: try: - bits.append('\033[%dm' % (_ansi_colors[fg])) + bits.append("\033[{}m".format(_ansi_colors[fg])) except KeyError: - raise TypeError('Unknown color %r' % fg) + raise TypeError("Unknown color '{}'".format(fg)) if bg: try: - bits.append('\033[%dm' % (_ansi_colors[bg] + 10)) + bits.append("\033[{}m".format(_ansi_colors[bg] + 10)) except KeyError: - raise TypeError('Unknown color %r' % bg) + raise TypeError("Unknown color '{}'".format(bg)) if bold is not None: - bits.append('\033[%dm' % (1 if bold else 22)) + bits.append("\033[{}m".format(1 if bold else 22)) if dim is not None: - bits.append('\033[%dm' % (2 if dim else 22)) + bits.append("\033[{}m".format(2 if dim else 22)) if underline is not None: - bits.append('\033[%dm' % (4 if underline else 24)) + bits.append("\033[{}m".format(4 if underline else 24)) if blink is not None: - bits.append('\033[%dm' % (5 if blink else 25)) + bits.append("\033[{}m".format(5 if blink else 25)) if reverse is not None: - bits.append('\033[%dm' % (7 if reverse else 27)) + bits.append("\033[{}m".format(7 if reverse else 27)) bits.append(text) if reset: bits.append(_ansi_reset_all) - return ''.join(bits) + return "".join(bits) def unstyle(text): @@ -478,8 +548,9 @@ def secho(message=None, file=None, nl=True, err=False, color=None, **styles): return echo(message, file=file, nl=nl, err=err, color=color) -def edit(text=None, editor=None, env=None, require_save=True, - extension='.txt', filename=None): +def edit( + text=None, editor=None, env=None, require_save=True, extension=".txt", filename=None +): r"""Edits the given text in the defined editor. If an editor is given (should be the full path to the executable but the regular operating system search path is used for finding the executable) it overrides @@ -508,8 +579,10 @@ def edit(text=None, editor=None, env=None, require_save=True, file as an indirection in that case. """ from ._termui_impl import Editor - editor = Editor(editor=editor, env=env, require_save=require_save, - extension=extension) + + editor = Editor( + editor=editor, env=env, require_save=require_save, extension=extension + ) if filename is None: return editor.edit(text) editor.edit_file(filename) @@ -538,6 +611,7 @@ def launch(url, wait=False, locate=False): the filesystem. """ from ._termui_impl import open_url + return open_url(url, wait=wait, locate=locate) @@ -574,10 +648,11 @@ def getchar(echo=False): def raw_terminal(): from ._termui_impl import raw_terminal as f + return f() -def pause(info='Press any key to continue ...', err=False): +def pause(info="Press any key to continue ...", err=False): """This command stops execution and waits for the user to press any key to continue. This is similar to the Windows batch "pause" command. If the program is not run through a terminal, this command diff --git a/pipenv/vendor/click/testing.py b/pipenv/vendor/click/testing.py index 1b2924e0..a3dba3b3 100644 --- a/pipenv/vendor/click/testing.py +++ b/pipenv/vendor/click/testing.py @@ -1,18 +1,16 @@ -import os -import sys -import shutil -import tempfile import contextlib +import os import shlex +import shutil +import sys +import tempfile -from ._compat import iteritems, PY2, string_types - - -# If someone wants to vendor click, we want to ensure the -# correct package is discovered. Ideally we could use a -# relative import here but unfortunately Python does not -# support that. -clickpkg = sys.modules[__name__.rsplit('.', 1)[0]] +from . import formatting +from . import termui +from . import utils +from ._compat import iteritems +from ._compat import PY2 +from ._compat import string_types if PY2: @@ -23,7 +21,6 @@ else: class EchoingStdin(object): - def __init__(self, input, output): self._input = input self._output = output @@ -53,16 +50,16 @@ class EchoingStdin(object): def make_input_stream(input, charset): # Is already an input stream. - if hasattr(input, 'read'): + if hasattr(input, "read"): if PY2: return input rv = _find_binary_reader(input) if rv is not None: return rv - raise TypeError('Could not find binary reader for input stream.') + raise TypeError("Could not find binary reader for input stream.") if input is None: - input = b'' + input = b"" elif not isinstance(input, bytes): input = input.encode(charset) if PY2: @@ -73,13 +70,14 @@ def make_input_stream(input, charset): class Result(object): """Holds the captured result of an invoked CLI script.""" - def __init__(self, runner, stdout_bytes, stderr_bytes, exit_code, - exception, exc_info=None): + def __init__( + self, runner, stdout_bytes, stderr_bytes, exit_code, exception, exc_info=None + ): #: The runner that created the result self.runner = runner #: The standard output as bytes. self.stdout_bytes = stdout_bytes - #: The standard error as bytes, or False(y) if not available + #: The standard error as bytes, or None if not available self.stderr_bytes = stderr_bytes #: The exit code as integer. self.exit_code = exit_code @@ -96,22 +94,22 @@ class Result(object): @property def stdout(self): """The standard output as unicode string.""" - return self.stdout_bytes.decode(self.runner.charset, 'replace') \ - .replace('\r\n', '\n') + return self.stdout_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) @property def stderr(self): """The standard error as unicode string.""" - if not self.stderr_bytes: + if self.stderr_bytes is None: raise ValueError("stderr not separately captured") - return self.stderr_bytes.decode(self.runner.charset, 'replace') \ - .replace('\r\n', '\n') - + return self.stderr_bytes.decode(self.runner.charset, "replace").replace( + "\r\n", "\n" + ) def __repr__(self): - return '<%s %s>' % ( - type(self).__name__, - self.exception and repr(self.exception) or 'okay', + return "<{} {}>".format( + type(self).__name__, repr(self.exception) if self.exception else "okay" ) @@ -136,10 +134,9 @@ class CliRunner(object): independently """ - def __init__(self, charset=None, env=None, echo_stdin=False, - mix_stderr=True): + def __init__(self, charset=None, env=None, echo_stdin=False, mix_stderr=True): if charset is None: - charset = 'utf-8' + charset = "utf-8" self.charset = charset self.env = env or {} self.echo_stdin = echo_stdin @@ -150,7 +147,7 @@ class CliRunner(object): for it. The default is the `name` attribute or ``"root"`` if not set. """ - return cli.name or 'root' + return cli.name or "root" def make_env(self, overrides=None): """Returns the environment overrides for invoking a script.""" @@ -182,8 +179,8 @@ class CliRunner(object): old_stdin = sys.stdin old_stdout = sys.stdout old_stderr = sys.stderr - old_forced_width = clickpkg.formatting.FORCED_WIDTH - clickpkg.formatting.FORCED_WIDTH = 80 + old_forced_width = formatting.FORCED_WIDTH + formatting.FORCED_WIDTH = 80 env = self.make_env(env) @@ -200,12 +197,10 @@ class CliRunner(object): if self.echo_stdin: input = EchoingStdin(input, bytes_output) input = io.TextIOWrapper(input, encoding=self.charset) - sys.stdout = io.TextIOWrapper( - bytes_output, encoding=self.charset) + sys.stdout = io.TextIOWrapper(bytes_output, encoding=self.charset) if not self.mix_stderr: bytes_error = io.BytesIO() - sys.stderr = io.TextIOWrapper( - bytes_error, encoding=self.charset) + sys.stderr = io.TextIOWrapper(bytes_error, encoding=self.charset) if self.mix_stderr: sys.stderr = sys.stdout @@ -213,16 +208,16 @@ class CliRunner(object): sys.stdin = input def visible_input(prompt=None): - sys.stdout.write(prompt or '') - val = input.readline().rstrip('\r\n') - sys.stdout.write(val + '\n') + sys.stdout.write(prompt or "") + val = input.readline().rstrip("\r\n") + sys.stdout.write("{}\n".format(val)) sys.stdout.flush() return val def hidden_input(prompt=None): - sys.stdout.write((prompt or '') + '\n') + sys.stdout.write("{}\n".format(prompt or "")) sys.stdout.flush() - return input.readline().rstrip('\r\n') + return input.readline().rstrip("\r\n") def _getchar(echo): char = sys.stdin.read(1) @@ -238,14 +233,14 @@ class CliRunner(object): return not default_color return not color - old_visible_prompt_func = clickpkg.termui.visible_prompt_func - old_hidden_prompt_func = clickpkg.termui.hidden_prompt_func - old__getchar_func = clickpkg.termui._getchar - old_should_strip_ansi = clickpkg.utils.should_strip_ansi - clickpkg.termui.visible_prompt_func = visible_input - clickpkg.termui.hidden_prompt_func = hidden_input - clickpkg.termui._getchar = _getchar - clickpkg.utils.should_strip_ansi = should_strip_ansi + old_visible_prompt_func = termui.visible_prompt_func + old_hidden_prompt_func = termui.hidden_prompt_func + old__getchar_func = termui._getchar + old_should_strip_ansi = utils.should_strip_ansi + termui.visible_prompt_func = visible_input + termui.hidden_prompt_func = hidden_input + termui._getchar = _getchar + utils.should_strip_ansi = should_strip_ansi old_env = {} try: @@ -271,14 +266,22 @@ class CliRunner(object): sys.stdout = old_stdout sys.stderr = old_stderr sys.stdin = old_stdin - clickpkg.termui.visible_prompt_func = old_visible_prompt_func - clickpkg.termui.hidden_prompt_func = old_hidden_prompt_func - clickpkg.termui._getchar = old__getchar_func - clickpkg.utils.should_strip_ansi = old_should_strip_ansi - clickpkg.formatting.FORCED_WIDTH = old_forced_width + termui.visible_prompt_func = old_visible_prompt_func + termui.hidden_prompt_func = old_hidden_prompt_func + termui._getchar = old__getchar_func + utils.should_strip_ansi = old_should_strip_ansi + formatting.FORCED_WIDTH = old_forced_width - def invoke(self, cli, args=None, input=None, env=None, - catch_exceptions=True, color=False, mix_stderr=False, **extra): + def invoke( + self, + cli, + args=None, + input=None, + env=None, + catch_exceptions=True, + color=False, + **extra + ): """Invokes a command in an isolated environment. The arguments are forwarded directly to the command line script, the `extra` keyword arguments are passed to the :meth:`~clickpkg.Command.main` function of @@ -335,7 +338,7 @@ class CliRunner(object): if not isinstance(exit_code, int): sys.stdout.write(str(exit_code)) - sys.stdout.write('\n') + sys.stdout.write("\n") exit_code = 1 except Exception as e: @@ -347,14 +350,19 @@ class CliRunner(object): finally: sys.stdout.flush() stdout = outstreams[0].getvalue() - stderr = outstreams[1] and outstreams[1].getvalue() + if self.mix_stderr: + stderr = None + else: + stderr = outstreams[1].getvalue() - return Result(runner=self, - stdout_bytes=stdout, - stderr_bytes=stderr, - exit_code=exit_code, - exception=exception, - exc_info=exc_info) + return Result( + runner=self, + stdout_bytes=stdout, + stderr_bytes=stderr, + exit_code=exit_code, + exception=exception, + exc_info=exc_info, + ) @contextlib.contextmanager def isolated_filesystem(self): @@ -370,5 +378,5 @@ class CliRunner(object): os.chdir(cwd) try: shutil.rmtree(t) - except (OSError, IOError): + except (OSError, IOError): # noqa: B014 pass diff --git a/pipenv/vendor/click/types.py b/pipenv/vendor/click/types.py index 1f88032f..505c39f8 100644 --- a/pipenv/vendor/click/types.py +++ b/pipenv/vendor/click/types.py @@ -2,10 +2,16 @@ import os import stat from datetime import datetime -from ._compat import open_stream, text_type, filename_to_ui, \ - get_filesystem_encoding, get_streerror, _get_argv_encoding, PY2 +from ._compat import _get_argv_encoding +from ._compat import filename_to_ui +from ._compat import get_filesystem_encoding +from ._compat import get_streerror +from ._compat import open_stream +from ._compat import PY2 +from ._compat import text_type from .exceptions import BadParameter -from .utils import safecall, LazyFile +from .utils import LazyFile +from .utils import safecall class ParamType(object): @@ -21,6 +27,7 @@ class ParamType(object): This can be the case when the object is used with prompt inputs. """ + is_composite = False #: the descriptive name of this type @@ -62,7 +69,7 @@ class ParamType(object): then leading and trailing whitespace is ignored. Otherwise, leading and trailing splitters usually lead to empty items being included. """ - return (rv or '').split(self.envvar_list_splitter) + return (rv or "").split(self.envvar_list_splitter) def fail(self, message, param=None, ctx=None): """Helper method to fail with an invalid value message.""" @@ -78,7 +85,6 @@ class CompositeParamType(ParamType): class FuncParamType(ParamType): - def __init__(self, func): self.name = func.__name__ self.func = func @@ -90,22 +96,22 @@ class FuncParamType(ParamType): try: value = text_type(value) except UnicodeError: - value = str(value).decode('utf-8', 'replace') + value = str(value).decode("utf-8", "replace") self.fail(value, param, ctx) class UnprocessedParamType(ParamType): - name = 'text' + name = "text" def convert(self, value, param, ctx): return value def __repr__(self): - return 'UNPROCESSED' + return "UNPROCESSED" class StringParamType(ParamType): - name = 'text' + name = "text" def convert(self, value, param, ctx): if isinstance(value, bytes): @@ -118,12 +124,14 @@ class StringParamType(ParamType): try: value = value.decode(fs_enc) except UnicodeError: - value = value.decode('utf-8', 'replace') + value = value.decode("utf-8", "replace") + else: + value = value.decode("utf-8", "replace") return value return value def __repr__(self): - return 'STRING' + return "STRING" class Choice(ParamType): @@ -133,54 +141,68 @@ class Choice(ParamType): You should only pass a list or tuple of choices. Other iterables (like generators) may lead to surprising results. + The resulting value will always be one of the originally passed choices + regardless of ``case_sensitive`` or any ``ctx.token_normalize_func`` + being specified. + See :ref:`choice-opts` for an example. :param case_sensitive: Set to false to make choices case insensitive. Defaults to true. """ - name = 'choice' + name = "choice" def __init__(self, choices, case_sensitive=True): self.choices = choices self.case_sensitive = case_sensitive def get_metavar(self, param): - return '[%s]' % '|'.join(self.choices) + return "[{}]".format("|".join(self.choices)) def get_missing_message(self, param): - return 'Choose from:\n\t%s.' % ',\n\t'.join(self.choices) + return "Choose from:\n\t{}.".format(",\n\t".join(self.choices)) def convert(self, value, param, ctx): - # Exact match - if value in self.choices: - return value - # Match through normalization and case sensitivity # first do token_normalize_func, then lowercase # preserve original `value` to produce an accurate message in # `self.fail` normed_value = value - normed_choices = self.choices + normed_choices = {choice: choice for choice in self.choices} - if ctx is not None and \ - ctx.token_normalize_func is not None: + if ctx is not None and ctx.token_normalize_func is not None: normed_value = ctx.token_normalize_func(value) - normed_choices = [ctx.token_normalize_func(choice) for choice in - self.choices] + normed_choices = { + ctx.token_normalize_func(normed_choice): original + for normed_choice, original in normed_choices.items() + } if not self.case_sensitive: - normed_value = normed_value.lower() - normed_choices = [choice.lower() for choice in normed_choices] + if PY2: + lower = str.lower + else: + lower = str.casefold + + normed_value = lower(normed_value) + normed_choices = { + lower(normed_choice): original + for normed_choice, original in normed_choices.items() + } if normed_value in normed_choices: - return normed_value + return normed_choices[normed_value] - self.fail('invalid choice: %s. (choose from %s)' % - (value, ', '.join(self.choices)), param, ctx) + self.fail( + "invalid choice: {}. (choose from {})".format( + value, ", ".join(self.choices) + ), + param, + ctx, + ) def __repr__(self): - return 'Choice(%r)' % list(self.choices) + return "Choice('{}')".format(list(self.choices)) class DateTime(ParamType): @@ -203,17 +225,14 @@ class DateTime(ParamType): ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``, ``'%Y-%m-%d %H:%M:%S'``. """ - name = 'datetime' + + name = "datetime" def __init__(self, formats=None): - self.formats = formats or [ - '%Y-%m-%d', - '%Y-%m-%dT%H:%M:%S', - '%Y-%m-%d %H:%M:%S' - ] + self.formats = formats or ["%Y-%m-%d", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d %H:%M:%S"] def get_metavar(self, param): - return '[{}]'.format('|'.join(self.formats)) + return "[{}]".format("|".join(self.formats)) def _try_to_convert_date(self, value, format): try: @@ -229,24 +248,26 @@ class DateTime(ParamType): return dtime self.fail( - 'invalid datetime format: {}. (choose from {})'.format( - value, ', '.join(self.formats))) + "invalid datetime format: {}. (choose from {})".format( + value, ", ".join(self.formats) + ) + ) def __repr__(self): - return 'DateTime' + return "DateTime" class IntParamType(ParamType): - name = 'integer' + name = "integer" def convert(self, value, param, ctx): try: return int(value) - except (ValueError, UnicodeError): - self.fail('%s is not a valid integer' % value, param, ctx) + except ValueError: + self.fail("{} is not a valid integer".format(value), param, ctx) def __repr__(self): - return 'INT' + return "INT" class IntRange(IntParamType): @@ -257,7 +278,8 @@ class IntRange(IntParamType): See :ref:`ranges` for an example. """ - name = 'integer range' + + name = "integer range" def __init__(self, min=None, max=None, clamp=False): self.min = min @@ -271,35 +293,55 @@ class IntRange(IntParamType): return self.min if self.max is not None and rv > self.max: return self.max - if self.min is not None and rv < self.min or \ - self.max is not None and rv > self.max: + if ( + self.min is not None + and rv < self.min + or self.max is not None + and rv > self.max + ): if self.min is None: - self.fail('%s is bigger than the maximum valid value ' - '%s.' % (rv, self.max), param, ctx) + self.fail( + "{} is bigger than the maximum valid value {}.".format( + rv, self.max + ), + param, + ctx, + ) elif self.max is None: - self.fail('%s is smaller than the minimum valid value ' - '%s.' % (rv, self.min), param, ctx) + self.fail( + "{} is smaller than the minimum valid value {}.".format( + rv, self.min + ), + param, + ctx, + ) else: - self.fail('%s is not in the valid range of %s to %s.' - % (rv, self.min, self.max), param, ctx) + self.fail( + "{} is not in the valid range of {} to {}.".format( + rv, self.min, self.max + ), + param, + ctx, + ) return rv def __repr__(self): - return 'IntRange(%r, %r)' % (self.min, self.max) + return "IntRange({}, {})".format(self.min, self.max) class FloatParamType(ParamType): - name = 'float' + name = "float" def convert(self, value, param, ctx): try: return float(value) - except (UnicodeError, ValueError): - self.fail('%s is not a valid floating point value' % - value, param, ctx) + except ValueError: + self.fail( + "{} is not a valid floating point value".format(value), param, ctx + ) def __repr__(self): - return 'FLOAT' + return "FLOAT" class FloatRange(FloatParamType): @@ -310,7 +352,8 @@ class FloatRange(FloatParamType): See :ref:`ranges` for an example. """ - name = 'float range' + + name = "float range" def __init__(self, min=None, max=None, clamp=False): self.min = min @@ -324,54 +367,74 @@ class FloatRange(FloatParamType): return self.min if self.max is not None and rv > self.max: return self.max - if self.min is not None and rv < self.min or \ - self.max is not None and rv > self.max: + if ( + self.min is not None + and rv < self.min + or self.max is not None + and rv > self.max + ): if self.min is None: - self.fail('%s is bigger than the maximum valid value ' - '%s.' % (rv, self.max), param, ctx) + self.fail( + "{} is bigger than the maximum valid value {}.".format( + rv, self.max + ), + param, + ctx, + ) elif self.max is None: - self.fail('%s is smaller than the minimum valid value ' - '%s.' % (rv, self.min), param, ctx) + self.fail( + "{} is smaller than the minimum valid value {}.".format( + rv, self.min + ), + param, + ctx, + ) else: - self.fail('%s is not in the valid range of %s to %s.' - % (rv, self.min, self.max), param, ctx) + self.fail( + "{} is not in the valid range of {} to {}.".format( + rv, self.min, self.max + ), + param, + ctx, + ) return rv def __repr__(self): - return 'FloatRange(%r, %r)' % (self.min, self.max) + return "FloatRange({}, {})".format(self.min, self.max) class BoolParamType(ParamType): - name = 'boolean' + name = "boolean" def convert(self, value, param, ctx): if isinstance(value, bool): return bool(value) value = value.lower() - if value in ('true', 't', '1', 'yes', 'y'): + if value in ("true", "t", "1", "yes", "y"): return True - elif value in ('false', 'f', '0', 'no', 'n'): + elif value in ("false", "f", "0", "no", "n"): return False - self.fail('%s is not a valid boolean' % value, param, ctx) + self.fail("{} is not a valid boolean".format(value), param, ctx) def __repr__(self): - return 'BOOL' + return "BOOL" class UUIDParameterType(ParamType): - name = 'uuid' + name = "uuid" def convert(self, value, param, ctx): import uuid + try: if PY2 and isinstance(value, text_type): - value = value.encode('ascii') + value = value.encode("ascii") return uuid.UUID(value) - except (UnicodeError, ValueError): - self.fail('%s is not a valid UUID value' % value, param, ctx) + except ValueError: + self.fail("{} is not a valid UUID value".format(value), param, ctx) def __repr__(self): - return 'UUID' + return "UUID" class File(ParamType): @@ -400,11 +463,13 @@ class File(ParamType): See :ref:`file-args` for more information. """ - name = 'filename' + + name = "filename" envvar_list_splitter = os.path.pathsep - def __init__(self, mode='r', encoding=None, errors='strict', lazy=None, - atomic=False): + def __init__( + self, mode="r", encoding=None, errors="strict", lazy=None, atomic=False + ): self.mode = mode self.encoding = encoding self.errors = errors @@ -414,29 +479,30 @@ class File(ParamType): def resolve_lazy_flag(self, value): if self.lazy is not None: return self.lazy - if value == '-': + if value == "-": return False - elif 'w' in self.mode: + elif "w" in self.mode: return True return False def convert(self, value, param, ctx): try: - if hasattr(value, 'read') or hasattr(value, 'write'): + if hasattr(value, "read") or hasattr(value, "write"): return value lazy = self.resolve_lazy_flag(value) if lazy: - f = LazyFile(value, self.mode, self.encoding, self.errors, - atomic=self.atomic) + f = LazyFile( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) if ctx is not None: ctx.call_on_close(f.close_intelligently) return f - f, should_close = open_stream(value, self.mode, - self.encoding, self.errors, - atomic=self.atomic) + f, should_close = open_stream( + value, self.mode, self.encoding, self.errors, atomic=self.atomic + ) # If a context is provided, we automatically close the file # at the end of the context execution (or flush out). If a # context does not exist, it's the caller's responsibility to @@ -448,11 +514,14 @@ class File(ParamType): else: ctx.call_on_close(safecall(f.flush)) return f - except (IOError, OSError) as e: - self.fail('Could not open file: %s: %s' % ( - filename_to_ui(value), - get_streerror(e), - ), param, ctx) + except (IOError, OSError) as e: # noqa: B014 + self.fail( + "Could not open file: {}: {}".format( + filename_to_ui(value), get_streerror(e) + ), + param, + ctx, + ) class Path(ParamType): @@ -485,11 +554,20 @@ class Path(ParamType): unicode depending on what makes most sense given the input data Click deals with. """ + envvar_list_splitter = os.path.pathsep - def __init__(self, exists=False, file_okay=True, dir_okay=True, - writable=False, readable=True, resolve_path=False, - allow_dash=False, path_type=None): + def __init__( + self, + exists=False, + file_okay=True, + dir_okay=True, + writable=False, + readable=True, + resolve_path=False, + allow_dash=False, + path_type=None, + ): self.exists = exists self.file_okay = file_okay self.dir_okay = dir_okay @@ -500,14 +578,14 @@ class Path(ParamType): self.type = path_type if self.file_okay and not self.dir_okay: - self.name = 'file' - self.path_type = 'File' + self.name = "file" + self.path_type = "File" elif self.dir_okay and not self.file_okay: - self.name = 'directory' - self.path_type = 'Directory' + self.name = "directory" + self.path_type = "Directory" else: - self.name = 'path' - self.path_type = 'Path' + self.name = "path" + self.path_type = "Path" def coerce_path_result(self, rv): if self.type is not None and not isinstance(rv, self.type): @@ -520,7 +598,7 @@ class Path(ParamType): def convert(self, value, param, ctx): rv = value - is_dash = self.file_okay and self.allow_dash and rv in (b'-', '-') + is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-") if not is_dash: if self.resolve_path: @@ -531,31 +609,44 @@ class Path(ParamType): except OSError: if not self.exists: return self.coerce_path_result(rv) - self.fail('%s "%s" does not exist.' % ( - self.path_type, - filename_to_ui(value) - ), param, ctx) + self.fail( + "{} '{}' does not exist.".format( + self.path_type, filename_to_ui(value) + ), + param, + ctx, + ) if not self.file_okay and stat.S_ISREG(st.st_mode): - self.fail('%s "%s" is a file.' % ( - self.path_type, - filename_to_ui(value) - ), param, ctx) + self.fail( + "{} '{}' is a file.".format(self.path_type, filename_to_ui(value)), + param, + ctx, + ) if not self.dir_okay and stat.S_ISDIR(st.st_mode): - self.fail('%s "%s" is a directory.' % ( - self.path_type, - filename_to_ui(value) - ), param, ctx) + self.fail( + "{} '{}' is a directory.".format( + self.path_type, filename_to_ui(value) + ), + param, + ctx, + ) if self.writable and not os.access(value, os.W_OK): - self.fail('%s "%s" is not writable.' % ( - self.path_type, - filename_to_ui(value) - ), param, ctx) + self.fail( + "{} '{}' is not writable.".format( + self.path_type, filename_to_ui(value) + ), + param, + ctx, + ) if self.readable and not os.access(value, os.R_OK): - self.fail('%s "%s" is not readable.' % ( - self.path_type, - filename_to_ui(value) - ), param, ctx) + self.fail( + "{} '{}' is not readable.".format( + self.path_type, filename_to_ui(value) + ), + param, + ctx, + ) return self.coerce_path_result(rv) @@ -579,7 +670,7 @@ class Tuple(CompositeParamType): @property def name(self): - return "<" + " ".join(ty.name for ty in self.types) + ">" + return "<{}>".format(" ".join(ty.name for ty in self.types)) @property def arity(self): @@ -587,14 +678,16 @@ class Tuple(CompositeParamType): def convert(self, value, param, ctx): if len(value) != len(self.types): - raise TypeError('It would appear that nargs is set to conflict ' - 'with the composite type arity.') + raise TypeError( + "It would appear that nargs is set to conflict with the" + " composite type arity." + ) return tuple(ty(x, param, ctx) for ty, x in zip(self.types, value)) def convert_type(ty, default=None): - """Converts a callable or python ty into the most appropriate param - ty. + """Converts a callable or python type into the most appropriate + param type. """ guessed_type = False if ty is None and default is not None: @@ -627,8 +720,9 @@ def convert_type(ty, default=None): if __debug__: try: if issubclass(ty, ParamType): - raise AssertionError('Attempted to use an uninstantiated ' - 'parameter type (%s).' % ty) + raise AssertionError( + "Attempted to use an uninstantiated parameter type ({}).".format(ty) + ) except TypeError: pass return FuncParamType(ty) diff --git a/pipenv/vendor/click/utils.py b/pipenv/vendor/click/utils.py index fc84369f..79265e73 100644 --- a/pipenv/vendor/click/utils.py +++ b/pipenv/vendor/click/utils.py @@ -1,34 +1,47 @@ import os import sys +from ._compat import _default_text_stderr +from ._compat import _default_text_stdout +from ._compat import auto_wrap_for_ansi +from ._compat import binary_streams +from ._compat import filename_to_ui +from ._compat import get_filesystem_encoding +from ._compat import get_streerror +from ._compat import is_bytes +from ._compat import open_stream +from ._compat import PY2 +from ._compat import should_strip_ansi +from ._compat import string_types +from ._compat import strip_ansi +from ._compat import text_streams +from ._compat import text_type +from ._compat import WIN from .globals import resolve_color_default -from ._compat import text_type, open_stream, get_filesystem_encoding, \ - get_streerror, string_types, PY2, binary_streams, text_streams, \ - filename_to_ui, auto_wrap_for_ansi, strip_ansi, should_strip_ansi, \ - _default_text_stdout, _default_text_stderr, is_bytes, WIN - if not PY2: from ._compat import _find_binary_writer elif WIN: - from ._winconsole import _get_windows_argv, \ - _hash_py_argv, _initial_argv_hash - + from ._winconsole import _get_windows_argv + from ._winconsole import _hash_py_argv + from ._winconsole import _initial_argv_hash echo_native_types = string_types + (bytes, bytearray) def _posixify(name): - return '-'.join(name.split()).lower() + return "-".join(name.split()).lower() def safecall(func): """Wraps a function so that it swallows exceptions.""" + def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception: pass + return wrapper @@ -38,7 +51,7 @@ def make_str(value): try: return value.decode(get_filesystem_encoding()) except UnicodeError: - return value.decode('utf-8', 'replace') + return value.decode("utf-8", "replace") return text_type(value) @@ -50,21 +63,21 @@ def make_default_short_help(help, max_length=45): done = False for word in words: - if word[-1:] == '.': + if word[-1:] == ".": done = True - new_length = result and 1 + len(word) or len(word) + new_length = 1 + len(word) if result else len(word) if total_length + new_length > max_length: - result.append('...') + result.append("...") done = True else: if result: - result.append(' ') + result.append(" ") result.append(word) if done: break total_length += new_length - return ''.join(result) + return "".join(result) class LazyFile(object): @@ -74,19 +87,19 @@ class LazyFile(object): files for writing. """ - def __init__(self, filename, mode='r', encoding=None, errors='strict', - atomic=False): + def __init__( + self, filename, mode="r", encoding=None, errors="strict", atomic=False + ): self.name = filename self.mode = mode self.encoding = encoding self.errors = errors self.atomic = atomic - if filename == '-': - self._f, self.should_close = open_stream(filename, mode, - encoding, errors) + if filename == "-": + self._f, self.should_close = open_stream(filename, mode, encoding, errors) else: - if 'r' in mode: + if "r" in mode: # Open and close the file in case we're opening it for # reading so that we can catch at least some errors in # some cases early. @@ -100,7 +113,7 @@ class LazyFile(object): def __repr__(self): if self._f is not None: return repr(self._f) - return '<unopened file %r %s>' % (self.name, self.mode) + return "<unopened file '{}' {}>".format(self.name, self.mode) def open(self): """Opens the file if it's not yet open. This call might fail with @@ -110,12 +123,12 @@ class LazyFile(object): if self._f is not None: return self._f try: - rv, self.should_close = open_stream(self.name, self.mode, - self.encoding, - self.errors, - atomic=self.atomic) - except (IOError, OSError) as e: + rv, self.should_close = open_stream( + self.name, self.mode, self.encoding, self.errors, atomic=self.atomic + ) + except (IOError, OSError) as e: # noqa: E402 from .exceptions import FileError + raise FileError(self.name, hint=get_streerror(e)) self._f = rv return rv @@ -144,7 +157,6 @@ class LazyFile(object): class KeepOpenFile(object): - def __init__(self, file): self._file = file @@ -222,11 +234,11 @@ def echo(message=None, file=None, nl=True, err=False, color=None): message = text_type(message) if nl: - message = message or u'' + message = message or u"" if isinstance(message, text_type): - message += u'\n' + message += u"\n" else: - message += b'\n' + message += b"\n" # If there is a message, and we're in Python 3, and the value looks # like bytes, we manually need to find the binary stream and write the @@ -273,11 +285,11 @@ def get_binary_stream(name): """ opener = binary_streams.get(name) if opener is None: - raise TypeError('Unknown standard stream %r' % name) + raise TypeError("Unknown standard stream '{}'".format(name)) return opener() -def get_text_stream(name, encoding=None, errors='strict'): +def get_text_stream(name, encoding=None, errors="strict"): """Returns a system stream for text processing. This usually returns a wrapped stream around a binary stream returned from :func:`get_binary_stream` but it also can take shortcuts on Python 3 @@ -290,12 +302,13 @@ def get_text_stream(name, encoding=None, errors='strict'): """ opener = text_streams.get(name) if opener is None: - raise TypeError('Unknown standard stream %r' % name) + raise TypeError("Unknown standard stream '{}'".format(name)) return opener(encoding, errors) -def open_file(filename, mode='r', encoding=None, errors='strict', - lazy=False, atomic=False): +def open_file( + filename, mode="r", encoding=None, errors="strict", lazy=False, atomic=False +): """This is similar to how the :class:`File` works but for manual usage. Files are opened non lazy by default. This can open regular files as well as stdin/stdout if ``'-'`` is passed. @@ -320,8 +333,7 @@ def open_file(filename, mode='r', encoding=None, errors='strict', """ if lazy: return LazyFile(filename, mode, encoding, errors, atomic=atomic) - f, should_close = open_stream(filename, mode, encoding, errors, - atomic=atomic) + f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic) if not should_close: f = KeepOpenFile(f) return f @@ -401,19 +413,21 @@ def get_app_dir(app_name, roaming=True, force_posix=False): application support folder. """ if WIN: - key = roaming and 'APPDATA' or 'LOCALAPPDATA' + key = "APPDATA" if roaming else "LOCALAPPDATA" folder = os.environ.get(key) if folder is None: - folder = os.path.expanduser('~') + folder = os.path.expanduser("~") return os.path.join(folder, app_name) if force_posix: - return os.path.join(os.path.expanduser('~/.' + _posixify(app_name))) - if sys.platform == 'darwin': - return os.path.join(os.path.expanduser( - '~/Library/Application Support'), app_name) + return os.path.join(os.path.expanduser("~/.{}".format(_posixify(app_name)))) + if sys.platform == "darwin": + return os.path.join( + os.path.expanduser("~/Library/Application Support"), app_name + ) return os.path.join( - os.environ.get('XDG_CONFIG_HOME', os.path.expanduser('~/.config')), - _posixify(app_name)) + os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), + _posixify(app_name), + ) class PacifyFlushWrapper(object): @@ -433,6 +447,7 @@ class PacifyFlushWrapper(object): self.wrapped.flush() except IOError as e: import errno + if e.errno != errno.EPIPE: raise diff --git a/pipenv/vendor/click_completion/__init__.py b/pipenv/vendor/click_completion/__init__.py index e30cc0e3..4d6444c7 100644 --- a/pipenv/vendor/click_completion/__init__.py +++ b/pipenv/vendor/click_completion/__init__.py @@ -19,7 +19,7 @@ from click_completion.core import completion_configuration, get_code, install, s from click_completion.lib import get_auto_shell from click_completion.patch import patch as _patch -__version__ = '0.5.0' +__version__ = '0.5.2' _initialized = False diff --git a/pipenv/vendor/click_completion/core.py b/pipenv/vendor/click_completion/core.py index dc47d471..74da12d0 100644 --- a/pipenv/vendor/click_completion/core.py +++ b/pipenv/vendor/click_completion/core.py @@ -121,6 +121,9 @@ def get_choices(cli, prog_name, args, incomplete): else: for param in ctx.command.get_params(ctx): if (completion_configuration.complete_options or incomplete and not incomplete[:1].isalnum()) and isinstance(param, Option): + # filter hidden click.Option + if getattr(param, 'hidden', False): + continue for opt in param.opts: if match(opt, incomplete): choices.append((opt, param.help)) @@ -201,7 +204,7 @@ def do_fish_complete(cli, prog_name): for item, help in get_choices(cli, prog_name, args, incomplete): if help: - echo("%s\t%s" % (item, re.sub('\s', ' ', help))) + echo("%s\t%s" % (item, re.sub(r'\s', ' ', help))) else: echo(item) @@ -232,11 +235,11 @@ def do_zsh_complete(cli, prog_name): incomplete = '' def escape(s): - return s.replace('"', '""').replace("'", "''").replace('$', '\\$') + return s.replace('"', '""').replace("'", "''").replace('$', '\\$').replace('`', '\\`') res = [] for item, help in get_choices(cli, prog_name, args, incomplete): if help: - res.append('"%s"\:"%s"' % (escape(item), escape(help))) + res.append(r'"%s"\:"%s"' % (escape(item), escape(help))) else: res.append('"%s"' % escape(item)) if res: @@ -349,13 +352,8 @@ def install(shell=None, prog_name=None, env_name=None, path=None, append=None, e path = path or os.path.expanduser('~') + '/.bash_completion' mode = mode or 'a' elif shell == 'zsh': - ohmyzsh = os.path.expanduser('~') + '/.oh-my-zsh' - if os.path.exists(ohmyzsh): - path = path or ohmyzsh + '/completions/_%s' % prog_name - mode = mode or 'w' - else: - path = path or os.path.expanduser('~') + '/.zshrc' - mode = mode or 'a' + path = path or os.path.expanduser('~') + '/.zshrc' + mode = mode or 'a' elif shell == 'powershell': subprocess.check_call(['powershell', 'Set-ExecutionPolicy Unrestricted -Scope CurrentUser']) path = path or subprocess.check_output(['powershell', '-NoProfile', 'echo $profile']).strip() if install else '' diff --git a/pipenv/vendor/click_completion/lib.py b/pipenv/vendor/click_completion/lib.py index 167ecfd8..fc195cd4 100644 --- a/pipenv/vendor/click_completion/lib.py +++ b/pipenv/vendor/click_completion/lib.py @@ -121,7 +121,5 @@ def split_args(line): def get_auto_shell(): - """Returns the current shell - - This feature depends on psutil and will not work if it is not available""" + """Returns the current shell""" return shellingham.detect_shell()[0] diff --git a/pipenv/vendor/click_completion/patch.py b/pipenv/vendor/click_completion/patch.py index ab351f45..409c192e 100644 --- a/pipenv/vendor/click_completion/patch.py +++ b/pipenv/vendor/click_completion/patch.py @@ -75,7 +75,29 @@ def multicommand_get_command_short_help(self, ctx, cmd_name): str The sub command short help """ - return self.get_command(ctx, cmd_name).short_help + return self.get_command(ctx, cmd_name).get_short_help_str() + + +def multicommand_get_command_hidden(self, ctx, cmd_name): + """Returns the short help of a subcommand + + It allows MultiCommand subclasses to implement more efficient ways to provide the subcommand hidden attribute, for + example by leveraging some caching. + + Parameters + ---------- + ctx : click.core.Context + The current context + cmd_name : + The sub command name + + Returns + ------- + bool + The sub command hidden status + """ + cmd = self.get_command(ctx, cmd_name) + return cmd.hidden if cmd else False def _shellcomplete(cli, prog_name, complete_var=None): @@ -139,4 +161,5 @@ def patch(): click.types.ParamType.complete = param_type_complete click.types.Choice.complete = choice_complete click.core.MultiCommand.get_command_short_help = multicommand_get_command_short_help + click.core.MultiCommand.get_command_hidden = multicommand_get_command_hidden click.core._bashcomplete = _shellcomplete diff --git a/pipenv/vendor/click_completion/zsh.j2 b/pipenv/vendor/click_completion/zsh.j2 index 9e1024a8..ac796615 100644 --- a/pipenv/vendor/click_completion/zsh.j2 +++ b/pipenv/vendor/click_completion/zsh.j2 @@ -3,6 +3,5 @@ _{{prog_name}}() { eval $(env COMMANDLINE="${words[1,$CURRENT]}" {{complete_var}}=complete-zsh {% for k, v in extra_env.items() %} {{k}}={{v}}{% endfor %} {{prog_name}}) } if [[ "$(basename -- ${(%):-%x})" != "_{{prog_name}}" ]]; then - autoload -U compinit && compinit compdef _{{prog_name}} {{prog_name}} fi diff --git a/pipenv/vendor/colorama/LICENSE.txt b/pipenv/vendor/colorama/LICENSE.txt index 5f567799..3105888e 100644 --- a/pipenv/vendor/colorama/LICENSE.txt +++ b/pipenv/vendor/colorama/LICENSE.txt @@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pipenv/vendor/colorama/__init__.py b/pipenv/vendor/colorama/__init__.py index f4d9ce21..34c263cc 100644 --- a/pipenv/vendor/colorama/__init__.py +++ b/pipenv/vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.3' diff --git a/pipenv/vendor/colorama/ansitowin32.py b/pipenv/vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/vendor/colorama/ansitowin32.py +++ b/pipenv/vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/vendor/colorama/initialise.py b/pipenv/vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/vendor/colorama/initialise.py +++ b/pipenv/vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/vendor/colorama/win32.py b/pipenv/vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/vendor/colorama/win32.py +++ b/pipenv/vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/vendor/colorama/winterm.py b/pipenv/vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/vendor/colorama/winterm.py +++ b/pipenv/vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/vendor/contextlib2.LICENSE.txt b/pipenv/vendor/contextlib2.LICENSE.txt new file mode 100644 index 00000000..5de20277 --- /dev/null +++ b/pipenv/vendor/contextlib2.LICENSE.txt @@ -0,0 +1,122 @@ + + +A. HISTORY OF THE SOFTWARE +========================== + +contextlib2 is a derivative of the contextlib module distributed by the PSF +as part of the Python standard library. According, it is itself redistributed +under the PSF license (reproduced in full below). As the contextlib module +was added only in Python 2.5, the licenses for earlier Python versions are +not applicable and have not been included. + +Python was created in the early 1990s by Guido van Rossum at Stichting +Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands +as a successor of a language called ABC. Guido remains Python's +principal author, although it includes many contributions from others. + +In 1995, Guido continued his work on Python at the Corporation for +National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) +in Reston, Virginia where he released several versions of the +software. + +In May 2000, Guido and the Python core development team moved to +BeOpen.com to form the BeOpen PythonLabs team. In October of the same +year, the PythonLabs team moved to Digital Creations (now Zope +Corporation, see http://www.zope.com). In 2001, the Python Software +Foundation (PSF, see http://www.python.org/psf/) was formed, a +non-profit organization created specifically to own Python-related +Intellectual Property. Zope Corporation is a sponsoring member of +the PSF. + +All Python releases are Open Source (see http://www.opensource.org for +the Open Source Definition). Historically, most, but not all, Python +releases have also been GPL-compatible; the table below summarizes +the various releases that included the contextlib module. + + Release Derived Year Owner GPL- + from compatible? (1) + + 2.5 2.4 2006 PSF yes + 2.5.1 2.5 2007 PSF yes + 2.5.2 2.5.1 2008 PSF yes + 2.5.3 2.5.2 2008 PSF yes + 2.6 2.5 2008 PSF yes + 2.6.1 2.6 2008 PSF yes + 2.6.2 2.6.1 2009 PSF yes + 2.6.3 2.6.2 2009 PSF yes + 2.6.4 2.6.3 2009 PSF yes + 2.6.5 2.6.4 2010 PSF yes + 3.0 2.6 2008 PSF yes + 3.0.1 3.0 2009 PSF yes + 3.1 3.0.1 2009 PSF yes + 3.1.1 3.1 2009 PSF yes + 3.1.2 3.1.1 2010 PSF yes + 3.1.3 3.1.2 2010 PSF yes + 3.1.4 3.1.3 2011 PSF yes + 3.2 3.1 2011 PSF yes + 3.2.1 3.2 2011 PSF yes + 3.2.2 3.2.1 2011 PSF yes + 3.3 3.2 2012 PSF yes + +Footnotes: + +(1) GPL-compatible doesn't mean that we're distributing Python under + the GPL. All Python licenses, unlike the GPL, let you distribute + a modified version without making your changes open source. The + GPL-compatible licenses make it possible to combine Python with + other software that is released under the GPL; the others don't. + +Thanks to the many outside volunteers who have worked under Guido's +direction to make these releases possible. + + +B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON +=============================================================== + +PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 +-------------------------------------------- + +1. This LICENSE AGREEMENT is between the Python Software Foundation +("PSF"), and the Individual or Organization ("Licensee") accessing and +otherwise using this software ("Python") in source or binary form and +its associated documentation. + +2. Subject to the terms and conditions of this License Agreement, PSF hereby +grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, +analyze, test, perform and/or display publicly, prepare derivative works, +distribute, and otherwise use Python alone or in any derivative version, +provided, however, that PSF's License Agreement and PSF's notice of copyright, +i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, +2011 Python Software Foundation; All Rights Reserved" are retained in Python +alone or in any derivative version prepared by Licensee. + +3. In the event Licensee prepares a derivative work that is based on +or incorporates Python or any part thereof, and wants to make +the derivative work available to others as provided herein, then +Licensee hereby agrees to include in any such work a brief summary of +the changes made to Python. + +4. PSF is making Python available to Licensee on an "AS IS" +basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR +IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND +DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS +FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT +INFRINGE ANY THIRD PARTY RIGHTS. + +5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON +FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS +A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, +OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. + +6. This License Agreement will automatically terminate upon a material +breach of its terms and conditions. + +7. Nothing in this License Agreement shall be deemed to create any +relationship of agency, partnership, or joint venture between PSF and +Licensee. This License Agreement does not grant permission to use PSF +trademarks or trade name in a trademark sense to endorse or promote +products or services of Licensee, or any third party. + +8. By copying, installing or otherwise using Python, Licensee +agrees to be bound by the terms and conditions of this License +Agreement. diff --git a/pipenv/vendor/contextlib2.py b/pipenv/vendor/contextlib2.py new file mode 100644 index 00000000..3aae8f41 --- /dev/null +++ b/pipenv/vendor/contextlib2.py @@ -0,0 +1,518 @@ +"""contextlib2 - backports and enhancements to the contextlib module""" + +import abc +import sys +import warnings +from collections import deque +from functools import wraps + +__all__ = ["contextmanager", "closing", "nullcontext", + "AbstractContextManager", + "ContextDecorator", "ExitStack", + "redirect_stdout", "redirect_stderr", "suppress"] + +# Backwards compatibility +__all__ += ["ContextStack"] + + +# Backport abc.ABC +if sys.version_info[:2] >= (3, 4): + _abc_ABC = abc.ABC +else: + _abc_ABC = abc.ABCMeta('ABC', (object,), {'__slots__': ()}) + + +# Backport classic class MRO +def _classic_mro(C, result): + if C in result: + return + result.append(C) + for B in C.__bases__: + _classic_mro(B, result) + return result + + +# Backport _collections_abc._check_methods +def _check_methods(C, *methods): + try: + mro = C.__mro__ + except AttributeError: + mro = tuple(_classic_mro(C, [])) + + for method in methods: + for B in mro: + if method in B.__dict__: + if B.__dict__[method] is None: + return NotImplemented + break + else: + return NotImplemented + return True + + +class AbstractContextManager(_abc_ABC): + """An abstract base class for context managers.""" + + def __enter__(self): + """Return `self` upon entering the runtime context.""" + return self + + @abc.abstractmethod + def __exit__(self, exc_type, exc_value, traceback): + """Raise any exception triggered within the runtime context.""" + return None + + @classmethod + def __subclasshook__(cls, C): + """Check whether subclass is considered a subclass of this ABC.""" + if cls is AbstractContextManager: + return _check_methods(C, "__enter__", "__exit__") + return NotImplemented + + +class ContextDecorator(object): + """A base class or mixin that enables context managers to work as decorators.""" + + def refresh_cm(self): + """Returns the context manager used to actually wrap the call to the + decorated function. + + The default implementation just returns *self*. + + Overriding this method allows otherwise one-shot context managers + like _GeneratorContextManager to support use as decorators via + implicit recreation. + + DEPRECATED: refresh_cm was never added to the standard library's + ContextDecorator API + """ + warnings.warn("refresh_cm was never added to the standard library", + DeprecationWarning) + return self._recreate_cm() + + def _recreate_cm(self): + """Return a recreated instance of self. + + Allows an otherwise one-shot context manager like + _GeneratorContextManager to support use as + a decorator via implicit recreation. + + This is a private interface just for _GeneratorContextManager. + See issue #11647 for details. + """ + return self + + def __call__(self, func): + @wraps(func) + def inner(*args, **kwds): + with self._recreate_cm(): + return func(*args, **kwds) + return inner + + +class _GeneratorContextManager(ContextDecorator): + """Helper for @contextmanager decorator.""" + + def __init__(self, func, args, kwds): + self.gen = func(*args, **kwds) + self.func, self.args, self.kwds = func, args, kwds + # Issue 19330: ensure context manager instances have good docstrings + doc = getattr(func, "__doc__", None) + if doc is None: + doc = type(self).__doc__ + self.__doc__ = doc + # Unfortunately, this still doesn't provide good help output when + # inspecting the created context manager instances, since pydoc + # currently bypasses the instance docstring and shows the docstring + # for the class instead. + # See http://bugs.python.org/issue19404 for more details. + + def _recreate_cm(self): + # _GCM instances are one-shot context managers, so the + # CM must be recreated each time a decorated function is + # called + return self.__class__(self.func, self.args, self.kwds) + + def __enter__(self): + try: + return next(self.gen) + except StopIteration: + raise RuntimeError("generator didn't yield") + + def __exit__(self, type, value, traceback): + if type is None: + try: + next(self.gen) + except StopIteration: + return + else: + raise RuntimeError("generator didn't stop") + else: + if value is None: + # Need to force instantiation so we can reliably + # tell if we get the same exception back + value = type() + try: + self.gen.throw(type, value, traceback) + raise RuntimeError("generator didn't stop after throw()") + except StopIteration as exc: + # Suppress StopIteration *unless* it's the same exception that + # was passed to throw(). This prevents a StopIteration + # raised inside the "with" statement from being suppressed. + return exc is not value + except RuntimeError as exc: + # Don't re-raise the passed in exception + if exc is value: + return False + # Likewise, avoid suppressing if a StopIteration exception + # was passed to throw() and later wrapped into a RuntimeError + # (see PEP 479). + if _HAVE_EXCEPTION_CHAINING and exc.__cause__ is value: + return False + raise + except: + # only re-raise if it's *not* the exception that was + # passed to throw(), because __exit__() must not raise + # an exception unless __exit__() itself failed. But throw() + # has to raise the exception to signal propagation, so this + # fixes the impedance mismatch between the throw() protocol + # and the __exit__() protocol. + # + if sys.exc_info()[1] is not value: + raise + + +def contextmanager(func): + """@contextmanager decorator. + + Typical usage: + + @contextmanager + def some_generator(<arguments>): + <setup> + try: + yield <value> + finally: + <cleanup> + + This makes this: + + with some_generator(<arguments>) as <variable>: + <body> + + equivalent to this: + + <setup> + try: + <variable> = <value> + <body> + finally: + <cleanup> + + """ + @wraps(func) + def helper(*args, **kwds): + return _GeneratorContextManager(func, args, kwds) + return helper + + +class closing(object): + """Context to automatically close something at the end of a block. + + Code like this: + + with closing(<module>.open(<arguments>)) as f: + <block> + + is equivalent to this: + + f = <module>.open(<arguments>) + try: + <block> + finally: + f.close() + + """ + def __init__(self, thing): + self.thing = thing + + def __enter__(self): + return self.thing + + def __exit__(self, *exc_info): + self.thing.close() + + +class _RedirectStream(object): + + _stream = None + + def __init__(self, new_target): + self._new_target = new_target + # We use a list of old targets to make this CM re-entrant + self._old_targets = [] + + def __enter__(self): + self._old_targets.append(getattr(sys, self._stream)) + setattr(sys, self._stream, self._new_target) + return self._new_target + + def __exit__(self, exctype, excinst, exctb): + setattr(sys, self._stream, self._old_targets.pop()) + + +class redirect_stdout(_RedirectStream): + """Context manager for temporarily redirecting stdout to another file. + + # How to send help() to stderr + with redirect_stdout(sys.stderr): + help(dir) + + # How to write help() to a file + with open('help.txt', 'w') as f: + with redirect_stdout(f): + help(pow) + """ + + _stream = "stdout" + + +class redirect_stderr(_RedirectStream): + """Context manager for temporarily redirecting stderr to another file.""" + + _stream = "stderr" + + +class suppress(object): + """Context manager to suppress specified exceptions + + After the exception is suppressed, execution proceeds with the next + statement following the with statement. + + with suppress(FileNotFoundError): + os.remove(somefile) + # Execution still resumes here if the file was already removed + """ + + def __init__(self, *exceptions): + self._exceptions = exceptions + + def __enter__(self): + pass + + def __exit__(self, exctype, excinst, exctb): + # Unlike isinstance and issubclass, CPython exception handling + # currently only looks at the concrete type hierarchy (ignoring + # the instance and subclass checking hooks). While Guido considers + # that a bug rather than a feature, it's a fairly hard one to fix + # due to various internal implementation details. suppress provides + # the simpler issubclass based semantics, rather than trying to + # exactly reproduce the limitations of the CPython interpreter. + # + # See http://bugs.python.org/issue12029 for more details + return exctype is not None and issubclass(exctype, self._exceptions) + + +# Context manipulation is Python 3 only +_HAVE_EXCEPTION_CHAINING = sys.version_info[0] >= 3 +if _HAVE_EXCEPTION_CHAINING: + def _make_context_fixer(frame_exc): + def _fix_exception_context(new_exc, old_exc): + # Context may not be correct, so find the end of the chain + while 1: + exc_context = new_exc.__context__ + if exc_context is old_exc: + # Context is already set correctly (see issue 20317) + return + if exc_context is None or exc_context is frame_exc: + break + new_exc = exc_context + # Change the end of the chain to point to the exception + # we expect it to reference + new_exc.__context__ = old_exc + return _fix_exception_context + + def _reraise_with_existing_context(exc_details): + try: + # bare "raise exc_details[1]" replaces our carefully + # set-up context + fixed_ctx = exc_details[1].__context__ + raise exc_details[1] + except BaseException: + exc_details[1].__context__ = fixed_ctx + raise +else: + # No exception context in Python 2 + def _make_context_fixer(frame_exc): + return lambda new_exc, old_exc: None + + # Use 3 argument raise in Python 2, + # but use exec to avoid SyntaxError in Python 3 + def _reraise_with_existing_context(exc_details): + exc_type, exc_value, exc_tb = exc_details + exec("raise exc_type, exc_value, exc_tb") + +# Handle old-style classes if they exist +try: + from types import InstanceType +except ImportError: + # Python 3 doesn't have old-style classes + _get_type = type +else: + # Need to handle old-style context managers on Python 2 + def _get_type(obj): + obj_type = type(obj) + if obj_type is InstanceType: + return obj.__class__ # Old-style class + return obj_type # New-style class + + +# Inspired by discussions on http://bugs.python.org/issue13585 +class ExitStack(object): + """Context manager for dynamic management of a stack of exit callbacks + + For example: + + with ExitStack() as stack: + files = [stack.enter_context(open(fname)) for fname in filenames] + # All opened files will automatically be closed at the end of + # the with statement, even if attempts to open files later + # in the list raise an exception + + """ + def __init__(self): + self._exit_callbacks = deque() + + def pop_all(self): + """Preserve the context stack by transferring it to a new instance""" + new_stack = type(self)() + new_stack._exit_callbacks = self._exit_callbacks + self._exit_callbacks = deque() + return new_stack + + def _push_cm_exit(self, cm, cm_exit): + """Helper to correctly register callbacks to __exit__ methods""" + def _exit_wrapper(*exc_details): + return cm_exit(cm, *exc_details) + _exit_wrapper.__self__ = cm + self.push(_exit_wrapper) + + def push(self, exit): + """Registers a callback with the standard __exit__ method signature + + Can suppress exceptions the same way __exit__ methods can. + + Also accepts any object with an __exit__ method (registering a call + to the method instead of the object itself) + """ + # We use an unbound method rather than a bound method to follow + # the standard lookup behaviour for special methods + _cb_type = _get_type(exit) + try: + exit_method = _cb_type.__exit__ + except AttributeError: + # Not a context manager, so assume its a callable + self._exit_callbacks.append(exit) + else: + self._push_cm_exit(exit, exit_method) + return exit # Allow use as a decorator + + def callback(self, callback, *args, **kwds): + """Registers an arbitrary callback and arguments. + + Cannot suppress exceptions. + """ + def _exit_wrapper(exc_type, exc, tb): + callback(*args, **kwds) + # We changed the signature, so using @wraps is not appropriate, but + # setting __wrapped__ may still help with introspection + _exit_wrapper.__wrapped__ = callback + self.push(_exit_wrapper) + return callback # Allow use as a decorator + + def enter_context(self, cm): + """Enters the supplied context manager + + If successful, also pushes its __exit__ method as a callback and + returns the result of the __enter__ method. + """ + # We look up the special methods on the type to match the with statement + _cm_type = _get_type(cm) + _exit = _cm_type.__exit__ + result = _cm_type.__enter__(cm) + self._push_cm_exit(cm, _exit) + return result + + def close(self): + """Immediately unwind the context stack""" + self.__exit__(None, None, None) + + def __enter__(self): + return self + + def __exit__(self, *exc_details): + received_exc = exc_details[0] is not None + + # We manipulate the exception state so it behaves as though + # we were actually nesting multiple with statements + frame_exc = sys.exc_info()[1] + _fix_exception_context = _make_context_fixer(frame_exc) + + # Callbacks are invoked in LIFO order to match the behaviour of + # nested context managers + suppressed_exc = False + pending_raise = False + while self._exit_callbacks: + cb = self._exit_callbacks.pop() + try: + if cb(*exc_details): + suppressed_exc = True + pending_raise = False + exc_details = (None, None, None) + except: + new_exc_details = sys.exc_info() + # simulate the stack of exceptions by setting the context + _fix_exception_context(new_exc_details[1], exc_details[1]) + pending_raise = True + exc_details = new_exc_details + if pending_raise: + _reraise_with_existing_context(exc_details) + return received_exc and suppressed_exc + + +# Preserve backwards compatibility +class ContextStack(ExitStack): + """Backwards compatibility alias for ExitStack""" + + def __init__(self): + warnings.warn("ContextStack has been renamed to ExitStack", + DeprecationWarning) + super(ContextStack, self).__init__() + + def register_exit(self, callback): + return self.push(callback) + + def register(self, callback, *args, **kwds): + return self.callback(callback, *args, **kwds) + + def preserve(self): + return self.pop_all() + + +class nullcontext(AbstractContextManager): + """Context manager that does no additional processing. + Used as a stand-in for a normal context manager, when a particular + block of code is only sometimes used with a normal context manager: + cm = optional_cm if condition else nullcontext() + with cm: + # Perform operation, using optional_cm if condition is True + """ + + def __init__(self, enter_result=None): + self.enter_result = enter_result + + def __enter__(self): + return self.enter_result + + def __exit__(self, *excinfo): + pass diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE deleted file mode 100644 index 00023c80..00000000 --- a/pipenv/vendor/cursor/LICENSE +++ /dev/null @@ -1,5 +0,0 @@ -This work is licensed under the Creative Commons -Attribution-ShareAlike 2.5 International License. To view a copy of -this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -send a letter to Creative Commons, PO Box 1866, Mountain View, -CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py deleted file mode 100644 index 76a4f671..00000000 --- a/pipenv/vendor/cursor/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cursor import hide, show, HiddenCursor - -__all__ = ["hide", "show", "HiddenCursor"] - diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py deleted file mode 100644 index e4407c02..00000000 --- a/pipenv/vendor/cursor/cursor.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- - -## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer -## Packager: Gijs TImmers: https://github.com/GijsTimmers - -## Based on James Spencer's answer on StackOverflow: -## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window - -## Licence: CC-BY-SA-2.5 -## http://creativecommons.org/licenses/by-sa/2.5/ - -## This work is licensed under the Creative Commons -## Attribution-ShareAlike 2.5 International License. To view a copy of -## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -## send a letter to Creative Commons, PO Box 1866, Mountain View, -## CA 94042, USA. - -import sys -import os - -if os.name == 'nt': - import ctypes - - class _CursorInfo(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("visible", ctypes.c_byte)] - -def hide(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = False - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25l") - stream.flush() - -def show(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = True - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25h") - stream.flush() - -class HiddenCursor(object): - def __init__(self, stream=sys.stdout): - self._stream = stream - def __enter__(self): - hide(stream=self._stream) - def __exit__(self, type, value, traceback): - show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/distlib/__init__.py b/pipenv/vendor/distlib/__init__.py index a786b4d3..e19aebdc 100644 --- a/pipenv/vendor/distlib/__init__.py +++ b/pipenv/vendor/distlib/__init__.py @@ -1,12 +1,12 @@ # -*- coding: utf-8 -*- # -# Copyright (C) 2012-2017 Vinay Sajip. +# Copyright (C) 2012-2019 Vinay Sajip. # Licensed to the Python Software Foundation under a contributor agreement. # See LICENSE.txt and CONTRIBUTORS.txt. # import logging -__version__ = '0.2.8' +__version__ = '0.3.0' class DistlibException(Exception): pass diff --git a/pipenv/vendor/distlib/_backport/sysconfig.py b/pipenv/vendor/distlib/_backport/sysconfig.py index 1df3aba1..b470a373 100644 --- a/pipenv/vendor/distlib/_backport/sysconfig.py +++ b/pipenv/vendor/distlib/_backport/sysconfig.py @@ -119,11 +119,9 @@ def _expand_globals(config): #_expand_globals(_SCHEMES) - # FIXME don't rely on sys.version here, its format is an implementation detail - # of CPython, use sys.version_info or sys.hexversion -_PY_VERSION = sys.version.split()[0] -_PY_VERSION_SHORT = sys.version[:3] -_PY_VERSION_SHORT_NO_DOT = _PY_VERSION[0] + _PY_VERSION[2] +_PY_VERSION = '%s.%s.%s' % sys.version_info[:3] +_PY_VERSION_SHORT = '%s.%s' % sys.version_info[:2] +_PY_VERSION_SHORT_NO_DOT = '%s%s' % sys.version_info[:2] _PREFIX = os.path.normpath(sys.prefix) _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _CONFIG_VARS = None diff --git a/pipenv/vendor/distlib/database.py b/pipenv/vendor/distlib/database.py index b13cdac9..c16c0c8d 100644 --- a/pipenv/vendor/distlib/database.py +++ b/pipenv/vendor/distlib/database.py @@ -567,7 +567,7 @@ class InstalledDistribution(BaseInstalledDistribution): p = os.path.join(path, 'top_level.txt') if os.path.exists(p): with open(p, 'rb') as f: - data = f.read() + data = f.read().decode('utf-8') self.modules = data.splitlines() def __repr__(self): diff --git a/pipenv/vendor/distlib/index.py b/pipenv/vendor/distlib/index.py index 2406be21..7a87cdcf 100644 --- a/pipenv/vendor/distlib/index.py +++ b/pipenv/vendor/distlib/index.py @@ -22,7 +22,7 @@ from .util import cached_property, zip_dir, ServerProxy logger = logging.getLogger(__name__) -DEFAULT_INDEX = 'https://pypi.python.org/pypi' +DEFAULT_INDEX = 'https://pypi.org/pypi' DEFAULT_REALM = 'pypi' class PackageIndex(object): diff --git a/pipenv/vendor/distlib/locators.py b/pipenv/vendor/distlib/locators.py index 5c655c3e..12a1d063 100644 --- a/pipenv/vendor/distlib/locators.py +++ b/pipenv/vendor/distlib/locators.py @@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) HASHER_HASH = re.compile(r'^(\w+)=([a-f0-9]+)') CHARSET = re.compile(r';\s*charset\s*=\s*(.*)\s*$', re.I) HTML_CONTENT_TYPE = re.compile('text/html|application/x(ht)?ml') -DEFAULT_INDEX = 'https://pypi.python.org/pypi' +DEFAULT_INDEX = 'https://pypi.org/pypi' def get_all_distribution_names(url=None): """ @@ -197,7 +197,7 @@ class Locator(object): is_downloadable = basename.endswith(self.downloadable_extensions) if is_wheel: compatible = is_compatible(Wheel(basename), self.wheel_tags) - return (t.scheme == 'https', 'pypi.python.org' in t.netloc, + return (t.scheme == 'https', 'pypi.org' in t.netloc, is_downloadable, is_wheel, compatible, basename) def prefer_url(self, url1, url2): @@ -304,18 +304,25 @@ class Locator(object): def _get_digest(self, info): """ - Get a digest from a dictionary by looking at keys of the form - 'algo_digest'. + Get a digest from a dictionary by looking at a "digests" dictionary + or keys of the form 'algo_digest'. Returns a 2-tuple (algo, digest) if found, else None. Currently looks only for SHA256, then MD5. """ result = None - for algo in ('sha256', 'md5'): - key = '%s_digest' % algo - if key in info: - result = (algo, info[key]) - break + if 'digests' in info: + digests = info['digests'] + for algo in ('sha256', 'md5'): + if algo in digests: + result = (algo, digests[algo]) + break + if not result: + for algo in ('sha256', 'md5'): + key = '%s_digest' % algo + if key in info: + result = (algo, info[key]) + break return result def _update_version_data(self, result, info): @@ -1049,7 +1056,7 @@ class AggregatingLocator(Locator): # versions which don't conform to PEP 426 / PEP 440. default_locator = AggregatingLocator( JSONLocator(), - SimpleScrapingLocator('https://pypi.python.org/simple/', + SimpleScrapingLocator('https://pypi.org/simple/', timeout=3.0), scheme='legacy') diff --git a/pipenv/vendor/distlib/metadata.py b/pipenv/vendor/distlib/metadata.py index 77eed7f9..2d61378e 100644 --- a/pipenv/vendor/distlib/metadata.py +++ b/pipenv/vendor/distlib/metadata.py @@ -91,9 +91,11 @@ _426_FIELDS = ('Metadata-Version', 'Name', 'Version', 'Platform', _426_MARKERS = ('Private-Version', 'Provides-Extra', 'Obsoleted-By', 'Setup-Requires-Dist', 'Extension') -# See issue #106: Sometimes 'Requires' occurs wrongly in the metadata. Include -# it in the tuple literal below to allow it (for now) -_566_FIELDS = _426_FIELDS + ('Description-Content-Type', 'Requires') +# See issue #106: Sometimes 'Requires' and 'Provides' occur wrongly in +# the metadata. Include them in the tuple literal below to allow them +# (for now). +_566_FIELDS = _426_FIELDS + ('Description-Content-Type', + 'Requires', 'Provides') _566_MARKERS = ('Description-Content-Type',) diff --git a/pipenv/vendor/distlib/scripts.py b/pipenv/vendor/distlib/scripts.py index 8e22cb91..51859741 100644 --- a/pipenv/vendor/distlib/scripts.py +++ b/pipenv/vendor/distlib/scripts.py @@ -39,27 +39,12 @@ _DEFAULT_MANIFEST = ''' # check if Python is called on the first line with this expression FIRST_LINE_RE = re.compile(b'^#!.*pythonw?[0-9.]*([ \t].*)?$') SCRIPT_TEMPLATE = r'''# -*- coding: utf-8 -*- +import re +import sys +from %(module)s import %(import_name)s if __name__ == '__main__': - import sys, re - - def _resolve(module, func): - __import__(module) - mod = sys.modules[module] - parts = func.split('.') - result = getattr(mod, parts.pop(0)) - for p in parts: - result = getattr(result, p) - return result - - try: - sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) - - func = _resolve('%(module)s', '%(func)s') - rc = func() # None interpreted as 0 - except Exception as e: # only supporting Python >= 2.6 - sys.stderr.write('%%s\n' %% e) - rc = 1 - sys.exit(rc) + sys.argv[0] = re.sub(r'(-script\.pyw|\.exe)?$', '', sys.argv[0]) + sys.exit(%(func)s()) ''' @@ -187,8 +172,16 @@ class ScriptMaker(object): if sys.platform.startswith('java'): # pragma: no cover executable = self._fix_jython_executable(executable) - # Normalise case for Windows - executable = os.path.normcase(executable) + + # Normalise case for Windows - COMMENTED OUT + # executable = os.path.normcase(executable) + # N.B. The normalising operation above has been commented out: See + # issue #124. Although paths in Windows are generally case-insensitive, + # they aren't always. For example, a path containing a ẞ (which is a + # LATIN CAPITAL LETTER SHARP S - U+1E9E) is normcased to ß (which is a + # LATIN SMALL LETTER SHARP S' - U+00DF). The two are not considered by + # Windows as equivalent in path names. + # If the user didn't specify an executable, it may be necessary to # cater for executable paths with spaces (not uncommon on Windows) if enquote: @@ -225,6 +218,7 @@ class ScriptMaker(object): def _get_script_text(self, entry): return self.script_template % dict(module=entry.prefix, + import_name=entry.suffix.split('.')[0], func=entry.suffix) manifest = _DEFAULT_MANIFEST @@ -299,9 +293,10 @@ class ScriptMaker(object): if '' in self.variants: scriptnames.add(name) if 'X' in self.variants: - scriptnames.add('%s%s' % (name, sys.version[0])) + scriptnames.add('%s%s' % (name, sys.version_info[0])) if 'X.Y' in self.variants: - scriptnames.add('%s-%s' % (name, sys.version[:3])) + scriptnames.add('%s-%s.%s' % (name, sys.version_info[0], + sys.version_info[1])) if options and options.get('gui', False): ext = 'pyw' else: @@ -381,8 +376,12 @@ class ScriptMaker(object): # Issue 31: don't hardcode an absolute package name, but # determine it relative to the current package distlib_package = __name__.rsplit('.', 1)[0] - result = finder(distlib_package).find(name).bytes - return result + resource = finder(distlib_package).find(name) + if not resource: + msg = ('Unable to find resource %s in package %s' % (name, + distlib_package)) + raise ValueError(msg) + return resource.bytes # Public API follows diff --git a/pipenv/vendor/distlib/t32.exe b/pipenv/vendor/distlib/t32.exe index a09d9268..8932a18e 100644 Binary files a/pipenv/vendor/distlib/t32.exe and b/pipenv/vendor/distlib/t32.exe differ diff --git a/pipenv/vendor/distlib/t64.exe b/pipenv/vendor/distlib/t64.exe index 9da9b40d..325b8057 100644 Binary files a/pipenv/vendor/distlib/t64.exe and b/pipenv/vendor/distlib/t64.exe differ diff --git a/pipenv/vendor/distlib/util.py b/pipenv/vendor/distlib/util.py index 9d4bfd3b..01324eae 100644 --- a/pipenv/vendor/distlib/util.py +++ b/pipenv/vendor/distlib/util.py @@ -703,7 +703,7 @@ class ExportEntry(object): ENTRY_RE = re.compile(r'''(?P<name>(\w|[-.+])+) \s*=\s*(?P<callable>(\w+)([:\.]\w+)*) - \s*(\[\s*(?P<flags>\w+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])? + \s*(\[\s*(?P<flags>[\w-]+(=\w+)?(,\s*\w+(=\w+)?)*)\s*\])? ''', re.VERBOSE) def get_export_entry(specification): @@ -804,11 +804,15 @@ def ensure_slash(s): def parse_credentials(netloc): username = password = None if '@' in netloc: - prefix, netloc = netloc.split('@', 1) + prefix, netloc = netloc.rsplit('@', 1) if ':' not in prefix: username = prefix else: username, password = prefix.split(':', 1) + if username: + username = unquote(username) + if password: + password = unquote(password) return username, password, netloc @@ -1434,7 +1438,8 @@ if ssl: ca_certs=self.ca_certs) else: # pragma: no cover context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) - context.options |= ssl.OP_NO_SSLv2 + if hasattr(ssl, 'OP_NO_SSLv2'): + context.options |= ssl.OP_NO_SSLv2 if self.cert_file: context.load_cert_chain(self.cert_file, self.key_file) kwargs = {} diff --git a/pipenv/vendor/distlib/w32.exe b/pipenv/vendor/distlib/w32.exe index 732215a9..e6439e9e 100644 Binary files a/pipenv/vendor/distlib/w32.exe and b/pipenv/vendor/distlib/w32.exe differ diff --git a/pipenv/vendor/distlib/w64.exe b/pipenv/vendor/distlib/w64.exe index c41bd0a0..46139dbf 100644 Binary files a/pipenv/vendor/distlib/w64.exe and b/pipenv/vendor/distlib/w64.exe differ diff --git a/pipenv/vendor/distlib/wheel.py b/pipenv/vendor/distlib/wheel.py index b04bfaef..bd179383 100644 --- a/pipenv/vendor/distlib/wheel.py +++ b/pipenv/vendor/distlib/wheel.py @@ -433,6 +433,22 @@ class Wheel(object): self.build_zip(pathname, archive_paths) return pathname + def skip_entry(self, arcname): + """ + Determine whether an archive entry should be skipped when verifying + or installing. + """ + # The signature file won't be in RECORD, + # and we don't currently don't do anything with it + # We also skip directories, as they won't be in RECORD + # either. See: + # + # https://github.com/pypa/wheel/issues/294 + # https://github.com/pypa/wheel/issues/287 + # https://github.com/pypa/wheel/pull/289 + # + return arcname.endswith(('/', '/RECORD.jws')) + def install(self, paths, maker, **kwargs): """ Install a wheel to the specified paths. If kwarg ``warner`` is @@ -514,9 +530,7 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: @@ -670,7 +684,7 @@ class Wheel(object): if cache is None: # Use native string to avoid issues on 2.x: see Python #20140. base = os.path.join(get_cache_base(), str('dylib-cache'), - sys.version[:3]) + '%s.%s' % sys.version_info[:2]) cache = Cache(base) return cache @@ -786,13 +800,15 @@ class Wheel(object): u_arcname = arcname else: u_arcname = arcname.decode('utf-8') - if '..' in u_arcname: + # See issue #115: some wheels have .. in their entries, but + # in the filename ... e.g. __main__..py ! So the check is + # updated to look for .. in the directory portions + p = u_arcname.split('/') + if '..' in p: raise DistlibException('invalid entry in ' 'wheel: %r' % u_arcname) - # The signature file won't be in RECORD, - # and we don't currently don't do anything with it - if u_arcname.endswith('/RECORD.jws'): + if self.skip_entry(u_arcname): continue row = records[u_arcname] if row[2] and str(zinfo.file_size) != row[2]: diff --git a/pipenv/vendor/dotenv/__init__.py b/pipenv/vendor/dotenv/__init__.py index 50f27cd4..b88d9bc2 100644 --- a/pipenv/vendor/dotenv/__init__.py +++ b/pipenv/vendor/dotenv/__init__.py @@ -1,12 +1,18 @@ +from .compat import IS_TYPE_CHECKING from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values +if IS_TYPE_CHECKING: + from typing import Any, Optional + def load_ipython_extension(ipython): + # type: (Any) -> None from .ipython import load_ipython_extension load_ipython_extension(ipython) def get_cli_string(path=None, action=None, key=None, value=None, quote=None): + # type: (Optional[str], Optional[str], Optional[str], Optional[str], Optional[str]) -> str """Returns a string suitable for running as a shell script. Useful for converting a arguments passed to a fabric task diff --git a/pipenv/vendor/dotenv/cli.py b/pipenv/vendor/dotenv/cli.py index 4e03c12a..d2a021a5 100644 --- a/pipenv/vendor/dotenv/cli.py +++ b/pipenv/vendor/dotenv/cli.py @@ -1,5 +1,6 @@ import os import sys +from subprocess import Popen try: import click @@ -8,9 +9,13 @@ except ImportError: 'Run pip install "python-dotenv[cli]" to fix this.') sys.exit(1) -from .main import dotenv_values, get_key, set_key, unset_key, run_command +from .compat import IS_TYPE_CHECKING +from .main import dotenv_values, get_key, set_key, unset_key from .version import __version__ +if IS_TYPE_CHECKING: + from typing import Any, List, Dict + @click.group() @click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), @@ -22,6 +27,7 @@ from .version import __version__ @click.version_option(version=__version__) @click.pass_context def cli(ctx, file, quote): + # type: (click.Context, Any, Any) -> None '''This script is used to set, get or unset values from a .env file.''' ctx.obj = {} ctx.obj['FILE'] = file @@ -31,6 +37,7 @@ def cli(ctx, file, quote): @cli.command() @click.pass_context def list(ctx): + # type: (click.Context) -> None '''Display all the stored key/value.''' file = ctx.obj['FILE'] dotenv_as_dict = dotenv_values(file) @@ -43,6 +50,7 @@ def list(ctx): @click.argument('key', required=True) @click.argument('value', required=True) def set(ctx, key, value): + # type: (click.Context, Any, Any) -> None '''Store the given key/value.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] @@ -57,6 +65,7 @@ def set(ctx, key, value): @click.pass_context @click.argument('key', required=True) def get(ctx, key): + # type: (click.Context, Any) -> None '''Retrieve the value for the given key.''' file = ctx.obj['FILE'] stored_value = get_key(file, key) @@ -70,6 +79,7 @@ def get(ctx, key): @click.pass_context @click.argument('key', required=True) def unset(ctx, key): + # type: (click.Context, Any) -> None '''Removes the given key.''' file = ctx.obj['FILE'] quote = ctx.obj['QUOTE'] @@ -84,15 +94,51 @@ def unset(ctx, key): @click.pass_context @click.argument('commandline', nargs=-1, type=click.UNPROCESSED) def run(ctx, commandline): + # type: (click.Context, List[str]) -> None """Run command with environment variables present.""" file = ctx.obj['FILE'] dotenv_as_dict = dotenv_values(file) if not commandline: click.echo('No command given.') exit(1) - ret = run_command(commandline, dotenv_as_dict) + ret = run_command(commandline, dotenv_as_dict) # type: ignore exit(ret) +def run_command(command, env): + # type: (List[str], Dict[str, str]) -> int + """Run command in sub process. + + Runs the command in a sub process with the variables from `env` + added in the current environment variables. + + Parameters + ---------- + command: List[str] + The command and it's parameters + env: Dict + The additional environment variables + + Returns + ------- + int + The return code of the command + + """ + # copy the current environment variables and add the vales from + # `env` + cmd_env = os.environ.copy() + cmd_env.update(env) + + p = Popen(command, + universal_newlines=True, + bufsize=0, + shell=False, + env=cmd_env) + _, _ = p.communicate() + + return p.returncode + + if __name__ == "__main__": cli() diff --git a/pipenv/vendor/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py index c4a481e6..61f555df 100644 --- a/pipenv/vendor/dotenv/compat.py +++ b/pipenv/vendor/dotenv/compat.py @@ -1,4 +1,49 @@ -try: +import sys + +PY2 = sys.version_info[0] == 2 # type: bool + +if PY2: from StringIO import StringIO # noqa -except ImportError: +else: from io import StringIO # noqa + + +def is_type_checking(): + # type: () -> bool + try: + from typing import TYPE_CHECKING + except ImportError: # pragma: no cover + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = is_type_checking() + + +if IS_TYPE_CHECKING: + from typing import Text + + +def to_env(text): + # type: (Text) -> str + """ + Encode a string the same way whether it comes from the environment or a `.env` file. + """ + if PY2: + return text.encode(sys.getfilesystemencoding() or "utf-8") + else: + return text + + +def to_text(string): + # type: (str) -> Text + """ + Make a string Unicode if it isn't already. + + This is useful for defining raw unicode strings because `ur"foo"` isn't valid in + Python 3. + """ + if PY2: + return string.decode("utf-8") + else: + return string diff --git a/pipenv/vendor/dotenv/ipython.py b/pipenv/vendor/dotenv/ipython.py index 06252f1e..7f1b13d6 100644 --- a/pipenv/vendor/dotenv/ipython.py +++ b/pipenv/vendor/dotenv/ipython.py @@ -1,8 +1,8 @@ from __future__ import print_function -from IPython.core.magic import Magics, line_magic, magics_class -from IPython.core.magic_arguments import (argument, magic_arguments, - parse_argstring) +from IPython.core.magic import Magics, line_magic, magics_class # type: ignore +from IPython.core.magic_arguments import (argument, magic_arguments, # type: ignore + parse_argstring) # type: ignore from .main import find_dotenv, load_dotenv diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 6ba28bbb..06a210e1 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -1,72 +1,60 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import codecs -import fileinput import io import os import re +import shutil import sys -from subprocess import Popen, PIPE, STDOUT +import tempfile import warnings from collections import OrderedDict +from contextlib import contextmanager -from .compat import StringIO +from .compat import StringIO, PY2, to_env, IS_TYPE_CHECKING +from .parser import parse_stream -__escape_decoder = codecs.getdecoder('unicode_escape') -__posix_variable = re.compile('\$\{[^\}]*\}') +if IS_TYPE_CHECKING: + from typing import ( + Dict, Iterator, Match, Optional, Pattern, Union, Text, IO, Tuple + ) + if sys.version_info >= (3, 6): + _PathLike = os.PathLike + else: + _PathLike = Text + if sys.version_info >= (3, 0): + _StringIO = StringIO + else: + _StringIO = StringIO[Text] -def decode_escaped(escaped): - return __escape_decoder(escaped)[0] - - -def parse_line(line): - line = line.strip() - - # Ignore lines with `#` or which doesn't have `=` in it. - if not line or line.startswith('#') or '=' not in line: - return None, None - - k, v = line.split('=', 1) - - if k.startswith('export '): - (_, _, k) = k.partition('export ') - - # Remove any leading and trailing spaces in key, value - k, v = k.strip(), v.strip() - - if v: - v = v.encode('unicode-escape').decode('ascii') - quoted = v[0] == v[-1] in ['"', "'"] - if quoted: - v = decode_escaped(v[1:-1]) - - return k, v +__posix_variable = re.compile(r'\$\{[^\}]*\}') # type: Pattern[Text] class DotEnv(): - def __init__(self, dotenv_path, verbose=False): - self.dotenv_path = dotenv_path - self._dict = None - self.verbose = verbose + def __init__(self, dotenv_path, verbose=False, encoding=None): + # type: (Union[Text, _PathLike, _StringIO], bool, Union[None, Text]) -> None + self.dotenv_path = dotenv_path # type: Union[Text,_PathLike, _StringIO] + self._dict = None # type: Optional[Dict[Text, Text]] + self.verbose = verbose # type: bool + self.encoding = encoding # type: Union[None, Text] + @contextmanager def _get_stream(self): - self._is_file = False + # type: () -> Iterator[IO[Text]] if isinstance(self.dotenv_path, StringIO): - return self.dotenv_path - - if os.path.exists(self.dotenv_path): - self._is_file = True - return io.open(self.dotenv_path) - - if self.verbose: - warnings.warn("File doesn't exist {}".format(self.dotenv_path)) - - return StringIO('') + yield self.dotenv_path + elif os.path.isfile(self.dotenv_path): + with io.open(self.dotenv_path, encoding=self.encoding) as stream: + yield stream + else: + if self.verbose: + warnings.warn("File doesn't exist {}".format(self.dotenv_path)) # type: ignore + yield StringIO('') def dict(self): + # type: () -> Dict[Text, Text] """Return dotenv as dict""" if self._dict: return self._dict @@ -76,37 +64,26 @@ class DotEnv(): return self._dict def parse(self): - f = self._get_stream() - - for line in f: - key, value = parse_line(line) - if not key: - continue - - yield key, value - - if self._is_file: - f.close() + # type: () -> Iterator[Tuple[Text, Text]] + with self._get_stream() as stream: + for mapping in parse_stream(stream): + if mapping.key is not None and mapping.value is not None: + yield mapping.key, mapping.value def set_as_environment_variables(self, override=False): + # type: (bool) -> bool """ Load the current dotenv as system environemt variable. """ for k, v in self.dict().items(): if k in os.environ and not override: continue - # With Python 2 on Windows, ensuree environment variables are - # system strings to avoid "TypeError: environment can only contain - # strings" in Python's subprocess module. - if sys.version_info.major < 3 and sys.platform == 'win32': - from pipenv.utils import fs_str - k = fs_str(k) - v = fs_str(v) - os.environ[k] = v + os.environ[to_env(k)] = to_env(v) return True def get(self, key): + # type: (Text) -> Optional[Text] """ """ data = self.dict() @@ -115,10 +92,13 @@ class DotEnv(): return data[key] if self.verbose: - warnings.warn("key %s not found in %s." % (key, self.dotenv_path)) + warnings.warn("key %s not found in %s." % (key, self.dotenv_path)) # type: ignore + + return None def get_key(dotenv_path, key_to_get): + # type: (Union[Text, _PathLike], Text) -> Optional[Text] """ Gets the value of a given key from the given .env @@ -127,7 +107,23 @@ def get_key(dotenv_path, key_to_get): return DotEnv(dotenv_path, verbose=True).get(key_to_get) +@contextmanager +def rewrite(path): + # type: (_PathLike) -> Iterator[Tuple[IO[Text], IO[Text]]] + try: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest: + with io.open(path) as source: + yield (source, dest) # type: ignore + except BaseException: + if os.path.isfile(dest.name): + os.unlink(dest.name) + raise + else: + shutil.move(dest.name, path) + + def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): + # type: (_PathLike, Text, Text, Text) -> Tuple[Optional[bool], Text, Text] """ Adds or Updates a key/value to the given .env @@ -136,81 +132,86 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): """ value_to_set = value_to_set.strip("'").strip('"') if not os.path.exists(dotenv_path): - warnings.warn("can't write to %s - it doesn't exist." % dotenv_path) + warnings.warn("can't write to %s - it doesn't exist." % dotenv_path) # type: ignore return None, key_to_set, value_to_set if " " in value_to_set: quote_mode = "always" - line_template = '{}="{}"' if quote_mode == "always" else '{}={}' + line_template = '{}="{}"\n' if quote_mode == "always" else '{}={}\n' line_out = line_template.format(key_to_set, value_to_set) - replaced = False - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_set: - replaced = True - line = line_out - print(line, end='') - - if not replaced: - with io.open(dotenv_path, "a") as f: - f.write("{}\n".format(line_out)) + with rewrite(dotenv_path) as (source, dest): + replaced = False + for mapping in parse_stream(source): + if mapping.key == key_to_set: + dest.write(line_out) + replaced = True + else: + dest.write(mapping.original) + if not replaced: + dest.write(line_out) return True, key_to_set, value_to_set def unset_key(dotenv_path, key_to_unset, quote_mode="always"): + # type: (_PathLike, Text, Text) -> Tuple[Optional[bool], Text] """ Removes a given key from the given .env If the .env path given doesn't exist, fails If the given key doesn't exist in the .env, fails """ - removed = False - if not os.path.exists(dotenv_path): - warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) + warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) # type: ignore return None, key_to_unset - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_unset: - removed = True - line = '' - print(line, end='') + removed = False + with rewrite(dotenv_path) as (source, dest): + for mapping in parse_stream(source): + if mapping.key == key_to_unset: + removed = True + else: + dest.write(mapping.original) if not removed: - warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) + warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) # type: ignore return None, key_to_unset return removed, key_to_unset def resolve_nested_variables(values): + # type: (Dict[Text, Text]) -> Dict[Text, Text] def _replacement(name): + # type: (Text) -> Text """ get appropriate value for a variable name. first search in environ, if not found, then look into the dotenv variables """ - ret = os.getenv(name, values.get(name, "")) + ret = os.getenv(name, new_values.get(name, "")) return ret def _re_sub_callback(match_object): + # type: (Match[Text]) -> Text """ From a match object gets the variable name and returns the correct replacement """ return _replacement(match_object.group()[2:-1]) - for k, v in values.items(): - values[k] = __posix_variable.sub(_re_sub_callback, v) + new_values = {} - return values + for k, v in values.items(): + new_values[k] = __posix_variable.sub(_re_sub_callback, v) + + return new_values def _walk_to_root(path): + # type: (Text) -> Iterator[Text] """ Yield directories starting from the given directory up to the root """ @@ -229,26 +230,40 @@ def _walk_to_root(path): def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): + # type: (Text, bool, bool) -> Text """ Search in increasingly higher folders for the given file Returns path to the file if found, or an empty string otherwise """ - if usecwd or '__file__' not in globals(): - # should work without __file__, e.g. in REPL or IPython notebook + + def _is_interactive(): + """ Decide whether this is running in a REPL or IPython notebook """ + main = __import__('__main__', None, None, fromlist=['__file__']) + return not hasattr(main, '__file__') + + if usecwd or _is_interactive(): + # Should work without __file__, e.g. in REPL or IPython notebook. path = os.getcwd() else: # will work for .py files frame = sys._getframe() # find first frame that is outside of this file - while frame.f_code.co_filename == __file__: + if PY2 and not __file__.endswith('.py'): + # in Python2 __file__ extension could be .pyc or .pyo (this doesn't account + # for edge case of Python compiled for non-standard extension) + current_file = __file__.rsplit('.', 1)[0] + '.py' + else: + current_file = __file__ + + while frame.f_code.co_filename == current_file: frame = frame.f_back frame_filename = frame.f_code.co_filename path = os.path.dirname(os.path.abspath(frame_filename)) for dirname in _walk_to_root(path): check_path = os.path.join(dirname, filename) - if os.path.exists(check_path): + if os.path.isfile(check_path): return check_path if raise_error_if_not_found: @@ -257,54 +272,21 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): return '' -def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False): - f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose).set_as_environment_variables(override=override) - - -def dotenv_values(dotenv_path=None, stream=None, verbose=False): - f = dotenv_path or stream or find_dotenv() - return DotEnv(f, verbose=verbose).dict() - - -def run_command(command, env): - """Run command in sub process. - - Runs the command in a sub process with the variables from `env` - added in the current environment variables. - - Parameters - ---------- - command: List[str] - The command and it's parameters - env: Dict - The additional environment variables - - Returns - ------- - int - The return code of the command +def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, **kwargs): + # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> bool + """Parse a .env file and then load all the variables found as environment variables. + - *dotenv_path*: absolute or relative path to .env file. + - *stream*: `StringIO` object with .env content. + - *verbose*: whether to output the warnings related to missing .env file etc. Defaults to `False`. + - *override*: where to override the system environment variables with the variables in `.env` file. + Defaults to `False`. """ - # copy the current environment variables and add the vales from - # `env` - cmd_env = os.environ.copy() - cmd_env.update(env) + f = dotenv_path or stream or find_dotenv() + return DotEnv(f, verbose=verbose, **kwargs).set_as_environment_variables(override=override) - p = Popen(command, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, - universal_newlines=True, - bufsize=0, - shell=False, - env=cmd_env) - try: - out, _ = p.communicate() - print(out) - except Exception: - warnings.warn('An error occured, running the command:') - out, _ = p.communicate() - warnings.warn(out) - return p.returncode +def dotenv_values(dotenv_path=None, stream=None, verbose=False, **kwargs): + # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, Union[None, Text]) -> Dict[Text, Text] + f = dotenv_path or stream or find_dotenv() + return DotEnv(f, verbose=verbose, **kwargs).dict() diff --git a/pipenv/vendor/dotenv/parser.py b/pipenv/vendor/dotenv/parser.py new file mode 100644 index 00000000..034ebfde --- /dev/null +++ b/pipenv/vendor/dotenv/parser.py @@ -0,0 +1,163 @@ +import codecs +import re + +from .compat import to_text, IS_TYPE_CHECKING + + +if IS_TYPE_CHECKING: + from typing import ( # noqa:F401 + IO, Iterator, Match, NamedTuple, Optional, Pattern, Sequence, Text, + Tuple + ) + + +def make_regex(string, extra_flags=0): + # type: (str, int) -> Pattern[Text] + return re.compile(to_text(string), re.UNICODE | extra_flags) + + +_whitespace = make_regex(r"\s*", extra_flags=re.MULTILINE) +_export = make_regex(r"(?:export[^\S\r\n]+)?") +_single_quoted_key = make_regex(r"'([^']+)'") +_unquoted_key = make_regex(r"([^=\#\s]+)") +_equal_sign = make_regex(r"[^\S\r\n]*=[^\S\r\n]*") +_single_quoted_value = make_regex(r"'((?:\\'|[^'])*)'") +_double_quoted_value = make_regex(r'"((?:\\"|[^"])*)"') +_unquoted_value_part = make_regex(r"([^ \r\n]*)") +_comment = make_regex(r"(?:\s*#[^\r\n]*)?") +_end_of_line = make_regex(r"[^\S\r\n]*(?:\r\n|\n|\r)?") +_rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?") +_double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]") +_single_quote_escapes = make_regex(r"\\[\\']") + + +try: + # this is necessary because we only import these from typing + # when we are type checking, and the linter is upset if we + # re-import + import typing + Binding = typing.NamedTuple("Binding", [("key", typing.Optional[typing.Text]), + ("value", typing.Optional[typing.Text]), + ("original", typing.Text)]) +except ImportError: # pragma: no cover + from collections import namedtuple + Binding = namedtuple("Binding", ["key", # type: ignore + "value", + "original"]) # type: Tuple[Optional[Text], Optional[Text], Text] + + +class Error(Exception): + pass + + +class Reader: + def __init__(self, stream): + # type: (IO[Text]) -> None + self.string = stream.read() + self.position = 0 + self.mark = 0 + + def has_next(self): + # type: () -> bool + return self.position < len(self.string) + + def set_mark(self): + # type: () -> None + self.mark = self.position + + def get_marked(self): + # type: () -> Text + return self.string[self.mark:self.position] + + def peek(self, count): + # type: (int) -> Text + return self.string[self.position:self.position + count] + + def read(self, count): + # type: (int) -> Text + result = self.string[self.position:self.position + count] + if len(result) < count: + raise Error("read: End of string") + self.position += count + return result + + def read_regex(self, regex): + # type: (Pattern[Text]) -> Sequence[Text] + match = regex.match(self.string, self.position) + if match is None: + raise Error("read_regex: Pattern not found") + self.position = match.end() + return match.groups() + + +def decode_escapes(regex, string): + # type: (Pattern[Text], Text) -> Text + def decode_match(match): + # type: (Match[Text]) -> Text + return codecs.decode(match.group(0), 'unicode-escape') # type: ignore + + return regex.sub(decode_match, string) + + +def parse_key(reader): + # type: (Reader) -> Text + char = reader.peek(1) + if char == "'": + (key,) = reader.read_regex(_single_quoted_key) + else: + (key,) = reader.read_regex(_unquoted_key) + return key + + +def parse_unquoted_value(reader): + # type: (Reader) -> Text + value = u"" + while True: + (part,) = reader.read_regex(_unquoted_value_part) + value += part + after = reader.peek(2) + if len(after) < 2 or after[0] in u"\r\n" or after[1] in u" #\r\n": + return value + value += reader.read(2) + + +def parse_value(reader): + # type: (Reader) -> Text + char = reader.peek(1) + if char == u"'": + (value,) = reader.read_regex(_single_quoted_value) + return decode_escapes(_single_quote_escapes, value) + elif char == u'"': + (value,) = reader.read_regex(_double_quoted_value) + return decode_escapes(_double_quote_escapes, value) + elif char in (u"", u"\n", u"\r"): + return u"" + else: + return parse_unquoted_value(reader) + + +def parse_binding(reader): + # type: (Reader) -> Binding + reader.set_mark() + try: + reader.read_regex(_whitespace) + reader.read_regex(_export) + key = parse_key(reader) + reader.read_regex(_equal_sign) + value = parse_value(reader) + reader.read_regex(_comment) + reader.read_regex(_end_of_line) + return Binding(key=key, value=value, original=reader.get_marked()) + except Error: + reader.read_regex(_rest_of_line) + return Binding(key=None, value=None, original=reader.get_marked()) + + +def parse_stream(stream): + # type:(IO[Text]) -> Iterator[Binding] + reader = Reader(stream) + while reader.has_next(): + try: + yield parse_binding(reader) + except Error: + return diff --git a/pipenv/vendor/dotenv/py.typed b/pipenv/vendor/dotenv/py.typed new file mode 100644 index 00000000..7632ecf7 --- /dev/null +++ b/pipenv/vendor/dotenv/py.typed @@ -0,0 +1 @@ +# Marker file for PEP 561 diff --git a/pipenv/vendor/dotenv/version.py b/pipenv/vendor/dotenv/version.py index d69d16e9..b2385cb4 100644 --- a/pipenv/vendor/dotenv/version.py +++ b/pipenv/vendor/dotenv/version.py @@ -1 +1 @@ -__version__ = "0.9.1" +__version__ = "0.10.3" diff --git a/pipenv/vendor/dparse/LICENSE b/pipenv/vendor/dparse/LICENSE new file mode 100644 index 00000000..49d08568 --- /dev/null +++ b/pipenv/vendor/dparse/LICENSE @@ -0,0 +1,11 @@ + +MIT License + +Copyright (c) 2017, Jannis Gebauer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + diff --git a/pipenv/vendor/dparse/__init__.py b/pipenv/vendor/dparse/__init__.py new file mode 100644 index 00000000..295f38ee --- /dev/null +++ b/pipenv/vendor/dparse/__init__.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import +"""Top-level package for Dependency Parser.""" + +__author__ = """Jannis Gebauer""" +__email__ = 'jay@pyup.io' +__version__ = '0.5.0' + +from .parser import parse diff --git a/pipenv/vendor/dparse/dependencies.py b/pipenv/vendor/dparse/dependencies.py new file mode 100644 index 00000000..8881a9bc --- /dev/null +++ b/pipenv/vendor/dparse/dependencies.py @@ -0,0 +1,196 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +import json + +from . import filetypes, errors + + +class Dependency(object): + """ + + """ + + def __init__(self, name, specs, line, source="pypi", meta={}, extras=[], line_numbers=None, index_server=None, hashes=(), dependency_type=None, section=None): + """ + + :param name: + :param specs: + :param line: + :param source: + :param extras: + :param line_numbers: + :param index_server: + :param hashes: + :param dependency_type: + """ + self.name = name + self.key = name.lower().replace("_", "-") + self.specs = specs + self.line = line + self.source = source + self.meta = meta + self.line_numbers = line_numbers + self.index_server = index_server + self.hashes = hashes + self.dependency_type = dependency_type + self.extras = extras + self.section = section + + def __str__(self): # pragma: no cover + """ + + :return: + """ + return "Dependency({name}, {specs}, {line})".format( + name=self.name, + specs=self.specs, + line=self.line + ) + + def serialize(self): + """ + + :return: + """ + return { + "name": self.name, + "specs": self.specs, + "line": self.line, + "source": self.source, + "meta": self.meta, + "line_numbers": self.line_numbers, + "index_server": self.index_server, + "hashes": self.hashes, + "dependency_type": self.dependency_type, + "extras": self.extras, + "section": self.section + } + + @classmethod + def deserialize(cls, d): + """ + + :param d: + :return: + """ + return cls(**d) + + @property + def full_name(self): + """ + + :return: + """ + if self.extras: + return "{}[{}]".format(self.name, ",".join(self.extras)) + return self.name + + +class DependencyFile(object): + """ + + """ + + def __init__(self, content, path=None, sha=None, file_type=None, marker=((), ()), parser=None): + """ + + :param content: + :param path: + :param sha: + :param marker: + :param file_type: + :param parser: + """ + self.content = content + self.file_type = file_type + self.path = path + self.sha = sha + self.marker = marker + + self.dependencies = [] + self.resolved_files = [] + self.is_valid = False + self.file_marker, self.line_marker = marker + + if parser: + self.parser = parser + else: + from . import parser as parser_class + if file_type is not None: + if file_type == filetypes.requirements_txt: + self.parser = parser_class.RequirementsTXTParser + elif file_type == filetypes.tox_ini: + self.parser = parser_class.ToxINIParser + elif file_type == filetypes.conda_yml: + self.parser = parser_class.CondaYMLParser + elif file_type == filetypes.pipfile: + self.parser = parser_class.PipfileParser + elif file_type == filetypes.pipfile_lock: + self.parser = parser_class.PipfileLockParser + elif file_type == filetypes.setup_cfg: + self.parser = parser_class.SetupCfgParser + + elif path is not None: + if path.endswith(".txt"): + self.parser = parser_class.RequirementsTXTParser + elif path.endswith(".yml"): + self.parser = parser_class.CondaYMLParser + elif path.endswith(".ini"): + self.parser = parser_class.ToxINIParser + elif path.endswith("Pipfile"): + self.parser = parser_class.PipfileParser + elif path.endswith("Pipfile.lock"): + self.parser = parser_class.PipfileLockParser + elif path.endswith("setup.cfg"): + self.parser = parser_class.SetupCfgParser + + if not hasattr(self, "parser"): + raise errors.UnknownDependencyFileError + + self.parser = self.parser(self) + + def serialize(self): + """ + + :return: + """ + return { + "file_type": self.file_type, + "content": self.content, + "path": self.path, + "sha": self.sha, + "dependencies": [dep.serialize() for dep in self.dependencies] + } + + @classmethod + def deserialize(cls, d): + """ + + :param d: + :return: + """ + dependencies = [Dependency.deserialize(dep) for dep in d.pop("dependencies", [])] + instance = cls(**d) + instance.dependencies = dependencies + return instance + + def json(self): # pragma: no cover + """ + + :return: + """ + return json.dumps(self.serialize(), indent=2) + + def parse(self): + """ + + :return: + """ + if self.parser.is_marked_file: + self.is_valid = False + return self + self.parser.parse() + + self.is_valid = len(self.dependencies) > 0 or len(self.resolved_files) > 0 + return self diff --git a/pipenv/vendor/dparse/errors.py b/pipenv/vendor/dparse/errors.py new file mode 100644 index 00000000..b7a65430 --- /dev/null +++ b/pipenv/vendor/dparse/errors.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +class UnknownDependencyFileError(Exception): + """ + + """ + pass diff --git a/pipenv/vendor/dparse/filetypes.py b/pipenv/vendor/dparse/filetypes.py new file mode 100644 index 00000000..df8a913e --- /dev/null +++ b/pipenv/vendor/dparse/filetypes.py @@ -0,0 +1,9 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import + +requirements_txt = "requirements.txt" +conda_yml = "conda.yml" +setup_cfg = "setup.cfg" +tox_ini = "tox.ini" +pipfile = "Pipfile" +pipfile_lock = "Pipfile.lock" diff --git a/pipenv/vendor/dparse/parser.py b/pipenv/vendor/dparse/parser.py new file mode 100644 index 00000000..9b2a0728 --- /dev/null +++ b/pipenv/vendor/dparse/parser.py @@ -0,0 +1,427 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals, absolute_import +from collections import OrderedDict +import re +import yaml + +from io import StringIO + +from six.moves.configparser import SafeConfigParser, NoOptionError + + +from .regex import URL_REGEX, HASH_REGEX + +from .dependencies import DependencyFile, Dependency +from packaging.requirements import Requirement as PackagingRequirement, InvalidRequirement +from . import filetypes +import toml +from packaging.specifiers import SpecifierSet +import json + + +# this is a backport from setuptools 26.1 +def setuptools_parse_requirements_backport(strs): # pragma: no cover + # Copyright (C) 2016 Jason R Coombs <jaraco@jaraco.com> + # + # Permission is hereby granted, free of charge, to any person obtaining a copy of + # this software and associated documentation files (the "Software"), to deal in + # the Software without restriction, including without limitation the rights to + # use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies + # of the Software, and to permit persons to whom the Software is furnished to do + # so, subject to the following conditions: + # + # The above copyright notice and this permission notice shall be included in all + # copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + # SOFTWARE. + """Yield ``Requirement`` objects for each specification in `strs` + + `strs` must be a string, or a (possibly-nested) iterable thereof. + """ + # create a steppable iterator, so we can handle \-continuations + def yield_lines(strs): + """Yield non-empty/non-comment lines of a string or sequence""" + if isinstance(strs, str): + for s in strs.splitlines(): + s = s.strip() + # skip blank lines/comments + if s and not s.startswith('#'): + yield s + else: + for ss in strs: + for s in yield_lines(ss): + yield s + lines = iter(yield_lines(strs)) + + for line in lines: + # Drop comments -- a hash without a space may be in a URL. + if ' #' in line: + line = line[:line.find(' #')] + # If there is a line continuation, drop it, and append the next line. + if line.endswith('\\'): + line = line[:-2].strip() + line += next(lines) + yield PackagingRequirement(line) + + +class RequirementsTXTLineParser(object): + """ + + """ + + @classmethod + def parse(cls, line): + """ + + :param line: + :return: + """ + try: + # setuptools requires a space before the comment. If this isn't the case, add it. + if "\t#" in line: + parsed, = setuptools_parse_requirements_backport(line.replace("\t#", "\t #")) + else: + parsed, = setuptools_parse_requirements_backport(line) + except InvalidRequirement: + return None + dep = Dependency( + name=parsed.name, + specs=parsed.specifier, + line=line, + extras=parsed.extras, + dependency_type=filetypes.requirements_txt + ) + return dep + + +class Parser(object): + """ + + """ + + def __init__(self, obj): + """ + + :param obj: + """ + self.obj = obj + self._lines = None + + def iter_lines(self, lineno=0): + """ + + :param lineno: + :return: + """ + for line in self.lines[lineno:]: + yield line + + @property + def lines(self): + """ + + :return: + """ + if self._lines is None: + self._lines = self.obj.content.splitlines() + return self._lines + + @property + def is_marked_file(self): + """ + + :return: + """ + for n, line in enumerate(self.iter_lines()): + for marker in self.obj.file_marker: + if marker in line: + return True + if n >= 2: + break + return False + + def is_marked_line(self, line): + """ + + :param line: + :return: + """ + for marker in self.obj.line_marker: + if marker in line: + return True + return False + + @classmethod + def parse_hashes(cls, line): + """ + + :param line: + :return: + """ + hashes = [] + for match in re.finditer(HASH_REGEX, line): + hashes.append(line[match.start():match.end()]) + return re.sub(HASH_REGEX, "", line).strip(), hashes + + @classmethod + def parse_index_server(cls, line): + """ + + :param line: + :return: + """ + matches = URL_REGEX.findall(line) + if matches: + url = matches[0] + return url if url.endswith("/") else url + "/" + return None + + @classmethod + def resolve_file(cls, file_path, line): + """ + + :param file_path: + :param line: + :return: + """ + line = line.replace("-r ", "").replace("--requirement ", "") + parts = file_path.split("/") + if " #" in line: + line = line.split("#")[0].strip() + if len(parts) == 1: + return line + return "/".join(parts[:-1]) + "/" + line + + +class RequirementsTXTParser(Parser): + """ + + """ + + def parse(self): + """ + Parses a requirements.txt-like file + """ + index_server = None + for num, line in enumerate(self.iter_lines()): + line = line.rstrip() + if not line: + continue + if line.startswith('#'): + # comments are lines that start with # only + continue + if line.startswith('-i') or \ + line.startswith('--index-url') or \ + line.startswith('--extra-index-url'): + # this file is using a private index server, try to parse it + index_server = self.parse_index_server(line) + continue + elif self.obj.path and (line.startswith('-r') or line.startswith('--requirement')): + self.obj.resolved_files.append(self.resolve_file(self.obj.path, line)) + elif line.startswith('-f') or line.startswith('--find-links') or \ + line.startswith('--no-index') or line.startswith('--allow-external') or \ + line.startswith('--allow-unverified') or line.startswith('-Z') or \ + line.startswith('--always-unzip'): + continue + elif self.is_marked_line(line): + continue + else: + try: + + parseable_line = line + + # multiline requirements are not parseable + if "\\" in line: + parseable_line = line.replace("\\", "") + for next_line in self.iter_lines(num + 1): + parseable_line += next_line.strip().replace("\\", "") + line += "\n" + next_line + if "\\" in next_line: + continue + break + # ignore multiline requirements if they are marked + if self.is_marked_line(parseable_line): + continue + + hashes = [] + if "--hash" in parseable_line: + parseable_line, hashes = Parser.parse_hashes(parseable_line) + + req = RequirementsTXTLineParser.parse(parseable_line) + if req: + req.hashes = hashes + req.index_server = index_server + # replace the requirements line with the 'real' line + req.line = line + self.obj.dependencies.append(req) + except ValueError: + continue + + +class ToxINIParser(Parser): + """ + + """ + + def parse(self): + """ + + :return: + """ + parser = SafeConfigParser() + parser.readfp(StringIO(self.obj.content)) + for section in parser.sections(): + try: + content = parser.get(section=section, option="deps") + for n, line in enumerate(content.splitlines()): + if self.is_marked_line(line): + continue + if line: + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + except NoOptionError: + pass + + +class CondaYMLParser(Parser): + """ + + """ + + def parse(self): + """ + + :return: + """ + try: + data = yaml.safe_load(self.obj.content) + if data and 'dependencies' in data and isinstance(data['dependencies'], list): + for dep in data['dependencies']: + if isinstance(dep, dict) and 'pip' in dep: + for n, line in enumerate(dep['pip']): + if self.is_marked_line(line): + continue + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + except yaml.YAMLError: + pass + + +class PipfileParser(Parser): + + def parse(self): + """ + Parse a Pipfile (as seen in pipenv) + :return: + """ + try: + data = toml.loads(self.obj.content, _dict=OrderedDict) + if data: + for package_type in ['packages', 'dev-packages']: + if package_type in data: + for name, specs in data[package_type].items(): + # skip on VCS dependencies + if not isinstance(specs, str): + continue + if specs == '*': + specs = '' + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(specs), + dependency_type=filetypes.pipfile, + line=''.join([name, specs]), + section=package_type + ) + ) + except (toml.TomlDecodeError, IndexError) as e: + pass + +class PipfileLockParser(Parser): + + def parse(self): + """ + Parse a Pipfile.lock (as seen in pipenv) + :return: + """ + try: + data = json.loads(self.obj.content, object_pairs_hook=OrderedDict) + if data: + for package_type in ['default', 'develop']: + if package_type in data: + for name, meta in data[package_type].items(): + # skip VCS dependencies + if 'version' not in meta: + continue + specs = meta['version'] + hashes = meta['hashes'] + self.obj.dependencies.append( + Dependency( + name=name, specs=SpecifierSet(specs), + dependency_type=filetypes.pipfile_lock, + hashes=hashes, + line=''.join([name, specs]), + section=package_type + ) + ) + except ValueError: + pass + + +class SetupCfgParser(Parser): + def parse(self): + parser = SafeConfigParser() + parser.readfp(StringIO(self.obj.content)) + for section in parser.values(): + if section.name == 'options': + options = 'install_requires', 'setup_requires', 'test_require' + for name in options: + content = section.get(name) + if not content: + continue + self._parse_content(content) + elif section.name == 'options.extras_require': + for content in section.values(): + self._parse_content(content) + + def _parse_content(self, content): + for n, line in enumerate(content.splitlines()): + if self.is_marked_line(line): + continue + if line: + req = RequirementsTXTLineParser.parse(line) + if req: + req.dependency_type = self.obj.file_type + self.obj.dependencies.append(req) + + +def parse(content, file_type=None, path=None, sha=None, marker=((), ()), parser=None): + """ + + :param content: + :param file_type: + :param path: + :param sha: + :param marker: + :param parser: + :return: + """ + dep_file = DependencyFile( + content=content, + path=path, + sha=sha, + marker=marker, + file_type=file_type, + parser=parser + ) + + return dep_file.parse() diff --git a/pipenv/vendor/dparse/regex.py b/pipenv/vendor/dparse/regex.py new file mode 100644 index 00000000..40cc4091 --- /dev/null +++ b/pipenv/vendor/dparse/regex.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals + +import re +# see https://gist.github.com/dperini/729294 +URL_REGEX = re.compile( + # protocol identifier + "(?:(?:https?|ftp)://)" + # user:pass authentication + "(?:\S+(?::\S*)?@)?" + "(?:" + # IP address exclusion + # private & local networks + "(?!(?:10|127)(?:\.\d{1,3}){3})" + "(?!(?:169\.254|192\.168)(?:\.\d{1,3}){2})" + "(?!172\.(?:1[6-9]|2\d|3[0-1])(?:\.\d{1,3}){2})" + # IP address dotted notation octets + # excludes loopback network 0.0.0.0 + # excludes reserved space >= 224.0.0.0 + # excludes network & broadcast addresses + # (first & last IP address of each class) + "(?:[1-9]\d?|1\d\d|2[01]\d|22[0-3])" + "(?:\.(?:1?\d{1,2}|2[0-4]\d|25[0-5])){2}" + "(?:\.(?:[1-9]\d?|1\d\d|2[0-4]\d|25[0-4]))" + "|" + # host name + "(?:(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)" + # domain name + "(?:\.(?:[a-z\u00a1-\uffff0-9]-?)*[a-z\u00a1-\uffff0-9]+)*" + # TLD identifier + "(?:\.(?:[a-z\u00a1-\uffff]{2,}))" + ")" + # port number + "(?::\d{2,5})?" + # resource path + "(?:/\S*)?", + re.UNICODE) + +HASH_REGEX = r"--hash[=| ][\w]+:[\w]+" diff --git a/pipenv/vendor/dparse/updater.py b/pipenv/vendor/dparse/updater.py new file mode 100644 index 00000000..77b5ae66 --- /dev/null +++ b/pipenv/vendor/dparse/updater.py @@ -0,0 +1,119 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function, unicode_literals +import re +import json +import tempfile +import toml +import os + + +class RequirementsTXTUpdater(object): + + SUB_REGEX = r"^{}(?=\s*\r?\n?$)" + + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + """ + Updates the requirement to the latest version for the given content and adds hashes + if neccessary. + :param content: str, content + :return: str, updated content + """ + new_line = "{name}{spec}{version}".format(name=dependency.full_name, spec=spec, version=version) + appendix = '' + # leave environment markers intact + if ";" in dependency.line: + # condense multiline, split out the env marker, strip comments and --hashes + new_line += ";" + dependency.line.splitlines()[0].split(";", 1)[1] \ + .split("#")[0].split("--hash")[0].rstrip() + # add the comment + if "#" in dependency.line: + # split the line into parts: requirement and comment + parts = dependency.line.split("#") + requirement, comment = parts[0], "#".join(parts[1:]) + # find all whitespaces between the requirement and the comment + whitespaces = (hex(ord('\t')), hex(ord(' '))) + trailing_whitespace = '' + for c in requirement[::-1]: + if hex(ord(c)) in whitespaces: + trailing_whitespace += c + else: + break + appendix += trailing_whitespace + "#" + comment + # if this is a hashed requirement, add a multiline break before the comment + if dependency.hashes and not new_line.endswith("\\"): + new_line += " \\" + # if this is a hashed requirement, add the hashes + if hashes: + for n, new_hash in enumerate(hashes): + new_line += "\n --hash={method}:{hash}".format( + method=new_hash['method'], + hash=new_hash['hash'] + ) + # append a new multiline break if this is not the last line + if len(hashes) > n + 1: + new_line += " \\" + new_line += appendix + + regex = cls.SUB_REGEX.format(re.escape(dependency.line)) + + return re.sub(regex, new_line, content, flags=re.MULTILINE) + + +class CondaYMLUpdater(RequirementsTXTUpdater): + + SUB_REGEX = r"{}(?=\s*\r?\n?$)" + + +class ToxINIUpdater(CondaYMLUpdater): + pass + + +class SetupCFGUpdater(CondaYMLUpdater): + pass + + +class PipfileUpdater(object): + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + data = toml.loads(content) + if data: + for package_type in ['packages', 'dev-packages']: + if package_type in data: + if dependency.full_name in data[package_type]: + data[package_type][dependency.full_name] = "{spec}{version}".format( + spec=spec, version=version + ) + try: + from pipenv.project import Project + except ImportError: + raise ImportError("Updating a Pipfile requires the pipenv extra to be installed. Install it with " + "pip install dparse[pipenv]") + pipfile = tempfile.NamedTemporaryFile(delete=False) + p = Project(chdir=False) + p.write_toml(data=data, path=pipfile.name) + data = open(pipfile.name).read() + os.remove(pipfile.name) + return data + + +class PipfileLockUpdater(object): + @classmethod + def update(cls, content, dependency, version, spec="==", hashes=()): + data = json.loads(content) + if data: + for package_type in ['default', 'develop']: + if package_type in data: + if dependency.full_name in data[package_type]: + data[package_type][dependency.full_name] = { + 'hashes': [ + "{method}:{hash}".format( + hash=h['hash'], + method=h['method'] + ) for h in hashes + ], + 'version': "{spec}{version}".format( + spec=spec, version=version + ) + } + return json.dumps(data, indent=4, separators=(',', ': ')) + "\n" diff --git a/pipenv/vendor/funcsigs/LICENSE b/pipenv/vendor/funcsigs/LICENSE new file mode 100644 index 00000000..3e563d6f --- /dev/null +++ b/pipenv/vendor/funcsigs/LICENSE @@ -0,0 +1,13 @@ +Copyright 2013 Aaron Iles + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/pipenv/vendor/funcsigs/__init__.py b/pipenv/vendor/funcsigs/__init__.py new file mode 100644 index 00000000..5f5378b4 --- /dev/null +++ b/pipenv/vendor/funcsigs/__init__.py @@ -0,0 +1,829 @@ +# Copyright 2001-2013 Python Software Foundation; All Rights Reserved +"""Function signature objects for callables + +Back port of Python 3.3's function signature tools from the inspect module, +modified to be compatible with Python 2.6, 2.7 and 3.3+. +""" +from __future__ import absolute_import, division, print_function +import itertools +import functools +import re +import types + +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict + +from funcsigs.version import __version__ + +__all__ = ['BoundArguments', 'Parameter', 'Signature', 'signature'] + + +_WrapperDescriptor = type(type.__call__) +_MethodWrapper = type(all.__call__) + +_NonUserDefinedCallables = (_WrapperDescriptor, + _MethodWrapper, + types.BuiltinFunctionType) + + +def formatannotation(annotation, base_module=None): + if isinstance(annotation, type): + if annotation.__module__ in ('builtins', '__builtin__', base_module): + return annotation.__name__ + return annotation.__module__+'.'+annotation.__name__ + return repr(annotation) + + +def _get_user_defined_method(cls, method_name, *nested): + try: + if cls is type: + return + meth = getattr(cls, method_name) + for name in nested: + meth = getattr(meth, name, meth) + except AttributeError: + return + else: + if not isinstance(meth, _NonUserDefinedCallables): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return meth + + +def signature(obj): + '''Get a signature object for the passed callable.''' + + if not callable(obj): + raise TypeError('{0!r} is not a callable object'.format(obj)) + + if isinstance(obj, types.MethodType): + sig = signature(obj.__func__) + if obj.__self__ is None: + # Unbound method - preserve as-is. + return sig + else: + # Bound method. Eat self - if we can. + params = tuple(sig.parameters.values()) + + if not params or params[0].kind in (_VAR_KEYWORD, _KEYWORD_ONLY): + raise ValueError('invalid method signature') + + kind = params[0].kind + if kind in (_POSITIONAL_OR_KEYWORD, _POSITIONAL_ONLY): + # Drop first parameter: + # '(p1, p2[, ...])' -> '(p2[, ...])' + params = params[1:] + else: + if kind is not _VAR_POSITIONAL: + # Unless we add a new parameter type we never + # get here + raise ValueError('invalid argument type') + # It's a var-positional parameter. + # Do nothing. '(*args[, ...])' -> '(*args[, ...])' + + return sig.replace(parameters=params) + + try: + sig = obj.__signature__ + except AttributeError: + pass + else: + if sig is not None: + return sig + + try: + # Was this function wrapped by a decorator? + wrapped = obj.__wrapped__ + except AttributeError: + pass + else: + return signature(wrapped) + + if isinstance(obj, types.FunctionType): + return Signature.from_function(obj) + + if isinstance(obj, functools.partial): + sig = signature(obj.func) + + new_params = OrderedDict(sig.parameters.items()) + + partial_args = obj.args or () + partial_keywords = obj.keywords or {} + try: + ba = sig.bind_partial(*partial_args, **partial_keywords) + except TypeError as ex: + msg = 'partial object {0!r} has incorrect arguments'.format(obj) + raise ValueError(msg) + + for arg_name, arg_value in ba.arguments.items(): + param = new_params[arg_name] + if arg_name in partial_keywords: + # We set a new default value, because the following code + # is correct: + # + # >>> def foo(a): print(a) + # >>> print(partial(partial(foo, a=10), a=20)()) + # 20 + # >>> print(partial(partial(foo, a=10), a=20)(a=30)) + # 30 + # + # So, with 'partial' objects, passing a keyword argument is + # like setting a new default value for the corresponding + # parameter + # + # We also mark this parameter with '_partial_kwarg' + # flag. Later, in '_bind', the 'default' value of this + # parameter will be added to 'kwargs', to simulate + # the 'functools.partial' real call. + new_params[arg_name] = param.replace(default=arg_value, + _partial_kwarg=True) + + elif (param.kind not in (_VAR_KEYWORD, _VAR_POSITIONAL) and + not param._partial_kwarg): + new_params.pop(arg_name) + + return sig.replace(parameters=new_params.values()) + + sig = None + if isinstance(obj, type): + # obj is a class or a metaclass + + # First, let's see if it has an overloaded __call__ defined + # in its metaclass + call = _get_user_defined_method(type(obj), '__call__') + if call is not None: + sig = signature(call) + else: + # Now we check if the 'obj' class has a '__new__' method + new = _get_user_defined_method(obj, '__new__') + if new is not None: + sig = signature(new) + else: + # Finally, we should have at least __init__ implemented + init = _get_user_defined_method(obj, '__init__') + if init is not None: + sig = signature(init) + elif not isinstance(obj, _NonUserDefinedCallables): + # An object with __call__ + # We also check that the 'obj' is not an instance of + # _WrapperDescriptor or _MethodWrapper to avoid + # infinite recursion (and even potential segfault) + call = _get_user_defined_method(type(obj), '__call__', 'im_func') + if call is not None: + sig = signature(call) + + if sig is not None: + # For classes and objects we skip the first parameter of their + # __call__, __new__, or __init__ methods + return sig.replace(parameters=tuple(sig.parameters.values())[1:]) + + if isinstance(obj, types.BuiltinFunctionType): + # Raise a nicer error message for builtins + msg = 'no signature found for builtin function {0!r}'.format(obj) + raise ValueError(msg) + + raise ValueError('callable {0!r} is not supported by signature'.format(obj)) + + +class _void(object): + '''A private marker - used in Parameter & Signature''' + + +class _empty(object): + pass + + +class _ParameterKind(int): + def __new__(self, *args, **kwargs): + obj = int.__new__(self, *args) + obj._name = kwargs['name'] + return obj + + def __str__(self): + return self._name + + def __repr__(self): + return '<_ParameterKind: {0!r}>'.format(self._name) + + +_POSITIONAL_ONLY = _ParameterKind(0, name='POSITIONAL_ONLY') +_POSITIONAL_OR_KEYWORD = _ParameterKind(1, name='POSITIONAL_OR_KEYWORD') +_VAR_POSITIONAL = _ParameterKind(2, name='VAR_POSITIONAL') +_KEYWORD_ONLY = _ParameterKind(3, name='KEYWORD_ONLY') +_VAR_KEYWORD = _ParameterKind(4, name='VAR_KEYWORD') + + +class Parameter(object): + '''Represents a parameter in a function signature. + + Has the following public attributes: + + * name : str + The name of the parameter as a string. + * default : object + The default value for the parameter if specified. If the + parameter has no default value, this attribute is not set. + * annotation + The annotation for the parameter if specified. If the + parameter has no annotation, this attribute is not set. + * kind : str + Describes how argument values are bound to the parameter. + Possible values: `Parameter.POSITIONAL_ONLY`, + `Parameter.POSITIONAL_OR_KEYWORD`, `Parameter.VAR_POSITIONAL`, + `Parameter.KEYWORD_ONLY`, `Parameter.VAR_KEYWORD`. + ''' + + __slots__ = ('_name', '_kind', '_default', '_annotation', '_partial_kwarg') + + POSITIONAL_ONLY = _POSITIONAL_ONLY + POSITIONAL_OR_KEYWORD = _POSITIONAL_OR_KEYWORD + VAR_POSITIONAL = _VAR_POSITIONAL + KEYWORD_ONLY = _KEYWORD_ONLY + VAR_KEYWORD = _VAR_KEYWORD + + empty = _empty + + def __init__(self, name, kind, default=_empty, annotation=_empty, + _partial_kwarg=False): + + if kind not in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD, + _VAR_POSITIONAL, _KEYWORD_ONLY, _VAR_KEYWORD): + raise ValueError("invalid value for 'Parameter.kind' attribute") + self._kind = kind + + if default is not _empty: + if kind in (_VAR_POSITIONAL, _VAR_KEYWORD): + msg = '{0} parameters cannot have default values'.format(kind) + raise ValueError(msg) + self._default = default + self._annotation = annotation + + if name is None: + if kind != _POSITIONAL_ONLY: + raise ValueError("None is not a valid name for a " + "non-positional-only parameter") + self._name = name + else: + name = str(name) + if kind != _POSITIONAL_ONLY and not re.match(r'[a-z_]\w*$', name, re.I): + msg = '{0!r} is not a valid parameter name'.format(name) + raise ValueError(msg) + self._name = name + + self._partial_kwarg = _partial_kwarg + + @property + def name(self): + return self._name + + @property + def default(self): + return self._default + + @property + def annotation(self): + return self._annotation + + @property + def kind(self): + return self._kind + + def replace(self, name=_void, kind=_void, annotation=_void, + default=_void, _partial_kwarg=_void): + '''Creates a customized copy of the Parameter.''' + + if name is _void: + name = self._name + + if kind is _void: + kind = self._kind + + if annotation is _void: + annotation = self._annotation + + if default is _void: + default = self._default + + if _partial_kwarg is _void: + _partial_kwarg = self._partial_kwarg + + return type(self)(name, kind, default=default, annotation=annotation, + _partial_kwarg=_partial_kwarg) + + def __str__(self): + kind = self.kind + + formatted = self._name + if kind == _POSITIONAL_ONLY: + if formatted is None: + formatted = '' + formatted = '<{0}>'.format(formatted) + + # Add annotation and default value + if self._annotation is not _empty: + formatted = '{0}:{1}'.format(formatted, + formatannotation(self._annotation)) + + if self._default is not _empty: + formatted = '{0}={1}'.format(formatted, repr(self._default)) + + if kind == _VAR_POSITIONAL: + formatted = '*' + formatted + elif kind == _VAR_KEYWORD: + formatted = '**' + formatted + + return formatted + + def __repr__(self): + return '<{0} at {1:#x} {2!r}>'.format(self.__class__.__name__, + id(self), self.name) + + def __hash__(self): + msg = "unhashable type: '{0}'".format(self.__class__.__name__) + raise TypeError(msg) + + def __eq__(self, other): + return (issubclass(other.__class__, Parameter) and + self._name == other._name and + self._kind == other._kind and + self._default == other._default and + self._annotation == other._annotation) + + def __ne__(self, other): + return not self.__eq__(other) + + +class BoundArguments(object): + '''Result of `Signature.bind` call. Holds the mapping of arguments + to the function's parameters. + + Has the following public attributes: + + * arguments : OrderedDict + An ordered mutable mapping of parameters' names to arguments' values. + Does not contain arguments' default values. + * signature : Signature + The Signature object that created this instance. + * args : tuple + Tuple of positional arguments values. + * kwargs : dict + Dict of keyword arguments values. + ''' + + def __init__(self, signature, arguments): + self.arguments = arguments + self._signature = signature + + @property + def signature(self): + return self._signature + + @property + def args(self): + args = [] + for param_name, param in self._signature.parameters.items(): + if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or + param._partial_kwarg): + # Keyword arguments mapped by 'functools.partial' + # (Parameter._partial_kwarg is True) are mapped + # in 'BoundArguments.kwargs', along with VAR_KEYWORD & + # KEYWORD_ONLY + break + + try: + arg = self.arguments[param_name] + except KeyError: + # We're done here. Other arguments + # will be mapped in 'BoundArguments.kwargs' + break + else: + if param.kind == _VAR_POSITIONAL: + # *args + args.extend(arg) + else: + # plain argument + args.append(arg) + + return tuple(args) + + @property + def kwargs(self): + kwargs = {} + kwargs_started = False + for param_name, param in self._signature.parameters.items(): + if not kwargs_started: + if (param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY) or + param._partial_kwarg): + kwargs_started = True + else: + if param_name not in self.arguments: + kwargs_started = True + continue + + if not kwargs_started: + continue + + try: + arg = self.arguments[param_name] + except KeyError: + pass + else: + if param.kind == _VAR_KEYWORD: + # **kwargs + kwargs.update(arg) + else: + # plain keyword argument + kwargs[param_name] = arg + + return kwargs + + def __hash__(self): + msg = "unhashable type: '{0}'".format(self.__class__.__name__) + raise TypeError(msg) + + def __eq__(self, other): + return (issubclass(other.__class__, BoundArguments) and + self.signature == other.signature and + self.arguments == other.arguments) + + def __ne__(self, other): + return not self.__eq__(other) + + +class Signature(object): + '''A Signature object represents the overall signature of a function. + It stores a Parameter object for each parameter accepted by the + function, as well as information specific to the function itself. + + A Signature object has the following public attributes and methods: + + * parameters : OrderedDict + An ordered mapping of parameters' names to the corresponding + Parameter objects (keyword-only arguments are in the same order + as listed in `code.co_varnames`). + * return_annotation : object + The annotation for the return type of the function if specified. + If the function has no annotation for its return type, this + attribute is not set. + * bind(*args, **kwargs) -> BoundArguments + Creates a mapping from positional and keyword arguments to + parameters. + * bind_partial(*args, **kwargs) -> BoundArguments + Creates a partial mapping from positional and keyword arguments + to parameters (simulating 'functools.partial' behavior.) + ''' + + __slots__ = ('_return_annotation', '_parameters') + + _parameter_cls = Parameter + _bound_arguments_cls = BoundArguments + + empty = _empty + + def __init__(self, parameters=None, return_annotation=_empty, + __validate_parameters__=True): + '''Constructs Signature from the given list of Parameter + objects and 'return_annotation'. All arguments are optional. + ''' + + if parameters is None: + params = OrderedDict() + else: + if __validate_parameters__: + params = OrderedDict() + top_kind = _POSITIONAL_ONLY + + for idx, param in enumerate(parameters): + kind = param.kind + if kind < top_kind: + msg = 'wrong parameter order: {0} before {1}' + msg = msg.format(top_kind, param.kind) + raise ValueError(msg) + else: + top_kind = kind + + name = param.name + if name is None: + name = str(idx) + param = param.replace(name=name) + + if name in params: + msg = 'duplicate parameter name: {0!r}'.format(name) + raise ValueError(msg) + params[name] = param + else: + params = OrderedDict(((param.name, param) + for param in parameters)) + + self._parameters = params + self._return_annotation = return_annotation + + @classmethod + def from_function(cls, func): + '''Constructs Signature for the given python function''' + + if not isinstance(func, types.FunctionType): + raise TypeError('{0!r} is not a Python function'.format(func)) + + Parameter = cls._parameter_cls + + # Parameter information. + func_code = func.__code__ + pos_count = func_code.co_argcount + arg_names = func_code.co_varnames + positional = tuple(arg_names[:pos_count]) + keyword_only_count = getattr(func_code, 'co_kwonlyargcount', 0) + keyword_only = arg_names[pos_count:(pos_count + keyword_only_count)] + annotations = getattr(func, '__annotations__', {}) + defaults = func.__defaults__ + kwdefaults = getattr(func, '__kwdefaults__', None) + + if defaults: + pos_default_count = len(defaults) + else: + pos_default_count = 0 + + parameters = [] + + # Non-keyword-only parameters w/o defaults. + non_default_count = pos_count - pos_default_count + for name in positional[:non_default_count]: + annotation = annotations.get(name, _empty) + parameters.append(Parameter(name, annotation=annotation, + kind=_POSITIONAL_OR_KEYWORD)) + + # ... w/ defaults. + for offset, name in enumerate(positional[non_default_count:]): + annotation = annotations.get(name, _empty) + parameters.append(Parameter(name, annotation=annotation, + kind=_POSITIONAL_OR_KEYWORD, + default=defaults[offset])) + + # *args + if func_code.co_flags & 0x04: + name = arg_names[pos_count + keyword_only_count] + annotation = annotations.get(name, _empty) + parameters.append(Parameter(name, annotation=annotation, + kind=_VAR_POSITIONAL)) + + # Keyword-only parameters. + for name in keyword_only: + default = _empty + if kwdefaults is not None: + default = kwdefaults.get(name, _empty) + + annotation = annotations.get(name, _empty) + parameters.append(Parameter(name, annotation=annotation, + kind=_KEYWORD_ONLY, + default=default)) + # **kwargs + if func_code.co_flags & 0x08: + index = pos_count + keyword_only_count + if func_code.co_flags & 0x04: + index += 1 + + name = arg_names[index] + annotation = annotations.get(name, _empty) + parameters.append(Parameter(name, annotation=annotation, + kind=_VAR_KEYWORD)) + + return cls(parameters, + return_annotation=annotations.get('return', _empty), + __validate_parameters__=False) + + @property + def parameters(self): + try: + return types.MappingProxyType(self._parameters) + except AttributeError: + return OrderedDict(self._parameters.items()) + + @property + def return_annotation(self): + return self._return_annotation + + def replace(self, parameters=_void, return_annotation=_void): + '''Creates a customized copy of the Signature. + Pass 'parameters' and/or 'return_annotation' arguments + to override them in the new copy. + ''' + + if parameters is _void: + parameters = self.parameters.values() + + if return_annotation is _void: + return_annotation = self._return_annotation + + return type(self)(parameters, + return_annotation=return_annotation) + + def __hash__(self): + msg = "unhashable type: '{0}'".format(self.__class__.__name__) + raise TypeError(msg) + + def __eq__(self, other): + if (not issubclass(type(other), Signature) or + self.return_annotation != other.return_annotation or + len(self.parameters) != len(other.parameters)): + return False + + other_positions = dict((param, idx) + for idx, param in enumerate(other.parameters.keys())) + + for idx, (param_name, param) in enumerate(self.parameters.items()): + if param.kind == _KEYWORD_ONLY: + try: + other_param = other.parameters[param_name] + except KeyError: + return False + else: + if param != other_param: + return False + else: + try: + other_idx = other_positions[param_name] + except KeyError: + return False + else: + if (idx != other_idx or + param != other.parameters[param_name]): + return False + + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def _bind(self, args, kwargs, partial=False): + '''Private method. Don't use directly.''' + + arguments = OrderedDict() + + parameters = iter(self.parameters.values()) + parameters_ex = () + arg_vals = iter(args) + + if partial: + # Support for binding arguments to 'functools.partial' objects. + # See 'functools.partial' case in 'signature()' implementation + # for details. + for param_name, param in self.parameters.items(): + if (param._partial_kwarg and param_name not in kwargs): + # Simulating 'functools.partial' behavior + kwargs[param_name] = param.default + + while True: + # Let's iterate through the positional arguments and corresponding + # parameters + try: + arg_val = next(arg_vals) + except StopIteration: + # No more positional arguments + try: + param = next(parameters) + except StopIteration: + # No more parameters. That's it. Just need to check that + # we have no `kwargs` after this while loop + break + else: + if param.kind == _VAR_POSITIONAL: + # That's OK, just empty *args. Let's start parsing + # kwargs + break + elif param.name in kwargs: + if param.kind == _POSITIONAL_ONLY: + msg = '{arg!r} parameter is positional only, ' \ + 'but was passed as a keyword' + msg = msg.format(arg=param.name) + raise TypeError(msg) + parameters_ex = (param,) + break + elif (param.kind == _VAR_KEYWORD or + param.default is not _empty): + # That's fine too - we have a default value for this + # parameter. So, lets start parsing `kwargs`, starting + # with the current parameter + parameters_ex = (param,) + break + else: + if partial: + parameters_ex = (param,) + break + else: + msg = '{arg!r} parameter lacking default value' + msg = msg.format(arg=param.name) + raise TypeError(msg) + else: + # We have a positional argument to process + try: + param = next(parameters) + except StopIteration: + raise TypeError('too many positional arguments') + else: + if param.kind in (_VAR_KEYWORD, _KEYWORD_ONLY): + # Looks like we have no parameter for this positional + # argument + raise TypeError('too many positional arguments') + + if param.kind == _VAR_POSITIONAL: + # We have an '*args'-like argument, let's fill it with + # all positional arguments we have left and move on to + # the next phase + values = [arg_val] + values.extend(arg_vals) + arguments[param.name] = tuple(values) + break + + if param.name in kwargs: + raise TypeError('multiple values for argument ' + '{arg!r}'.format(arg=param.name)) + + arguments[param.name] = arg_val + + # Now, we iterate through the remaining parameters to process + # keyword arguments + kwargs_param = None + for param in itertools.chain(parameters_ex, parameters): + if param.kind == _POSITIONAL_ONLY: + # This should never happen in case of a properly built + # Signature object (but let's have this check here + # to ensure correct behaviour just in case) + raise TypeError('{arg!r} parameter is positional only, ' + 'but was passed as a keyword'. \ + format(arg=param.name)) + + if param.kind == _VAR_KEYWORD: + # Memorize that we have a '**kwargs'-like parameter + kwargs_param = param + continue + + param_name = param.name + try: + arg_val = kwargs.pop(param_name) + except KeyError: + # We have no value for this parameter. It's fine though, + # if it has a default value, or it is an '*args'-like + # parameter, left alone by the processing of positional + # arguments. + if (not partial and param.kind != _VAR_POSITIONAL and + param.default is _empty): + raise TypeError('{arg!r} parameter lacking default value'. \ + format(arg=param_name)) + + else: + arguments[param_name] = arg_val + + if kwargs: + if kwargs_param is not None: + # Process our '**kwargs'-like parameter + arguments[kwargs_param.name] = kwargs + else: + raise TypeError('too many keyword arguments %r' % kwargs) + + return self._bound_arguments_cls(self, arguments) + + def bind(*args, **kwargs): + '''Get a BoundArguments object, that maps the passed `args` + and `kwargs` to the function's signature. Raises `TypeError` + if the passed arguments can not be bound. + ''' + return args[0]._bind(args[1:], kwargs) + + def bind_partial(self, *args, **kwargs): + '''Get a BoundArguments object, that partially maps the + passed `args` and `kwargs` to the function's signature. + Raises `TypeError` if the passed arguments can not be bound. + ''' + return self._bind(args, kwargs, partial=True) + + def __str__(self): + result = [] + render_kw_only_separator = True + for idx, param in enumerate(self.parameters.values()): + formatted = str(param) + + kind = param.kind + if kind == _VAR_POSITIONAL: + # OK, we have an '*args'-like parameter, so we won't need + # a '*' to separate keyword-only arguments + render_kw_only_separator = False + elif kind == _KEYWORD_ONLY and render_kw_only_separator: + # We have a keyword-only parameter to render and we haven't + # rendered an '*args'-like parameter before, so add a '*' + # separator to the parameters list ("foo(arg1, *, arg2)" case) + result.append('*') + # This condition should be only triggered once, so + # reset the flag + render_kw_only_separator = False + + result.append(formatted) + + rendered = '({0})'.format(', '.join(result)) + + if self.return_annotation is not _empty: + anno = formatannotation(self.return_annotation) + rendered += ' -> {0}'.format(anno) + + return rendered diff --git a/pipenv/vendor/funcsigs/version.py b/pipenv/vendor/funcsigs/version.py new file mode 100644 index 00000000..7863915f --- /dev/null +++ b/pipenv/vendor/funcsigs/version.py @@ -0,0 +1 @@ +__version__ = "1.0.2" diff --git a/pipenv/vendor/idna/LICENSE.rst b/pipenv/vendor/idna/LICENSE.rst index 3ee64fba..63664b82 100644 --- a/pipenv/vendor/idna/LICENSE.rst +++ b/pipenv/vendor/idna/LICENSE.rst @@ -1,7 +1,9 @@ License ------- -Copyright (c) 2013-2018, Kim Davies. All rights reserved. +License: bsd-3-clause + +Copyright (c) 2013-2020, Kim Davies. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -30,51 +32,3 @@ modification, are permitted provided that the following conditions are met: (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -Portions of the codec implementation and unit tests are derived from the -Python standard library, which carries the `Python Software Foundation -License <https://docs.python.org/2/license.html>`_: - - Copyright (c) 2001-2014 Python Software Foundation; All Rights Reserved - -Portions of the unit tests are derived from the Unicode standard, which -is subject to the Unicode, Inc. License Agreement: - - Copyright (c) 1991-2014 Unicode, Inc. All rights reserved. - Distributed under the Terms of Use in - <http://www.unicode.org/copyright.html>. - - Permission is hereby granted, free of charge, to any person obtaining - a copy of the Unicode data files and any associated documentation - (the "Data Files") or Unicode software and any associated documentation - (the "Software") to deal in the Data Files or Software - without restriction, including without limitation the rights to use, - copy, modify, merge, publish, distribute, and/or sell copies of - the Data Files or Software, and to permit persons to whom the Data Files - or Software are furnished to do so, provided that - - (a) this copyright and permission notice appear with all copies - of the Data Files or Software, - - (b) this copyright and permission notice appear in associated - documentation, and - - (c) there is clear notice in each modified Data File or in the Software - as well as in the documentation associated with the Data File(s) or - Software that the data or software has been modified. - - THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF - ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE - WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - NONINFRINGEMENT OF THIRD PARTY RIGHTS. - IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS - NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL - DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, - DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER - TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR - PERFORMANCE OF THE DATA FILES OR SOFTWARE. - - Except as contained in this notice, the name of a copyright holder - shall not be used in advertising or otherwise to promote the sale, - use or other dealings in these Data Files or Software without prior - written authorization of the copyright holder. diff --git a/pipenv/vendor/idna/__init__.py b/pipenv/vendor/idna/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/codec.py b/pipenv/vendor/idna/codec.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/compat.py b/pipenv/vendor/idna/compat.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/core.py b/pipenv/vendor/idna/core.py old mode 100755 new mode 100644 index 090c2c18..9c3bba2a --- a/pipenv/vendor/idna/core.py +++ b/pipenv/vendor/idna/core.py @@ -9,7 +9,7 @@ _virama_combining_class = 9 _alabel_prefix = b'xn--' _unicode_dots_re = re.compile(u'[\u002e\u3002\uff0e\uff61]') -if sys.version_info[0] == 3: +if sys.version_info[0] >= 3: unicode = str unichr = chr @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label @@ -303,6 +300,8 @@ def ulabel(label): label = label.lower() if label.startswith(_alabel_prefix): label = label[len(_alabel_prefix):] + if label.decode('ascii')[-1] == '-': + raise IDNAError('A-label must not end with a hyphen') else: check_label(label) return label.decode('ascii') diff --git a/pipenv/vendor/idna/idnadata.py b/pipenv/vendor/idna/idnadata.py old mode 100755 new mode 100644 index 17974e23..2b81c522 --- a/pipenv/vendor/idna/idnadata.py +++ b/pipenv/vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "12.1.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -74,6 +74,7 @@ scripts = { 0x304100003097, 0x309d000030a0, 0x1b0010001b11f, + 0x1b1500001b153, 0x1f2000001f201, ), 'Katakana': ( @@ -85,6 +86,7 @@ scripts = { 0xff660000ff70, 0xff710000ff9e, 0x1b0000001b001, + 0x1b1640001b168, ), } joining_types = { @@ -248,6 +250,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +525,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +694,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -758,6 +826,7 @@ joining_types = { 0x1e941: 68, 0x1e942: 68, 0x1e943: 68, + 0x1e94b: 84, } codepoint_classes = { 'PVALID': ( @@ -1034,14 +1103,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1124,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1148,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1208,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1190,18 +1261,11 @@ codepoint_classes = { 0xe5000000e5a, 0xe8100000e83, 0xe8400000e85, - 0xe8700000e89, - 0xe8a00000e8b, - 0xe8d00000e8e, - 0xe9400000e98, - 0xe9900000ea0, - 0xea100000ea4, + 0xe8600000e8b, + 0xe8c00000ea4, 0xea500000ea6, - 0xea700000ea8, - 0xeaa00000eac, - 0xead00000eb3, - 0xeb400000eba, - 0xebb00000ebe, + 0xea700000eb3, + 0xeb400000ebe, 0xec000000ec5, 0xec600000ec7, 0xec800000ece, @@ -1276,7 +1340,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1302,7 +1366,7 @@ codepoint_classes = { 0x1c4000001c4a, 0x1c4d00001c7e, 0x1cd000001cd3, - 0x1cd400001cfa, + 0x1cd400001cfb, 0x1d0000001d2c, 0x1d2f00001d30, 0x1d3b00001d3c, @@ -1544,11 +1608,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1719,14 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, + 0xa7bb0000a7bc, + 0xa7bd0000a7be, + 0xa7bf0000a7c0, + 0xa7c30000a7c4, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1684,7 +1753,7 @@ codepoint_classes = { 0xab200000ab27, 0xab280000ab2f, 0xab300000ab5b, - 0xab600000ab66, + 0xab600000ab68, 0xabc00000abeb, 0xabec0000abee, 0xabf00000abfa, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,12 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, + 0x10fe000010ff7, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1838,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1862,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1872,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e00011460, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1805,17 +1882,22 @@ codepoint_classes = { 0x1160000011641, 0x1164400011645, 0x116500001165a, - 0x11680000116b8, + 0x11680000116b9, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, + 0x119a0000119a8, + 0x119aa000119d8, + 0x119da000119e2, + 0x119e3000119e5, 0x11a0000011a3f, 0x11a4700011a48, - 0x11a5000011a84, - 0x11a8600011a9a, + 0x11a5000011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1913,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,13 +1934,17 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, - 0x16f0000016f45, - 0x16f5000016f7f, + 0x16e6000016e80, + 0x16f0000016f4b, + 0x16f4f00016f88, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x16fe300016fe4, + 0x17000000187f8, 0x1880000018af3, 0x1b0000001b11f, + 0x1b1500001b153, + 0x1b1640001b168, 0x1b1700001b2fc, 0x1bc000001bc6b, 0x1bc700001bc7d, @@ -1869,9 +1962,14 @@ codepoint_classes = { 0x1e01b0001e022, 0x1e0230001e025, 0x1e0260001e02b, + 0x1e1000001e12d, + 0x1e1300001e13e, + 0x1e1400001e14a, + 0x1e14e0001e14f, + 0x1e2c00001e2fa, 0x1e8000001e8c5, 0x1e8d00001e8d7, - 0x1e9220001e94b, + 0x1e9220001e94c, 0x1e9500001e95a, 0x200000002a6d7, 0x2a7000002b735, diff --git a/pipenv/vendor/idna/intranges.py b/pipenv/vendor/idna/intranges.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/package_data.py b/pipenv/vendor/idna/package_data.py old mode 100755 new mode 100644 index 39c192ba..b5d82165 --- a/pipenv/vendor/idna/package_data.py +++ b/pipenv/vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.9' diff --git a/pipenv/vendor/idna/uts46data.py b/pipenv/vendor/idna/uts46data.py old mode 100755 new mode 100644 index 79731cb9..2711136d --- a/pipenv/vendor/idna/uts46data.py +++ b/pipenv/vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "12.1.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1277,9 +1272,7 @@ def _seg_12(): (0xC64, 'X'), (0xC66, 'V'), (0xC70, 'X'), - (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), + (0xC77, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,33 +1348,19 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ - (0xE87, 'V'), - (0xE89, 'X'), - (0xE8A, 'V'), + (0xE86, 'V'), (0xE8B, 'X'), - (0xE8D, 'V'), - (0xE8E, 'X'), - (0xE94, 'V'), - (0xE98, 'X'), - (0xE99, 'V'), - (0xEA0, 'X'), - (0xEA1, 'V'), + (0xE8C, 'V'), (0xEA4, 'X'), (0xEA5, 'V'), (0xEA6, 'X'), (0xEA7, 'V'), - (0xEA8, 'X'), - (0xEAA, 'V'), - (0xEAC, 'X'), - (0xEAD, 'V'), + ] + +def _seg_13(): + return [ (0xEB3, 'M', u'ໍາ'), (0xEB4, 'V'), - (0xEBA, 'X'), - (0xEBB, 'V'), (0xEBE, 'X'), (0xEC0, 'V'), (0xEC5, 'X'), @@ -1459,10 +1438,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1484,6 +1459,10 @@ def _seg_14(): (0x12C8, 'V'), (0x12D7, 'X'), (0x12D8, 'V'), + ] + +def _seg_14(): + return [ (0x1311, 'X'), (0x1312, 'V'), (0x1316, 'X'), @@ -1538,7 +1517,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1542,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1588,6 +1563,10 @@ def _seg_15(): (0x1BFC, 'V'), (0x1C38, 'X'), (0x1C3B, 'V'), + ] + +def _seg_15(): + return [ (0x1C4A, 'X'), (0x1C4D, 'V'), (0x1C80, 'M', u'в'), @@ -1599,10 +1578,57 @@ def _seg_15(): (0x1C87, 'M', u'ѣ'), (0x1C88, 'M', u'ꙋ'), (0x1C89, 'X'), + (0x1C90, 'M', u'ა'), + (0x1C91, 'M', u'ბ'), + (0x1C92, 'M', u'გ'), + (0x1C93, 'M', u'დ'), + (0x1C94, 'M', u'ე'), + (0x1C95, 'M', u'ვ'), + (0x1C96, 'M', u'ზ'), + (0x1C97, 'M', u'თ'), + (0x1C98, 'M', u'ი'), + (0x1C99, 'M', u'კ'), + (0x1C9A, 'M', u'ლ'), + (0x1C9B, 'M', u'მ'), + (0x1C9C, 'M', u'ნ'), + (0x1C9D, 'M', u'ო'), + (0x1C9E, 'M', u'პ'), + (0x1C9F, 'M', u'ჟ'), + (0x1CA0, 'M', u'რ'), + (0x1CA1, 'M', u'ს'), + (0x1CA2, 'M', u'ტ'), + (0x1CA3, 'M', u'უ'), + (0x1CA4, 'M', u'ფ'), + (0x1CA5, 'M', u'ქ'), + (0x1CA6, 'M', u'ღ'), + (0x1CA7, 'M', u'ყ'), + (0x1CA8, 'M', u'შ'), + (0x1CA9, 'M', u'ჩ'), + (0x1CAA, 'M', u'ც'), + (0x1CAB, 'M', u'ძ'), + (0x1CAC, 'M', u'წ'), + (0x1CAD, 'M', u'ჭ'), + (0x1CAE, 'M', u'ხ'), + (0x1CAF, 'M', u'ჯ'), + (0x1CB0, 'M', u'ჰ'), + (0x1CB1, 'M', u'ჱ'), + (0x1CB2, 'M', u'ჲ'), + (0x1CB3, 'M', u'ჳ'), + (0x1CB4, 'M', u'ჴ'), + (0x1CB5, 'M', u'ჵ'), + (0x1CB6, 'M', u'ჶ'), + (0x1CB7, 'M', u'ჷ'), + (0x1CB8, 'M', u'ჸ'), + (0x1CB9, 'M', u'ჹ'), + (0x1CBA, 'M', u'ჺ'), + (0x1CBB, 'X'), + (0x1CBD, 'M', u'ჽ'), + (0x1CBE, 'M', u'ჾ'), + (0x1CBF, 'M', u'ჿ'), (0x1CC0, 'V'), (0x1CC8, 'X'), (0x1CD0, 'V'), - (0x1CFA, 'X'), + (0x1CFB, 'X'), (0x1D00, 'V'), (0x1D2C, 'M', u'a'), (0x1D2D, 'M', u'æ'), @@ -1641,6 +1667,10 @@ def _seg_15(): (0x1D4E, 'V'), (0x1D4F, 'M', u'k'), (0x1D50, 'M', u'm'), + ] + +def _seg_16(): + return [ (0x1D51, 'M', u'ŋ'), (0x1D52, 'M', u'o'), (0x1D53, 'M', u'ɔ'), @@ -1667,10 +1697,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1745,6 +1771,10 @@ def _seg_16(): (0x1E1C, 'M', u'ḝ'), (0x1E1D, 'V'), (0x1E1E, 'M', u'ḟ'), + ] + +def _seg_17(): + return [ (0x1E1F, 'V'), (0x1E20, 'M', u'ḡ'), (0x1E21, 'V'), @@ -1771,10 +1801,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1849,6 +1875,10 @@ def _seg_17(): (0x1E80, 'M', u'ẁ'), (0x1E81, 'V'), (0x1E82, 'M', u'ẃ'), + ] + +def _seg_18(): + return [ (0x1E83, 'V'), (0x1E84, 'M', u'ẅ'), (0x1E85, 'V'), @@ -1875,10 +1905,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1953,6 +1979,10 @@ def _seg_18(): (0x1EE9, 'V'), (0x1EEA, 'M', u'ừ'), (0x1EEB, 'V'), + ] + +def _seg_19(): + return [ (0x1EEC, 'M', u'ử'), (0x1EED, 'V'), (0x1EEE, 'M', u'ữ'), @@ -1979,10 +2009,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -2057,6 +2083,10 @@ def _seg_19(): (0x1F80, 'M', u'ἀι'), (0x1F81, 'M', u'ἁι'), (0x1F82, 'M', u'ἂι'), + ] + +def _seg_20(): + return [ (0x1F83, 'M', u'ἃι'), (0x1F84, 'M', u'ἄι'), (0x1F85, 'M', u'ἅι'), @@ -2083,10 +2113,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2161,6 +2187,10 @@ def _seg_20(): (0x1FEE, '3', u' ̈́'), (0x1FEF, '3', u'`'), (0x1FF0, 'X'), + ] + +def _seg_21(): + return [ (0x1FF2, 'M', u'ὼι'), (0x1FF3, 'M', u'ωι'), (0x1FF4, 'M', u'ώι'), @@ -2187,10 +2217,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2265,6 +2291,10 @@ def _seg_21(): (0x20C0, 'X'), (0x20D0, 'V'), (0x20F1, 'X'), + ] + +def _seg_22(): + return [ (0x2100, '3', u'a/c'), (0x2101, '3', u'a/s'), (0x2102, 'M', u'c'), @@ -2291,10 +2321,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2369,6 +2395,10 @@ def _seg_22(): (0x2175, 'M', u'vi'), (0x2176, 'M', u'vii'), (0x2177, 'M', u'viii'), + ] + +def _seg_23(): + return [ (0x2178, 'M', u'ix'), (0x2179, 'M', u'x'), (0x217A, 'M', u'xi'), @@ -2395,10 +2425,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2473,6 +2499,10 @@ def _seg_23(): (0x24B5, '3', u'(z)'), (0x24B6, 'M', u'a'), (0x24B7, 'M', u'b'), + ] + +def _seg_24(): + return [ (0x24B8, 'M', u'c'), (0x24B9, 'M', u'd'), (0x24BA, 'M', u'e'), @@ -2499,10 +2529,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2541,13 +2567,6 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), - (0x2BC9, 'X'), - (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2584,6 +2603,10 @@ def _seg_24(): (0x2C21, 'M', u'ⱑ'), (0x2C22, 'M', u'ⱒ'), (0x2C23, 'M', u'ⱓ'), + ] + +def _seg_25(): + return [ (0x2C24, 'M', u'ⱔ'), (0x2C25, 'M', u'ⱕ'), (0x2C26, 'M', u'ⱖ'), @@ -2603,10 +2626,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2688,6 +2707,10 @@ def _seg_25(): (0x2CBA, 'M', u'ⲻ'), (0x2CBB, 'V'), (0x2CBC, 'M', u'ⲽ'), + ] + +def _seg_26(): + return [ (0x2CBD, 'V'), (0x2CBE, 'M', u'ⲿ'), (0x2CBF, 'V'), @@ -2707,10 +2730,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2768,7 +2787,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E50, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2792,6 +2811,10 @@ def _seg_26(): (0x2F0D, 'M', u'冖'), (0x2F0E, 'M', u'冫'), (0x2F0F, 'M', u'几'), + ] + +def _seg_27(): + return [ (0x2F10, 'M', u'凵'), (0x2F11, 'M', u'刀'), (0x2F12, 'M', u'力'), @@ -2811,10 +2834,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2896,6 +2915,10 @@ def _seg_27(): (0x2F71, 'M', u'禸'), (0x2F72, 'M', u'禾'), (0x2F73, 'M', u'穴'), + ] + +def _seg_28(): + return [ (0x2F74, 'M', u'立'), (0x2F75, 'M', u'竹'), (0x2F76, 'M', u'米'), @@ -2915,10 +2938,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -3000,6 +3019,10 @@ def _seg_28(): (0x2FD5, 'M', u'龠'), (0x2FD6, 'X'), (0x3000, '3', u' '), + ] + +def _seg_29(): + return [ (0x3001, 'V'), (0x3002, 'M', u'.'), (0x3003, 'V'), @@ -3019,13 +3042,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3104,6 +3123,10 @@ def _seg_29(): (0x317C, 'M', u'ᄯ'), (0x317D, 'M', u'ᄲ'), (0x317E, 'M', u'ᄶ'), + ] + +def _seg_30(): + return [ (0x317F, 'M', u'ᅀ'), (0x3180, 'M', u'ᅇ'), (0x3181, 'M', u'ᅌ'), @@ -3123,10 +3146,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3208,6 +3227,10 @@ def _seg_30(): (0x323C, '3', u'(監)'), (0x323D, '3', u'(企)'), (0x323E, '3', u'(資)'), + ] + +def _seg_31(): + return [ (0x323F, '3', u'(協)'), (0x3240, '3', u'(祭)'), (0x3241, '3', u'(休)'), @@ -3227,10 +3250,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3312,6 +3331,10 @@ def _seg_31(): (0x32A7, 'M', u'左'), (0x32A8, 'M', u'右'), (0x32A9, 'M', u'医'), + ] + +def _seg_32(): + return [ (0x32AA, 'M', u'宗'), (0x32AB, 'M', u'学'), (0x32AC, 'M', u'監'), @@ -3331,10 +3354,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3401,7 +3420,7 @@ def _seg_32(): (0x32FC, 'M', u'ヰ'), (0x32FD, 'M', u'ヱ'), (0x32FE, 'M', u'ヲ'), - (0x32FF, 'X'), + (0x32FF, 'M', u'令和'), (0x3300, 'M', u'アパート'), (0x3301, 'M', u'アルファ'), (0x3302, 'M', u'アンペア'), @@ -3416,6 +3435,10 @@ def _seg_32(): (0x330B, 'M', u'カイリ'), (0x330C, 'M', u'カラット'), (0x330D, 'M', u'カロリー'), + ] + +def _seg_33(): + return [ (0x330E, 'M', u'ガロン'), (0x330F, 'M', u'ガンマ'), (0x3310, 'M', u'ギガ'), @@ -3435,10 +3458,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3520,6 +3539,10 @@ def _seg_33(): (0x336F, 'M', u'23点'), (0x3370, 'M', u'24点'), (0x3371, 'M', u'hpa'), + ] + +def _seg_34(): + return [ (0x3372, 'M', u'da'), (0x3373, 'M', u'au'), (0x3374, 'M', u'bar'), @@ -3539,10 +3562,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3624,6 +3643,10 @@ def _seg_34(): (0x33D3, 'M', u'lx'), (0x33D4, 'M', u'mb'), (0x33D5, 'M', u'mil'), + ] + +def _seg_35(): + return [ (0x33D6, 'M', u'mol'), (0x33D7, 'M', u'ph'), (0x33D8, 'X'), @@ -3643,10 +3666,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3673,7 +3692,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3728,6 +3747,10 @@ def _seg_35(): (0xA66D, 'V'), (0xA680, 'M', u'ꚁ'), (0xA681, 'V'), + ] + +def _seg_36(): + return [ (0xA682, 'M', u'ꚃ'), (0xA683, 'V'), (0xA684, 'M', u'ꚅ'), @@ -3747,10 +3770,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3832,6 +3851,10 @@ def _seg_36(): (0xA766, 'M', u'ꝧ'), (0xA767, 'V'), (0xA768, 'M', u'ꝩ'), + ] + +def _seg_37(): + return [ (0xA769, 'V'), (0xA76A, 'M', u'ꝫ'), (0xA76B, 'V'), @@ -3851,10 +3874,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3893,7 +3912,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3902,7 +3921,21 @@ def _seg_37(): (0xA7B5, 'V'), (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), - (0xA7B8, 'X'), + (0xA7B8, 'M', u'ꞹ'), + (0xA7B9, 'V'), + (0xA7BA, 'M', u'ꞻ'), + (0xA7BB, 'V'), + (0xA7BC, 'M', u'ꞽ'), + (0xA7BD, 'V'), + (0xA7BE, 'M', u'ꞿ'), + (0xA7BF, 'V'), + (0xA7C0, 'X'), + (0xA7C2, 'M', u'ꟃ'), + (0xA7C3, 'V'), + (0xA7C4, 'M', u'ꞔ'), + (0xA7C5, 'M', u'ʂ'), + (0xA7C6, 'M', u'ᶎ'), + (0xA7C7, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,13 +3950,15 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), (0xA980, 'V'), (0xA9CE, 'X'), + ] + +def _seg_38(): + return [ (0xA9CF, 'V'), (0xA9DA, 'X'), (0xA9DE, 'V'), @@ -3954,11 +3989,7 @@ def _seg_37(): (0xAB5E, 'M', u'ɫ'), (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), - (0xAB66, 'X'), - ] - -def _seg_38(): - return [ + (0xAB68, 'X'), (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -4028,6 +4059,10 @@ def _seg_38(): (0xABB2, 'M', u'Ꮲ'), (0xABB3, 'M', u'Ꮳ'), (0xABB4, 'M', u'Ꮴ'), + ] + +def _seg_39(): + return [ (0xABB5, 'M', u'Ꮵ'), (0xABB6, 'M', u'Ꮶ'), (0xABB7, 'M', u'Ꮷ'), @@ -4059,10 +4094,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4132,6 +4163,10 @@ def _seg_39(): (0xF94D, 'M', u'淚'), (0xF94E, 'M', u'漏'), (0xF94F, 'M', u'累'), + ] + +def _seg_40(): + return [ (0xF950, 'M', u'縷'), (0xF951, 'M', u'陋'), (0xF952, 'M', u'勒'), @@ -4163,10 +4198,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4236,6 +4267,10 @@ def _seg_40(): (0xF9B1, 'M', u'鈴'), (0xF9B2, 'M', u'零'), (0xF9B3, 'M', u'靈'), + ] + +def _seg_41(): + return [ (0xF9B4, 'M', u'領'), (0xF9B5, 'M', u'例'), (0xF9B6, 'M', u'禮'), @@ -4267,10 +4302,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4340,6 +4371,10 @@ def _seg_41(): (0xFA17, 'M', u'益'), (0xFA18, 'M', u'礼'), (0xFA19, 'M', u'神'), + ] + +def _seg_42(): + return [ (0xFA1A, 'M', u'祥'), (0xFA1B, 'M', u'福'), (0xFA1C, 'M', u'靖'), @@ -4371,10 +4406,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4444,6 +4475,10 @@ def _seg_42(): (0xFA80, 'M', u'婢'), (0xFA81, 'M', u'嬨'), (0xFA82, 'M', u'廒'), + ] + +def _seg_43(): + return [ (0xFA83, 'M', u'廙'), (0xFA84, 'M', u'彩'), (0xFA85, 'M', u'徭'), @@ -4475,10 +4510,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4548,6 +4579,10 @@ def _seg_43(): (0xFB15, 'M', u'մի'), (0xFB16, 'M', u'վն'), (0xFB17, 'M', u'մխ'), + ] + +def _seg_44(): + return [ (0xFB18, 'X'), (0xFB1D, 'M', u'יִ'), (0xFB1E, 'V'), @@ -4579,10 +4614,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4652,6 +4683,10 @@ def _seg_44(): (0xFBF0, 'M', u'ئۇ'), (0xFBF2, 'M', u'ئۆ'), (0xFBF4, 'M', u'ئۈ'), + ] + +def _seg_45(): + return [ (0xFBF6, 'M', u'ئې'), (0xFBF9, 'M', u'ئى'), (0xFBFC, 'M', u'ی'), @@ -4683,10 +4718,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4756,6 +4787,10 @@ def _seg_45(): (0xFC5E, '3', u' ٌّ'), (0xFC5F, '3', u' ٍّ'), (0xFC60, '3', u' َّ'), + ] + +def _seg_46(): + return [ (0xFC61, '3', u' ُّ'), (0xFC62, '3', u' ِّ'), (0xFC63, '3', u' ّٰ'), @@ -4787,10 +4822,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4860,6 +4891,10 @@ def _seg_46(): (0xFCC2, 'M', u'قح'), (0xFCC3, 'M', u'قم'), (0xFCC4, 'M', u'كج'), + ] + +def _seg_47(): + return [ (0xFCC5, 'M', u'كح'), (0xFCC6, 'M', u'كخ'), (0xFCC7, 'M', u'كل'), @@ -4891,10 +4926,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4964,6 +4995,10 @@ def _seg_47(): (0xFD26, 'M', u'شح'), (0xFD27, 'M', u'شخ'), (0xFD28, 'M', u'شم'), + ] + +def _seg_48(): + return [ (0xFD29, 'M', u'شر'), (0xFD2A, 'M', u'سر'), (0xFD2B, 'M', u'صر'), @@ -4995,10 +5030,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5068,6 +5099,10 @@ def _seg_48(): (0xFDAD, 'M', u'لمي'), (0xFDAE, 'M', u'يحي'), (0xFDAF, 'M', u'يجي'), + ] + +def _seg_49(): + return [ (0xFDB0, 'M', u'يمي'), (0xFDB1, 'M', u'ممي'), (0xFDB2, 'M', u'قمي'), @@ -5099,10 +5134,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5172,6 +5203,10 @@ def _seg_49(): (0xFE65, '3', u'>'), (0xFE66, '3', u'='), (0xFE67, 'X'), + ] + +def _seg_50(): + return [ (0xFE68, '3', u'\\'), (0xFE69, '3', u'$'), (0xFE6A, '3', u'%'), @@ -5203,10 +5238,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5276,6 +5307,10 @@ def _seg_50(): (0xFF22, 'M', u'b'), (0xFF23, 'M', u'c'), (0xFF24, 'M', u'd'), + ] + +def _seg_51(): + return [ (0xFF25, 'M', u'e'), (0xFF26, 'M', u'f'), (0xFF27, 'M', u'g'), @@ -5307,10 +5342,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5380,6 +5411,10 @@ def _seg_51(): (0xFF86, 'M', u'ニ'), (0xFF87, 'M', u'ヌ'), (0xFF88, 'M', u'ネ'), + ] + +def _seg_52(): + return [ (0xFF89, 'M', u'ノ'), (0xFF8A, 'M', u'ハ'), (0xFF8B, 'M', u'ヒ'), @@ -5411,10 +5446,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5484,6 +5515,10 @@ def _seg_52(): (0x1000C, 'X'), (0x1000D, 'V'), (0x10027, 'X'), + ] + +def _seg_53(): + return [ (0x10028, 'V'), (0x1003B, 'X'), (0x1003C, 'V'), @@ -5515,10 +5550,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5588,6 +5619,10 @@ def _seg_53(): (0x104BD, 'M', u'𐓥'), (0x104BE, 'M', u'𐓦'), (0x104BF, 'M', u'𐓧'), + ] + +def _seg_54(): + return [ (0x104C0, 'M', u'𐓨'), (0x104C1, 'M', u'𐓩'), (0x104C2, 'M', u'𐓪'), @@ -5619,10 +5654,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5666,11 +5697,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5692,6 +5723,10 @@ def _seg_54(): (0x10BA9, 'V'), (0x10BB0, 'X'), (0x10C00, 'V'), + ] + +def _seg_55(): + return [ (0x10C49, 'X'), (0x10C80, 'M', u'𐳀'), (0x10C81, 'M', u'𐳁'), @@ -5723,10 +5758,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5752,9 +5783,17 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), + (0x10FE0, 'V'), + (0x10FF7, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5809,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5788,6 +5827,10 @@ def _seg_55(): (0x11288, 'V'), (0x11289, 'X'), (0x1128A, 'V'), + ] + +def _seg_56(): + return [ (0x1128E, 'X'), (0x1128F, 'V'), (0x1129E, 'X'), @@ -5811,7 +5854,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5870,12 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + (0x11460, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5852,15 +5891,17 @@ def _seg_56(): (0x11660, 'V'), (0x1166D, 'X'), (0x11680, 'V'), - (0x116B8, 'X'), + (0x116B9, 'X'), (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5890,6 +5931,10 @@ def _seg_56(): (0x118BA, 'M', u'𑣚'), (0x118BB, 'M', u'𑣛'), (0x118BC, 'M', u'𑣜'), + ] + +def _seg_57(): + return [ (0x118BD, 'M', u'𑣝'), (0x118BE, 'M', u'𑣞'), (0x118BF, 'M', u'𑣟'), @@ -5897,13 +5942,15 @@ def _seg_56(): (0x118F3, 'X'), (0x118FF, 'V'), (0x11900, 'X'), + (0x119A0, 'V'), + (0x119A8, 'X'), + (0x119AA, 'V'), + (0x119D8, 'X'), + (0x119DA, 'V'), + (0x119E5, 'X'), (0x11A00, 'V'), (0x11A48, 'X'), (0x11A50, 'V'), - (0x11A84, 'X'), - (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,15 +5978,27 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), - (0x12000, 'V'), + (0x11D60, 'V'), + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), + (0x11FC0, 'V'), + (0x11FF2, 'X'), + (0x11FFF, 'V'), (0x1239A, 'X'), (0x12400, 'V'), (0x1246F, 'X'), @@ -5973,20 +6032,62 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E40, 'M', u'𖹠'), + (0x16E41, 'M', u'𖹡'), + (0x16E42, 'M', u'𖹢'), + ] + +def _seg_58(): + return [ + (0x16E43, 'M', u'𖹣'), + (0x16E44, 'M', u'𖹤'), + (0x16E45, 'M', u'𖹥'), + (0x16E46, 'M', u'𖹦'), + (0x16E47, 'M', u'𖹧'), + (0x16E48, 'M', u'𖹨'), + (0x16E49, 'M', u'𖹩'), + (0x16E4A, 'M', u'𖹪'), + (0x16E4B, 'M', u'𖹫'), + (0x16E4C, 'M', u'𖹬'), + (0x16E4D, 'M', u'𖹭'), + (0x16E4E, 'M', u'𖹮'), + (0x16E4F, 'M', u'𖹯'), + (0x16E50, 'M', u'𖹰'), + (0x16E51, 'M', u'𖹱'), + (0x16E52, 'M', u'𖹲'), + (0x16E53, 'M', u'𖹳'), + (0x16E54, 'M', u'𖹴'), + (0x16E55, 'M', u'𖹵'), + (0x16E56, 'M', u'𖹶'), + (0x16E57, 'M', u'𖹷'), + (0x16E58, 'M', u'𖹸'), + (0x16E59, 'M', u'𖹹'), + (0x16E5A, 'M', u'𖹺'), + (0x16E5B, 'M', u'𖹻'), + (0x16E5C, 'M', u'𖹼'), + (0x16E5D, 'M', u'𖹽'), + (0x16E5E, 'M', u'𖹾'), + (0x16E5F, 'M', u'𖹿'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), - (0x16F45, 'X'), - (0x16F50, 'V'), - (0x16F7F, 'X'), + (0x16F4B, 'X'), + (0x16F4F, 'V'), + (0x16F88, 'X'), (0x16F8F, 'V'), (0x16FA0, 'X'), (0x16FE0, 'V'), - (0x16FE2, 'X'), + (0x16FE4, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F8, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), (0x1B11F, 'X'), + (0x1B150, 'V'), + (0x1B153, 'X'), + (0x1B164, 'V'), + (0x1B168, 'X'), (0x1B170, 'V'), (0x1B2FC, 'X'), (0x1BC00, 'V'), @@ -6025,21 +6126,23 @@ def _seg_57(): (0x1D1E9, 'X'), (0x1D200, 'V'), (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), + (0x1D406, 'M', u'g'), ] -def _seg_58(): +def _seg_59(): return [ - (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), (0x1D409, 'M', u'j'), @@ -6139,11 +6242,11 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), + (0x1D46A, 'M', u'c'), ] -def _seg_59(): +def _seg_60(): return [ - (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), (0x1D46D, 'M', u'f'), @@ -6243,11 +6346,11 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), + (0x1D4D1, 'M', u'b'), ] -def _seg_60(): +def _seg_61(): return [ - (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), (0x1D4D4, 'M', u'e'), @@ -6347,11 +6450,11 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), + (0x1D536, 'M', u'y'), ] -def _seg_61(): +def _seg_62(): return [ - (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), (0x1D539, 'M', u'b'), @@ -6451,11 +6554,11 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), + (0x1D59C, 'M', u'w'), ] -def _seg_62(): +def _seg_63(): return [ - (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), (0x1D59F, 'M', u'z'), @@ -6555,11 +6658,11 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), + (0x1D600, 'M', u's'), ] -def _seg_63(): +def _seg_64(): return [ - (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), (0x1D603, 'M', u'v'), @@ -6659,11 +6762,11 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), + (0x1D664, 'M', u'o'), ] -def _seg_64(): +def _seg_65(): return [ - (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), (0x1D667, 'M', u'r'), @@ -6763,11 +6866,11 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), + (0x1D6C9, 'M', u'θ'), ] -def _seg_65(): +def _seg_66(): return [ - (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), (0x1D6CC, 'M', u'λ'), @@ -6867,11 +6970,11 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), + (0x1D72F, 'M', u'τ'), ] -def _seg_66(): +def _seg_67(): return [ - (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), (0x1D732, 'M', u'χ'), @@ -6971,11 +7074,11 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), + (0x1D795, 'M', u'ζ'), ] -def _seg_67(): +def _seg_68(): return [ - (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), (0x1D798, 'M', u'ι'), @@ -7075,11 +7178,11 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), + (0x1D7FC, 'M', u'6'), ] -def _seg_68(): +def _seg_69(): return [ - (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), (0x1D7FF, 'M', u'9'), @@ -7099,6 +7202,18 @@ def _seg_68(): (0x1E025, 'X'), (0x1E026, 'V'), (0x1E02B, 'X'), + (0x1E100, 'V'), + (0x1E12D, 'X'), + (0x1E130, 'V'), + (0x1E13E, 'X'), + (0x1E140, 'V'), + (0x1E14A, 'X'), + (0x1E14E, 'V'), + (0x1E150, 'X'), + (0x1E2C0, 'V'), + (0x1E2FA, 'X'), + (0x1E2FF, 'V'), + (0x1E300, 'X'), (0x1E800, 'V'), (0x1E8C5, 'X'), (0x1E8C7, 'V'), @@ -7138,11 +7253,15 @@ def _seg_68(): (0x1E920, 'M', u'𞥂'), (0x1E921, 'M', u'𞥃'), (0x1E922, 'V'), - (0x1E94B, 'X'), + (0x1E94C, 'X'), (0x1E950, 'V'), (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), + (0x1ED01, 'V'), + (0x1ED3E, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7283,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_70(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7302,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7387,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_71(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7406,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7491,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_72(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7505,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7422,7 +7541,8 @@ def _seg_71(): (0x1F150, 'V'), (0x1F16A, 'M', u'mc'), (0x1F16B, 'M', u'md'), - (0x1F16C, 'X'), + (0x1F16C, 'M', u'mr'), + (0x1F16D, 'X'), (0x1F170, 'V'), (0x1F190, 'M', u'dj'), (0x1F191, 'V'), @@ -7475,6 +7595,10 @@ def _seg_71(): (0x1F238, 'M', u'申'), (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), + ] + +def _seg_73(): + return [ (0x1F23B, 'M', u'配'), (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), @@ -7491,21 +7615,19 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), - (0x1F6D5, 'X'), + (0x1F6D6, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FB, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), + (0x1F7E0, 'V'), + (0x1F7EC, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7518,18 +7640,28 @@ def _seg_72(): (0x1F8AE, 'X'), (0x1F900, 'V'), (0x1F90C, 'X'), - (0x1F910, 'V'), - (0x1F93F, 'X'), - (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), - (0x1F9C0, 'V'), - (0x1F9C1, 'X'), - (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1F90D, 'V'), + (0x1F972, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F9A3, 'X'), + (0x1F9A5, 'V'), + (0x1F9AB, 'X'), + (0x1F9AE, 'V'), + (0x1F9CB, 'X'), + (0x1F9CD, 'V'), + (0x1FA54, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), + (0x1FA70, 'V'), + (0x1FA74, 'X'), + (0x1FA78, 'V'), + (0x1FA7B, 'X'), + (0x1FA80, 'V'), + (0x1FA83, 'X'), + (0x1FA90, 'V'), + (0x1FA96, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7567,6 +7699,10 @@ def _seg_72(): (0x2F818, 'M', u'冤'), (0x2F819, 'M', u'仌'), (0x2F81A, 'M', u'冬'), + ] + +def _seg_74(): + return [ (0x2F81B, 'M', u'况'), (0x2F81C, 'M', u'𩇟'), (0x2F81D, 'M', u'凵'), @@ -7595,10 +7731,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7671,6 +7803,10 @@ def _seg_73(): (0x2F880, 'M', u'嵼'), (0x2F881, 'M', u'巡'), (0x2F882, 'M', u'巢'), + ] + +def _seg_75(): + return [ (0x2F883, 'M', u'㠯'), (0x2F884, 'M', u'巽'), (0x2F885, 'M', u'帨'), @@ -7699,10 +7835,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7775,6 +7907,10 @@ def _seg_74(): (0x2F8E6, 'M', u'椔'), (0x2F8E7, 'M', u'㮝'), (0x2F8E8, 'M', u'楂'), + ] + +def _seg_76(): + return [ (0x2F8E9, 'M', u'榣'), (0x2F8EA, 'M', u'槪'), (0x2F8EB, 'M', u'檨'), @@ -7803,10 +7939,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7879,6 +8011,10 @@ def _seg_75(): (0x2F94C, 'M', u'䂖'), (0x2F94D, 'M', u'𥐝'), (0x2F94E, 'M', u'硎'), + ] + +def _seg_77(): + return [ (0x2F94F, 'M', u'碌'), (0x2F950, 'M', u'磌'), (0x2F951, 'M', u'䃣'), @@ -7907,10 +8043,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7983,6 +8115,10 @@ def _seg_76(): (0x2F9B1, 'M', u'𧃒'), (0x2F9B2, 'M', u'䕫'), (0x2F9B3, 'M', u'虐'), + ] + +def _seg_78(): + return [ (0x2F9B4, 'M', u'虜'), (0x2F9B5, 'M', u'虧'), (0x2F9B6, 'M', u'虩'), @@ -8011,10 +8147,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8087,6 +8219,10 @@ def _seg_77(): (0x2FA16, 'M', u'䵖'), (0x2FA17, 'M', u'黹'), (0x2FA18, 'M', u'黾'), + ] + +def _seg_79(): + return [ (0x2FA19, 'M', u'鼅'), (0x2FA1A, 'M', u'鼏'), (0x2FA1B, 'M', u'鼖'), @@ -8176,4 +8312,6 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() + + _seg_79() ) diff --git a/pipenv/vendor/importlib_metadata/LICENSE b/pipenv/vendor/importlib_metadata/LICENSE new file mode 100644 index 00000000..be7e092b --- /dev/null +++ b/pipenv/vendor/importlib_metadata/LICENSE @@ -0,0 +1,13 @@ +Copyright 2017-2019 Jason R. Coombs, Barry Warsaw + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/pipenv/vendor/importlib_metadata/__init__.py b/pipenv/vendor/importlib_metadata/__init__.py new file mode 100644 index 00000000..95a08ba0 --- /dev/null +++ b/pipenv/vendor/importlib_metadata/__init__.py @@ -0,0 +1,602 @@ +from __future__ import unicode_literals, absolute_import + +import io +import os +import re +import abc +import csv +import sys +import zipp +import operator +import functools +import itertools +import posixpath +import collections + +from ._compat import ( + install, + NullFinder, + ConfigParser, + suppress, + map, + FileNotFoundError, + IsADirectoryError, + NotADirectoryError, + PermissionError, + pathlib, + ModuleNotFoundError, + MetaPathFinder, + email_message_from_string, + PyPy_repr, + unique_ordered, + ) +from importlib import import_module +from itertools import starmap + + +__metaclass__ = type + + +__all__ = [ + 'Distribution', + 'DistributionFinder', + 'PackageNotFoundError', + 'distribution', + 'distributions', + 'entry_points', + 'files', + 'metadata', + 'requires', + 'version', + ] + + +class PackageNotFoundError(ModuleNotFoundError): + """The package was not found.""" + + +class EntryPoint( + PyPy_repr, + collections.namedtuple('EntryPointBase', 'name value group')): + """An entry point as defined by Python packaging conventions. + + See `the packaging docs on entry points + <https://packaging.python.org/specifications/entry-points/>`_ + for more information. + """ + + pattern = re.compile( + r'(?P<module>[\w.]+)\s*' + r'(:\s*(?P<attr>[\w.]+))?\s*' + r'(?P<extras>\[.*\])?\s*$' + ) + """ + A regular expression describing the syntax for an entry point, + which might look like: + + - module + - package.module + - package.module:attribute + - package.module:object.attribute + - package.module:attr [extra1, extra2] + + Other combinations are possible as well. + + The expression is lenient about whitespace around the ':', + following the attr, and following any extras. + """ + + def load(self): + """Load the entry point from its definition. If only a module + is indicated by the value, return that module. Otherwise, + return the named object. + """ + match = self.pattern.match(self.value) + module = import_module(match.group('module')) + attrs = filter(None, (match.group('attr') or '').split('.')) + return functools.reduce(getattr, attrs, module) + + @property + def module(self): + match = self.pattern.match(self.value) + return match.group('module') + + @property + def attr(self): + match = self.pattern.match(self.value) + return match.group('attr') + + @property + def extras(self): + match = self.pattern.match(self.value) + return list(re.finditer(r'\w+', match.group('extras') or '')) + + @classmethod + def _from_config(cls, config): + return [ + cls(name, value, group) + for group in config.sections() + for name, value in config.items(group) + ] + + @classmethod + def _from_text(cls, text): + config = ConfigParser(delimiters='=') + # case sensitive: https://stackoverflow.com/q/1611799/812183 + config.optionxform = str + try: + config.read_string(text) + except AttributeError: # pragma: nocover + # Python 2 has no read_string + config.readfp(io.StringIO(text)) + return EntryPoint._from_config(config) + + def __iter__(self): + """ + Supply iter so one may construct dicts of EntryPoints easily. + """ + return iter((self.name, self)) + + def __reduce__(self): + return ( + self.__class__, + (self.name, self.value, self.group), + ) + + +class PackagePath(pathlib.PurePosixPath): + """A reference to a path in a package""" + + def read_text(self, encoding='utf-8'): + with self.locate().open(encoding=encoding) as stream: + return stream.read() + + def read_binary(self): + with self.locate().open('rb') as stream: + return stream.read() + + def locate(self): + """Return a path-like object for this path""" + return self.dist.locate_file(self) + + +class FileHash: + def __init__(self, spec): + self.mode, _, self.value = spec.partition('=') + + def __repr__(self): + return '<FileHash mode: {} value: {}>'.format(self.mode, self.value) + + +class Distribution: + """A Python distribution package.""" + + @abc.abstractmethod + def read_text(self, filename): + """Attempt to load metadata file given by the name. + + :param filename: The name of the file in the distribution info. + :return: The text if found, otherwise None. + """ + + @abc.abstractmethod + def locate_file(self, path): + """ + Given a path to a file in this distribution, return a path + to it. + """ + + @classmethod + def from_name(cls, name): + """Return the Distribution for the given package name. + + :param name: The name of the distribution package to search for. + :return: The Distribution instance (or subclass thereof) for the named + package, if found. + :raises PackageNotFoundError: When the named package's distribution + metadata cannot be found. + """ + for resolver in cls._discover_resolvers(): + dists = resolver(DistributionFinder.Context(name=name)) + dist = next(dists, None) + if dist is not None: + return dist + else: + raise PackageNotFoundError(name) + + @classmethod + def discover(cls, **kwargs): + """Return an iterable of Distribution objects for all packages. + + Pass a ``context`` or pass keyword arguments for constructing + a context. + + :context: A ``DistributionFinder.Context`` object. + :return: Iterable of Distribution objects for all packages. + """ + context = kwargs.pop('context', None) + if context and kwargs: + raise ValueError("cannot accept context and kwargs") + context = context or DistributionFinder.Context(**kwargs) + return itertools.chain.from_iterable( + resolver(context) + for resolver in cls._discover_resolvers() + ) + + @staticmethod + def at(path): + """Return a Distribution for the indicated metadata path + + :param path: a string or path-like object + :return: a concrete Distribution instance for the path + """ + return PathDistribution(pathlib.Path(path)) + + @staticmethod + def _discover_resolvers(): + """Search the meta_path for resolvers.""" + declared = ( + getattr(finder, 'find_distributions', None) + for finder in sys.meta_path + ) + return filter(None, declared) + + @property + def metadata(self): + """Return the parsed metadata for this Distribution. + + The returned object will have keys that name the various bits of + metadata. See PEP 566 for details. + """ + text = ( + self.read_text('METADATA') + or self.read_text('PKG-INFO') + # This last clause is here to support old egg-info files. Its + # effect is to just end up using the PathDistribution's self._path + # (which points to the egg-info file) attribute unchanged. + or self.read_text('') + ) + return email_message_from_string(text) + + @property + def version(self): + """Return the 'Version' metadata for the distribution package.""" + return self.metadata['Version'] + + @property + def entry_points(self): + return EntryPoint._from_text(self.read_text('entry_points.txt')) + + @property + def files(self): + """Files in this distribution. + + :return: List of PackagePath for this distribution or None + + Result is `None` if the metadata file that enumerates files + (i.e. RECORD for dist-info or SOURCES.txt for egg-info) is + missing. + Result may be empty if the metadata exists but is empty. + """ + file_lines = self._read_files_distinfo() or self._read_files_egginfo() + + def make_file(name, hash=None, size_str=None): + result = PackagePath(name) + result.hash = FileHash(hash) if hash else None + result.size = int(size_str) if size_str else None + result.dist = self + return result + + return file_lines and list(starmap(make_file, csv.reader(file_lines))) + + def _read_files_distinfo(self): + """ + Read the lines of RECORD + """ + text = self.read_text('RECORD') + return text and text.splitlines() + + def _read_files_egginfo(self): + """ + SOURCES.txt might contain literal commas, so wrap each line + in quotes. + """ + text = self.read_text('SOURCES.txt') + return text and map('"{}"'.format, text.splitlines()) + + @property + def requires(self): + """Generated requirements specified for this Distribution""" + reqs = self._read_dist_info_reqs() or self._read_egg_info_reqs() + return reqs and list(reqs) + + def _read_dist_info_reqs(self): + return self.metadata.get_all('Requires-Dist') + + def _read_egg_info_reqs(self): + source = self.read_text('requires.txt') + return source and self._deps_from_requires_text(source) + + @classmethod + def _deps_from_requires_text(cls, source): + section_pairs = cls._read_sections(source.splitlines()) + sections = { + section: list(map(operator.itemgetter('line'), results)) + for section, results in + itertools.groupby(section_pairs, operator.itemgetter('section')) + } + return cls._convert_egg_info_reqs_to_simple_reqs(sections) + + @staticmethod + def _read_sections(lines): + section = None + for line in filter(None, lines): + section_match = re.match(r'\[(.*)\]$', line) + if section_match: + section = section_match.group(1) + continue + yield locals() + + @staticmethod + def _convert_egg_info_reqs_to_simple_reqs(sections): + """ + Historically, setuptools would solicit and store 'extra' + requirements, including those with environment markers, + in separate sections. More modern tools expect each + dependency to be defined separately, with any relevant + extras and environment markers attached directly to that + requirement. This method converts the former to the + latter. See _test_deps_from_requires_text for an example. + """ + def make_condition(name): + return name and 'extra == "{name}"'.format(name=name) + + def parse_condition(section): + section = section or '' + extra, sep, markers = section.partition(':') + if extra and markers: + markers = '({markers})'.format(markers=markers) + conditions = list(filter(None, [markers, make_condition(extra)])) + return '; ' + ' and '.join(conditions) if conditions else '' + + for section, deps in sections.items(): + for dep in deps: + yield dep + parse_condition(section) + + +class DistributionFinder(MetaPathFinder): + """ + A MetaPathFinder capable of discovering installed distributions. + """ + + class Context: + """ + Keyword arguments presented by the caller to + ``distributions()`` or ``Distribution.discover()`` + to narrow the scope of a search for distributions + in all DistributionFinders. + + Each DistributionFinder may expect any parameters + and should attempt to honor the canonical + parameters defined below when appropriate. + """ + + name = None + """ + Specific name for which a distribution finder should match. + A name of ``None`` matches all distributions. + """ + + def __init__(self, **kwargs): + vars(self).update(kwargs) + + @property + def path(self): + """ + The path that a distribution finder should search. + + Typically refers to Python package paths and defaults + to ``sys.path``. + """ + return vars(self).get('path', sys.path) + + @abc.abstractmethod + def find_distributions(self, context=Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching the ``context``, + a DistributionFinder.Context instance. + """ + + +class FastPath: + """ + Micro-optimized class for searching a path for + children. + """ + + def __init__(self, root): + self.root = root + self.base = os.path.basename(root).lower() + + def joinpath(self, child): + return pathlib.Path(self.root, child) + + def children(self): + with suppress(Exception): + return os.listdir(self.root or '') + with suppress(Exception): + return self.zip_children() + return [] + + def zip_children(self): + zip_path = zipp.Path(self.root) + names = zip_path.root.namelist() + self.joinpath = zip_path.joinpath + + return unique_ordered( + child.split(posixpath.sep, 1)[0] + for child in names + ) + + def is_egg(self, search): + base = self.base + return ( + base == search.versionless_egg_name + or base.startswith(search.prefix) + and base.endswith('.egg')) + + def search(self, name): + for child in self.children(): + n_low = child.lower() + if (n_low in name.exact_matches + or n_low.startswith(name.prefix) + and n_low.endswith(name.suffixes) + # legacy case: + or self.is_egg(name) and n_low == 'egg-info'): + yield self.joinpath(child) + + +class Prepared: + """ + A prepared search for metadata on a possibly-named package. + """ + normalized = '' + prefix = '' + suffixes = '.dist-info', '.egg-info' + exact_matches = [''][:0] + versionless_egg_name = '' + + def __init__(self, name): + self.name = name + if name is None: + return + self.normalized = name.lower().replace('-', '_') + self.prefix = self.normalized + '-' + self.exact_matches = [ + self.normalized + suffix for suffix in self.suffixes] + self.versionless_egg_name = self.normalized + '.egg' + + +@install +class MetadataPathFinder(NullFinder, DistributionFinder): + """A degenerate finder for distribution packages on the file system. + + This finder supplies only a find_distributions() method for versions + of Python that do not have a PathFinder find_distributions(). + """ + + def find_distributions(self, context=DistributionFinder.Context()): + """ + Find distributions. + + Return an iterable of all Distribution instances capable of + loading the metadata for packages matching ``context.name`` + (or all names if ``None`` indicated) along the paths in the list + of directories ``context.path``. + """ + found = self._search_paths(context.name, context.path) + return map(PathDistribution, found) + + @classmethod + def _search_paths(cls, name, paths): + """Find metadata directories in paths heuristically.""" + return itertools.chain.from_iterable( + path.search(Prepared(name)) + for path in map(FastPath, paths) + ) + + +class PathDistribution(Distribution): + def __init__(self, path): + """Construct a distribution from a path to the metadata directory. + + :param path: A pathlib.Path or similar object supporting + .joinpath(), __div__, .parent, and .read_text(). + """ + self._path = path + + def read_text(self, filename): + with suppress(FileNotFoundError, IsADirectoryError, KeyError, + NotADirectoryError, PermissionError): + return self._path.joinpath(filename).read_text(encoding='utf-8') + read_text.__doc__ = Distribution.read_text.__doc__ + + def locate_file(self, path): + return self._path.parent / path + + +def distribution(distribution_name): + """Get the ``Distribution`` instance for the named package. + + :param distribution_name: The name of the distribution package as a string. + :return: A ``Distribution`` instance (or subclass thereof). + """ + return Distribution.from_name(distribution_name) + + +def distributions(**kwargs): + """Get all ``Distribution`` instances in the current environment. + + :return: An iterable of ``Distribution`` instances. + """ + return Distribution.discover(**kwargs) + + +def metadata(distribution_name): + """Get the metadata for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: An email.Message containing the parsed metadata. + """ + return Distribution.from_name(distribution_name).metadata + + +def version(distribution_name): + """Get the version string for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: The version string for the package as defined in the package's + "Version" metadata key. + """ + return distribution(distribution_name).version + + +def entry_points(): + """Return EntryPoint objects for all installed packages. + + :return: EntryPoint objects for all installed packages. + """ + eps = itertools.chain.from_iterable( + dist.entry_points for dist in distributions()) + by_group = operator.attrgetter('group') + ordered = sorted(eps, key=by_group) + grouped = itertools.groupby(ordered, by_group) + return { + group: tuple(eps) + for group, eps in grouped + } + + +def files(distribution_name): + """Return a list of files for the named package. + + :param distribution_name: The name of the distribution package to query. + :return: List of files composing the distribution. + """ + return distribution(distribution_name).files + + +def requires(distribution_name): + """ + Return a list of requirements for the named package. + + :return: An iterator of requirements, suitable for + packaging.requirement.Requirement. + """ + return distribution(distribution_name).requires + + +__version__ = version(__name__) diff --git a/pipenv/vendor/importlib_metadata/_compat.py b/pipenv/vendor/importlib_metadata/_compat.py new file mode 100644 index 00000000..f59b57db --- /dev/null +++ b/pipenv/vendor/importlib_metadata/_compat.py @@ -0,0 +1,150 @@ +from __future__ import absolute_import + +import io +import abc +import sys +import email + + +if sys.version_info > (3,): # pragma: nocover + import builtins + from configparser import ConfigParser + import contextlib + FileNotFoundError = builtins.FileNotFoundError + IsADirectoryError = builtins.IsADirectoryError + NotADirectoryError = builtins.NotADirectoryError + PermissionError = builtins.PermissionError + map = builtins.map + from itertools import filterfalse +else: # pragma: nocover + from backports.configparser import ConfigParser + from itertools import imap as map # type: ignore + from itertools import ifilterfalse as filterfalse + import contextlib2 as contextlib + FileNotFoundError = IOError, OSError + IsADirectoryError = IOError, OSError + NotADirectoryError = IOError, OSError + PermissionError = IOError, OSError + +suppress = contextlib.suppress + +if sys.version_info > (3, 5): # pragma: nocover + import pathlib +else: # pragma: nocover + import pathlib2 as pathlib + +try: + ModuleNotFoundError = builtins.FileNotFoundError +except (NameError, AttributeError): # pragma: nocover + ModuleNotFoundError = ImportError # type: ignore + + +if sys.version_info >= (3,): # pragma: nocover + from importlib.abc import MetaPathFinder +else: # pragma: nocover + class MetaPathFinder(object): + __metaclass__ = abc.ABCMeta + + +__metaclass__ = type +__all__ = [ + 'install', 'NullFinder', 'MetaPathFinder', 'ModuleNotFoundError', + 'pathlib', 'ConfigParser', 'map', 'suppress', 'FileNotFoundError', + 'NotADirectoryError', 'email_message_from_string', + ] + + +def install(cls): + """ + Class decorator for installation on sys.meta_path. + + Adds the backport DistributionFinder to sys.meta_path and + attempts to disable the finder functionality of the stdlib + DistributionFinder. + """ + sys.meta_path.append(cls()) + disable_stdlib_finder() + return cls + + +def disable_stdlib_finder(): + """ + Give the backport primacy for discovering path-based distributions + by monkey-patching the stdlib O_O. + + See #91 for more background for rationale on this sketchy + behavior. + """ + def matches(finder): + return ( + getattr(finder, '__module__', None) == '_frozen_importlib_external' + and hasattr(finder, 'find_distributions') + ) + for finder in filter(matches, sys.meta_path): # pragma: nocover + del finder.find_distributions + + +class NullFinder: + """ + A "Finder" (aka "MetaClassFinder") that never finds any modules, + but may find distributions. + """ + @staticmethod + def find_spec(*args, **kwargs): + return None + + # In Python 2, the import system requires finders + # to have a find_module() method, but this usage + # is deprecated in Python 3 in favor of find_spec(). + # For the purposes of this finder (i.e. being present + # on sys.meta_path but having no other import + # system functionality), the two methods are identical. + find_module = find_spec + + +def py2_message_from_string(text): # nocoverpy3 + # Work around https://bugs.python.org/issue25545 where + # email.message_from_string cannot handle Unicode on Python 2. + io_buffer = io.StringIO(text) + return email.message_from_file(io_buffer) + + +email_message_from_string = ( + py2_message_from_string + if sys.version_info < (3,) else + email.message_from_string + ) + + +class PyPy_repr: + """ + Override repr for EntryPoint objects on PyPy to avoid __iter__ access. + Ref #97, #102. + """ + affected = hasattr(sys, 'pypy_version_info') + + def __compat_repr__(self): # pragma: nocover + def make_param(name): + value = getattr(self, name) + return '{name}={value!r}'.format(**locals()) + params = ', '.join(map(make_param, self._fields)) + return 'EntryPoint({params})'.format(**locals()) + + if affected: # pragma: nocover + __repr__ = __compat_repr__ + del affected + + +# from itertools recipes +def unique_everseen(iterable): # pragma: nocover + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + seen_add = seen.add + + for element in filterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + + +unique_ordered = ( + unique_everseen if sys.version_info < (3, 7) else dict.fromkeys) diff --git a/pipenv/vendor/importlib_metadata/docs/__init__.py b/pipenv/vendor/importlib_metadata/docs/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_metadata/docs/changelog.rst b/pipenv/vendor/importlib_metadata/docs/changelog.rst new file mode 100644 index 00000000..d638e38d --- /dev/null +++ b/pipenv/vendor/importlib_metadata/docs/changelog.rst @@ -0,0 +1,275 @@ +========================= + importlib_metadata NEWS +========================= + +v1.6.0 +====== + +* Added ``module`` and ``attr`` attributes to ``EntryPoint`` + +v1.5.2 +====== + +* Fix redundant entries from ``FastPath.zip_children``. + Closes #117. + +v1.5.1 +====== + +* Improve reliability and consistency of compatibility + imports for contextlib and pathlib when running tests. + Closes #116. + +v1.5.0 +====== + +* Additional performance optimizations in FastPath now + saves an additional 20% on a typical call. +* Correct for issue where PyOxidizer finder has no + ``__module__`` attribute. Closes #110. + +v1.4.0 +====== + +* Through careful optimization, ``distribution()`` is + 3-4x faster. Thanks to Antony Lee for the + contribution. Closes #95. + +* When searching through ``sys.path``, if any error + occurs attempting to list a path entry, that entry + is skipped, making the system much more lenient + to errors. Closes #94. + +v1.3.0 +====== + +* Improve custom finders documentation. Closes #105. + +v1.2.0 +====== + +* Once again, drop support for Python 3.4. Ref #104. + +v1.1.3 +====== + +* Restored support for Python 3.4 due to improper version + compatibility declarations in the v1.1.0 and v1.1.1 + releases. Closes #104. + +v1.1.2 +====== + +* Repaired project metadata to correctly declare the + ``python_requires`` directive. Closes #103. + +v1.1.1 +====== + +* Fixed ``repr(EntryPoint)`` on PyPy 3 also. Closes #102. + +v1.1.0 +====== + +* Dropped support for Python 3.4. +* EntryPoints are now pickleable. Closes #96. +* Fixed ``repr(EntryPoint)`` on PyPy 2. Closes #97. + +v1.0.0 +====== + +* Project adopts semver for versioning. + +* Removed compatibility shim introduced in 0.23. + +* For better compatibility with the stdlib implementation and to + avoid the same distributions being discovered by the stdlib and + backport implementations, the backport now disables the + stdlib DistributionFinder during initialization (import time). + Closes #91 and closes #100. + +0.23 +==== +* Added a compatibility shim to prevent failures on beta releases + of Python before the signature changed to accept the + "context" parameter on find_distributions. This workaround + will have a limited lifespan, not to extend beyond release of + Python 3.8 final. + +0.22 +==== +* Renamed ``package`` parameter to ``distribution_name`` + as `recommended <https://bugs.python.org/issue34632#msg349423>`_ + in the following functions: ``distribution``, ``metadata``, + ``version``, ``files``, and ``requires``. This + backward-incompatible change is expected to have little impact + as these functions are assumed to be primarily used with + positional parameters. + +0.21 +==== +* ``importlib.metadata`` now exposes the ``DistributionFinder`` + metaclass and references it in the docs for extending the + search algorithm. +* Add ``Distribution.at`` for constructing a Distribution object + from a known metadata directory on the file system. Closes #80. +* Distribution finders now receive a context object that + supplies ``.path`` and ``.name`` properties. This change + introduces a fundamental backward incompatibility for + any projects implementing a ``find_distributions`` method + on a ``MetaPathFinder``. This new layer of abstraction + allows this context to be supplied directly or constructed + on demand and opens the opportunity for a + ``find_distributions`` method to solicit additional + context from the caller. Closes #85. + +0.20 +==== +* Clarify in the docs that calls to ``.files`` could return + ``None`` when the metadata is not present. Closes #69. +* Return all requirements and not just the first for dist-info + packages. Closes #67. + +0.19 +==== +* Restrain over-eager egg metadata resolution. +* Add support for entry points with colons in the name. Closes #75. + +0.18 +==== +* Parse entry points case sensitively. Closes #68 +* Add a version constraint on the backport configparser package. Closes #66 + +0.17 +==== +* Fix a permission problem in the tests on Windows. + +0.16 +==== +* Don't crash if there exists an EGG-INFO directory on sys.path. + +0.15 +==== +* Fix documentation. + +0.14 +==== +* Removed ``local_distribution`` function from the API. + **This backward-incompatible change removes this + behavior summarily**. Projects should remove their + reliance on this behavior. A replacement behavior is + under review in the `pep517 project + <https://github.com/pypa/pep517>`_. Closes #42. + +0.13 +==== +* Update docstrings to match PEP 8. Closes #63. +* Merged modules into one module. Closes #62. + +0.12 +==== +* Add support for eggs. !65; Closes #19. + +0.11 +==== +* Support generic zip files (not just wheels). Closes #59 +* Support zip files with multiple distributions in them. Closes #60 +* Fully expose the public API in ``importlib_metadata.__all__``. + +0.10 +==== +* The ``Distribution`` ABC is now officially part of the public API. + Closes #37. +* Fixed support for older single file egg-info formats. Closes #43. +* Fixed a testing bug when ``$CWD`` has spaces in the path. Closes #50. +* Add Python 3.8 to the ``tox`` testing matrix. + +0.9 +=== +* Fixed issue where entry points without an attribute would raise an + Exception. Closes #40. +* Removed unused ``name`` parameter from ``entry_points()``. Closes #44. +* ``DistributionFinder`` classes must now be instantiated before + being placed on ``sys.meta_path``. + +0.8 +=== +* This library can now discover/enumerate all installed packages. **This + backward-incompatible change alters the protocol finders must + implement to support distribution package discovery.** Closes #24. +* The signature of ``find_distributions()`` on custom installer finders + should now accept two parameters, ``name`` and ``path`` and + these parameters must supply defaults. +* The ``entry_points()`` method no longer accepts a package name + but instead returns all entry points in a dictionary keyed by the + ``EntryPoint.group``. The ``resolve`` method has been removed. Instead, + call ``EntryPoint.load()``, which has the same semantics as + ``pkg_resources`` and ``entrypoints``. **This is a backward incompatible + change.** +* Metadata is now always returned as Unicode text regardless of + Python version. Closes #29. +* This library can now discover metadata for a 'local' package (found + in the current-working directory). Closes #27. +* Added ``files()`` function for resolving files from a distribution. +* Added a new ``requires()`` function, which returns the requirements + for a package suitable for parsing by + ``packaging.requirements.Requirement``. Closes #18. +* The top-level ``read_text()`` function has been removed. Use + ``PackagePath.read_text()`` on instances returned by the ``files()`` + function. **This is a backward incompatible change.** +* Release dates are now automatically injected into the changelog + based on SCM tags. + +0.7 +=== +* Fixed issue where packages with dashes in their names would + not be discovered. Closes #21. +* Distribution lookup is now case-insensitive. Closes #20. +* Wheel distributions can no longer be discovered by their module + name. Like Path distributions, they must be indicated by their + distribution package name. + +0.6 +=== +* Removed ``importlib_metadata.distribution`` function. Now + the public interface is primarily the utility functions exposed + in ``importlib_metadata.__all__``. Closes #14. +* Added two new utility functions ``read_text`` and + ``metadata``. + +0.5 +=== +* Updated README and removed details about Distribution + class, now considered private. Closes #15. +* Added test suite support for Python 3.4+. +* Fixed SyntaxErrors on Python 3.4 and 3.5. !12 +* Fixed errors on Windows joining Path elements. !15 + +0.4 +=== +* Housekeeping. + +0.3 +=== +* Added usage documentation. Closes #8 +* Add support for getting metadata from wheels on ``sys.path``. Closes #9 + +0.2 +=== +* Added ``importlib_metadata.entry_points()``. Closes #1 +* Added ``importlib_metadata.resolve()``. Closes #12 +* Add support for Python 2.7. Closes #4 + +0.1 +=== +* Initial release. + + +.. + Local Variables: + mode: change-log-mode + indent-tabs-mode: nil + sentence-end-double-space: t + fill-column: 78 + coding: utf-8 + End: diff --git a/pipenv/vendor/importlib_metadata/docs/conf.py b/pipenv/vendor/importlib_metadata/docs/conf.py new file mode 100644 index 00000000..129a7a4e --- /dev/null +++ b/pipenv/vendor/importlib_metadata/docs/conf.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +# +# importlib_metadata documentation build configuration file, created by +# sphinx-quickstart on Thu Nov 30 10:21:00 2017. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +# import os +# import sys +# sys.path.insert(0, os.path.abspath('.')) + + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'rst.linker', + 'sphinx.ext.autodoc', + 'sphinx.ext.coverage', + 'sphinx.ext.doctest', + 'sphinx.ext.intersphinx', + 'sphinx.ext.viewcode', + ] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = 'importlib_metadata' +copyright = '2017-2019, Jason R. Coombs, Barry Warsaw' +author = 'Jason R. Coombs, Barry Warsaw' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = '0.1' +# The full version, including alpha/beta/rc tags. +release = '0.1' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This patterns also effect to html_static_path and html_extra_path +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'default' + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# This is required for the alabaster theme +# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars +html_sidebars = { + '**': [ + 'relations.html', # needs 'show_related': True theme option to display + 'searchbox.html', + ] + } + + +# -- Options for HTMLHelp output ------------------------------------------ + +# Output file base name for HTML help builder. +htmlhelp_basename = 'importlib_metadatadoc' + + +# -- Options for LaTeX output --------------------------------------------- + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', + } + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'importlib_metadata.tex', + 'importlib\\_metadata Documentation', + 'Brett Cannon, Barry Warsaw', 'manual'), + ] + + +# -- Options for manual page output --------------------------------------- + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'importlib_metadata', 'importlib_metadata Documentation', + [author], 1) + ] + + +# -- Options for Texinfo output ------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'importlib_metadata', 'importlib_metadata Documentation', + author, 'importlib_metadata', 'One line description of project.', + 'Miscellaneous'), + ] + + +# Example configuration for intersphinx: refer to the Python standard library. +intersphinx_mapping = { + 'python': ('https://docs.python.org/3', None), + 'importlib_resources': ( + 'https://importlib-resources.readthedocs.io/en/latest/', None + ), + } + + +# For rst.linker, inject release dates into changelog.rst +link_files = { + 'changelog.rst': dict( + replace=[ + dict( + pattern=r'^(?m)((?P<scm_version>v?\d+(\.\d+){1,2}))\n[-=]+\n', + with_scm='{text}\n{rev[timestamp]:%Y-%m-%d}\n\n', + ), + ], + ), + } diff --git a/pipenv/vendor/importlib_metadata/docs/index.rst b/pipenv/vendor/importlib_metadata/docs/index.rst new file mode 100644 index 00000000..530197cf --- /dev/null +++ b/pipenv/vendor/importlib_metadata/docs/index.rst @@ -0,0 +1,50 @@ +=============================== + Welcome to importlib_metadata +=============================== + +``importlib_metadata`` is a library which provides an API for accessing an +installed package's metadata (see :pep:`566`), such as its entry points or its top-level +name. This functionality intends to replace most uses of ``pkg_resources`` +`entry point API`_ and `metadata API`_. Along with :mod:`importlib.resources` in +Python 3.7 and newer (backported as :doc:`importlib_resources <importlib_resources:index>` for older +versions of Python), this can eliminate the need to use the older and less +efficient ``pkg_resources`` package. + +``importlib_metadata`` is a backport of Python 3.8's standard library +:doc:`importlib.metadata <library/importlib.metadata>` module for Python 2.7, and 3.4 through 3.7. Users of +Python 3.8 and beyond are encouraged to use the standard library module. +When imported on Python 3.8 and later, ``importlib_metadata`` replaces the +DistributionFinder behavior from the stdlib, but leaves the API in tact. +Developers looking for detailed API descriptions should refer to the Python +3.8 standard library documentation. + +The documentation here includes a general :ref:`usage <using>` guide. + + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + using.rst + changelog (links).rst + + +Project details +=============== + + * Project home: https://gitlab.com/python-devs/importlib_metadata + * Report bugs at: https://gitlab.com/python-devs/importlib_metadata/issues + * Code hosting: https://gitlab.com/python-devs/importlib_metadata.git + * Documentation: http://importlib_metadata.readthedocs.io/ + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api diff --git a/pipenv/vendor/importlib_metadata/docs/using.rst b/pipenv/vendor/importlib_metadata/docs/using.rst new file mode 100644 index 00000000..d1ca7658 --- /dev/null +++ b/pipenv/vendor/importlib_metadata/docs/using.rst @@ -0,0 +1,260 @@ +.. _using: + +================================= + Using :mod:`!importlib_metadata` +================================= + +``importlib_metadata`` is a library that provides for access to installed +package metadata. Built in part on Python's import system, this library +intends to replace similar functionality in the `entry point +API`_ and `metadata API`_ of ``pkg_resources``. Along with +:mod:`importlib.resources` in Python 3.7 +and newer (backported as :doc:`importlib_resources <importlib_resources:index>` for older versions of +Python), this can eliminate the need to use the older and less efficient +``pkg_resources`` package. + +By "installed package" we generally mean a third-party package installed into +Python's ``site-packages`` directory via tools such as `pip +<https://pypi.org/project/pip/>`_. Specifically, +it means a package with either a discoverable ``dist-info`` or ``egg-info`` +directory, and metadata defined by :pep:`566` or its older specifications. +By default, package metadata can live on the file system or in zip archives on +:data:`sys.path`. Through an extension mechanism, the metadata can live almost +anywhere. + + +Overview +======== + +Let's say you wanted to get the version string for a package you've installed +using ``pip``. We start by creating a virtual environment and installing +something into it:: + + $ python3 -m venv example + $ source example/bin/activate + (example) $ pip install importlib_metadata + (example) $ pip install wheel + +You can get the version string for ``wheel`` by running the following:: + + (example) $ python + >>> from importlib_metadata import version + >>> version('wheel') + '0.32.3' + +You can also get the set of entry points keyed by group, such as +``console_scripts``, ``distutils.commands`` and others. Each group contains a +sequence of :ref:`EntryPoint <entry-points>` objects. + +You can get the :ref:`metadata for a distribution <metadata>`:: + + >>> list(metadata('wheel')) + ['Metadata-Version', 'Name', 'Version', 'Summary', 'Home-page', 'Author', 'Author-email', 'Maintainer', 'Maintainer-email', 'License', 'Project-URL', 'Project-URL', 'Project-URL', 'Keywords', 'Platform', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Classifier', 'Requires-Python', 'Provides-Extra', 'Requires-Dist', 'Requires-Dist'] + +You can also get a :ref:`distribution's version number <version>`, list its +:ref:`constituent files <files>`, and get a list of the distribution's +:ref:`requirements`. + + +Functional API +============== + +This package provides the following functionality via its public API. + + +.. _entry-points: + +Entry points +------------ + +The ``entry_points()`` function returns a dictionary of all entry points, +keyed by group. Entry points are represented by ``EntryPoint`` instances; +each ``EntryPoint`` has a ``.name``, ``.group``, and ``.value`` attributes and +a ``.load()`` method to resolve the value. There are also ``.module``, +``.attr``, and ``.extras`` attributes for getting the components of the +``.value`` attribute:: + + >>> eps = entry_points() + >>> list(eps) + ['console_scripts', 'distutils.commands', 'distutils.setup_keywords', 'egg_info.writers', 'setuptools.installation'] + >>> scripts = eps['console_scripts'] + >>> wheel = [ep for ep in scripts if ep.name == 'wheel'][0] + >>> wheel + EntryPoint(name='wheel', value='wheel.cli:main', group='console_scripts') + >>> wheel.module + 'wheel.cli' + >>> wheel.attr + 'main' + >>> wheel.extras + [] + >>> main = wheel.load() + >>> main + <function main at 0x103528488> + +The ``group`` and ``name`` are arbitrary values defined by the package author +and usually a client will wish to resolve all entry points for a particular +group. Read `the setuptools docs +<https://setuptools.readthedocs.io/en/latest/setuptools.html#dynamic-discovery-of-services-and-plugins>`_ +for more information on entrypoints, their definition, and usage. + + +.. _metadata: + +Distribution metadata +--------------------- + +Every distribution includes some metadata, which you can extract using the +``metadata()`` function:: + + >>> wheel_metadata = metadata('wheel') + +The keys of the returned data structure [#f1]_ name the metadata keywords, and +their values are returned unparsed from the distribution metadata:: + + >>> wheel_metadata['Requires-Python'] + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + + +.. _version: + +Distribution versions +--------------------- + +The ``version()`` function is the quickest way to get a distribution's version +number, as a string:: + + >>> version('wheel') + '0.32.3' + + +.. _files: + +Distribution files +------------------ + +You can also get the full set of files contained within a distribution. The +``files()`` function takes a distribution package name and returns all of the +files installed by this distribution. Each file object returned is a +``PackagePath``, a :class:`pathlib.Path` derived object with additional ``dist``, +``size``, and ``hash`` properties as indicated by the metadata. For example:: + + >>> util = [p for p in files('wheel') if 'util.py' in str(p)][0] + >>> util + PackagePath('wheel/util.py') + >>> util.size + 859 + >>> util.dist + <importlib_metadata._hooks.PathDistribution object at 0x101e0cef0> + >>> util.hash + <FileHash mode: sha256 value: bYkw5oMccfazVCoYQwKkkemoVyMAFoR34mmKBx8R1NI> + +Once you have the file, you can also read its contents:: + + >>> print(util.read_text()) + import base64 + import sys + ... + def as_bytes(s): + if isinstance(s, text_type): + return s.encode('utf-8') + return s + +In the case where the metadata file listing files +(RECORD or SOURCES.txt) is missing, ``files()`` will +return ``None``. The caller may wish to wrap calls to +``files()`` in `always_iterable +<https://more-itertools.readthedocs.io/en/stable/api.html#more_itertools.always_iterable>`_ +or otherwise guard against this condition if the target +distribution is not known to have the metadata present. + +.. _requirements: + +Distribution requirements +------------------------- + +To get the full set of requirements for a distribution, use the ``requires()`` +function:: + + >>> requires('wheel') + ["pytest (>=3.0.0) ; extra == 'test'", "pytest-cov ; extra == 'test'"] + + +Distributions +============= + +While the above API is the most common and convenient usage, you can get all +of that information from the ``Distribution`` class. A ``Distribution`` is an +abstract object that represents the metadata for a Python package. You can +get the ``Distribution`` instance:: + + >>> from importlib_metadata import distribution + >>> dist = distribution('wheel') + +Thus, an alternative way to get the version number is through the +``Distribution`` instance:: + + >>> dist.version + '0.32.3' + +There are all kinds of additional metadata available on the ``Distribution`` +instance:: + + >>> d.metadata['Requires-Python'] + '>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*' + >>> d.metadata['License'] + 'MIT' + +The full set of available metadata is not described here. See :pep:`566` +for additional details. + + +Extending the search algorithm +============================== + +Because package metadata is not available through :data:`sys.path` searches, or +package loaders directly, the metadata for a package is found through import +system `finders`_. To find a distribution package's metadata, +``importlib.metadata`` queries the list of :term:`meta path finders <meta path finder>` on +:data:`sys.meta_path`. + +By default ``importlib_metadata`` installs a finder for distribution packages +found on the file system. This finder doesn't actually find any *packages*, +but it can find the packages' metadata. + +The abstract class :py:class:`importlib.abc.MetaPathFinder` defines the +interface expected of finders by Python's import system. +``importlib_metadata`` extends this protocol by looking for an optional +``find_distributions`` callable on the finders from +:data:`sys.meta_path` and presents this extended interface as the +``DistributionFinder`` abstract base class, which defines this abstract +method:: + + @abc.abstractmethod + def find_distributions(context=DistributionFinder.Context()): + """Return an iterable of all Distribution instances capable of + loading the metadata for packages for the indicated ``context``. + """ + +The ``DistributionFinder.Context`` object provides ``.path`` and ``.name`` +properties indicating the path to search and names to match and may +supply other relevant context. + +What this means in practice is that to support finding distribution package +metadata in locations other than the file system, subclass +``Distribution`` and implement the abstract methods. Then from +a custom finder, return instances of this derived ``Distribution`` in the +``find_distributions()`` method. + + +.. _`entry point API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#entry-points +.. _`metadata API`: https://setuptools.readthedocs.io/en/latest/pkg_resources.html#metadata-api +.. _`finders`: https://docs.python.org/3/reference/import.html#finders-and-loaders + + +.. rubric:: Footnotes + +.. [#f1] Technically, the returned distribution metadata object is an + :class:`email.message.EmailMessage` + instance, but this is an implementation detail, and not part of the + stable API. You should only use dictionary-like methods and syntax + to access the metadata contents. diff --git a/pipenv/vendor/importlib_metadata/tests/__init__.py b/pipenv/vendor/importlib_metadata/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_metadata/tests/data/__init__.py b/pipenv/vendor/importlib_metadata/tests/data/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_metadata/tests/data/example-21.12-py3-none-any.whl b/pipenv/vendor/importlib_metadata/tests/data/example-21.12-py3-none-any.whl new file mode 100644 index 00000000..641ab07f Binary files /dev/null and b/pipenv/vendor/importlib_metadata/tests/data/example-21.12-py3-none-any.whl differ diff --git a/pipenv/vendor/importlib_metadata/tests/fixtures.py b/pipenv/vendor/importlib_metadata/tests/fixtures.py new file mode 100644 index 00000000..218b699c --- /dev/null +++ b/pipenv/vendor/importlib_metadata/tests/fixtures.py @@ -0,0 +1,210 @@ +from __future__ import unicode_literals + +import os +import sys +import shutil +import tempfile +import textwrap + +from .._compat import pathlib, contextlib + + +__metaclass__ = type + + +@contextlib.contextmanager +def tempdir(): + tmpdir = tempfile.mkdtemp() + try: + yield pathlib.Path(tmpdir) + finally: + shutil.rmtree(tmpdir) + + +@contextlib.contextmanager +def save_cwd(): + orig = os.getcwd() + try: + yield + finally: + os.chdir(orig) + + +@contextlib.contextmanager +def tempdir_as_cwd(): + with tempdir() as tmp: + with save_cwd(): + os.chdir(str(tmp)) + yield tmp + + +@contextlib.contextmanager +def install_finder(finder): + sys.meta_path.append(finder) + try: + yield + finally: + sys.meta_path.remove(finder) + + +class Fixtures: + def setUp(self): + self.fixtures = contextlib.ExitStack() + self.addCleanup(self.fixtures.close) + + +class SiteDir(Fixtures): + def setUp(self): + super(SiteDir, self).setUp() + self.site_dir = self.fixtures.enter_context(tempdir()) + + +class OnSysPath(Fixtures): + @staticmethod + @contextlib.contextmanager + def add_sys_path(dir): + sys.path[:0] = [str(dir)] + try: + yield + finally: + sys.path.remove(str(dir)) + + def setUp(self): + super(OnSysPath, self).setUp() + self.fixtures.enter_context(self.add_sys_path(self.site_dir)) + + +class DistInfoPkg(OnSysPath, SiteDir): + files = { + "distinfo_pkg-1.0.0.dist-info": { + "METADATA": """ + Name: distinfo-pkg + Author: Steven Ma + Version: 1.0.0 + Requires-Dist: wheel >= 1.0 + Requires-Dist: pytest; extra == 'test' + """, + "RECORD": "mod.py,sha256=abc,20\n", + "entry_points.txt": """ + [entries] + main = mod:main + ns:sub = mod:main + """ + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(DistInfoPkg, self).setUp() + build_files(DistInfoPkg.files, self.site_dir) + + +class DistInfoPkgOffPath(SiteDir): + def setUp(self): + super(DistInfoPkgOffPath, self).setUp() + build_files(DistInfoPkg.files, self.site_dir) + + +class EggInfoPkg(OnSysPath, SiteDir): + files = { + "egginfo_pkg.egg-info": { + "PKG-INFO": """ + Name: egginfo-pkg + Author: Steven Ma + License: Unknown + Version: 1.0.0 + Classifier: Intended Audience :: Developers + Classifier: Topic :: Software Development :: Libraries + """, + "SOURCES.txt": """ + mod.py + egginfo_pkg.egg-info/top_level.txt + """, + "entry_points.txt": """ + [entries] + main = mod:main + """, + "requires.txt": """ + wheel >= 1.0; python_version >= "2.7" + [test] + pytest + """, + "top_level.txt": "mod\n" + }, + "mod.py": """ + def main(): + print("hello world") + """, + } + + def setUp(self): + super(EggInfoPkg, self).setUp() + build_files(EggInfoPkg.files, prefix=self.site_dir) + + +class EggInfoFile(OnSysPath, SiteDir): + files = { + "egginfo_file.egg-info": """ + Metadata-Version: 1.0 + Name: egginfo_file + Version: 0.1 + Summary: An example package + Home-page: www.example.com + Author: Eric Haffa-Vee + Author-email: eric@example.coms + License: UNKNOWN + Description: UNKNOWN + Platform: UNKNOWN + """, + } + + def setUp(self): + super(EggInfoFile, self).setUp() + build_files(EggInfoFile.files, prefix=self.site_dir) + + +def build_files(file_defs, prefix=pathlib.Path()): + """Build a set of files/directories, as described by the + + file_defs dictionary. Each key/value pair in the dictionary is + interpreted as a filename/contents pair. If the contents value is a + dictionary, a directory is created, and the dictionary interpreted + as the files within it, recursively. + + For example: + + {"README.txt": "A README file", + "foo": { + "__init__.py": "", + "bar": { + "__init__.py": "", + }, + "baz.py": "# Some code", + } + } + """ + for name, contents in file_defs.items(): + full_name = prefix / name + if isinstance(contents, dict): + full_name.mkdir() + build_files(contents, prefix=full_name) + else: + if isinstance(contents, bytes): + with full_name.open('wb') as f: + f.write(contents) + else: + with full_name.open('w') as f: + f.write(DALS(contents)) + + +def DALS(str): + "Dedent and left-strip" + return textwrap.dedent(str).lstrip() + + +class NullFinder: + def find_module(self, name): + pass diff --git a/pipenv/vendor/importlib_metadata/tests/test_api.py b/pipenv/vendor/importlib_metadata/tests/test_api.py new file mode 100644 index 00000000..aa346ddb --- /dev/null +++ b/pipenv/vendor/importlib_metadata/tests/test_api.py @@ -0,0 +1,176 @@ +import re +import textwrap +import unittest + +from . import fixtures +from .. import ( + Distribution, PackageNotFoundError, __version__, distribution, + entry_points, files, metadata, requires, version, + ) + +try: + from collections.abc import Iterator +except ImportError: + from collections import Iterator # noqa: F401 + +try: + from builtins import str as text +except ImportError: + from __builtin__ import unicode as text + + +class APITests( + fixtures.EggInfoPkg, + fixtures.DistInfoPkg, + fixtures.EggInfoFile, + unittest.TestCase): + + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + pkg_version = version('egginfo-pkg') + assert isinstance(pkg_version, text) + assert re.match(self.version_pattern, pkg_version) + + def test_retrieves_version_of_distinfo_pkg(self): + pkg_version = version('distinfo-pkg') + assert isinstance(pkg_version, text) + assert re.match(self.version_pattern, pkg_version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + distribution('does-not-exist') + + def test_for_top_level(self): + self.assertEqual( + distribution('egginfo-pkg').read_text('top_level.txt').strip(), + 'mod') + + def test_read_text(self): + top_level = [ + path for path in files('egginfo-pkg') + if path.name == 'top_level.txt' + ][0] + self.assertEqual(top_level.read_text(), 'mod\n') + + def test_entry_points(self): + entries = dict(entry_points()['entries']) + ep = entries['main'] + self.assertEqual(ep.value, 'mod:main') + self.assertEqual(ep.extras, []) + + def test_metadata_for_this_package(self): + md = metadata('egginfo-pkg') + assert md['author'] == 'Steven Ma' + assert md['LICENSE'] == 'Unknown' + assert md['Name'] == 'egginfo-pkg' + classifiers = md.get_all('Classifier') + assert 'Topic :: Software Development :: Libraries' in classifiers + + def test_importlib_metadata_version(self): + assert re.match(self.version_pattern, __version__) + + @staticmethod + def _test_files(files): + root = files[0].root + for file in files: + assert file.root == root + assert not file.hash or file.hash.value + assert not file.hash or file.hash.mode == 'sha256' + assert not file.size or file.size >= 0 + assert file.locate().exists() + assert isinstance(file.read_binary(), bytes) + if file.name.endswith('.py'): + file.read_text() + + def test_file_hash_repr(self): + try: + assertRegex = self.assertRegex + except AttributeError: + # Python 2 + assertRegex = self.assertRegexpMatches + + util = [ + p for p in files('distinfo-pkg') + if p.name == 'mod.py' + ][0] + assertRegex( + repr(util.hash), + '<FileHash mode: sha256 value: .*>') + + def test_files_dist_info(self): + self._test_files(files('distinfo-pkg')) + + def test_files_egg_info(self): + self._test_files(files('egginfo-pkg')) + + def test_version_egg_info_file(self): + self.assertEqual(version('egginfo-file'), '0.1') + + def test_requires_egg_info_file(self): + requirements = requires('egginfo-file') + self.assertIsNone(requirements) + + def test_requires_egg_info(self): + deps = requires('egginfo-pkg') + assert len(deps) == 2 + assert any( + dep == 'wheel >= 1.0; python_version >= "2.7"' + for dep in deps + ) + + def test_requires_dist_info(self): + deps = requires('distinfo-pkg') + assert len(deps) == 2 + assert all(deps) + assert 'wheel >= 1.0' in deps + assert "pytest; extra == 'test'" in deps + + def test_more_complex_deps_requires_text(self): + requires = textwrap.dedent(""" + dep1 + dep2 + + [:python_version < "3"] + dep3 + + [extra1] + dep4 + + [extra2:python_version < "3"] + dep5 + """) + deps = sorted(Distribution._deps_from_requires_text(requires)) + expected = [ + 'dep1', + 'dep2', + 'dep3; python_version < "3"', + 'dep4; extra == "extra1"', + 'dep5; (python_version < "3") and extra == "extra2"', + ] + # It's important that the environment marker expression be + # wrapped in parentheses to avoid the following 'and' binding more + # tightly than some other part of the environment expression. + + assert deps == expected + + +class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase): + def test_find_distributions_specified_path(self): + dists = Distribution.discover(path=[str(self.site_dir)]) + assert any( + dist.metadata['Name'] == 'distinfo-pkg' + for dist in dists + ) + + def test_distribution_at_pathlib(self): + """Demonstrate how to load metadata direct from a directory. + """ + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(dist_info_path) + assert dist.version == '1.0.0' + + def test_distribution_at_str(self): + dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info' + dist = Distribution.at(str(dist_info_path)) + assert dist.version == '1.0.0' diff --git a/pipenv/vendor/importlib_metadata/tests/test_integration.py b/pipenv/vendor/importlib_metadata/tests/test_integration.py new file mode 100644 index 00000000..c881927d --- /dev/null +++ b/pipenv/vendor/importlib_metadata/tests/test_integration.py @@ -0,0 +1,42 @@ +import unittest +import packaging.requirements +import packaging.version + +from . import fixtures +from .. import ( + _compat, + version, + ) + + +class IntegrationTests(fixtures.DistInfoPkg, unittest.TestCase): + + def test_package_spec_installed(self): + """ + Illustrate the recommended procedure to determine if + a specified version of a package is installed. + """ + def is_installed(package_spec): + req = packaging.requirements.Requirement(package_spec) + return version(req.name) in req.specifier + + assert is_installed('distinfo-pkg==1.0') + assert is_installed('distinfo-pkg>=1.0,<2.0') + assert not is_installed('distinfo-pkg<1.0') + + +class FinderTests(fixtures.Fixtures, unittest.TestCase): + + def test_finder_without_module(self): + class ModuleFreeFinder(fixtures.NullFinder): + """ + A finder without an __module__ attribute + """ + def __getattribute__(self, name): + if name == '__module__': + raise AttributeError(name) + return super().__getattribute__(name) + + self.fixtures.enter_context( + fixtures.install_finder(ModuleFreeFinder())) + _compat.disable_stdlib_finder() diff --git a/pipenv/vendor/importlib_metadata/tests/test_main.py b/pipenv/vendor/importlib_metadata/tests/test_main.py new file mode 100644 index 00000000..876203f6 --- /dev/null +++ b/pipenv/vendor/importlib_metadata/tests/test_main.py @@ -0,0 +1,258 @@ +# coding: utf-8 +from __future__ import unicode_literals + +import re +import json +import pickle +import textwrap +import unittest +import importlib +import importlib_metadata +import pyfakefs.fake_filesystem_unittest as ffs + +from . import fixtures +from .. import ( + Distribution, EntryPoint, MetadataPathFinder, + PackageNotFoundError, distributions, + entry_points, metadata, version, + ) + +try: + from builtins import str as text +except ImportError: + from __builtin__ import unicode as text + + +class BasicTests(fixtures.DistInfoPkg, unittest.TestCase): + version_pattern = r'\d+\.\d+(\.\d)?' + + def test_retrieves_version_of_self(self): + dist = Distribution.from_name('distinfo-pkg') + assert isinstance(dist.version, text) + assert re.match(self.version_pattern, dist.version) + + def test_for_name_does_not_exist(self): + with self.assertRaises(PackageNotFoundError): + Distribution.from_name('does-not-exist') + + def test_new_style_classes(self): + self.assertIsInstance(Distribution, type) + self.assertIsInstance(MetadataPathFinder, type) + + +class ImportTests(fixtures.DistInfoPkg, unittest.TestCase): + def test_import_nonexistent_module(self): + # Ensure that the MetadataPathFinder does not crash an import of a + # non-existent module. + with self.assertRaises(ImportError): + importlib.import_module('does_not_exist') + + def test_resolve(self): + entries = dict(entry_points()['entries']) + ep = entries['main'] + self.assertEqual(ep.load().__name__, "main") + + def test_entrypoint_with_colon_in_name(self): + entries = dict(entry_points()['entries']) + ep = entries['ns:sub'] + self.assertEqual(ep.value, 'mod:main') + + def test_resolve_without_attr(self): + ep = EntryPoint( + name='ep', + value='importlib_metadata', + group='grp', + ) + assert ep.load() is importlib_metadata + + +class NameNormalizationTests( + fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_dashes(site_dir): + """ + Create minimal metadata for a package with dashes + in the name (and thus underscores in the filename). + """ + metadata_dir = site_dir / 'my_pkg.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w') as strm: + strm.write('Version: 1.0\n') + return 'my-pkg' + + def test_dashes_in_dist_name_found_as_underscores(self): + """ + For a package with a dash in the name, the dist-info metadata + uses underscores in the name. Ensure the metadata loads. + """ + pkg_name = self.pkg_with_dashes(self.site_dir) + assert version(pkg_name) == '1.0' + + @staticmethod + def pkg_with_mixed_case(site_dir): + """ + Create minimal metadata for a package with mixed case + in the name. + """ + metadata_dir = site_dir / 'CherryPy.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w') as strm: + strm.write('Version: 1.0\n') + return 'CherryPy' + + def test_dist_name_found_as_any_case(self): + """ + Ensure the metadata loads when queried with any case. + """ + pkg_name = self.pkg_with_mixed_case(self.site_dir) + assert version(pkg_name) == '1.0' + assert version(pkg_name.lower()) == '1.0' + assert version(pkg_name.upper()) == '1.0' + + +class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + @staticmethod + def pkg_with_non_ascii_description(site_dir): + """ + Create minimal metadata for a package with non-ASCII in + the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write('Description: pôrˈtend\n') + return 'portend' + + @staticmethod + def pkg_with_non_ascii_description_egg_info(site_dir): + """ + Create minimal metadata for an egg-info package with + non-ASCII in the description. + """ + metadata_dir = site_dir / 'portend.dist-info' + metadata_dir.mkdir() + metadata = metadata_dir / 'METADATA' + with metadata.open('w', encoding='utf-8') as fp: + fp.write(textwrap.dedent(""" + Name: portend + + pôrˈtend + """).lstrip()) + return 'portend' + + def test_metadata_loads(self): + pkg_name = self.pkg_with_non_ascii_description(self.site_dir) + meta = metadata(pkg_name) + assert meta['Description'] == 'pôrˈtend' + + def test_metadata_loads_egg_info(self): + pkg_name = self.pkg_with_non_ascii_description_egg_info(self.site_dir) + meta = metadata(pkg_name) + assert meta.get_payload() == 'pôrˈtend\n' + + +class DiscoveryTests(fixtures.EggInfoPkg, + fixtures.DistInfoPkg, + unittest.TestCase): + + def test_package_discovery(self): + dists = list(distributions()) + assert all( + isinstance(dist, Distribution) + for dist in dists + ) + assert any( + dist.metadata['Name'] == 'egginfo-pkg' + for dist in dists + ) + assert any( + dist.metadata['Name'] == 'distinfo-pkg' + for dist in dists + ) + + def test_invalid_usage(self): + with self.assertRaises(ValueError): + list(distributions(context='something', name='else')) + + +class DirectoryTest(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase): + def test_egg_info(self): + # make an `EGG-INFO` directory that's unrelated + self.site_dir.joinpath('EGG-INFO').mkdir() + # used to crash with `IsADirectoryError` + with self.assertRaises(PackageNotFoundError): + version('unknown-package') + + def test_egg(self): + egg = self.site_dir.joinpath('foo-3.6.egg') + egg.mkdir() + with self.add_sys_path(egg): + with self.assertRaises(PackageNotFoundError): + version('foo') + + +class MissingSysPath(fixtures.OnSysPath, unittest.TestCase): + site_dir = '/does-not-exist' + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + importlib_metadata.distributions() + + +class InaccessibleSysPath(fixtures.OnSysPath, ffs.TestCase): + site_dir = '/access-denied' + + def setUp(self): + super(InaccessibleSysPath, self).setUp() + self.setUpPyfakefs() + self.fs.create_dir(self.site_dir, perm_bits=000) + + def test_discovery(self): + """ + Discovering distributions should succeed even if + there is an invalid path on sys.path. + """ + list(importlib_metadata.distributions()) + + +class TestEntryPoints(unittest.TestCase): + def __init__(self, *args): + super(TestEntryPoints, self).__init__(*args) + self.ep = importlib_metadata.EntryPoint('name', 'value', 'group') + + def test_entry_point_pickleable(self): + revived = pickle.loads(pickle.dumps(self.ep)) + assert revived == self.ep + + def test_immutable(self): + """EntryPoints should be immutable""" + with self.assertRaises(AttributeError): + self.ep.name = 'badactor' + + def test_repr(self): + assert 'EntryPoint' in repr(self.ep) + assert 'name=' in repr(self.ep) + assert "'name'" in repr(self.ep) + + def test_hashable(self): + """EntryPoints should be hashable""" + hash(self.ep) + + def test_json_dump(self): + """ + json should not expect to be able to dump an EntryPoint + """ + with self.assertRaises(Exception): + json.dumps(self.ep) + + def test_module(self): + assert self.ep.module == 'value' + + def test_attr(self): + assert self.ep.attr is None diff --git a/pipenv/vendor/importlib_metadata/tests/test_zip.py b/pipenv/vendor/importlib_metadata/tests/test_zip.py new file mode 100644 index 00000000..515f593d --- /dev/null +++ b/pipenv/vendor/importlib_metadata/tests/test_zip.py @@ -0,0 +1,77 @@ +import sys +import unittest + +from .. import ( + distribution, entry_points, files, PackageNotFoundError, + version, distributions, + ) + +try: + from importlib.resources import path +except ImportError: + from importlib_resources import path + +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + + +class TestZip(unittest.TestCase): + root = 'importlib_metadata.tests.data' + + def setUp(self): + # Find the path to the example-*.whl so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + wheel = self.resources.enter_context( + path(self.root, 'example-21.12-py3-none-any.whl')) + sys.path.insert(0, str(wheel)) + self.resources.callback(sys.path.pop, 0) + + def test_zip_version(self): + self.assertEqual(version('example'), '21.12') + + def test_zip_version_does_not_match(self): + with self.assertRaises(PackageNotFoundError): + version('definitely-not-installed') + + def test_zip_entry_points(self): + scripts = dict(entry_points()['console_scripts']) + entry_point = scripts['example'] + self.assertEqual(entry_point.value, 'example:main') + entry_point = scripts['Example'] + self.assertEqual(entry_point.value, 'example:main') + + def test_missing_metadata(self): + self.assertIsNone(distribution('example').read_text('does not exist')) + + def test_case_insensitive(self): + self.assertEqual(version('Example'), '21.12') + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.whl/' in path, path + + def test_one_distribution(self): + dists = list(distributions(path=sys.path[:1])) + assert len(dists) == 1 + + +class TestEgg(TestZip): + def setUp(self): + # Find the path to the example-*.egg so we can add it to the front of + # sys.path, where we'll then try to find the metadata thereof. + self.resources = ExitStack() + self.addCleanup(self.resources.close) + egg = self.resources.enter_context( + path(self.root, 'example-21.12-py3.6.egg')) + sys.path.insert(0, str(egg)) + self.resources.callback(sys.path.pop, 0) + + def test_files(self): + for file in files('example'): + path = str(file.dist.locate_file(file)) + assert '.egg/' in path, path diff --git a/pipenv/vendor/importlib_resources/LICENSE b/pipenv/vendor/importlib_resources/LICENSE new file mode 100644 index 00000000..378b991a --- /dev/null +++ b/pipenv/vendor/importlib_resources/LICENSE @@ -0,0 +1,13 @@ +Copyright 2017-2019 Brett Cannon, Barry Warsaw + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/pipenv/vendor/importlib_resources/__init__.py b/pipenv/vendor/importlib_resources/__init__.py new file mode 100644 index 00000000..4bce94b0 --- /dev/null +++ b/pipenv/vendor/importlib_resources/__init__.py @@ -0,0 +1,56 @@ +"""Read resources contained within a package.""" + +import sys + +from ._compat import metadata +from ._common import as_file + +# for compatibility. Ref #88 +__import__('importlib_resources.trees') + + +__all__ = [ + 'Package', + 'Resource', + 'ResourceReader', + 'as_file', + 'contents', + 'files', + 'is_resource', + 'open_binary', + 'open_text', + 'path', + 'read_binary', + 'read_text', + ] + + +if sys.version_info >= (3,): + from importlib_resources._py3 import ( + Package, + Resource, + contents, + files, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, + ) + from importlib_resources.abc import ResourceReader +else: + from importlib_resources._py2 import ( + contents, + files, + is_resource, + open_binary, + open_text, + path, + read_binary, + read_text, + ) + del __all__[:3] + + +__version__ = metadata.version('importlib_resources') diff --git a/pipenv/vendor/importlib_resources/_common.py b/pipenv/vendor/importlib_resources/_common.py new file mode 100644 index 00000000..3a5b7e44 --- /dev/null +++ b/pipenv/vendor/importlib_resources/_common.py @@ -0,0 +1,76 @@ +from __future__ import absolute_import + +import os +import tempfile +import contextlib + +from ._compat import ( + Path, package_spec, FileNotFoundError, ZipPath, + singledispatch, suppress, + ) + + +def from_package(package): + """ + Return a Traversable object for the given package. + + """ + spec = package_spec(package) + return from_traversable_resources(spec) or fallback_resources(spec) + + +def from_traversable_resources(spec): + """ + If the spec.loader implements TraversableResources, + directly or implicitly, it will have a ``files()`` method. + """ + with suppress(AttributeError): + return spec.loader.files() + + +def fallback_resources(spec): + package_directory = Path(spec.origin).parent + try: + archive_path = spec.loader.archive + rel_path = package_directory.relative_to(archive_path) + return ZipPath(archive_path, str(rel_path) + '/') + except Exception: + pass + return package_directory + + +@contextlib.contextmanager +def _tempfile(reader, suffix=''): + # Not using tempfile.NamedTemporaryFile as it leads to deeper 'try' + # blocks due to the need to close the temporary file to work on Windows + # properly. + fd, raw_path = tempfile.mkstemp(suffix=suffix) + try: + os.write(fd, reader()) + os.close(fd) + yield Path(raw_path) + finally: + try: + os.remove(raw_path) + except FileNotFoundError: + pass + + +@singledispatch +@contextlib.contextmanager +def as_file(path): + """ + Given a Traversable object, return that object as a + path on the local file system in a context manager. + """ + with _tempfile(path.read_bytes, suffix=path.name) as local: + yield local + + +@as_file.register(Path) +@contextlib.contextmanager +def _(path): + """ + Degenerate behavior for pathlib.Path objects. + """ + yield path diff --git a/pipenv/vendor/importlib_resources/_compat.py b/pipenv/vendor/importlib_resources/_compat.py new file mode 100644 index 00000000..48ca6afd --- /dev/null +++ b/pipenv/vendor/importlib_resources/_compat.py @@ -0,0 +1,60 @@ +from __future__ import absolute_import + +# flake8: noqa + +try: + from pathlib import Path, PurePath +except ImportError: + from pathlib2 import Path, PurePath # type: ignore + + +try: + from contextlib import suppress +except ImportError: + from contextlib2 import suppress # type: ignore + + +try: + from functools import singledispatch +except ImportError: + from singledispatch import singledispatch # type: ignore + + +try: + from abc import ABC # type: ignore +except ImportError: + from abc import ABCMeta + + class ABC(object): # type: ignore + __metaclass__ = ABCMeta + + +try: + FileNotFoundError = FileNotFoundError # type: ignore +except NameError: + FileNotFoundError = OSError # type: ignore + + +try: + from importlib import metadata +except ImportError: + import importlib_metadata as metadata # type: ignore + + +try: + from zipfile import Path as ZipPath # type: ignore +except ImportError: + from zipp import Path as ZipPath # type: ignore + + +class PackageSpec(object): + def __init__(self, **kwargs): + vars(self).update(kwargs) + + +def package_spec(package): + return getattr(package, '__spec__', None) or \ + PackageSpec( + origin=package.__file__, + loader=getattr(package, '__loader__', None), + ) diff --git a/pipenv/vendor/importlib_resources/_py2.py b/pipenv/vendor/importlib_resources/_py2.py new file mode 100644 index 00000000..26ce45d2 --- /dev/null +++ b/pipenv/vendor/importlib_resources/_py2.py @@ -0,0 +1,142 @@ +import os +import errno + +from . import _common +from ._compat import FileNotFoundError +from importlib import import_module +from io import BytesIO, TextIOWrapper, open as io_open + + +def _resolve(name): + """If name is a string, resolve to a module.""" + if not isinstance(name, basestring): # noqa: F821 + return name + return import_module(name) + + +def _get_package(package): + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + module = _resolve(package) + if not hasattr(module, '__path__'): + raise TypeError("{!r} is not a package".format(package)) + return module + + +def _normalize_path(path): + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError("{!r} must be only a file name".format(path)) + return file_name + + +def open_binary(package, resource): + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + # Using pathlib doesn't work well here due to the lack of 'strict' argument + # for pathlib.Path.resolve() prior to Python 3.6. + package_path = os.path.dirname(package.__file__) + relative_path = os.path.join(package_path, resource) + full_path = os.path.abspath(relative_path) + try: + return io_open(full_path, 'rb') + except IOError: + # This might be a package in a zip file. zipimport provides a loader + # with a functioning get_data() method, however we have to strip the + # archive (i.e. the .zip file's name) off the front of the path. This + # is because the zipimport loader in Python 2 doesn't actually follow + # PEP 302. It should allow the full path, but actually requires that + # the path be relative to the zip file. + try: + loader = package.__loader__ + full_path = relative_path[len(loader.archive)+1:] + data = loader.get_data(full_path) + except (IOError, AttributeError): + package_name = package.__name__ + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + return BytesIO(data) + + +def open_text(package, resource, encoding='utf-8', errors='strict'): + """Return a file-like object opened for text reading of the resource.""" + return TextIOWrapper( + open_binary(package, resource), encoding=encoding, errors=errors) + + +def read_binary(package, resource): + """Return the binary contents of the resource.""" + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package, resource, encoding='utf-8', errors='strict'): + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +def files(package): + return _common.from_package(_get_package(package)) + + +def path(package, resource): + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + path = files(package).joinpath(_normalize_path(resource)) + if not path.is_file(): + raise FileNotFoundError(path) + return _common.as_file(path) + + +def is_resource(package, name): + """True if name is a resource inside package. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + try: + package_contents = set(contents(package)) + except OSError as error: + if error.errno not in (errno.ENOENT, errno.ENOTDIR): + # We won't hit this in the Python 2 tests, so it'll appear + # uncovered. We could mock os.listdir() to return a non-ENOENT or + # ENOTDIR, but then we'd have to depend on another external + # library since Python 2 doesn't have unittest.mock. It's not + # worth it. + raise # pragma: nocover + return False + if name not in package_contents: + return False + return (_common.from_package(package) / name).is_file() + + +def contents(package): + """Return an iterable of entries in `package`. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + return list(item.name for item in _common.from_package(package).iterdir()) diff --git a/pipenv/vendor/importlib_resources/_py3.py b/pipenv/vendor/importlib_resources/_py3.py new file mode 100644 index 00000000..8dedde4c --- /dev/null +++ b/pipenv/vendor/importlib_resources/_py3.py @@ -0,0 +1,203 @@ +import os +import sys + +from . import abc as resources_abc +from . import _common +from contextlib import contextmanager, suppress +from importlib import import_module +from importlib.abc import ResourceLoader +from io import BytesIO, TextIOWrapper +from pathlib import Path +from types import ModuleType +from typing import Iterable, Iterator, Optional, Set, Union # noqa: F401 +from typing import cast +from typing.io import BinaryIO, TextIO + +if False: # TYPE_CHECKING + from typing import ContextManager + +Package = Union[ModuleType, str] +if sys.version_info >= (3, 6): + Resource = Union[str, os.PathLike] # pragma: <=35 +else: + Resource = str # pragma: >=36 + + +def _resolve(name) -> ModuleType: + """If name is a string, resolve to a module.""" + if hasattr(name, '__spec__'): + return name + return import_module(name) + + +def _get_package(package) -> ModuleType: + """Take a package name or module object and return the module. + + If a name, the module is imported. If the resolved module + object is not a package, raise an exception. + """ + module = _resolve(package) + if module.__spec__.submodule_search_locations is None: + raise TypeError('{!r} is not a package'.format(package)) + return module + + +def _normalize_path(path) -> str: + """Normalize a path by ensuring it is a string. + + If the resulting string contains path separators, an exception is raised. + """ + str_path = str(path) + parent, file_name = os.path.split(str_path) + if parent: + raise ValueError('{!r} must be only a file name'.format(path)) + return file_name + + +def _get_resource_reader( + package: ModuleType) -> Optional[resources_abc.ResourceReader]: + # Return the package's loader if it's a ResourceReader. We can't use + # a issubclass() check here because apparently abc.'s __subclasscheck__() + # hook wants to create a weak reference to the object, but + # zipimport.zipimporter does not support weak references, resulting in a + # TypeError. That seems terrible. + spec = package.__spec__ + reader = getattr(spec.loader, 'get_resource_reader', None) + if reader is None: + return None + return cast(resources_abc.ResourceReader, reader(spec.name)) + + +def open_binary(package: Package, resource: Resource) -> BinaryIO: + """Return a file-like object opened for binary reading of the resource.""" + resource = _normalize_path(resource) + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.open_resource(resource) + # Using pathlib doesn't work well here due to the lack of 'strict' + # argument for pathlib.Path.resolve() prior to Python 3.6. + absolute_package_path = os.path.abspath( + package.__spec__.origin or 'non-existent file') + package_path = os.path.dirname(absolute_package_path) + full_path = os.path.join(package_path, resource) + try: + return open(full_path, mode='rb') + except OSError: + # Just assume the loader is a resource loader; all the relevant + # importlib.machinery loaders are and an AttributeError for + # get_data() will make it clear what is needed from the loader. + loader = cast(ResourceLoader, package.__spec__.loader) + data = None + if hasattr(package.__spec__.loader, 'get_data'): + with suppress(OSError): + data = loader.get_data(full_path) + if data is None: + package_name = package.__spec__.name + message = '{!r} resource not found in {!r}'.format( + resource, package_name) + raise FileNotFoundError(message) + return BytesIO(data) + + +def open_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> TextIO: + """Return a file-like object opened for text reading of the resource.""" + return TextIOWrapper( + open_binary(package, resource), encoding=encoding, errors=errors) + + +def read_binary(package: Package, resource: Resource) -> bytes: + """Return the binary contents of the resource.""" + with open_binary(package, resource) as fp: + return fp.read() + + +def read_text(package: Package, + resource: Resource, + encoding: str = 'utf-8', + errors: str = 'strict') -> str: + """Return the decoded string of the resource. + + The decoding-related arguments have the same semantics as those of + bytes.decode(). + """ + with open_text(package, resource, encoding, errors) as fp: + return fp.read() + + +def files(package: Package) -> resources_abc.Traversable: + """ + Get a Traversable resource from a package + """ + return _common.from_package(_get_package(package)) + + +def path( + package: Package, resource: Resource, + ) -> 'ContextManager[Path]': + """A context manager providing a file path object to the resource. + + If the resource does not already exist on its own on the file system, + a temporary file will be created. If the file was created, the file + will be deleted upon exiting the context manager (no exception is + raised if the file was deleted prior to the context manager + exiting). + """ + reader = _get_resource_reader(_get_package(package)) + return ( + _path_from_reader(reader, resource) + if reader else + _common.as_file(files(package).joinpath(_normalize_path(resource))) + ) + + +@contextmanager +def _path_from_reader(reader, resource): + norm_resource = _normalize_path(resource) + with suppress(FileNotFoundError): + yield Path(reader.resource_path(norm_resource)) + return + opener_reader = reader.open_resource(norm_resource) + with _common._tempfile(opener_reader.read, suffix=norm_resource) as res: + yield res + + +def is_resource(package: Package, name: str) -> bool: + """True if `name` is a resource inside `package`. + + Directories are *not* resources. + """ + package = _get_package(package) + _normalize_path(name) + reader = _get_resource_reader(package) + if reader is not None: + return reader.is_resource(name) + package_contents = set(contents(package)) + if name not in package_contents: + return False + return (_common.from_package(package) / name).is_file() + + +def contents(package: Package) -> Iterable[str]: + """Return an iterable of entries in `package`. + + Note that not all entries are resources. Specifically, directories are + not considered resources. Use `is_resource()` on each entry returned here + to check if it is a resource or not. + """ + package = _get_package(package) + reader = _get_resource_reader(package) + if reader is not None: + return reader.contents() + # Is the package a namespace package? By definition, namespace packages + # cannot have resources. + namespace = ( + package.__spec__.origin is None or + package.__spec__.origin == 'namespace' + ) + if namespace or not package.__spec__.has_location: + return () + return list(item.name for item in _common.from_package(package).iterdir()) diff --git a/pipenv/vendor/importlib_resources/abc.py b/pipenv/vendor/importlib_resources/abc.py new file mode 100644 index 00000000..28596a4a --- /dev/null +++ b/pipenv/vendor/importlib_resources/abc.py @@ -0,0 +1,134 @@ +from __future__ import absolute_import + +import abc + +from ._compat import ABC, FileNotFoundError + +# Use mypy's comment syntax for Python 2 compatibility +try: + from typing import BinaryIO, Iterable, Text +except ImportError: + pass + + +class ResourceReader(ABC): + """Abstract base class for loaders to provide resource reading support.""" + + @abc.abstractmethod + def open_resource(self, resource): + # type: (Text) -> BinaryIO + """Return an opened, file-like object for binary reading. + + The 'resource' argument is expected to represent only a file name. + If the resource cannot be found, FileNotFoundError is raised. + """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. + raise FileNotFoundError + + @abc.abstractmethod + def resource_path(self, resource): + # type: (Text) -> Text + """Return the file system path to the specified resource. + + The 'resource' argument is expected to represent only a file name. + If the resource does not exist on the file system, raise + FileNotFoundError. + """ + # This deliberately raises FileNotFoundError instead of + # NotImplementedError so that if this method is accidentally called, + # it'll still do the right thing. + raise FileNotFoundError + + @abc.abstractmethod + def is_resource(self, path): + # type: (Text) -> bool + """Return True if the named 'path' is a resource. + + Files are resources, directories are not. + """ + raise FileNotFoundError + + @abc.abstractmethod + def contents(self): + # type: () -> Iterable[str] + """Return an iterable of entries in `package`.""" + raise FileNotFoundError + + +class Traversable(ABC): + """ + An object with a subset of pathlib.Path methods suitable for + traversing directories and opening files. + """ + + @abc.abstractmethod + def iterdir(self): + """ + Yield Traversable objects in self + """ + + @abc.abstractmethod + def read_bytes(self): + """ + Read contents of self as bytes + """ + + @abc.abstractmethod + def read_text(self, encoding=None): + """ + Read contents of self as bytes + """ + + @abc.abstractmethod + def is_dir(self): + """ + Return True if self is a dir + """ + + @abc.abstractmethod + def is_file(self): + """ + Return True if self is a file + """ + + @abc.abstractmethod + def joinpath(self, child): + """ + Return Traversable child in self + """ + + @abc.abstractmethod + def __truediv__(self, child): + """ + Return Traversable child in self + """ + + @abc.abstractmethod + def open(self, mode='r', *args, **kwargs): + """ + mode may be 'r' or 'rb' to open as text or binary. Return a handle + suitable for reading (same as pathlib.Path.open). + + When opening as text, accepts encoding parameters such as those + accepted by io.TextIOWrapper. + """ + + +class TraversableResources(ResourceReader): + @abc.abstractmethod + def files(self): + """Return a Traversable object for the loaded package.""" + + def open_resource(self, resource): + return self.files().joinpath(resource).open('rb') + + def resource_path(self, resource): + raise FileNotFoundError(resource) + + def is_resource(self, path): + return self.files().joinpath(path).isfile() + + def contents(self): + return (item.name for item in self.files().iterdir()) diff --git a/pipenv/vendor/importlib_resources/tests/__init__.py b/pipenv/vendor/importlib_resources/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data01/__init__.py b/pipenv/vendor/importlib_resources/tests/data01/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data01/binary.file b/pipenv/vendor/importlib_resources/tests/data01/binary.file new file mode 100644 index 00000000..eaf36c1d Binary files /dev/null and b/pipenv/vendor/importlib_resources/tests/data01/binary.file differ diff --git a/pipenv/vendor/importlib_resources/tests/data01/subdirectory/__init__.py b/pipenv/vendor/importlib_resources/tests/data01/subdirectory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data01/subdirectory/binary.file b/pipenv/vendor/importlib_resources/tests/data01/subdirectory/binary.file new file mode 100644 index 00000000..eaf36c1d Binary files /dev/null and b/pipenv/vendor/importlib_resources/tests/data01/subdirectory/binary.file differ diff --git a/pipenv/vendor/importlib_resources/tests/data01/utf-16.file b/pipenv/vendor/importlib_resources/tests/data01/utf-16.file new file mode 100644 index 00000000..2cb77229 Binary files /dev/null and b/pipenv/vendor/importlib_resources/tests/data01/utf-16.file differ diff --git a/pipenv/vendor/importlib_resources/tests/data01/utf-8.file b/pipenv/vendor/importlib_resources/tests/data01/utf-8.file new file mode 100644 index 00000000..1c0132ad --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/data01/utf-8.file @@ -0,0 +1 @@ +Hello, UTF-8 world! diff --git a/pipenv/vendor/importlib_resources/tests/data02/__init__.py b/pipenv/vendor/importlib_resources/tests/data02/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data02/one/__init__.py b/pipenv/vendor/importlib_resources/tests/data02/one/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data02/one/resource1.txt b/pipenv/vendor/importlib_resources/tests/data02/one/resource1.txt new file mode 100644 index 00000000..61a813e4 --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/data02/one/resource1.txt @@ -0,0 +1 @@ +one resource diff --git a/pipenv/vendor/importlib_resources/tests/data02/two/__init__.py b/pipenv/vendor/importlib_resources/tests/data02/two/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data02/two/resource2.txt b/pipenv/vendor/importlib_resources/tests/data02/two/resource2.txt new file mode 100644 index 00000000..a80ce46e --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/data02/two/resource2.txt @@ -0,0 +1 @@ +two resource diff --git a/pipenv/vendor/importlib_resources/tests/data03/__init__.py b/pipenv/vendor/importlib_resources/tests/data03/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/data03/namespace/resource1.txt b/pipenv/vendor/importlib_resources/tests/data03/namespace/resource1.txt new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/test_open.py b/pipenv/vendor/importlib_resources/tests/test_open.py new file mode 100644 index 00000000..8a3429f2 --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/test_open.py @@ -0,0 +1,73 @@ +import unittest + +import importlib_resources as resources +from . import data01 +from . import util +from .._compat import FileNotFoundError + + +class CommonBinaryTests(util.CommonTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_binary(package, path): + pass + + +class CommonTextTests(util.CommonTests, unittest.TestCase): + def execute(self, package, path): + with resources.open_text(package, path): + pass + + +class OpenTests: + def test_open_binary(self): + with resources.open_binary(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, b'Hello, UTF-8 world!\n') + + def test_open_text_default_encoding(self): + with resources.open_text(self.data, 'utf-8.file') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_open_text_given_encoding(self): + with resources.open_text( + self.data, 'utf-16.file', 'utf-16', 'strict') as fp: + result = fp.read() + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_open_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'strict') as fp: + self.assertRaises(UnicodeError, fp.read) + with resources.open_text( + self.data, 'utf-16.file', 'utf-8', 'ignore') as fp: + result = fp.read() + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + def test_open_binary_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_binary, self.data, 'does-not-exist') + + def test_open_text_FileNotFoundError(self): + self.assertRaises( + FileNotFoundError, + resources.open_text, self.data, 'does-not-exist') + + +class OpenDiskTests(OpenTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class OpenZipTests(OpenTests, util.ZipSetup, unittest.TestCase): + pass + + +if __name__ == '__main__': + unittest.main() diff --git a/pipenv/vendor/importlib_resources/tests/test_path.py b/pipenv/vendor/importlib_resources/tests/test_path.py new file mode 100644 index 00000000..f6004756 --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/test_path.py @@ -0,0 +1,42 @@ +import unittest + +import importlib_resources as resources +from . import data01 +from . import util + + +class CommonTests(util.CommonTests, unittest.TestCase): + + def execute(self, package, path): + with resources.path(package, path): + pass + + +class PathTests: + + def test_reading(self): + # Path should be readable. + # Test also implicitly verifies the returned object is a pathlib.Path + # instance. + with resources.path(self.data, 'utf-8.file') as path: + self.assertTrue(path.name.endswith("utf-8.file"), repr(path)) + # pathlib.Path.read_text() was introduced in Python 3.5. + with path.open('r', encoding='utf-8') as file: + text = file.read() + self.assertEqual('Hello, UTF-8 world!\n', text) + + +class PathDiskTests(PathTests, unittest.TestCase): + data = data01 + + +class PathZipTests(PathTests, util.ZipSetup, unittest.TestCase): + def test_remove_in_context_manager(self): + # It is not an error if the file that was temporarily stashed on the + # file system is removed inside the `with` stanza. + with resources.path(self.data, 'utf-8.file') as path: + path.unlink() + + +if __name__ == '__main__': + unittest.main() diff --git a/pipenv/vendor/importlib_resources/tests/test_read.py b/pipenv/vendor/importlib_resources/tests/test_read.py new file mode 100644 index 00000000..ee94d8ad --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/test_read.py @@ -0,0 +1,63 @@ +import unittest +import importlib_resources as resources + +from . import data01 +from . import util +from importlib import import_module + + +class CommonBinaryTests(util.CommonTests, unittest.TestCase): + def execute(self, package, path): + resources.read_binary(package, path) + + +class CommonTextTests(util.CommonTests, unittest.TestCase): + def execute(self, package, path): + resources.read_text(package, path) + + +class ReadTests: + def test_read_binary(self): + result = resources.read_binary(self.data, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_text_default_encoding(self): + result = resources.read_text(self.data, 'utf-8.file') + self.assertEqual(result, 'Hello, UTF-8 world!\n') + + def test_read_text_given_encoding(self): + result = resources.read_text( + self.data, 'utf-16.file', encoding='utf-16') + self.assertEqual(result, 'Hello, UTF-16 world!\n') + + def test_read_text_with_errors(self): + # Raises UnicodeError without the 'errors' argument. + self.assertRaises( + UnicodeError, resources.read_text, self.data, 'utf-16.file') + result = resources.read_text(self.data, 'utf-16.file', errors='ignore') + self.assertEqual( + result, + 'H\x00e\x00l\x00l\x00o\x00,\x00 ' + '\x00U\x00T\x00F\x00-\x001\x006\x00 ' + '\x00w\x00o\x00r\x00l\x00d\x00!\x00\n\x00') + + +class ReadDiskTests(ReadTests, unittest.TestCase): + data = data01 + + +class ReadZipTests(ReadTests, util.ZipSetup, unittest.TestCase): + def test_read_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + result = resources.read_binary( + submodule, 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + def test_read_submodule_resource_by_name(self): + result = resources.read_binary( + 'ziptestdata.subdirectory', 'binary.file') + self.assertEqual(result, b'\0\1\2\3') + + +if __name__ == '__main__': + unittest.main() diff --git a/pipenv/vendor/importlib_resources/tests/test_resource.py b/pipenv/vendor/importlib_resources/tests/test_resource.py new file mode 100644 index 00000000..8c5a72cb --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/test_resource.py @@ -0,0 +1,170 @@ +import sys +import unittest +import importlib_resources as resources + +from . import data01 +from . import zipdata01, zipdata02 +from . import util +from importlib import import_module + + +class ResourceTests: + # Subclasses are expected to set the `data` attribute. + + def test_is_resource_good_path(self): + self.assertTrue(resources.is_resource(self.data, 'binary.file')) + + def test_is_resource_missing(self): + self.assertFalse(resources.is_resource(self.data, 'not-a-file')) + + def test_is_resource_subresource_directory(self): + # Directories are not resources. + self.assertFalse(resources.is_resource(self.data, 'subdirectory')) + + def test_contents(self): + contents = set(resources.contents(self.data)) + # There may be cruft in the directory listing of the data directory. + # Under Python 3 we could have a __pycache__ directory, and under + # Python 2 we could have .pyc files. These are both artifacts of the + # test suite importing these modules and writing these caches. They + # aren't germane to this test, so just filter them out. + contents.discard('__pycache__') + contents.discard('__init__.pyc') + contents.discard('__init__.pyo') + self.assertEqual(contents, { + '__init__.py', + 'subdirectory', + 'utf-8.file', + 'binary.file', + 'utf-16.file', + }) + + +class ResourceDiskTests(ResourceTests, unittest.TestCase): + def setUp(self): + self.data = data01 + + +class ResourceZipTests(ResourceTests, util.ZipSetup, unittest.TestCase): + pass + + +@unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') +class ResourceLoaderTests(unittest.TestCase): + def test_resource_contents(self): + package = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + self.assertEqual( + set(resources.contents(package)), + {'A', 'B', 'C'}) + + def test_resource_is_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertTrue(resources.is_resource(package, 'B')) + + def test_resource_directory_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'D')) + + def test_resource_missing_is_not_resource(self): + package = util.create_package( + file=data01, path=data01.__file__, + contents=['A', 'B', 'C', 'D/E', 'D/F']) + self.assertFalse(resources.is_resource(package, 'Z')) + + +class ResourceCornerCaseTests(unittest.TestCase): + def test_package_has_no_reader_fallback(self): + # Test odd ball packages which: + # 1. Do not have a ResourceReader as a loader + # 2. Are not on the file system + # 3. Are not in a zip file + module = util.create_package( + file=data01, path=data01.__file__, contents=['A', 'B', 'C']) + # Give the module a dummy loader. + module.__loader__ = object() + # Give the module a dummy origin. + module.__file__ = '/path/which/shall/not/be/named' + if sys.version_info >= (3,): + module.__spec__.loader = module.__loader__ + module.__spec__.origin = module.__file__ + self.assertFalse(resources.is_resource(module, 'A')) + + +class ResourceFromZipsTest01(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata01 # type: ignore + + def test_is_submodule_resource(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertTrue( + resources.is_resource(submodule, 'binary.file')) + + def test_read_submodule_resource_by_name(self): + self.assertTrue( + resources.is_resource('ziptestdata.subdirectory', 'binary.file')) + + def test_submodule_contents(self): + submodule = import_module('ziptestdata.subdirectory') + self.assertEqual( + set(resources.contents(submodule)), + {'__init__.py', 'binary.file'}) + + def test_submodule_contents_by_name(self): + self.assertEqual( + set(resources.contents('ziptestdata.subdirectory')), + {'__init__.py', 'binary.file'}) + + +class ResourceFromZipsTest02(util.ZipSetupBase, unittest.TestCase): + ZIP_MODULE = zipdata02 # type: ignore + + def test_unrelated_contents(self): + # https://gitlab.com/python-devs/importlib_resources/issues/44 + # + # Here we have a zip file with two unrelated subpackages. The bug + # reports that getting the contents of a resource returns unrelated + # files. + self.assertEqual( + set(resources.contents('ziptestdata.one')), + {'__init__.py', 'resource1.txt'}) + self.assertEqual( + set(resources.contents('ziptestdata.two')), + {'__init__.py', 'resource2.txt'}) + + +@unittest.skipIf(sys.version_info < (3,), 'No namespace packages in Python 2') +class NamespaceTest(unittest.TestCase): + def test_namespaces_cannot_have_resources(self): + contents = resources.contents( + 'importlib_resources.tests.data03.namespace') + self.assertFalse(list(contents)) + # Even though there is a file in the namespace directory, it is not + # considered a resource, since namespace packages can't have them. + self.assertFalse(resources.is_resource( + 'importlib_resources.tests.data03.namespace', + 'resource1.txt')) + # We should get an exception if we try to read it or open it. + self.assertRaises( + FileNotFoundError, + resources.open_text, + 'importlib_resources.tests.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.open_binary, + 'importlib_resources.tests.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_text, + 'importlib_resources.tests.data03.namespace', 'resource1.txt') + self.assertRaises( + FileNotFoundError, + resources.read_binary, + 'importlib_resources.tests.data03.namespace', 'resource1.txt') + + +if __name__ == '__main__': + unittest.main() diff --git a/pipenv/vendor/importlib_resources/tests/util.py b/pipenv/vendor/importlib_resources/tests/util.py new file mode 100644 index 00000000..8c26496d --- /dev/null +++ b/pipenv/vendor/importlib_resources/tests/util.py @@ -0,0 +1,213 @@ +import abc +import importlib +import io +import sys +import types +import unittest + +from . import data01 +from . import zipdata01 +from .._compat import ABC, Path, PurePath, FileNotFoundError +from ..abc import ResourceReader + +try: + from test.support import modules_setup, modules_cleanup +except ImportError: + # Python 2.7. + def modules_setup(): + return sys.modules.copy(), + + def modules_cleanup(oldmodules): + # Encoders/decoders are registered permanently within the internal + # codec cache. If we destroy the corresponding modules their + # globals will be set to None which will trip up the cached functions. + encodings = [(k, v) for k, v in sys.modules.items() + if k.startswith('encodings.')] + sys.modules.clear() + sys.modules.update(encodings) + # XXX: This kind of problem can affect more than just encodings. In + # particular extension modules (such as _ssl) don't cope with reloading + # properly. Really, test modules should be cleaning out the test + # specific modules they know they added (ala test_runpy) rather than + # relying on this function (as test_importhooks and test_pkg do + # currently). Implicitly imported *real* modules should be left alone + # (see issue 10556). + sys.modules.update(oldmodules) + + +try: + from importlib.machinery import ModuleSpec +except ImportError: + ModuleSpec = None # type: ignore + + +def create_package(file, path, is_package=True, contents=()): + class Reader(ResourceReader): + def get_resource_reader(self, package): + return self + + def open_resource(self, path): + self._path = path + if isinstance(file, Exception): + raise file + else: + return file + + def resource_path(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + else: + return path + + def is_resource(self, path_): + self._path = path_ + if isinstance(path, Exception): + raise path + for entry in contents: + parts = entry.split('/') + if len(parts) == 1 and parts[0] == path_: + return True + return False + + def contents(self): + if isinstance(path, Exception): + raise path + # There's no yield from in baseball, er, Python 2. + for entry in contents: + yield entry + + name = 'testingpackage' + # Unforunately importlib.util.module_from_spec() was not introduced until + # Python 3.5. + module = types.ModuleType(name) + if ModuleSpec is None: + # Python 2. + module.__name__ = name + module.__file__ = 'does-not-exist' + if is_package: + module.__path__ = [] + else: + # Python 3. + loader = Reader() + spec = ModuleSpec( + name, loader, + origin='does-not-exist', + is_package=is_package) + module.__spec__ = spec + module.__loader__ = loader + return module + + +class CommonTests(ABC): + + @abc.abstractmethod + def execute(self, package, path): + raise NotImplementedError + + def test_package_name(self): + # Passing in the package name should succeed. + self.execute(data01.__name__, 'utf-8.file') + + def test_package_object(self): + # Passing in the package itself should succeed. + self.execute(data01, 'utf-8.file') + + def test_string_path(self): + # Passing in a string for the path should succeed. + path = 'utf-8.file' + self.execute(data01, path) + + @unittest.skipIf(sys.version_info < (3, 6), 'requires os.PathLike support') + def test_pathlib_path(self): + # Passing in a pathlib.PurePath object for the path should succeed. + path = PurePath('utf-8.file') + self.execute(data01, path) + + def test_absolute_path(self): + # An absolute path is a ValueError. + path = Path(__file__) + full_path = path.parent/'utf-8.file' + with self.assertRaises(ValueError): + self.execute(data01, full_path) + + def test_relative_path(self): + # A reative path is a ValueError. + with self.assertRaises(ValueError): + self.execute(data01, '../data01/utf-8.file') + + def test_importing_module_as_side_effect(self): + # The anchor package can already be imported. + del sys.modules[data01.__name__] + self.execute(data01.__name__, 'utf-8.file') + + def test_non_package_by_name(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + self.execute(__name__, 'utf-8.file') + + def test_non_package_by_package(self): + # The anchor package cannot be a module. + with self.assertRaises(TypeError): + module = sys.modules['importlib_resources.tests.util'] + self.execute(module, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_opener(self): + bytes_data = io.BytesIO(b'Hello, world!') + package = create_package(file=bytes_data, path=FileNotFoundError()) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + @unittest.skipIf(sys.version_info < (3,), 'No ResourceReader in Python 2') + def test_resource_path(self): + bytes_data = io.BytesIO(b'Hello, world!') + path = __file__ + package = create_package(file=bytes_data, path=path) + self.execute(package, 'utf-8.file') + self.assertEqual(package.__loader__._path, 'utf-8.file') + + def test_useless_loader(self): + package = create_package(file=FileNotFoundError(), + path=FileNotFoundError()) + with self.assertRaises(FileNotFoundError): + self.execute(package, 'utf-8.file') + + +class ZipSetupBase: + ZIP_MODULE = None + + @classmethod + def setUpClass(cls): + data_path = Path(cls.ZIP_MODULE.__file__) + data_dir = data_path.parent + cls._zip_path = str(data_dir / 'ziptestdata.zip') + sys.path.append(cls._zip_path) + cls.data = importlib.import_module('ziptestdata') + + @classmethod + def tearDownClass(cls): + try: + sys.path.remove(cls._zip_path) + except ValueError: + pass + + try: + del sys.path_importer_cache[cls._zip_path] + del sys.modules[cls.data.__name__] + except KeyError: + pass + + try: + del cls.data + del cls._zip_path + except AttributeError: + pass + + def setUp(self): + modules = modules_setup() + self.addCleanup(modules_cleanup, *modules) + + +class ZipSetup(ZipSetupBase): + ZIP_MODULE = zipdata01 # type: ignore diff --git a/pipenv/vendor/importlib_resources/tests/zipdata01/__init__.py b/pipenv/vendor/importlib_resources/tests/zipdata01/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/zipdata01/ziptestdata.zip b/pipenv/vendor/importlib_resources/tests/zipdata01/ziptestdata.zip new file mode 100644 index 00000000..ddcccfb3 Binary files /dev/null and b/pipenv/vendor/importlib_resources/tests/zipdata01/ziptestdata.zip differ diff --git a/pipenv/vendor/importlib_resources/tests/zipdata02/__init__.py b/pipenv/vendor/importlib_resources/tests/zipdata02/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/importlib_resources/tests/zipdata02/ziptestdata.zip b/pipenv/vendor/importlib_resources/tests/zipdata02/ziptestdata.zip new file mode 100644 index 00000000..93f4ede5 Binary files /dev/null and b/pipenv/vendor/importlib_resources/tests/zipdata02/ziptestdata.zip differ diff --git a/pipenv/vendor/importlib_resources/trees.py b/pipenv/vendor/importlib_resources/trees.py new file mode 100644 index 00000000..ba42bb55 --- /dev/null +++ b/pipenv/vendor/importlib_resources/trees.py @@ -0,0 +1,6 @@ +# for compatibility with 1.1, continue to expose as_file here. + +from ._common import as_file + + +__all__ = ['as_file'] diff --git a/pipenv/vendor/jinja2/LICENSE.rst b/pipenv/vendor/jinja2/LICENSE.rst new file mode 100644 index 00000000..c37cae49 --- /dev/null +++ b/pipenv/vendor/jinja2/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2007 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/jinja2/__init__.py b/pipenv/vendor/jinja2/__init__.py index 42aa763d..7f4a1c55 100644 --- a/pipenv/vendor/jinja2/__init__.py +++ b/pipenv/vendor/jinja2/__init__.py @@ -1,83 +1,44 @@ # -*- coding: utf-8 -*- +"""Jinja is a template engine written in pure Python. It provides a +non-XML syntax that supports inline expressions and an optional +sandboxed environment. """ - jinja2 - ~~~~~~ +from markupsafe import escape +from markupsafe import Markup - Jinja2 is a template engine written in pure Python. It provides a - Django inspired non-XML syntax but supports inline expressions and - an optional sandboxed environment. +from .bccache import BytecodeCache +from .bccache import FileSystemBytecodeCache +from .bccache import MemcachedBytecodeCache +from .environment import Environment +from .environment import Template +from .exceptions import TemplateAssertionError +from .exceptions import TemplateError +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .filters import contextfilter +from .filters import environmentfilter +from .filters import evalcontextfilter +from .loaders import BaseLoader +from .loaders import ChoiceLoader +from .loaders import DictLoader +from .loaders import FileSystemLoader +from .loaders import FunctionLoader +from .loaders import ModuleLoader +from .loaders import PackageLoader +from .loaders import PrefixLoader +from .runtime import ChainableUndefined +from .runtime import DebugUndefined +from .runtime import make_logging_undefined +from .runtime import StrictUndefined +from .runtime import Undefined +from .utils import clear_caches +from .utils import contextfunction +from .utils import environmentfunction +from .utils import evalcontextfunction +from .utils import is_undefined +from .utils import select_autoescape - Nutshell - -------- - - Here a small example of a Jinja2 template:: - - {% extends 'base.html' %} - {% block title %}Memberlist{% endblock %} - {% block content %} - <ul> - {% for user in users %} - <li><a href="{{ user.url }}">{{ user.username }}</a></li> - {% endfor %} - </ul> - {% endblock %} - - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -__docformat__ = 'restructuredtext en' -__version__ = '2.10' - -# high level interface -from jinja2.environment import Environment, Template - -# loaders -from jinja2.loaders import BaseLoader, FileSystemLoader, PackageLoader, \ - DictLoader, FunctionLoader, PrefixLoader, ChoiceLoader, \ - ModuleLoader - -# bytecode caches -from jinja2.bccache import BytecodeCache, FileSystemBytecodeCache, \ - MemcachedBytecodeCache - -# undefined types -from jinja2.runtime import Undefined, DebugUndefined, StrictUndefined, \ - make_logging_undefined - -# exceptions -from jinja2.exceptions import TemplateError, UndefinedError, \ - TemplateNotFound, TemplatesNotFound, TemplateSyntaxError, \ - TemplateAssertionError, TemplateRuntimeError - -# decorators and public utilities -from jinja2.filters import environmentfilter, contextfilter, \ - evalcontextfilter -from jinja2.utils import Markup, escape, clear_caches, \ - environmentfunction, evalcontextfunction, contextfunction, \ - is_undefined, select_autoescape - -__all__ = [ - 'Environment', 'Template', 'BaseLoader', 'FileSystemLoader', - 'PackageLoader', 'DictLoader', 'FunctionLoader', 'PrefixLoader', - 'ChoiceLoader', 'BytecodeCache', 'FileSystemBytecodeCache', - 'MemcachedBytecodeCache', 'Undefined', 'DebugUndefined', - 'StrictUndefined', 'TemplateError', 'UndefinedError', 'TemplateNotFound', - 'TemplatesNotFound', 'TemplateSyntaxError', 'TemplateAssertionError', - 'TemplateRuntimeError', - 'ModuleLoader', 'environmentfilter', 'contextfilter', 'Markup', 'escape', - 'environmentfunction', 'contextfunction', 'clear_caches', 'is_undefined', - 'evalcontextfilter', 'evalcontextfunction', 'make_logging_undefined', - 'select_autoescape', -] - - -def _patch_async(): - from jinja2.utils import have_async_gen - if have_async_gen: - from jinja2.asyncsupport import patch_all - patch_all() - - -_patch_async() -del _patch_async +__version__ = "2.11.1" diff --git a/pipenv/vendor/jinja2/_compat.py b/pipenv/vendor/jinja2/_compat.py index 61d85301..1f044954 100644 --- a/pipenv/vendor/jinja2/_compat.py +++ b/pipenv/vendor/jinja2/_compat.py @@ -1,22 +1,12 @@ # -*- coding: utf-8 -*- -""" - jinja2._compat - ~~~~~~~~~~~~~~ - - Some py2/py3 compatibility support based on a stripped down - version of six so we don't have to depend on a specific version - of it. - - :copyright: Copyright 2013 by the Jinja team, see AUTHORS. - :license: BSD, see LICENSE for details. -""" +# flake8: noqa +import marshal import sys PY2 = sys.version_info[0] == 2 -PYPY = hasattr(sys, 'pypy_translation_info') +PYPY = hasattr(sys, "pypy_translation_info") _identity = lambda x: x - if not PY2: unichr = chr range_type = range @@ -30,6 +20,7 @@ if not PY2: import pickle from io import BytesIO, StringIO + NativeStringIO = StringIO def reraise(tp, value, tb=None): @@ -46,6 +37,9 @@ if not PY2: implements_to_string = _identity encode_filename = _identity + marshal_dump = marshal.dump + marshal_load = marshal.load + else: unichr = unichr text_type = unicode @@ -59,11 +53,13 @@ else: import cPickle as pickle from cStringIO import StringIO as BytesIO, StringIO + NativeStringIO = BytesIO - exec('def reraise(tp, value, tb=None):\n raise tp, value, tb') + exec("def reraise(tp, value, tb=None):\n raise tp, value, tb") from itertools import imap, izip, ifilter + intern = intern def implements_iterator(cls): @@ -73,14 +69,25 @@ else: def implements_to_string(cls): cls.__unicode__ = cls.__str__ - cls.__str__ = lambda x: x.__unicode__().encode('utf-8') + cls.__str__ = lambda x: x.__unicode__().encode("utf-8") return cls def encode_filename(filename): if isinstance(filename, unicode): - return filename.encode('utf-8') + return filename.encode("utf-8") return filename + def marshal_dump(code, f): + if isinstance(f, file): + marshal.dump(code, f) + else: + f.write(marshal.dumps(code)) + + def marshal_load(f): + if isinstance(f, file): + return marshal.load(f) + return marshal.loads(f.read()) + def with_metaclass(meta, *bases): """Create a base class with a metaclass.""" @@ -90,10 +97,36 @@ def with_metaclass(meta, *bases): class metaclass(type): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) try: from urllib.parse import quote_from_bytes as url_quote except ImportError: from urllib import quote as url_quote + + +try: + from collections import abc +except ImportError: + import collections as abc + + +try: + from os import fspath +except ImportError: + try: + from pathlib import PurePath + except ImportError: + PurePath = None + + def fspath(path): + if hasattr(path, "__fspath__"): + return path.__fspath__() + + # Python 3.5 doesn't have __fspath__ yet, use str. + if PurePath is not None and isinstance(path, PurePath): + return str(path) + + return path diff --git a/pipenv/vendor/jinja2/_identifier.py b/pipenv/vendor/jinja2/_identifier.py index 2eac35d5..224d5449 100644 --- a/pipenv/vendor/jinja2/_identifier.py +++ b/pipenv/vendor/jinja2/_identifier.py @@ -1,2 +1,6 @@ +import re + # generated by scripts/generate_identifier_pattern.py -pattern = '·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯' +pattern = re.compile( + r"[\w·̀-ͯ·҃-֑҇-ׇֽֿׁׂׅׄؐ-ًؚ-ٰٟۖ-ۜ۟-۪ۤۧۨ-ܑۭܰ-݊ަ-ް߫-߳ࠖ-࠙ࠛ-ࠣࠥ-ࠧࠩ-࡙࠭-࡛ࣔ-ࣣ࣡-ःऺ-़ा-ॏ॑-ॗॢॣঁ-ঃ়া-ৄেৈো-্ৗৢৣਁ-ਃ਼ਾ-ੂੇੈੋ-੍ੑੰੱੵઁ-ઃ઼ા-ૅે-ૉો-્ૢૣଁ-ଃ଼ା-ୄେୈୋ-୍ୖୗୢୣஂா-ூெ-ைொ-்ௗఀ-ఃా-ౄె-ైొ-్ౕౖౢౣಁ-ಃ಼ಾ-ೄೆ-ೈೊ-್ೕೖೢೣഁ-ഃാ-ൄെ-ൈൊ-്ൗൢൣංඃ්ා-ුූෘ-ෟෲෳัิ-ฺ็-๎ັິ-ູົຼ່-ໍ༹༘༙༵༷༾༿ཱ-྄྆྇ྍ-ྗྙ-ྼ࿆ါ-ှၖ-ၙၞ-ၠၢ-ၤၧ-ၭၱ-ၴႂ-ႍႏႚ-ႝ፝-፟ᜒ-᜔ᜲ-᜴ᝒᝓᝲᝳ឴-៓៝᠋-᠍ᢅᢆᢩᤠ-ᤫᤰ-᤻ᨗ-ᨛᩕ-ᩞ᩠-᩿᩼᪰-᪽ᬀ-ᬄ᬴-᭄᭫-᭳ᮀ-ᮂᮡ-ᮭ᯦-᯳ᰤ-᰷᳐-᳔᳒-᳨᳭ᳲ-᳴᳸᳹᷀-᷵᷻-᷿‿⁀⁔⃐-⃥⃜⃡-⃰℘℮⳯-⵿⳱ⷠ-〪ⷿ-゙゚〯꙯ꙴ-꙽ꚞꚟ꛰꛱ꠂ꠆ꠋꠣ-ꠧꢀꢁꢴ-ꣅ꣠-꣱ꤦ-꤭ꥇ-꥓ꦀ-ꦃ꦳-꧀ꧥꨩ-ꨶꩃꩌꩍꩻ-ꩽꪰꪲ-ꪴꪷꪸꪾ꪿꫁ꫫ-ꫯꫵ꫶ꯣ-ꯪ꯬꯭ﬞ︀-️︠-︯︳︴﹍-﹏＿𐇽𐋠𐍶-𐍺𐨁-𐨃𐨅𐨆𐨌-𐨏𐨸-𐨿𐨺𐫦𐫥𑀀-𑀂𑀸-𑁆𑁿-𑂂𑂰-𑂺𑄀-𑄂𑄧-𑅳𑄴𑆀-𑆂𑆳-𑇊𑇀-𑇌𑈬-𑈷𑈾𑋟-𑋪𑌀-𑌃𑌼𑌾-𑍄𑍇𑍈𑍋-𑍍𑍗𑍢𑍣𑍦-𑍬𑍰-𑍴𑐵-𑑆𑒰-𑓃𑖯-𑖵𑖸-𑗀𑗜𑗝𑘰-𑙀𑚫-𑚷𑜝-𑜫𑰯-𑰶𑰸-𑰿𑲒-𑲧𑲩-𑲶𖫰-𖫴𖬰-𖬶𖽑-𖽾𖾏-𖾒𛲝𛲞𝅥-𝅩𝅭-𝅲𝅻-𝆂𝆅-𝆋𝆪-𝆭𝉂-𝉄𝨀-𝨶𝨻-𝩬𝩵𝪄𝪛-𝪟𝪡-𝪯𞀀-𞀆𞀈-𞀘𞀛-𞀡𞀣𞀤𞀦-𞣐𞀪-𞣖𞥄-𞥊󠄀-󠇯]+" # noqa: B950 +) diff --git a/pipenv/vendor/jinja2/asyncfilters.py b/pipenv/vendor/jinja2/asyncfilters.py index 5c1f46d7..d29f6c62 100644 --- a/pipenv/vendor/jinja2/asyncfilters.py +++ b/pipenv/vendor/jinja2/asyncfilters.py @@ -1,12 +1,13 @@ from functools import wraps -from jinja2.asyncsupport import auto_aiter -from jinja2 import filters +from . import filters +from .asyncsupport import auto_aiter +from .asyncsupport import auto_await async def auto_to_seq(value): seq = [] - if hasattr(value, '__aiter__'): + if hasattr(value, "__aiter__"): async for item in value: seq.append(item) else: @@ -16,8 +17,7 @@ async def auto_to_seq(value): async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): - seq, func = filters.prepare_select_or_reject( - args, kwargs, modfunc, lookup_attr) + seq, func = filters.prepare_select_or_reject(args, kwargs, modfunc, lookup_attr) if seq: async for item in auto_aiter(seq): if func(item): @@ -26,14 +26,20 @@ async def async_select_or_reject(args, kwargs, modfunc, lookup_attr): def dualfilter(normal_filter, async_filter): wrap_evalctx = False - if getattr(normal_filter, 'environmentfilter', False): - is_async = lambda args: args[0].is_async + if getattr(normal_filter, "environmentfilter", False): + + def is_async(args): + return args[0].is_async + wrap_evalctx = False else: - if not getattr(normal_filter, 'evalcontextfilter', False) and \ - not getattr(normal_filter, 'contextfilter', False): + if not getattr(normal_filter, "evalcontextfilter", False) and not getattr( + normal_filter, "contextfilter", False + ): wrap_evalctx = True - is_async = lambda args: args[0].environment.is_async + + def is_async(args): + return args[0].environment.is_async @wraps(normal_filter) def wrapper(*args, **kwargs): @@ -55,6 +61,7 @@ def dualfilter(normal_filter, async_filter): def asyncfiltervariant(original): def decorator(f): return dualfilter(original, f) + return decorator @@ -63,19 +70,22 @@ async def do_first(environment, seq): try: return await auto_aiter(seq).__anext__() except StopAsyncIteration: - return environment.undefined('No first item, sequence was empty.') + return environment.undefined("No first item, sequence was empty.") @asyncfiltervariant(filters.do_groupby) async def do_groupby(environment, value, attribute): expr = filters.make_attrgetter(environment, attribute) - return [filters._GroupTuple(key, await auto_to_seq(values)) - for key, values in filters.groupby(sorted( - await auto_to_seq(value), key=expr), expr)] + return [ + filters._GroupTuple(key, await auto_to_seq(values)) + for key, values in filters.groupby( + sorted(await auto_to_seq(value), key=expr), expr + ) + ] @asyncfiltervariant(filters.do_join) -async def do_join(eval_ctx, value, d=u'', attribute=None): +async def do_join(eval_ctx, value, d=u"", attribute=None): return filters.do_join(eval_ctx, await auto_to_seq(value), d, attribute) @@ -109,7 +119,7 @@ async def do_map(*args, **kwargs): seq, func = filters.prepare_map(args, kwargs) if seq: async for item in auto_aiter(seq): - yield func(item) + yield await auto_await(func(item)) @asyncfiltervariant(filters.do_sum) @@ -118,7 +128,10 @@ async def do_sum(environment, iterable, attribute=None, start=0): if attribute is not None: func = filters.make_attrgetter(environment, attribute) else: - func = lambda x: x + + def func(x): + return x + async for item in auto_aiter(iterable): rv += func(item) return rv @@ -130,17 +143,17 @@ async def do_slice(value, slices, fill_with=None): ASYNC_FILTERS = { - 'first': do_first, - 'groupby': do_groupby, - 'join': do_join, - 'list': do_list, + "first": do_first, + "groupby": do_groupby, + "join": do_join, + "list": do_list, # we intentionally do not support do_last because that would be # ridiculous - 'reject': do_reject, - 'rejectattr': do_rejectattr, - 'map': do_map, - 'select': do_select, - 'selectattr': do_selectattr, - 'sum': do_sum, - 'slice': do_slice, + "reject": do_reject, + "rejectattr": do_rejectattr, + "map": do_map, + "select": do_select, + "selectattr": do_selectattr, + "sum": do_sum, + "slice": do_slice, } diff --git a/pipenv/vendor/jinja2/asyncsupport.py b/pipenv/vendor/jinja2/asyncsupport.py index b1e7b5ce..78ba3739 100644 --- a/pipenv/vendor/jinja2/asyncsupport.py +++ b/pipenv/vendor/jinja2/asyncsupport.py @@ -1,29 +1,27 @@ # -*- coding: utf-8 -*- +"""The code for async support. Importing this patches Jinja on supported +Python versions. """ - jinja2.asyncsupport - ~~~~~~~~~~~~~~~~~~~ - - Has all the code for async support which is implemented as a patch - for supported Python versions. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import sys import asyncio import inspect from functools import update_wrapper -from jinja2.utils import concat, internalcode, Markup -from jinja2.environment import TemplateModule -from jinja2.runtime import LoopContextBase, _last_iteration +from markupsafe import Markup + +from .environment import TemplateModule +from .runtime import LoopContext +from .utils import concat +from .utils import internalcode +from .utils import missing async def concat_async(async_gen): rv = [] + async def collect(): async for event in async_gen: rv.append(event) + await collect() return concat(rv) @@ -34,10 +32,7 @@ async def generate_async(self, *args, **kwargs): async for event in self.root_render_func(self.new_context(vars)): yield event except Exception: - exc_info = sys.exc_info() - else: - return - yield self.environment.handle_exception(exc_info, True) + yield self.environment.handle_exception() def wrap_generate_func(original_generate): @@ -48,17 +43,18 @@ def wrap_generate_func(original_generate): yield loop.run_until_complete(async_gen.__anext__()) except StopAsyncIteration: pass + def generate(self, *args, **kwargs): if not self.environment.is_async: return original_generate(self, *args, **kwargs) return _convert_generator(self, asyncio.get_event_loop(), args, kwargs) + return update_wrapper(generate, original_generate) async def render_async(self, *args, **kwargs): if not self.environment.is_async: - raise RuntimeError('The environment was not created with async mode ' - 'enabled.') + raise RuntimeError("The environment was not created with async mode enabled.") vars = dict(*args, **kwargs) ctx = self.new_context(vars) @@ -66,8 +62,7 @@ async def render_async(self, *args, **kwargs): try: return await concat_async(self.root_render_func(ctx)) except Exception: - exc_info = sys.exc_info() - return self.environment.handle_exception(exc_info, True) + return self.environment.handle_exception() def wrap_render_func(original_render): @@ -76,6 +71,7 @@ def wrap_render_func(original_render): return original_render(self, *args, **kwargs) loop = asyncio.get_event_loop() return loop.run_until_complete(self.render_async(*args, **kwargs)) + return update_wrapper(render, original_render) @@ -109,6 +105,7 @@ def wrap_macro_invoke(original_invoke): if not self._environment.is_async: return original_invoke(self, arguments, autoescape) return async_invoke(self, arguments, autoescape) + return update_wrapper(_invoke, original_invoke) @@ -124,9 +121,9 @@ def wrap_default_module(original_default_module): @internalcode def _get_default_module(self): if self.environment.is_async: - raise RuntimeError('Template module attribute is unavailable ' - 'in async mode') + raise RuntimeError("Template module attribute is unavailable in async mode") return original_default_module(self) + return _get_default_module @@ -139,30 +136,30 @@ async def make_module_async(self, vars=None, shared=False, locals=None): def patch_template(): - from jinja2 import Template + from . import Template + Template.generate = wrap_generate_func(Template.generate) - Template.generate_async = update_wrapper( - generate_async, Template.generate_async) - Template.render_async = update_wrapper( - render_async, Template.render_async) + Template.generate_async = update_wrapper(generate_async, Template.generate_async) + Template.render_async = update_wrapper(render_async, Template.render_async) Template.render = wrap_render_func(Template.render) - Template._get_default_module = wrap_default_module( - Template._get_default_module) + Template._get_default_module = wrap_default_module(Template._get_default_module) Template._get_default_module_async = get_default_module_async Template.make_module_async = update_wrapper( - make_module_async, Template.make_module_async) + make_module_async, Template.make_module_async + ) def patch_runtime(): - from jinja2.runtime import BlockReference, Macro - BlockReference.__call__ = wrap_block_reference_call( - BlockReference.__call__) + from .runtime import BlockReference, Macro + + BlockReference.__call__ = wrap_block_reference_call(BlockReference.__call__) Macro._invoke = wrap_macro_invoke(Macro._invoke) def patch_filters(): - from jinja2.filters import FILTERS - from jinja2.asyncfilters import ASYNC_FILTERS + from .filters import FILTERS + from .asyncfilters import ASYNC_FILTERS + FILTERS.update(ASYNC_FILTERS) @@ -179,7 +176,7 @@ async def auto_await(value): async def auto_aiter(iterable): - if hasattr(iterable, '__aiter__'): + if hasattr(iterable, "__aiter__"): async for item in iterable: yield item return @@ -187,70 +184,81 @@ async def auto_aiter(iterable): yield item -class AsyncLoopContext(LoopContextBase): - - def __init__(self, async_iterator, undefined, after, length, recurse=None, - depth0=0): - LoopContextBase.__init__(self, undefined, recurse, depth0) - self._async_iterator = async_iterator - self._after = after - self._length = length +class AsyncLoopContext(LoopContext): + _to_iterator = staticmethod(auto_aiter) @property - def length(self): - if self._length is None: - raise TypeError('Loop length for some iterators cannot be ' - 'lazily calculated in async mode') + async def length(self): + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = [x async for x in self._iterator] + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + return self._length - def __aiter__(self): - return AsyncLoopContextIterator(self) + @property + async def revindex0(self): + return await self.length - self.index + @property + async def revindex(self): + return await self.length - self.index0 -class AsyncLoopContextIterator(object): - __slots__ = ('context',) + async def _peek_next(self): + if self._after is not missing: + return self._after - def __init__(self, context): - self.context = context + try: + self._after = await self._iterator.__anext__() + except StopAsyncIteration: + self._after = missing + + return self._after + + @property + async def last(self): + return await self._peek_next() is missing + + @property + async def nextitem(self): + rv = await self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv def __aiter__(self): return self async def __anext__(self): - ctx = self.context - ctx.index0 += 1 - if ctx._after is _last_iteration: - raise StopAsyncIteration() - ctx._before = ctx._current - ctx._current = ctx._after - try: - ctx._after = await ctx._async_iterator.__anext__() - except StopAsyncIteration: - ctx._after = _last_iteration - return ctx._current, ctx + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = await self._iterator.__anext__() + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self async def make_async_loop_context(iterable, undefined, recurse=None, depth0=0): - # Length is more complicated and less efficient in async mode. The - # reason for this is that we cannot know if length will be used - # upfront but because length is a property we cannot lazily execute it - # later. This means that we need to buffer it up and measure :( - # - # We however only do this for actual iterators, not for async - # iterators as blocking here does not seem like the best idea in the - # world. - try: - length = len(iterable) - except (TypeError, AttributeError): - if not hasattr(iterable, '__aiter__'): - iterable = tuple(iterable) - length = len(iterable) - else: - length = None - async_iterator = auto_aiter(iterable) - try: - after = await async_iterator.__anext__() - except StopAsyncIteration: - after = _last_iteration - return AsyncLoopContext(async_iterator, undefined, after, length, recurse, - depth0) + import warnings + + warnings.warn( + "This template must be recompiled with at least Jinja 2.11, or" + " it will fail in 3.0.", + DeprecationWarning, + stacklevel=2, + ) + return AsyncLoopContext(iterable, undefined, recurse, depth0) + + +patch_all() diff --git a/pipenv/vendor/jinja2/bccache.py b/pipenv/vendor/jinja2/bccache.py index 080e527c..9c066103 100644 --- a/pipenv/vendor/jinja2/bccache.py +++ b/pipenv/vendor/jinja2/bccache.py @@ -1,60 +1,37 @@ # -*- coding: utf-8 -*- +"""The optional bytecode cache system. This is useful if you have very +complex template situations and the compilation of all those templates +slows down your application too much. + +Situations where this is useful are often forking web applications that +are initialized on the first request. """ - jinja2.bccache - ~~~~~~~~~~~~~~ - - This module implements the bytecode cache system Jinja is optionally - using. This is useful if you have very complex template situations and - the compiliation of all those templates slow down your application too - much. - - Situations where this is useful are often forking web applications that - are initialized on the first request. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" -from os import path, listdir -import os -import sys -import stat import errno -import marshal -import tempfile import fnmatch +import os +import stat +import sys +import tempfile from hashlib import sha1 -from jinja2.utils import open_if_exists -from jinja2._compat import BytesIO, pickle, PY2, text_type +from os import listdir +from os import path +from ._compat import BytesIO +from ._compat import marshal_dump +from ._compat import marshal_load +from ._compat import pickle +from ._compat import text_type +from .utils import open_if_exists -# marshal works better on 3.x, one hack less required -if not PY2: - marshal_dump = marshal.dump - marshal_load = marshal.load -else: - - def marshal_dump(code, f): - if isinstance(f, file): - marshal.dump(code, f) - else: - f.write(marshal.dumps(code)) - - def marshal_load(f): - if isinstance(f, file): - return marshal.load(f) - return marshal.loads(f.read()) - - -bc_version = 3 - -# magic version used to only change with new jinja versions. With 2.6 -# we change this to also take Python version changes into account. The -# reason for this is that Python tends to segfault if fed earlier bytecode -# versions because someone thought it would be a good idea to reuse opcodes -# or make Python incompatible with earlier versions. -bc_magic = 'j2'.encode('ascii') + \ - pickle.dumps(bc_version, 2) + \ - pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1]) +bc_version = 4 +# Magic bytes to identify Jinja bytecode cache files. Contains the +# Python major and minor version to avoid loading incompatible bytecode +# if a project upgrades its Python version. +bc_magic = ( + b"j2" + + pickle.dumps(bc_version, 2) + + pickle.dumps((sys.version_info[0] << 24) | sys.version_info[1], 2) +) class Bucket(object): @@ -98,7 +75,7 @@ class Bucket(object): def write_bytecode(self, f): """Dump the bytecode into the file or file like object passed.""" if self.code is None: - raise TypeError('can\'t write empty bucket') + raise TypeError("can't write empty bucket") f.write(bc_magic) pickle.dump(self.checksum, f, 2) marshal_dump(self.code, f) @@ -140,7 +117,7 @@ class BytecodeCache(object): bucket.write_bytecode(f) A more advanced version of a filesystem based bytecode cache is part of - Jinja2. + Jinja. """ def load_bytecode(self, bucket): @@ -158,24 +135,24 @@ class BytecodeCache(object): raise NotImplementedError() def clear(self): - """Clears the cache. This method is not used by Jinja2 but should be + """Clears the cache. This method is not used by Jinja but should be implemented to allow applications to clear the bytecode cache used by a particular environment. """ def get_cache_key(self, name, filename=None): """Returns the unique hash key for this template name.""" - hash = sha1(name.encode('utf-8')) + hash = sha1(name.encode("utf-8")) if filename is not None: - filename = '|' + filename + filename = "|" + filename if isinstance(filename, text_type): - filename = filename.encode('utf-8') + filename = filename.encode("utf-8") hash.update(filename) return hash.hexdigest() def get_source_checksum(self, source): """Returns a checksum for the source.""" - return sha1(source.encode('utf-8')).hexdigest() + return sha1(source.encode("utf-8")).hexdigest() def get_bucket(self, environment, name, filename, source): """Return a cache bucket for the given template. All arguments are @@ -210,7 +187,7 @@ class FileSystemBytecodeCache(BytecodeCache): This bytecode cache supports clearing of the cache using the clear method. """ - def __init__(self, directory=None, pattern='__jinja2_%s.cache'): + def __init__(self, directory=None, pattern="__jinja2_%s.cache"): if directory is None: directory = self._get_default_cache_dir() self.directory = directory @@ -218,19 +195,21 @@ class FileSystemBytecodeCache(BytecodeCache): def _get_default_cache_dir(self): def _unsafe_dir(): - raise RuntimeError('Cannot determine safe temp directory. You ' - 'need to explicitly provide one.') + raise RuntimeError( + "Cannot determine safe temp directory. You " + "need to explicitly provide one." + ) tmpdir = tempfile.gettempdir() # On windows the temporary directory is used specific unless # explicitly forced otherwise. We can just use that. - if os.name == 'nt': + if os.name == "nt": return tmpdir - if not hasattr(os, 'getuid'): + if not hasattr(os, "getuid"): _unsafe_dir() - dirname = '_jinja2-cache-%d' % os.getuid() + dirname = "_jinja2-cache-%d" % os.getuid() actual_dir = os.path.join(tmpdir, dirname) try: @@ -241,18 +220,22 @@ class FileSystemBytecodeCache(BytecodeCache): try: os.chmod(actual_dir, stat.S_IRWXU) actual_dir_stat = os.lstat(actual_dir) - if actual_dir_stat.st_uid != os.getuid() \ - or not stat.S_ISDIR(actual_dir_stat.st_mode) \ - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): _unsafe_dir() except OSError as e: if e.errno != errno.EEXIST: raise actual_dir_stat = os.lstat(actual_dir) - if actual_dir_stat.st_uid != os.getuid() \ - or not stat.S_ISDIR(actual_dir_stat.st_mode) \ - or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU: + if ( + actual_dir_stat.st_uid != os.getuid() + or not stat.S_ISDIR(actual_dir_stat.st_mode) + or stat.S_IMODE(actual_dir_stat.st_mode) != stat.S_IRWXU + ): _unsafe_dir() return actual_dir @@ -261,7 +244,7 @@ class FileSystemBytecodeCache(BytecodeCache): return path.join(self.directory, self.pattern % bucket.key) def load_bytecode(self, bucket): - f = open_if_exists(self._get_cache_filename(bucket), 'rb') + f = open_if_exists(self._get_cache_filename(bucket), "rb") if f is not None: try: bucket.load_bytecode(f) @@ -269,7 +252,7 @@ class FileSystemBytecodeCache(BytecodeCache): f.close() def dump_bytecode(self, bucket): - f = open(self._get_cache_filename(bucket), 'wb') + f = open(self._get_cache_filename(bucket), "wb") try: bucket.write_bytecode(f) finally: @@ -280,7 +263,8 @@ class FileSystemBytecodeCache(BytecodeCache): # write access on the file system and the function does not exist # normally. from os import remove - files = fnmatch.filter(listdir(self.directory), self.pattern % '*') + + files = fnmatch.filter(listdir(self.directory), self.pattern % "*") for filename in files: try: remove(path.join(self.directory, filename)) @@ -296,9 +280,8 @@ class MemcachedBytecodeCache(BytecodeCache): Libraries compatible with this class: - - `werkzeug <http://werkzeug.pocoo.org/>`_.contrib.cache - - `python-memcached <https://www.tummy.com/Community/software/python-memcached/>`_ - - `cmemcache <http://gijsbert.org/cmemcache/>`_ + - `cachelib <https://github.com/pallets/cachelib>`_ + - `python-memcached <https://pypi.org/project/python-memcached/>`_ (Unfortunately the django cache interface is not compatible because it does not support storing binary data, only unicode. You can however pass @@ -334,8 +317,13 @@ class MemcachedBytecodeCache(BytecodeCache): `ignore_memcache_errors` parameter. """ - def __init__(self, client, prefix='jinja2/bytecode/', timeout=None, - ignore_memcache_errors=True): + def __init__( + self, + client, + prefix="jinja2/bytecode/", + timeout=None, + ignore_memcache_errors=True, + ): self.client = client self.prefix = prefix self.timeout = timeout diff --git a/pipenv/vendor/jinja2/compiler.py b/pipenv/vendor/jinja2/compiler.py index d534a827..f450ec6e 100644 --- a/pipenv/vendor/jinja2/compiler.py +++ b/pipenv/vendor/jinja2/compiler.py @@ -1,59 +1,62 @@ # -*- coding: utf-8 -*- -""" - jinja2.compiler - ~~~~~~~~~~~~~~~ - - Compiles nodes into python code. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from itertools import chain -from copy import deepcopy -from keyword import iskeyword as is_python_keyword +"""Compiles nodes from the parser into Python code.""" +from collections import namedtuple from functools import update_wrapper -from jinja2 import nodes -from jinja2.nodes import EvalContext -from jinja2.visitor import NodeVisitor -from jinja2.optimizer import Optimizer -from jinja2.exceptions import TemplateAssertionError -from jinja2.utils import Markup, concat, escape -from jinja2._compat import range_type, text_type, string_types, \ - iteritems, NativeStringIO, imap, izip -from jinja2.idtracking import Symbols, VAR_LOAD_PARAMETER, \ - VAR_LOAD_RESOLVE, VAR_LOAD_ALIAS, VAR_LOAD_UNDEFINED +from itertools import chain +from keyword import iskeyword as is_python_keyword +from markupsafe import escape +from markupsafe import Markup + +from . import nodes +from ._compat import imap +from ._compat import iteritems +from ._compat import izip +from ._compat import NativeStringIO +from ._compat import range_type +from ._compat import string_types +from ._compat import text_type +from .exceptions import TemplateAssertionError +from .idtracking import Symbols +from .idtracking import VAR_LOAD_ALIAS +from .idtracking import VAR_LOAD_PARAMETER +from .idtracking import VAR_LOAD_RESOLVE +from .idtracking import VAR_LOAD_UNDEFINED +from .nodes import EvalContext +from .optimizer import Optimizer +from .utils import concat +from .visitor import NodeVisitor operators = { - 'eq': '==', - 'ne': '!=', - 'gt': '>', - 'gteq': '>=', - 'lt': '<', - 'lteq': '<=', - 'in': 'in', - 'notin': 'not in' + "eq": "==", + "ne": "!=", + "gt": ">", + "gteq": ">=", + "lt": "<", + "lteq": "<=", + "in": "in", + "notin": "not in", } # what method to iterate over items do we want to use for dict iteration # in generated code? on 2.x let's go with iteritems, on 3.x with items -if hasattr(dict, 'iteritems'): - dict_item_iter = 'iteritems' +if hasattr(dict, "iteritems"): + dict_item_iter = "iteritems" else: - dict_item_iter = 'items' + dict_item_iter = "items" -code_features = ['division'] +code_features = ["division"] # does this python version support generator stops? (PEP 0479) try: - exec('from __future__ import generator_stop') - code_features.append('generator_stop') + exec("from __future__ import generator_stop") + code_features.append("generator_stop") except SyntaxError: pass # does this python version support yield from? try: - exec('def f(): yield from x()') + exec("def f(): yield from x()") except SyntaxError: supports_yield_from = False else: @@ -68,17 +71,19 @@ def optimizeconst(f): if new_node != node: return self.visit(new_node, frame) return f(self, node, frame, **kwargs) + return update_wrapper(new_func, f) -def generate(node, environment, name, filename, stream=None, - defer_init=False, optimized=True): +def generate( + node, environment, name, filename, stream=None, defer_init=False, optimized=True +): """Generate the python source for a node tree.""" if not isinstance(node, nodes.Template): - raise TypeError('Can\'t compile non template nodes') - generator = environment.code_generator_class(environment, name, filename, - stream, defer_init, - optimized) + raise TypeError("Can't compile non template nodes") + generator = environment.code_generator_class( + environment, name, filename, stream, defer_init, optimized + ) generator.visit(node) if stream is None: return generator.stream.getvalue() @@ -119,7 +124,6 @@ def find_undeclared(nodes, names): class MacroRef(object): - def __init__(self, node): self.node = node self.accesses_caller = False @@ -132,8 +136,7 @@ class Frame(object): def __init__(self, eval_ctx, parent=None, level=None): self.eval_ctx = eval_ctx - self.symbols = Symbols(parent and parent.symbols or None, - level=level) + self.symbols = Symbols(parent and parent.symbols or None, level=level) # a toplevel frame is the root + soft frames such as if conditions. self.toplevel = False @@ -223,7 +226,7 @@ class UndeclaredNameVisitor(NodeVisitor): self.undeclared = set() def visit_Name(self, node): - if node.ctx == 'load' and node.name in self.names: + if node.ctx == "load" and node.name in self.names: self.undeclared.add(node.name) if self.undeclared == self.names: raise VisitorExit() @@ -242,9 +245,9 @@ class CompilerExit(Exception): class CodeGenerator(NodeVisitor): - - def __init__(self, environment, name, filename, stream=None, - defer_init=False, optimized=True): + def __init__( + self, environment, name, filename, stream=None, defer_init=False, optimized=True + ): if stream is None: stream = NativeStringIO() self.environment = environment @@ -306,7 +309,7 @@ class CodeGenerator(NodeVisitor): self._param_def_block = [] # Tracks the current context. - self._context_reference_stack = ['context'] + self._context_reference_stack = ["context"] # -- Various compilation helpers @@ -317,30 +320,30 @@ class CodeGenerator(NodeVisitor): def temporary_identifier(self): """Get a new unique identifier.""" self._last_identifier += 1 - return 't_%d' % self._last_identifier + return "t_%d" % self._last_identifier def buffer(self, frame): """Enable buffering for the frame from that point onwards.""" frame.buffer = self.temporary_identifier() - self.writeline('%s = []' % frame.buffer) + self.writeline("%s = []" % frame.buffer) def return_buffer_contents(self, frame, force_unescaped=False): """Return the buffer contents of the frame.""" if not force_unescaped: if frame.eval_ctx.volatile: - self.writeline('if context.eval_ctx.autoescape:') + self.writeline("if context.eval_ctx.autoescape:") self.indent() - self.writeline('return Markup(concat(%s))' % frame.buffer) + self.writeline("return Markup(concat(%s))" % frame.buffer) self.outdent() - self.writeline('else:') + self.writeline("else:") self.indent() - self.writeline('return concat(%s)' % frame.buffer) + self.writeline("return concat(%s)" % frame.buffer) self.outdent() return elif frame.eval_ctx.autoescape: - self.writeline('return Markup(concat(%s))' % frame.buffer) + self.writeline("return Markup(concat(%s))" % frame.buffer) return - self.writeline('return concat(%s)' % frame.buffer) + self.writeline("return concat(%s)" % frame.buffer) def indent(self): """Indent by one.""" @@ -353,14 +356,14 @@ class CodeGenerator(NodeVisitor): def start_write(self, frame, node=None): """Yield or write into the frame buffer.""" if frame.buffer is None: - self.writeline('yield ', node) + self.writeline("yield ", node) else: - self.writeline('%s.append(' % frame.buffer, node) + self.writeline("%s.append(" % frame.buffer, node) def end_write(self, frame): """End the writing process started by `start_write`.""" if frame.buffer is not None: - self.write(')') + self.write(")") def simple_write(self, s, frame, node=None): """Simple shortcut for start_write + write + end_write.""" @@ -373,7 +376,7 @@ class CodeGenerator(NodeVisitor): is no buffer a dummy ``if 0: yield None`` is written automatically. """ try: - self.writeline('pass') + self.writeline("pass") for node in nodes: self.visit(node, frame) except CompilerExit: @@ -383,14 +386,13 @@ class CodeGenerator(NodeVisitor): """Write a string into the output stream.""" if self._new_lines: if not self._first_write: - self.stream.write('\n' * self._new_lines) + self.stream.write("\n" * self._new_lines) self.code_lineno += self._new_lines if self._write_debug_info is not None: - self.debug_info.append((self._write_debug_info, - self.code_lineno)) + self.debug_info.append((self._write_debug_info, self.code_lineno)) self._write_debug_info = None self._first_write = False - self.stream.write(' ' * self._indentation) + self.stream.write(" " * self._indentation) self._new_lines = 0 self.stream.write(x) @@ -410,7 +412,7 @@ class CodeGenerator(NodeVisitor): """Writes a function call to the stream for the current node. A leading comma is added automatically. The extra keyword arguments may not include python keywords otherwise a syntax - error could occour. The extra keyword arguments should be given + error could occur. The extra keyword arguments should be given as python dict. """ # if any of the given keyword arguments is a python keyword @@ -422,41 +424,41 @@ class CodeGenerator(NodeVisitor): break for arg in node.args: - self.write(', ') + self.write(", ") self.visit(arg, frame) if not kwarg_workaround: for kwarg in node.kwargs: - self.write(', ') + self.write(", ") self.visit(kwarg, frame) if extra_kwargs is not None: for key, value in iteritems(extra_kwargs): - self.write(', %s=%s' % (key, value)) + self.write(", %s=%s" % (key, value)) if node.dyn_args: - self.write(', *') + self.write(", *") self.visit(node.dyn_args, frame) if kwarg_workaround: if node.dyn_kwargs is not None: - self.write(', **dict({') + self.write(", **dict({") else: - self.write(', **{') + self.write(", **{") for kwarg in node.kwargs: - self.write('%r: ' % kwarg.key) + self.write("%r: " % kwarg.key) self.visit(kwarg.value, frame) - self.write(', ') + self.write(", ") if extra_kwargs is not None: for key, value in iteritems(extra_kwargs): - self.write('%r: %s, ' % (key, value)) + self.write("%r: %s, " % (key, value)) if node.dyn_kwargs is not None: - self.write('}, **') + self.write("}, **") self.visit(node.dyn_kwargs, frame) - self.write(')') + self.write(")") else: - self.write('}') + self.write("}") elif node.dyn_kwargs is not None: - self.write(', **') + self.write(", **") self.visit(node.dyn_kwargs, frame) def pull_dependencies(self, nodes): @@ -464,13 +466,14 @@ class CodeGenerator(NodeVisitor): visitor = DependencyFinderVisitor() for node in nodes: visitor.visit(node) - for dependency in 'filters', 'tests': + for dependency in "filters", "tests": mapping = getattr(self, dependency) for name in getattr(visitor, dependency): if name not in mapping: mapping[name] = self.temporary_identifier() - self.writeline('%s = environment.%s[%r]' % - (mapping[name], dependency, name)) + self.writeline( + "%s = environment.%s[%r]" % (mapping[name], dependency, name) + ) def enter_frame(self, frame): undefs = [] @@ -478,16 +481,15 @@ class CodeGenerator(NodeVisitor): if action == VAR_LOAD_PARAMETER: pass elif action == VAR_LOAD_RESOLVE: - self.writeline('%s = %s(%r)' % - (target, self.get_resolve_func(), param)) + self.writeline("%s = %s(%r)" % (target, self.get_resolve_func(), param)) elif action == VAR_LOAD_ALIAS: - self.writeline('%s = %s' % (target, param)) + self.writeline("%s = %s" % (target, param)) elif action == VAR_LOAD_UNDEFINED: undefs.append(target) else: - raise NotImplementedError('unknown load instruction') + raise NotImplementedError("unknown load instruction") if undefs: - self.writeline('%s = missing' % ' = '.join(undefs)) + self.writeline("%s = missing" % " = ".join(undefs)) def leave_frame(self, frame, with_python_scope=False): if not with_python_scope: @@ -495,12 +497,12 @@ class CodeGenerator(NodeVisitor): for target, _ in iteritems(frame.symbols.loads): undefs.append(target) if undefs: - self.writeline('%s = missing' % ' = '.join(undefs)) + self.writeline("%s = missing" % " = ".join(undefs)) def func(self, name): if self.environment.is_async: - return 'async def %s' % name - return 'def %s' % name + return "async def %s" % name + return "def %s" % name def macro_body(self, node, frame): """Dump the function def of a macro or call block.""" @@ -512,16 +514,16 @@ class CodeGenerator(NodeVisitor): skip_special_params = set() args = [] for idx, arg in enumerate(node.args): - if arg.name == 'caller': + if arg.name == "caller": explicit_caller = idx - if arg.name in ('kwargs', 'varargs'): + if arg.name in ("kwargs", "varargs"): skip_special_params.add(arg.name) args.append(frame.symbols.ref(arg.name)) - undeclared = find_undeclared(node.body, ('caller', 'kwargs', 'varargs')) + undeclared = find_undeclared(node.body, ("caller", "kwargs", "varargs")) - if 'caller' in undeclared: - # In older Jinja2 versions there was a bug that allowed caller + if "caller" in undeclared: + # In older Jinja versions there was a bug that allowed caller # to retain the special behavior even if it was mentioned in # the argument list. However thankfully this was only really # working if it was the last argument. So we are explicitly @@ -531,23 +533,26 @@ class CodeGenerator(NodeVisitor): try: node.defaults[explicit_caller - len(node.args)] except IndexError: - self.fail('When defining macros or call blocks the ' - 'special "caller" argument must be omitted ' - 'or be given a default.', node.lineno) + self.fail( + "When defining macros or call blocks the " + 'special "caller" argument must be omitted ' + "or be given a default.", + node.lineno, + ) else: - args.append(frame.symbols.declare_parameter('caller')) + args.append(frame.symbols.declare_parameter("caller")) macro_ref.accesses_caller = True - if 'kwargs' in undeclared and not 'kwargs' in skip_special_params: - args.append(frame.symbols.declare_parameter('kwargs')) + if "kwargs" in undeclared and "kwargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("kwargs")) macro_ref.accesses_kwargs = True - if 'varargs' in undeclared and not 'varargs' in skip_special_params: - args.append(frame.symbols.declare_parameter('varargs')) + if "varargs" in undeclared and "varargs" not in skip_special_params: + args.append(frame.symbols.declare_parameter("varargs")) macro_ref.accesses_varargs = True # macros are delayed, they never require output checks frame.require_output_check = False frame.symbols.analyze_node(node) - self.writeline('%s(%s):' % (self.func('macro'), ', '.join(args)), node) + self.writeline("%s(%s):" % (self.func("macro"), ", ".join(args)), node) self.indent() self.buffer(frame) @@ -556,17 +561,17 @@ class CodeGenerator(NodeVisitor): self.push_parameter_definitions(frame) for idx, arg in enumerate(node.args): ref = frame.symbols.ref(arg.name) - self.writeline('if %s is missing:' % ref) + self.writeline("if %s is missing:" % ref) self.indent() try: default = node.defaults[idx - len(node.args)] except IndexError: - self.writeline('%s = undefined(%r, name=%r)' % ( - ref, - 'parameter %r was not provided' % arg.name, - arg.name)) + self.writeline( + "%s = undefined(%r, name=%r)" + % (ref, "parameter %r was not provided" % arg.name, arg.name) + ) else: - self.writeline('%s = ' % ref) + self.writeline("%s = " % ref) self.visit(default, frame) self.mark_parameter_stored(ref) self.outdent() @@ -581,35 +586,46 @@ class CodeGenerator(NodeVisitor): def macro_def(self, macro_ref, frame): """Dump the macro definition for the def created by macro_body.""" - arg_tuple = ', '.join(repr(x.name) for x in macro_ref.node.args) - name = getattr(macro_ref.node, 'name', None) + arg_tuple = ", ".join(repr(x.name) for x in macro_ref.node.args) + name = getattr(macro_ref.node, "name", None) if len(macro_ref.node.args) == 1: - arg_tuple += ',' - self.write('Macro(environment, macro, %r, (%s), %r, %r, %r, ' - 'context.eval_ctx.autoescape)' % - (name, arg_tuple, macro_ref.accesses_kwargs, - macro_ref.accesses_varargs, macro_ref.accesses_caller)) + arg_tuple += "," + self.write( + "Macro(environment, macro, %r, (%s), %r, %r, %r, " + "context.eval_ctx.autoescape)" + % ( + name, + arg_tuple, + macro_ref.accesses_kwargs, + macro_ref.accesses_varargs, + macro_ref.accesses_caller, + ) + ) def position(self, node): """Return a human readable position for the node.""" - rv = 'line %d' % node.lineno + rv = "line %d" % node.lineno if self.name is not None: - rv += ' in ' + repr(self.name) + rv += " in " + repr(self.name) return rv def dump_local_context(self, frame): - return '{%s}' % ', '.join( - '%r: %s' % (name, target) for name, target - in iteritems(frame.symbols.dump_stores())) + return "{%s}" % ", ".join( + "%r: %s" % (name, target) + for name, target in iteritems(frame.symbols.dump_stores()) + ) def write_commons(self): """Writes a common preamble that is used by root and block functions. Primarily this sets up common local helpers and enforces a generator through a dead branch. """ - self.writeline('resolve = context.resolve_or_missing') - self.writeline('undefined = environment.undefined') - self.writeline('if 0: yield None') + self.writeline("resolve = context.resolve_or_missing") + self.writeline("undefined = environment.undefined") + # always use the standard Undefined class for the implicit else of + # conditional expressions + self.writeline("cond_expr_undefined = Undefined") + self.writeline("if 0: yield None") def push_parameter_definitions(self, frame): """Pushes all parameter targets from the given frame into a local @@ -642,12 +658,12 @@ class CodeGenerator(NodeVisitor): def get_resolve_func(self): target = self._context_reference_stack[-1] - if target == 'context': - return 'resolve' - return '%s.resolve' % target + if target == "context": + return "resolve" + return "%s.resolve" % target def derive_context(self, frame): - return '%s.derived(%s)' % ( + return "%s.derived(%s)" % ( self.get_context_ref(), self.dump_local_context(frame), ) @@ -669,44 +685,48 @@ class CodeGenerator(NodeVisitor): vars = self._assign_stack.pop() if not frame.toplevel or not vars: return - public_names = [x for x in vars if x[:1] != '_'] + public_names = [x for x in vars if x[:1] != "_"] if len(vars) == 1: name = next(iter(vars)) ref = frame.symbols.ref(name) - self.writeline('context.vars[%r] = %s' % (name, ref)) + self.writeline("context.vars[%r] = %s" % (name, ref)) else: - self.writeline('context.vars.update({') + self.writeline("context.vars.update({") for idx, name in enumerate(vars): if idx: - self.write(', ') + self.write(", ") ref = frame.symbols.ref(name) - self.write('%r: %s' % (name, ref)) - self.write('})') + self.write("%r: %s" % (name, ref)) + self.write("})") if public_names: if len(public_names) == 1: - self.writeline('context.exported_vars.add(%r)' % - public_names[0]) + self.writeline("context.exported_vars.add(%r)" % public_names[0]) else: - self.writeline('context.exported_vars.update((%s))' % - ', '.join(imap(repr, public_names))) + self.writeline( + "context.exported_vars.update((%s))" + % ", ".join(imap(repr, public_names)) + ) # -- Statement Visitors def visit_Template(self, node, frame=None): - assert frame is None, 'no root frame allowed' + assert frame is None, "no root frame allowed" eval_ctx = EvalContext(self.environment, self.name) - from jinja2.runtime import __all__ as exported - self.writeline('from __future__ import %s' % ', '.join(code_features)) - self.writeline('from jinja2.runtime import ' + ', '.join(exported)) + from .runtime import exported + + self.writeline("from __future__ import %s" % ", ".join(code_features)) + self.writeline("from jinja2.runtime import " + ", ".join(exported)) if self.environment.is_async: - self.writeline('from jinja2.asyncsupport import auto_await, ' - 'auto_aiter, make_async_loop_context') + self.writeline( + "from jinja2.asyncsupport import auto_await, " + "auto_aiter, AsyncLoopContext" + ) # if we want a deferred initialization we cannot move the # environment into a local name - envenv = not self.defer_init and ', environment=environment' or '' + envenv = not self.defer_init and ", environment=environment" or "" # do we have an extends tag at all? If not, we can save some # overhead by just not processing any inheritance code. @@ -715,7 +735,7 @@ class CodeGenerator(NodeVisitor): # find all blocks for block in node.find_all(nodes.Block): if block.name in self.blocks: - self.fail('block %r defined twice' % block.name, block.lineno) + self.fail("block %r defined twice" % block.name, block.lineno) self.blocks[block.name] = block # find all imports and import them @@ -723,32 +743,32 @@ class CodeGenerator(NodeVisitor): if import_.importname not in self.import_aliases: imp = import_.importname self.import_aliases[imp] = alias = self.temporary_identifier() - if '.' in imp: - module, obj = imp.rsplit('.', 1) - self.writeline('from %s import %s as %s' % - (module, obj, alias)) + if "." in imp: + module, obj = imp.rsplit(".", 1) + self.writeline("from %s import %s as %s" % (module, obj, alias)) else: - self.writeline('import %s as %s' % (imp, alias)) + self.writeline("import %s as %s" % (imp, alias)) # add the load name - self.writeline('name = %r' % self.name) + self.writeline("name = %r" % self.name) # generate the root render function. - self.writeline('%s(context, missing=missing%s):' % - (self.func('root'), envenv), extra=1) + self.writeline( + "%s(context, missing=missing%s):" % (self.func("root"), envenv), extra=1 + ) self.indent() self.write_commons() # process the root frame = Frame(eval_ctx) - if 'self' in find_undeclared(node.body, ('self',)): - ref = frame.symbols.declare_parameter('self') - self.writeline('%s = TemplateReference(context)' % ref) + if "self" in find_undeclared(node.body, ("self",)): + ref = frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) frame.symbols.analyze_node(node) frame.toplevel = frame.rootlevel = True frame.require_output_check = have_extends and not self.has_known_extends if have_extends: - self.writeline('parent_template = None') + self.writeline("parent_template = None") self.enter_frame(frame) self.pull_dependencies(node.body) self.blockvisit(node.body, frame) @@ -759,39 +779,42 @@ class CodeGenerator(NodeVisitor): if have_extends: if not self.has_known_extends: self.indent() - self.writeline('if parent_template is not None:') + self.writeline("if parent_template is not None:") self.indent() if supports_yield_from and not self.environment.is_async: - self.writeline('yield from parent_template.' - 'root_render_func(context)') + self.writeline("yield from parent_template.root_render_func(context)") else: - self.writeline('%sfor event in parent_template.' - 'root_render_func(context):' % - (self.environment.is_async and 'async ' or '')) + self.writeline( + "%sfor event in parent_template." + "root_render_func(context):" + % (self.environment.is_async and "async " or "") + ) self.indent() - self.writeline('yield event') + self.writeline("yield event") self.outdent() self.outdent(1 + (not self.has_known_extends)) # at this point we now have the blocks collected and can visit them too. for name, block in iteritems(self.blocks): - self.writeline('%s(context, missing=missing%s):' % - (self.func('block_' + name), envenv), - block, 1) + self.writeline( + "%s(context, missing=missing%s):" + % (self.func("block_" + name), envenv), + block, + 1, + ) self.indent() self.write_commons() # It's important that we do not make this frame a child of the # toplevel template. This would cause a variety of # interesting issues with identifier tracking. block_frame = Frame(eval_ctx) - undeclared = find_undeclared(block.body, ('self', 'super')) - if 'self' in undeclared: - ref = block_frame.symbols.declare_parameter('self') - self.writeline('%s = TemplateReference(context)' % ref) - if 'super' in undeclared: - ref = block_frame.symbols.declare_parameter('super') - self.writeline('%s = context.super(%r, ' - 'block_%s)' % (ref, name, name)) + undeclared = find_undeclared(block.body, ("self", "super")) + if "self" in undeclared: + ref = block_frame.symbols.declare_parameter("self") + self.writeline("%s = TemplateReference(context)" % ref) + if "super" in undeclared: + ref = block_frame.symbols.declare_parameter("super") + self.writeline("%s = context.super(%r, block_%s)" % (ref, name, name)) block_frame.symbols.analyze_node(block) block_frame.block = name self.enter_frame(block_frame) @@ -800,13 +823,15 @@ class CodeGenerator(NodeVisitor): self.leave_frame(block_frame, with_python_scope=True) self.outdent() - self.writeline('blocks = {%s}' % ', '.join('%r: block_%s' % (x, x) - for x in self.blocks), - extra=1) + self.writeline( + "blocks = {%s}" % ", ".join("%r: block_%s" % (x, x) for x in self.blocks), + extra=1, + ) # add a function that returns the debug info - self.writeline('debug_info = %r' % '&'.join('%s=%s' % x for x - in self.debug_info)) + self.writeline( + "debug_info = %r" % "&".join("%s=%s" % x for x in self.debug_info) + ) def visit_Block(self, node, frame): """Call a block and register it for the template.""" @@ -817,7 +842,7 @@ class CodeGenerator(NodeVisitor): if self.has_known_extends: return if self.extends_so_far > 0: - self.writeline('if parent_template is None:') + self.writeline("if parent_template is None:") self.indent() level += 1 @@ -826,16 +851,22 @@ class CodeGenerator(NodeVisitor): else: context = self.get_context_ref() - if supports_yield_from and not self.environment.is_async and \ - frame.buffer is None: - self.writeline('yield from context.blocks[%r][0](%s)' % ( - node.name, context), node) + if ( + supports_yield_from + and not self.environment.is_async + and frame.buffer is None + ): + self.writeline( + "yield from context.blocks[%r][0](%s)" % (node.name, context), node + ) else: - loop = self.environment.is_async and 'async for' or 'for' - self.writeline('%s event in context.blocks[%r][0](%s):' % ( - loop, node.name, context), node) + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in context.blocks[%r][0](%s):" % (loop, node.name, context), + node, + ) self.indent() - self.simple_write('event', frame) + self.simple_write("event", frame) self.outdent() self.outdent(level) @@ -843,8 +874,7 @@ class CodeGenerator(NodeVisitor): def visit_Extends(self, node, frame): """Calls the extender.""" if not frame.toplevel: - self.fail('cannot use extend from a non top-level scope', - node.lineno) + self.fail("cannot use extend from a non top-level scope", node.lineno) # if the number of extends statements in general is zero so # far, we don't have to add a check if something extended @@ -856,10 +886,9 @@ class CodeGenerator(NodeVisitor): # time too, but i welcome it not to confuse users by throwing the # same error at different times just "because we can". if not self.has_known_extends: - self.writeline('if parent_template is not None:') + self.writeline("if parent_template is not None:") self.indent() - self.writeline('raise TemplateRuntimeError(%r)' % - 'extended multiple times') + self.writeline("raise TemplateRuntimeError(%r)" % "extended multiple times") # if we have a known extends already we don't need that code here # as we know that the template execution will end here. @@ -868,14 +897,14 @@ class CodeGenerator(NodeVisitor): else: self.outdent() - self.writeline('parent_template = environment.get_template(', node) + self.writeline("parent_template = environment.get_template(", node) self.visit(node.template, frame) - self.write(', %r)' % self.name) - self.writeline('for name, parent_block in parent_template.' - 'blocks.%s():' % dict_item_iter) + self.write(", %r)" % self.name) + self.writeline( + "for name, parent_block in parent_template.blocks.%s():" % dict_item_iter + ) self.indent() - self.writeline('context.blocks.setdefault(name, []).' - 'append(parent_block)') + self.writeline("context.blocks.setdefault(name, []).append(parent_block)") self.outdent() # if this extends statement was in the root level we can take @@ -890,52 +919,56 @@ class CodeGenerator(NodeVisitor): def visit_Include(self, node, frame): """Handles includes.""" if node.ignore_missing: - self.writeline('try:') + self.writeline("try:") self.indent() - func_name = 'get_or_select_template' + func_name = "get_or_select_template" if isinstance(node.template, nodes.Const): if isinstance(node.template.value, string_types): - func_name = 'get_template' + func_name = "get_template" elif isinstance(node.template.value, (tuple, list)): - func_name = 'select_template' + func_name = "select_template" elif isinstance(node.template, (nodes.Tuple, nodes.List)): - func_name = 'select_template' + func_name = "select_template" - self.writeline('template = environment.%s(' % func_name, node) + self.writeline("template = environment.%s(" % func_name, node) self.visit(node.template, frame) - self.write(', %r)' % self.name) + self.write(", %r)" % self.name) if node.ignore_missing: self.outdent() - self.writeline('except TemplateNotFound:') + self.writeline("except TemplateNotFound:") self.indent() - self.writeline('pass') + self.writeline("pass") self.outdent() - self.writeline('else:') + self.writeline("else:") self.indent() skip_event_yield = False if node.with_context: - loop = self.environment.is_async and 'async for' or 'for' - self.writeline('%s event in template.root_render_func(' - 'template.new_context(context.get_all(), True, ' - '%s)):' % (loop, self.dump_local_context(frame))) + loop = self.environment.is_async and "async for" or "for" + self.writeline( + "%s event in template.root_render_func(" + "template.new_context(context.get_all(), True, " + "%s)):" % (loop, self.dump_local_context(frame)) + ) elif self.environment.is_async: - self.writeline('for event in (await ' - 'template._get_default_module_async())' - '._body_stream:') + self.writeline( + "for event in (await " + "template._get_default_module_async())" + "._body_stream:" + ) else: if supports_yield_from: - self.writeline('yield from template._get_default_module()' - '._body_stream') + self.writeline("yield from template._get_default_module()._body_stream") skip_event_yield = True else: - self.writeline('for event in template._get_default_module()' - '._body_stream:') + self.writeline( + "for event in template._get_default_module()._body_stream:" + ) if not skip_event_yield: self.indent() - self.simple_write('event', frame) + self.simple_write("event", frame) self.outdent() if node.ignore_missing: @@ -943,40 +976,50 @@ class CodeGenerator(NodeVisitor): def visit_Import(self, node, frame): """Visit regular imports.""" - self.writeline('%s = ' % frame.symbols.ref(node.target), node) + self.writeline("%s = " % frame.symbols.ref(node.target), node) if frame.toplevel: - self.write('context.vars[%r] = ' % node.target) + self.write("context.vars[%r] = " % node.target) if self.environment.is_async: - self.write('await ') - self.write('environment.get_template(') + self.write("await ") + self.write("environment.get_template(") self.visit(node.template, frame) - self.write(', %r).' % self.name) + self.write(", %r)." % self.name) if node.with_context: - self.write('make_module%s(context.get_all(), True, %s)' - % (self.environment.is_async and '_async' or '', - self.dump_local_context(frame))) + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) elif self.environment.is_async: - self.write('_get_default_module_async()') + self.write("_get_default_module_async()") else: - self.write('_get_default_module()') - if frame.toplevel and not node.target.startswith('_'): - self.writeline('context.exported_vars.discard(%r)' % node.target) + self.write("_get_default_module()") + if frame.toplevel and not node.target.startswith("_"): + self.writeline("context.exported_vars.discard(%r)" % node.target) def visit_FromImport(self, node, frame): """Visit named imports.""" self.newline(node) - self.write('included_template = %senvironment.get_template(' - % (self.environment.is_async and 'await ' or '')) + self.write( + "included_template = %senvironment.get_template(" + % (self.environment.is_async and "await " or "") + ) self.visit(node.template, frame) - self.write(', %r).' % self.name) + self.write(", %r)." % self.name) if node.with_context: - self.write('make_module%s(context.get_all(), True, %s)' - % (self.environment.is_async and '_async' or '', - self.dump_local_context(frame))) + self.write( + "make_module%s(context.get_all(), True, %s)" + % ( + self.environment.is_async and "_async" or "", + self.dump_local_context(frame), + ) + ) elif self.environment.is_async: - self.write('_get_default_module_async()') + self.write("_get_default_module_async()") else: - self.write('_get_default_module()') + self.write("_get_default_module()") var_names = [] discarded_names = [] @@ -985,41 +1028,51 @@ class CodeGenerator(NodeVisitor): name, alias = name else: alias = name - self.writeline('%s = getattr(included_template, ' - '%r, missing)' % (frame.symbols.ref(alias), name)) - self.writeline('if %s is missing:' % frame.symbols.ref(alias)) + self.writeline( + "%s = getattr(included_template, " + "%r, missing)" % (frame.symbols.ref(alias), name) + ) + self.writeline("if %s is missing:" % frame.symbols.ref(alias)) self.indent() - self.writeline('%s = undefined(%r %% ' - 'included_template.__name__, ' - 'name=%r)' % - (frame.symbols.ref(alias), - 'the template %%r (imported on %s) does ' - 'not export the requested name %s' % ( - self.position(node), - repr(name) - ), name)) + self.writeline( + "%s = undefined(%r %% " + "included_template.__name__, " + "name=%r)" + % ( + frame.symbols.ref(alias), + "the template %%r (imported on %s) does " + "not export the requested name %s" + % (self.position(node), repr(name)), + name, + ) + ) self.outdent() if frame.toplevel: var_names.append(alias) - if not alias.startswith('_'): + if not alias.startswith("_"): discarded_names.append(alias) if var_names: if len(var_names) == 1: name = var_names[0] - self.writeline('context.vars[%r] = %s' % - (name, frame.symbols.ref(name))) + self.writeline( + "context.vars[%r] = %s" % (name, frame.symbols.ref(name)) + ) else: - self.writeline('context.vars.update({%s})' % ', '.join( - '%r: %s' % (name, frame.symbols.ref(name)) for name in var_names - )) + self.writeline( + "context.vars.update({%s})" + % ", ".join( + "%r: %s" % (name, frame.symbols.ref(name)) for name in var_names + ) + ) if discarded_names: if len(discarded_names) == 1: - self.writeline('context.exported_vars.discard(%r)' % - discarded_names[0]) + self.writeline("context.exported_vars.discard(%r)" % discarded_names[0]) else: - self.writeline('context.exported_vars.difference_' - 'update((%s))' % ', '.join(imap(repr, discarded_names))) + self.writeline( + "context.exported_vars.difference_" + "update((%s))" % ", ".join(imap(repr, discarded_names)) + ) def visit_For(self, node, frame): loop_frame = frame.inner() @@ -1029,35 +1082,35 @@ class CodeGenerator(NodeVisitor): # try to figure out if we have an extended loop. An extended loop # is necessary if the loop is in recursive mode if the special loop # variable is accessed in the body. - extended_loop = node.recursive or 'loop' in \ - find_undeclared(node.iter_child_nodes( - only=('body',)), ('loop',)) + extended_loop = node.recursive or "loop" in find_undeclared( + node.iter_child_nodes(only=("body",)), ("loop",) + ) loop_ref = None if extended_loop: - loop_ref = loop_frame.symbols.declare_parameter('loop') + loop_ref = loop_frame.symbols.declare_parameter("loop") - loop_frame.symbols.analyze_node(node, for_branch='body') + loop_frame.symbols.analyze_node(node, for_branch="body") if node.else_: - else_frame.symbols.analyze_node(node, for_branch='else') + else_frame.symbols.analyze_node(node, for_branch="else") if node.test: loop_filter_func = self.temporary_identifier() - test_frame.symbols.analyze_node(node, for_branch='test') - self.writeline('%s(fiter):' % self.func(loop_filter_func), node.test) + test_frame.symbols.analyze_node(node, for_branch="test") + self.writeline("%s(fiter):" % self.func(loop_filter_func), node.test) self.indent() self.enter_frame(test_frame) - self.writeline(self.environment.is_async and 'async for ' or 'for ') + self.writeline(self.environment.is_async and "async for " or "for ") self.visit(node.target, loop_frame) - self.write(' in ') - self.write(self.environment.is_async and 'auto_aiter(fiter)' or 'fiter') - self.write(':') + self.write(" in ") + self.write(self.environment.is_async and "auto_aiter(fiter)" or "fiter") + self.write(":") self.indent() - self.writeline('if ', node.test) + self.writeline("if ", node.test) self.visit(node.test, test_frame) - self.write(':') + self.write(":") self.indent() - self.writeline('yield ') + self.writeline("yield ") self.visit(node.target, loop_frame) self.outdent(3) self.leave_frame(test_frame, with_python_scope=True) @@ -1066,8 +1119,9 @@ class CodeGenerator(NodeVisitor): # variables at that point. Because loops can be nested but the loop # variable is a special one we have to enforce aliasing for it. if node.recursive: - self.writeline('%s(reciter, loop_render_func, depth=0):' % - self.func('loop'), node) + self.writeline( + "%s(reciter, loop_render_func, depth=0):" % self.func("loop"), node + ) self.indent() self.buffer(loop_frame) @@ -1077,57 +1131,60 @@ class CodeGenerator(NodeVisitor): # make sure the loop variable is a special one and raise a template # assertion error if a loop tries to write to loop if extended_loop: - self.writeline('%s = missing' % loop_ref) + self.writeline("%s = missing" % loop_ref) for name in node.find_all(nodes.Name): - if name.ctx == 'store' and name.name == 'loop': - self.fail('Can\'t assign to special loop variable ' - 'in for-loop target', name.lineno) + if name.ctx == "store" and name.name == "loop": + self.fail( + "Can't assign to special loop variable in for-loop target", + name.lineno, + ) if node.else_: iteration_indicator = self.temporary_identifier() - self.writeline('%s = 1' % iteration_indicator) + self.writeline("%s = 1" % iteration_indicator) - self.writeline(self.environment.is_async and 'async for ' or 'for ', node) + self.writeline(self.environment.is_async and "async for " or "for ", node) self.visit(node.target, loop_frame) if extended_loop: if self.environment.is_async: - self.write(', %s in await make_async_loop_context(' % loop_ref) + self.write(", %s in AsyncLoopContext(" % loop_ref) else: - self.write(', %s in LoopContext(' % loop_ref) + self.write(", %s in LoopContext(" % loop_ref) else: - self.write(' in ') + self.write(" in ") if node.test: - self.write('%s(' % loop_filter_func) + self.write("%s(" % loop_filter_func) if node.recursive: - self.write('reciter') + self.write("reciter") else: if self.environment.is_async and not extended_loop: - self.write('auto_aiter(') + self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async and not extended_loop: - self.write(')') + self.write(")") if node.test: - self.write(')') + self.write(")") if node.recursive: - self.write(', undefined, loop_render_func, depth):') + self.write(", undefined, loop_render_func, depth):") else: - self.write(extended_loop and ', undefined):' or ':') + self.write(extended_loop and ", undefined):" or ":") self.indent() self.enter_frame(loop_frame) self.blockvisit(node.body, loop_frame) if node.else_: - self.writeline('%s = 0' % iteration_indicator) + self.writeline("%s = 0" % iteration_indicator) self.outdent() - self.leave_frame(loop_frame, with_python_scope=node.recursive - and not node.else_) + self.leave_frame( + loop_frame, with_python_scope=node.recursive and not node.else_ + ) if node.else_: - self.writeline('if %s:' % iteration_indicator) + self.writeline("if %s:" % iteration_indicator) self.indent() self.enter_frame(else_frame) self.blockvisit(node.else_, else_frame) @@ -1141,33 +1198,33 @@ class CodeGenerator(NodeVisitor): self.outdent() self.start_write(frame, node) if self.environment.is_async: - self.write('await ') - self.write('loop(') + self.write("await ") + self.write("loop(") if self.environment.is_async: - self.write('auto_aiter(') + self.write("auto_aiter(") self.visit(node.iter, frame) if self.environment.is_async: - self.write(')') - self.write(', loop)') + self.write(")") + self.write(", loop)") self.end_write(frame) def visit_If(self, node, frame): if_frame = frame.soft() - self.writeline('if ', node) + self.writeline("if ", node) self.visit(node.test, if_frame) - self.write(':') + self.write(":") self.indent() self.blockvisit(node.body, if_frame) self.outdent() for elif_ in node.elif_: - self.writeline('elif ', elif_) + self.writeline("elif ", elif_) self.visit(elif_.test, if_frame) - self.write(':') + self.write(":") self.indent() self.blockvisit(elif_.body, if_frame) self.outdent() if node.else_: - self.writeline('else:') + self.writeline("else:") self.indent() self.blockvisit(node.else_, if_frame) self.outdent() @@ -1176,16 +1233,15 @@ class CodeGenerator(NodeVisitor): macro_frame, macro_ref = self.macro_body(node, frame) self.newline() if frame.toplevel: - if not node.name.startswith('_'): - self.write('context.exported_vars.add(%r)' % node.name) - ref = frame.symbols.ref(node.name) - self.writeline('context.vars[%r] = ' % node.name) - self.write('%s = ' % frame.symbols.ref(node.name)) + if not node.name.startswith("_"): + self.write("context.exported_vars.add(%r)" % node.name) + self.writeline("context.vars[%r] = " % node.name) + self.write("%s = " % frame.symbols.ref(node.name)) self.macro_def(macro_ref, macro_frame) def visit_CallBlock(self, node, frame): call_frame, macro_ref = self.macro_body(node, frame) - self.writeline('caller = ') + self.writeline("caller = ") self.macro_def(macro_ref, call_frame) self.start_write(frame, node) self.visit_Call(node.call, frame, forward_caller=True) @@ -1206,10 +1262,10 @@ class CodeGenerator(NodeVisitor): with_frame = frame.inner() with_frame.symbols.analyze_node(node) self.enter_frame(with_frame) - for idx, (target, expr) in enumerate(izip(node.targets, node.values)): + for target, expr in izip(node.targets, node.values): self.newline() self.visit(target, with_frame) - self.write(' = ') + self.write(" = ") self.visit(expr, frame) self.blockvisit(node.body, with_frame) self.leave_frame(with_frame) @@ -1218,156 +1274,187 @@ class CodeGenerator(NodeVisitor): self.newline(node) self.visit(node.node, frame) - def visit_Output(self, node, frame): - # if we have a known extends statement, we don't output anything - # if we are in a require_output_check section - if self.has_known_extends and frame.require_output_check: - return + _FinalizeInfo = namedtuple("_FinalizeInfo", ("const", "src")) + #: The default finalize function if the environment isn't configured + #: with one. Or if the environment has one, this is called on that + #: function's output for constants. + _default_finalize = text_type + _finalize = None + + def _make_finalize(self): + """Build the finalize function to be used on constants and at + runtime. Cached so it's only created once for all output nodes. + + Returns a ``namedtuple`` with the following attributes: + + ``const`` + A function to finalize constant data at compile time. + + ``src`` + Source code to output around nodes to be evaluated at + runtime. + """ + if self._finalize is not None: + return self._finalize + + finalize = default = self._default_finalize + src = None - allow_constant_finalize = True if self.environment.finalize: - func = self.environment.finalize - if getattr(func, 'contextfunction', False) or \ - getattr(func, 'evalcontextfunction', False): - allow_constant_finalize = False - elif getattr(func, 'environmentfunction', False): - finalize = lambda x: text_type( - self.environment.finalize(self.environment, x)) - else: - finalize = lambda x: text_type(self.environment.finalize(x)) + src = "environment.finalize(" + env_finalize = self.environment.finalize + + def finalize(value): + return default(env_finalize(value)) + + if getattr(env_finalize, "contextfunction", False): + src += "context, " + finalize = None # noqa: F811 + elif getattr(env_finalize, "evalcontextfunction", False): + src += "context.eval_ctx, " + finalize = None + elif getattr(env_finalize, "environmentfunction", False): + src += "environment, " + + def finalize(value): + return default(env_finalize(self.environment, value)) + + self._finalize = self._FinalizeInfo(finalize, src) + return self._finalize + + def _output_const_repr(self, group): + """Given a group of constant values converted from ``Output`` + child nodes, produce a string to write to the template module + source. + """ + return repr(concat(group)) + + def _output_child_to_const(self, node, frame, finalize): + """Try to optimize a child of an ``Output`` node by trying to + convert it to constant, finalized data at compile time. + + If :exc:`Impossible` is raised, the node is not constant and + will be evaluated at runtime. Any other exception will also be + evaluated at runtime for easier debugging. + """ + const = node.as_const(frame.eval_ctx) + + if frame.eval_ctx.autoescape: + const = escape(const) + + # Template data doesn't go through finalize. + if isinstance(node, nodes.TemplateData): + return text_type(const) + + return finalize.const(const) + + def _output_child_pre(self, node, frame, finalize): + """Output extra source code before visiting a child of an + ``Output`` node. + """ + if frame.eval_ctx.volatile: + self.write("(escape if context.eval_ctx.autoescape else to_string)(") + elif frame.eval_ctx.autoescape: + self.write("escape(") else: - finalize = text_type + self.write("to_string(") - # if we are inside a frame that requires output checking, we do so - outdent_later = False + if finalize.src is not None: + self.write(finalize.src) + + def _output_child_post(self, node, frame, finalize): + """Output extra source code after visiting a child of an + ``Output`` node. + """ + self.write(")") + + if finalize.src is not None: + self.write(")") + + def visit_Output(self, node, frame): + # If an extends is active, don't render outside a block. if frame.require_output_check: - self.writeline('if parent_template is None:') - self.indent() - outdent_later = True + # A top-level extends is known to exist at compile time. + if self.has_known_extends: + return - # try to evaluate as many chunks as possible into a static - # string at compile time. + self.writeline("if parent_template is None:") + self.indent() + + finalize = self._make_finalize() body = [] + + # Evaluate constants at compile time if possible. Each item in + # body will be either a list of static data or a node to be + # evaluated at runtime. for child in node.nodes: try: - if not allow_constant_finalize: + if not ( + # If the finalize function requires runtime context, + # constants can't be evaluated at compile time. + finalize.const + # Unless it's basic template data that won't be + # finalized anyway. + or isinstance(child, nodes.TemplateData) + ): raise nodes.Impossible() - const = child.as_const(frame.eval_ctx) - except nodes.Impossible: - body.append(child) - continue - # the frame can't be volatile here, becaus otherwise the - # as_const() function would raise an Impossible exception - # at that point. - try: - if frame.eval_ctx.autoescape: - if hasattr(const, '__html__'): - const = const.__html__() - else: - const = escape(const) - const = finalize(const) - except Exception: - # if something goes wrong here we evaluate the node - # at runtime for easier debugging + + const = self._output_child_to_const(child, frame, finalize) + except (nodes.Impossible, Exception): + # The node was not constant and needs to be evaluated at + # runtime. Or another error was raised, which is easier + # to debug at runtime. body.append(child) continue + if body and isinstance(body[-1], list): body[-1].append(const) else: body.append([const]) - # if we have less than 3 nodes or a buffer we yield or extend/append - if len(body) < 3 or frame.buffer is not None: - if frame.buffer is not None: - # for one item we append, for more we extend - if len(body) == 1: - self.writeline('%s.append(' % frame.buffer) - else: - self.writeline('%s.extend((' % frame.buffer) - self.indent() - for item in body: - if isinstance(item, list): - val = repr(concat(item)) - if frame.buffer is None: - self.writeline('yield ' + val) - else: - self.writeline(val + ',') - else: - if frame.buffer is None: - self.writeline('yield ', item) - else: - self.newline(item) - close = 1 - if frame.eval_ctx.volatile: - self.write('(escape if context.eval_ctx.autoescape' - ' else to_string)(') - elif frame.eval_ctx.autoescape: - self.write('escape(') - else: - self.write('to_string(') - if self.environment.finalize is not None: - self.write('environment.finalize(') - if getattr(self.environment.finalize, - "contextfunction", False): - self.write('context, ') - close += 1 - self.visit(item, frame) - self.write(')' * close) - if frame.buffer is not None: - self.write(',') - if frame.buffer is not None: - # close the open parentheses - self.outdent() - self.writeline(len(body) == 1 and ')' or '))') + if frame.buffer is not None: + if len(body) == 1: + self.writeline("%s.append(" % frame.buffer) + else: + self.writeline("%s.extend((" % frame.buffer) - # otherwise we create a format string as this is faster in that case - else: - format = [] - arguments = [] - for item in body: - if isinstance(item, list): - format.append(concat(item).replace('%', '%%')) - else: - format.append('%s') - arguments.append(item) - self.writeline('yield ') - self.write(repr(concat(format)) + ' % (') self.indent() - for argument in arguments: - self.newline(argument) - close = 0 - if frame.eval_ctx.volatile: - self.write('(escape if context.eval_ctx.autoescape else' - ' to_string)(') - close += 1 - elif frame.eval_ctx.autoescape: - self.write('escape(') - close += 1 - if self.environment.finalize is not None: - self.write('environment.finalize(') - if getattr(self.environment.finalize, - 'contextfunction', False): - self.write('context, ') - elif getattr(self.environment.finalize, - 'evalcontextfunction', False): - self.write('context.eval_ctx, ') - elif getattr(self.environment.finalize, - 'environmentfunction', False): - self.write('environment, ') - close += 1 - self.visit(argument, frame) - self.write(')' * close + ', ') - self.outdent() - self.writeline(')') - if outdent_later: + for item in body: + if isinstance(item, list): + # A group of constant data to join and output. + val = self._output_const_repr(item) + + if frame.buffer is None: + self.writeline("yield " + val) + else: + self.writeline(val + ",") + else: + if frame.buffer is None: + self.writeline("yield ", item) + else: + self.newline(item) + + # A node to be evaluated at runtime. + self._output_child_pre(item, frame, finalize) + self.visit(item, frame) + self._output_child_post(item, frame, finalize) + + if frame.buffer is not None: + self.write(",") + + if frame.buffer is not None: + self.outdent() + self.writeline(")" if len(body) == 1 else "))") + + if frame.require_output_check: self.outdent() def visit_Assign(self, node, frame): self.push_assign_tracking() self.newline(node) self.visit(node.target, frame) - self.write(' = ') + self.write(" = ") self.visit(node.node, frame) self.pop_assign_tracking(frame) @@ -1384,20 +1471,19 @@ class CodeGenerator(NodeVisitor): self.blockvisit(node.body, block_frame) self.newline(node) self.visit(node.target, frame) - self.write(' = (Markup if context.eval_ctx.autoescape ' - 'else identity)(') + self.write(" = (Markup if context.eval_ctx.autoescape else identity)(") if node.filter is not None: self.visit_Filter(node.filter, block_frame) else: - self.write('concat(%s)' % block_frame.buffer) - self.write(')') + self.write("concat(%s)" % block_frame.buffer) + self.write(")") self.pop_assign_tracking(frame) self.leave_frame(block_frame) # -- Expression Visitors def visit_Name(self, node, frame): - if node.ctx == 'store' and frame.toplevel: + if node.ctx == "store" and frame.toplevel: if self._assign_stack: self._assign_stack[-1].add(node.name) ref = frame.symbols.ref(node.name) @@ -1405,12 +1491,17 @@ class CodeGenerator(NodeVisitor): # If we are looking up a variable we might have to deal with the # case where it's undefined. We can skip that case if the load # instruction indicates a parameter which are always defined. - if node.ctx == 'load': + if node.ctx == "load": load = frame.symbols.find_load(ref) - if not (load is not None and load[0] == VAR_LOAD_PARAMETER and \ - not self.parameter_is_undeclared(ref)): - self.write('(undefined(name=%r) if %s is missing else %s)' % - (node.name, ref, ref)) + if not ( + load is not None + and load[0] == VAR_LOAD_PARAMETER + and not self.parameter_is_undeclared(ref) + ): + self.write( + "(undefined(name=%r) if %s is missing else %s)" + % (node.name, ref, ref) + ) return self.write(ref) @@ -1420,12 +1511,14 @@ class CodeGenerator(NodeVisitor): # `foo.bar` notation they will be parsed as a normal attribute access # when used anywhere but in a `set` context ref = frame.symbols.ref(node.name) - self.writeline('if not isinstance(%s, Namespace):' % ref) + self.writeline("if not isinstance(%s, Namespace):" % ref) self.indent() - self.writeline('raise TemplateRuntimeError(%r)' % - 'cannot assign attribute on non-namespace object') + self.writeline( + "raise TemplateRuntimeError(%r)" + % "cannot assign attribute on non-namespace object" + ) self.outdent() - self.writeline('%s[%r]' % (ref, node.attr)) + self.writeline("%s[%r]" % (ref, node.attr)) def visit_Const(self, node, frame): val = node.as_const(frame.eval_ctx) @@ -1438,230 +1531,256 @@ class CodeGenerator(NodeVisitor): try: self.write(repr(node.as_const(frame.eval_ctx))) except nodes.Impossible: - self.write('(Markup if context.eval_ctx.autoescape else identity)(%r)' - % node.data) + self.write( + "(Markup if context.eval_ctx.autoescape else identity)(%r)" % node.data + ) def visit_Tuple(self, node, frame): - self.write('(') + self.write("(") idx = -1 for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item, frame) - self.write(idx == 0 and ',)' or ')') + self.write(idx == 0 and ",)" or ")") def visit_List(self, node, frame): - self.write('[') + self.write("[") for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item, frame) - self.write(']') + self.write("]") def visit_Dict(self, node, frame): - self.write('{') + self.write("{") for idx, item in enumerate(node.items): if idx: - self.write(', ') + self.write(", ") self.visit(item.key, frame) - self.write(': ') + self.write(": ") self.visit(item.value, frame) - self.write('}') + self.write("}") - def binop(operator, interceptable=True): + def binop(operator, interceptable=True): # noqa: B902 @optimizeconst def visitor(self, node, frame): - if self.environment.sandboxed and \ - operator in self.environment.intercepted_binops: - self.write('environment.call_binop(context, %r, ' % operator) + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_binops + ): + self.write("environment.call_binop(context, %r, " % operator) self.visit(node.left, frame) - self.write(', ') + self.write(", ") self.visit(node.right, frame) else: - self.write('(') + self.write("(") self.visit(node.left, frame) - self.write(' %s ' % operator) + self.write(" %s " % operator) self.visit(node.right, frame) - self.write(')') + self.write(")") + return visitor - def uaop(operator, interceptable=True): + def uaop(operator, interceptable=True): # noqa: B902 @optimizeconst def visitor(self, node, frame): - if self.environment.sandboxed and \ - operator in self.environment.intercepted_unops: - self.write('environment.call_unop(context, %r, ' % operator) + if ( + self.environment.sandboxed + and operator in self.environment.intercepted_unops + ): + self.write("environment.call_unop(context, %r, " % operator) self.visit(node.node, frame) else: - self.write('(' + operator) + self.write("(" + operator) self.visit(node.node, frame) - self.write(')') + self.write(")") + return visitor - visit_Add = binop('+') - visit_Sub = binop('-') - visit_Mul = binop('*') - visit_Div = binop('/') - visit_FloorDiv = binop('//') - visit_Pow = binop('**') - visit_Mod = binop('%') - visit_And = binop('and', interceptable=False) - visit_Or = binop('or', interceptable=False) - visit_Pos = uaop('+') - visit_Neg = uaop('-') - visit_Not = uaop('not ', interceptable=False) + visit_Add = binop("+") + visit_Sub = binop("-") + visit_Mul = binop("*") + visit_Div = binop("/") + visit_FloorDiv = binop("//") + visit_Pow = binop("**") + visit_Mod = binop("%") + visit_And = binop("and", interceptable=False) + visit_Or = binop("or", interceptable=False) + visit_Pos = uaop("+") + visit_Neg = uaop("-") + visit_Not = uaop("not ", interceptable=False) del binop, uaop @optimizeconst def visit_Concat(self, node, frame): if frame.eval_ctx.volatile: - func_name = '(context.eval_ctx.volatile and' \ - ' markup_join or unicode_join)' + func_name = "(context.eval_ctx.volatile and markup_join or unicode_join)" elif frame.eval_ctx.autoescape: - func_name = 'markup_join' + func_name = "markup_join" else: - func_name = 'unicode_join' - self.write('%s((' % func_name) + func_name = "unicode_join" + self.write("%s((" % func_name) for arg in node.nodes: self.visit(arg, frame) - self.write(', ') - self.write('))') + self.write(", ") + self.write("))") @optimizeconst def visit_Compare(self, node, frame): + self.write("(") self.visit(node.expr, frame) for op in node.ops: self.visit(op, frame) + self.write(")") def visit_Operand(self, node, frame): - self.write(' %s ' % operators[node.op]) + self.write(" %s " % operators[node.op]) self.visit(node.expr, frame) @optimizeconst def visit_Getattr(self, node, frame): - self.write('environment.getattr(') + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getattr(") self.visit(node.node, frame) - self.write(', %r)' % node.attr) + self.write(", %r)" % node.attr) + + if self.environment.is_async: + self.write("))") @optimizeconst def visit_Getitem(self, node, frame): # slices bypass the environment getitem method. if isinstance(node.arg, nodes.Slice): self.visit(node.node, frame) - self.write('[') + self.write("[") self.visit(node.arg, frame) - self.write(']') + self.write("]") else: - self.write('environment.getitem(') + if self.environment.is_async: + self.write("(await auto_await(") + + self.write("environment.getitem(") self.visit(node.node, frame) - self.write(', ') + self.write(", ") self.visit(node.arg, frame) - self.write(')') + self.write(")") + + if self.environment.is_async: + self.write("))") def visit_Slice(self, node, frame): if node.start is not None: self.visit(node.start, frame) - self.write(':') + self.write(":") if node.stop is not None: self.visit(node.stop, frame) if node.step is not None: - self.write(':') + self.write(":") self.visit(node.step, frame) @optimizeconst def visit_Filter(self, node, frame): if self.environment.is_async: - self.write('await auto_await(') - self.write(self.filters[node.name] + '(') + self.write("await auto_await(") + self.write(self.filters[node.name] + "(") func = self.environment.filters.get(node.name) if func is None: - self.fail('no filter named %r' % node.name, node.lineno) - if getattr(func, 'contextfilter', False): - self.write('context, ') - elif getattr(func, 'evalcontextfilter', False): - self.write('context.eval_ctx, ') - elif getattr(func, 'environmentfilter', False): - self.write('environment, ') + self.fail("no filter named %r" % node.name, node.lineno) + if getattr(func, "contextfilter", False): + self.write("context, ") + elif getattr(func, "evalcontextfilter", False): + self.write("context.eval_ctx, ") + elif getattr(func, "environmentfilter", False): + self.write("environment, ") # if the filter node is None we are inside a filter block # and want to write to the current buffer if node.node is not None: self.visit(node.node, frame) elif frame.eval_ctx.volatile: - self.write('(context.eval_ctx.autoescape and' - ' Markup(concat(%s)) or concat(%s))' % - (frame.buffer, frame.buffer)) + self.write( + "(context.eval_ctx.autoescape and" + " Markup(concat(%s)) or concat(%s))" % (frame.buffer, frame.buffer) + ) elif frame.eval_ctx.autoescape: - self.write('Markup(concat(%s))' % frame.buffer) + self.write("Markup(concat(%s))" % frame.buffer) else: - self.write('concat(%s)' % frame.buffer) + self.write("concat(%s)" % frame.buffer) self.signature(node, frame) - self.write(')') + self.write(")") if self.environment.is_async: - self.write(')') + self.write(")") @optimizeconst def visit_Test(self, node, frame): - self.write(self.tests[node.name] + '(') + self.write(self.tests[node.name] + "(") if node.name not in self.environment.tests: - self.fail('no test named %r' % node.name, node.lineno) + self.fail("no test named %r" % node.name, node.lineno) self.visit(node.node, frame) self.signature(node, frame) - self.write(')') + self.write(")") @optimizeconst def visit_CondExpr(self, node, frame): def write_expr2(): if node.expr2 is not None: return self.visit(node.expr2, frame) - self.write('undefined(%r)' % ('the inline if-' - 'expression on %s evaluated to false and ' - 'no else section was defined.' % self.position(node))) + self.write( + "cond_expr_undefined(%r)" + % ( + "the inline if-" + "expression on %s evaluated to false and " + "no else section was defined." % self.position(node) + ) + ) - self.write('(') + self.write("(") self.visit(node.expr1, frame) - self.write(' if ') + self.write(" if ") self.visit(node.test, frame) - self.write(' else ') + self.write(" else ") write_expr2() - self.write(')') + self.write(")") @optimizeconst def visit_Call(self, node, frame, forward_caller=False): if self.environment.is_async: - self.write('await auto_await(') + self.write("await auto_await(") if self.environment.sandboxed: - self.write('environment.call(context, ') + self.write("environment.call(context, ") else: - self.write('context.call(') + self.write("context.call(") self.visit(node.node, frame) - extra_kwargs = forward_caller and {'caller': 'caller'} or None + extra_kwargs = forward_caller and {"caller": "caller"} or None self.signature(node, frame, extra_kwargs) - self.write(')') + self.write(")") if self.environment.is_async: - self.write(')') + self.write(")") def visit_Keyword(self, node, frame): - self.write(node.key + '=') + self.write(node.key + "=") self.visit(node.value, frame) # -- Unused nodes for extensions def visit_MarkSafe(self, node, frame): - self.write('Markup(') + self.write("Markup(") self.visit(node.expr, frame) - self.write(')') + self.write(")") def visit_MarkSafeIfAutoescape(self, node, frame): - self.write('(context.eval_ctx.autoescape and Markup or identity)(') + self.write("(context.eval_ctx.autoescape and Markup or identity)(") self.visit(node.expr, frame) - self.write(')') + self.write(")") def visit_EnvironmentAttribute(self, node, frame): - self.write('environment.' + node.name) + self.write("environment." + node.name) def visit_ExtensionAttribute(self, node, frame): - self.write('environment.extensions[%r].%s' % (node.identifier, node.name)) + self.write("environment.extensions[%r].%s" % (node.identifier, node.name)) def visit_ImportedName(self, node, frame): self.write(self.import_aliases[node.importname]) @@ -1670,13 +1789,16 @@ class CodeGenerator(NodeVisitor): self.write(node.name) def visit_ContextReference(self, node, frame): - self.write('context') + self.write("context") + + def visit_DerivedContextReference(self, node, frame): + self.write(self.derive_context(frame)) def visit_Continue(self, node, frame): - self.writeline('continue', node) + self.writeline("continue", node) def visit_Break(self, node, frame): - self.writeline('break', node) + self.writeline("break", node) def visit_Scope(self, node, frame): scope_frame = frame.inner() @@ -1687,8 +1809,8 @@ class CodeGenerator(NodeVisitor): def visit_OverlayScope(self, node, frame): ctx = self.temporary_identifier() - self.writeline('%s = %s' % (ctx, self.derive_context(frame))) - self.writeline('%s.vars = ' % ctx) + self.writeline("%s = %s" % (ctx, self.derive_context(frame))) + self.writeline("%s.vars = " % ctx) self.visit(node.context, frame) self.push_context_reference(ctx) @@ -1701,7 +1823,7 @@ class CodeGenerator(NodeVisitor): def visit_EvalContextModifier(self, node, frame): for keyword in node.options: - self.writeline('context.eval_ctx.%s = ' % keyword.key) + self.writeline("context.eval_ctx.%s = " % keyword.key) self.visit(keyword.value, frame) try: val = keyword.value.as_const(frame.eval_ctx) @@ -1713,9 +1835,9 @@ class CodeGenerator(NodeVisitor): def visit_ScopedEvalContextModifier(self, node, frame): old_ctx_name = self.temporary_identifier() saved_ctx = frame.eval_ctx.save() - self.writeline('%s = context.eval_ctx.save()' % old_ctx_name) + self.writeline("%s = context.eval_ctx.save()" % old_ctx_name) self.visit_EvalContextModifier(node, frame) for child in node.body: self.visit(child, frame) frame.eval_ctx.revert(saved_ctx) - self.writeline('context.eval_ctx.revert(%s)' % old_ctx_name) + self.writeline("context.eval_ctx.revert(%s)" % old_ctx_name) diff --git a/pipenv/vendor/jinja2/constants.py b/pipenv/vendor/jinja2/constants.py index 11efd1ed..bf7f2ca7 100644 --- a/pipenv/vendor/jinja2/constants.py +++ b/pipenv/vendor/jinja2/constants.py @@ -1,17 +1,6 @@ # -*- coding: utf-8 -*- -""" - jinja.constants - ~~~~~~~~~~~~~~~ - - Various constants. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" - - #: list of lorem ipsum words used by the lipsum() helper function -LOREM_IPSUM_WORDS = u'''\ +LOREM_IPSUM_WORDS = u"""\ a ac accumsan ad adipiscing aenean aliquam aliquet amet ante aptent arcu at auctor augue bibendum blandit class commodo condimentum congue consectetuer consequat conubia convallis cras cubilia cum curabitur curae cursus dapibus @@ -29,4 +18,4 @@ ridiculus risus rutrum sagittis sapien scelerisque sed sem semper senectus sit sociis sociosqu sodales sollicitudin suscipit suspendisse taciti tellus tempor tempus tincidunt torquent tortor tristique turpis ullamcorper ultrices ultricies urna ut varius vehicula vel velit venenatis vestibulum vitae vivamus -viverra volutpat vulputate''' +viverra volutpat vulputate""" diff --git a/pipenv/vendor/jinja2/debug.py b/pipenv/vendor/jinja2/debug.py index b61139f0..d2c5a06b 100644 --- a/pipenv/vendor/jinja2/debug.py +++ b/pipenv/vendor/jinja2/debug.py @@ -1,372 +1,271 @@ -# -*- coding: utf-8 -*- -""" - jinja2.debug - ~~~~~~~~~~~~ - - Implements the debug interface for Jinja. This module does some pretty - ugly stuff with the Python traceback system in order to achieve tracebacks - with correct line numbers, locals and contents. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" import sys -import traceback -from types import TracebackType, CodeType -from jinja2.utils import missing, internal_code -from jinja2.exceptions import TemplateSyntaxError -from jinja2._compat import iteritems, reraise, PY2 +from types import CodeType -# on pypy we can take advantage of transparent proxies -try: - from __pypy__ import tproxy -except ImportError: - tproxy = None +from . import TemplateSyntaxError +from ._compat import PYPY +from .utils import internal_code +from .utils import missing -# how does the raise helper look like? -try: - exec("raise TypeError, 'foo'") -except SyntaxError: - raise_helper = 'raise __jinja_exception__[1]' -except TypeError: - raise_helper = 'raise __jinja_exception__[0], __jinja_exception__[1]' +def rewrite_traceback_stack(source=None): + """Rewrite the current exception to replace any tracebacks from + within compiled template code with tracebacks that look like they + came from the template source. + This must be called within an ``except`` block. -class TracebackFrameProxy(object): - """Proxies a traceback frame.""" - - def __init__(self, tb): - self.tb = tb - self._tb_next = None - - @property - def tb_next(self): - return self._tb_next - - def set_next(self, next): - if tb_set_next is not None: - try: - tb_set_next(self.tb, next and next.tb or None) - except Exception: - # this function can fail due to all the hackery it does - # on various python implementations. We just catch errors - # down and ignore them if necessary. - pass - self._tb_next = next - - @property - def is_jinja_frame(self): - return '__jinja_template__' in self.tb.tb_frame.f_globals - - def __getattr__(self, name): - return getattr(self.tb, name) - - -def make_frame_proxy(frame): - proxy = TracebackFrameProxy(frame) - if tproxy is None: - return proxy - def operation_handler(operation, *args, **kwargs): - if operation in ('__getattribute__', '__getattr__'): - return getattr(proxy, args[0]) - elif operation == '__setattr__': - proxy.__setattr__(*args, **kwargs) - else: - return getattr(proxy, operation)(*args, **kwargs) - return tproxy(TracebackType, operation_handler) - - -class ProcessedTraceback(object): - """Holds a Jinja preprocessed traceback for printing or reraising.""" - - def __init__(self, exc_type, exc_value, frames): - assert frames, 'no frames for this traceback?' - self.exc_type = exc_type - self.exc_value = exc_value - self.frames = frames - - # newly concatenate the frames (which are proxies) - prev_tb = None - for tb in self.frames: - if prev_tb is not None: - prev_tb.set_next(tb) - prev_tb = tb - prev_tb.set_next(None) - - def render_as_text(self, limit=None): - """Return a string with the traceback.""" - lines = traceback.format_exception(self.exc_type, self.exc_value, - self.frames[0], limit=limit) - return ''.join(lines).rstrip() - - def render_as_html(self, full=False): - """Return a unicode string with the traceback as rendered HTML.""" - from jinja2.debugrenderer import render_traceback - return u'%s\n\n<!--\n%s\n-->' % ( - render_traceback(self, full=full), - self.render_as_text().decode('utf-8', 'replace') - ) - - @property - def is_template_syntax_error(self): - """`True` if this is a template syntax error.""" - return isinstance(self.exc_value, TemplateSyntaxError) - - @property - def exc_info(self): - """Exception info tuple with a proxy around the frame objects.""" - return self.exc_type, self.exc_value, self.frames[0] - - @property - def standard_exc_info(self): - """Standard python exc_info for re-raising""" - tb = self.frames[0] - # the frame will be an actual traceback (or transparent proxy) if - # we are on pypy or a python implementation with support for tproxy - if type(tb) is not TracebackType: - tb = tb.tb - return self.exc_type, self.exc_value, tb - - -def make_traceback(exc_info, source_hint=None): - """Creates a processed traceback object from the exc_info.""" - exc_type, exc_value, tb = exc_info - if isinstance(exc_value, TemplateSyntaxError): - exc_info = translate_syntax_error(exc_value, source_hint) - initial_skip = 0 - else: - initial_skip = 1 - return translate_exception(exc_info, initial_skip) - - -def translate_syntax_error(error, source=None): - """Rewrites a syntax error to please traceback systems.""" - error.source = source - error.translated = True - exc_info = (error.__class__, error, None) - filename = error.filename - if filename is None: - filename = '<unknown>' - return fake_exc_info(exc_info, filename, error.lineno) - - -def translate_exception(exc_info, initial_skip=0): - """If passed an exc_info it will automatically rewrite the exceptions - all the way down to the correct line numbers and frames. + :param exc_info: A :meth:`sys.exc_info` tuple. If not provided, + the current ``exc_info`` is used. + :param source: For ``TemplateSyntaxError``, the original source if + known. + :return: A :meth:`sys.exc_info` tuple that can be re-raised. """ - tb = exc_info[2] - frames = [] + exc_type, exc_value, tb = sys.exc_info() - # skip some internal frames if wanted - for x in range(initial_skip): - if tb is not None: - tb = tb.tb_next - initial_tb = tb + if isinstance(exc_value, TemplateSyntaxError) and not exc_value.translated: + exc_value.translated = True + exc_value.source = source + try: + # Remove the old traceback on Python 3, otherwise the frames + # from the compiler still show up. + exc_value.with_traceback(None) + except AttributeError: + pass + + # Outside of runtime, so the frame isn't executing template + # code, but it still needs to point at the template. + tb = fake_traceback( + exc_value, None, exc_value.filename or "<unknown>", exc_value.lineno + ) + else: + # Skip the frame for the render function. + tb = tb.tb_next + + stack = [] + + # Build the stack of traceback object, replacing any in template + # code with the source file and line information. while tb is not None: - # skip frames decorated with @internalcode. These are internal - # calls we can't avoid and that are useless in template debugging - # output. + # Skip frames decorated with @internalcode. These are internal + # calls that aren't useful in template debugging output. if tb.tb_frame.f_code in internal_code: tb = tb.tb_next continue - # save a reference to the next frame if we override the current - # one with a faked one. - next = tb.tb_next + template = tb.tb_frame.f_globals.get("__jinja_template__") - # fake template exceptions - template = tb.tb_frame.f_globals.get('__jinja_template__') if template is not None: lineno = template.get_corresponding_lineno(tb.tb_lineno) - tb = fake_exc_info(exc_info[:2] + (tb,), template.filename, - lineno)[2] + fake_tb = fake_traceback(exc_value, tb, template.filename, lineno) + stack.append(fake_tb) + else: + stack.append(tb) - frames.append(make_frame_proxy(tb)) - tb = next + tb = tb.tb_next - # if we don't have any exceptions in the frames left, we have to - # reraise it unchanged. - # XXX: can we backup here? when could this happen? - if not frames: - reraise(exc_info[0], exc_info[1], exc_info[2]) + tb_next = None - return ProcessedTraceback(exc_info[0], exc_info[1], frames) + # Assign tb_next in reverse to avoid circular references. + for tb in reversed(stack): + tb_next = tb_set_next(tb, tb_next) + + return exc_type, exc_value, tb_next -def get_jinja_locals(real_locals): - ctx = real_locals.get('context') - if ctx: - locals = ctx.get_all().copy() +def fake_traceback(exc_value, tb, filename, lineno): + """Produce a new traceback object that looks like it came from the + template source instead of the compiled code. The filename, line + number, and location name will point to the template, and the local + variables will be the current template context. + + :param exc_value: The original exception to be re-raised to create + the new traceback. + :param tb: The original traceback to get the local variables and + code info from. + :param filename: The template filename. + :param lineno: The line number in the template source. + """ + if tb is not None: + # Replace the real locals with the context that would be + # available at that point in the template. + locals = get_template_locals(tb.tb_frame.f_locals) + locals.pop("__jinja_exception__", None) else: locals = {} + globals = { + "__name__": filename, + "__file__": filename, + "__jinja_exception__": exc_value, + } + # Raise an exception at the correct line number. + code = compile("\n" * (lineno - 1) + "raise __jinja_exception__", filename, "exec") + + # Build a new code object that points to the template file and + # replaces the location with a block name. + try: + location = "template" + + if tb is not None: + function = tb.tb_frame.f_code.co_name + + if function == "root": + location = "top-level template code" + elif function.startswith("block_"): + location = 'block "%s"' % function[6:] + + # Collect arguments for the new code object. CodeType only + # accepts positional arguments, and arguments were inserted in + # new Python versions. + code_args = [] + + for attr in ( + "argcount", + "posonlyargcount", # Python 3.8 + "kwonlyargcount", # Python 3 + "nlocals", + "stacksize", + "flags", + "code", # codestring + "consts", # constants + "names", + "varnames", + ("filename", filename), + ("name", location), + "firstlineno", + "lnotab", + "freevars", + "cellvars", + ): + if isinstance(attr, tuple): + # Replace with given value. + code_args.append(attr[1]) + continue + + try: + # Copy original value if it exists. + code_args.append(getattr(code, "co_" + attr)) + except AttributeError: + # Some arguments were added later. + continue + + code = CodeType(*code_args) + except Exception: + # Some environments such as Google App Engine don't support + # modifying code objects. + pass + + # Execute the new code, which is guaranteed to raise, and return + # the new traceback without this frame. + try: + exec(code, globals, locals) + except BaseException: + return sys.exc_info()[2].tb_next + + +def get_template_locals(real_locals): + """Based on the runtime locals, get the context that would be + available at that point in the template. + """ + # Start with the current template context. + ctx = real_locals.get("context") + + if ctx: + data = ctx.get_all().copy() + else: + data = {} + + # Might be in a derived context that only sets local variables + # rather than pushing a context. Local variables follow the scheme + # l_depth_name. Find the highest-depth local that has a value for + # each name. local_overrides = {} - for name, value in iteritems(real_locals): - if not name.startswith('l_') or value is missing: + for name, value in real_locals.items(): + if not name.startswith("l_") or value is missing: + # Not a template variable, or no longer relevant. continue + try: - _, depth, name = name.split('_', 2) + _, depth, name = name.split("_", 2) depth = int(depth) except ValueError: continue + cur_depth = local_overrides.get(name, (-1,))[0] + if cur_depth < depth: local_overrides[name] = (depth, value) - for name, (_, value) in iteritems(local_overrides): + # Modify the context with any derived context. + for name, (_, value) in local_overrides.items(): if value is missing: - locals.pop(name, None) + data.pop(name, None) else: - locals[name] = value + data[name] = value - return locals + return data -def fake_exc_info(exc_info, filename, lineno): - """Helper for `translate_exception`.""" - exc_type, exc_value, tb = exc_info +if sys.version_info >= (3, 7): + # tb_next is directly assignable as of Python 3.7 + def tb_set_next(tb, tb_next): + tb.tb_next = tb_next + return tb - # figure the real context out - if tb is not None: - locals = get_jinja_locals(tb.tb_frame.f_locals) - # if there is a local called __jinja_exception__, we get - # rid of it to not break the debug functionality. - locals.pop('__jinja_exception__', None) +elif PYPY: + # PyPy might have special support, and won't work with ctypes. + try: + import tputil + except ImportError: + # Without tproxy support, use the original traceback. + def tb_set_next(tb, tb_next): + return tb + else: - locals = {} + # With tproxy support, create a proxy around the traceback that + # returns the new tb_next. + def tb_set_next(tb, tb_next): + def controller(op): + if op.opname == "__getattribute__" and op.args[0] == "tb_next": + return tb_next - # assamble fake globals we need - globals = { - '__name__': filename, - '__file__': filename, - '__jinja_exception__': exc_info[:2], + return op.delegate() - # we don't want to keep the reference to the template around - # to not cause circular dependencies, but we mark it as Jinja - # frame for the ProcessedTraceback - '__jinja_template__': None - } - - # and fake the exception - code = compile('\n' * (lineno - 1) + raise_helper, filename, 'exec') - - # if it's possible, change the name of the code. This won't work - # on some python environments such as google appengine - try: - if tb is None: - location = 'template' - else: - function = tb.tb_frame.f_code.co_name - if function == 'root': - location = 'top-level template code' - elif function.startswith('block_'): - location = 'block "%s"' % function[6:] - else: - location = 'template' - - if PY2: - code = CodeType(0, code.co_nlocals, code.co_stacksize, - code.co_flags, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - location, code.co_firstlineno, - code.co_lnotab, (), ()) - else: - code = CodeType(0, code.co_kwonlyargcount, - code.co_nlocals, code.co_stacksize, - code.co_flags, code.co_code, code.co_consts, - code.co_names, code.co_varnames, filename, - location, code.co_firstlineno, - code.co_lnotab, (), ()) - except Exception as e: - pass - - # execute the code and catch the new traceback - try: - exec(code, globals, locals) - except: - exc_info = sys.exc_info() - new_tb = exc_info[2].tb_next - - # return without this frame - return exc_info[:2] + (new_tb,) + return tputil.make_proxy(controller, obj=tb) -def _init_ugly_crap(): - """This function implements a few ugly things so that we can patch the - traceback objects. The function returned allows resetting `tb_next` on - any python traceback object. Do not attempt to use this on non cpython - interpreters - """ +else: + # Use ctypes to assign tb_next at the C level since it's read-only + # from Python. import ctypes - from types import TracebackType - if PY2: - # figure out size of _Py_ssize_t for Python 2: - if hasattr(ctypes.pythonapi, 'Py_InitModule4_64'): - _Py_ssize_t = ctypes.c_int64 - else: - _Py_ssize_t = ctypes.c_int - else: - # platform ssize_t on Python 3 - _Py_ssize_t = ctypes.c_ssize_t - - # regular python - class _PyObject(ctypes.Structure): - pass - _PyObject._fields_ = [ - ('ob_refcnt', _Py_ssize_t), - ('ob_type', ctypes.POINTER(_PyObject)) - ] - - # python with trace - if hasattr(sys, 'getobjects'): - class _PyObject(ctypes.Structure): - pass - _PyObject._fields_ = [ - ('_ob_next', ctypes.POINTER(_PyObject)), - ('_ob_prev', ctypes.POINTER(_PyObject)), - ('ob_refcnt', _Py_ssize_t), - ('ob_type', ctypes.POINTER(_PyObject)) + class _CTraceback(ctypes.Structure): + _fields_ = [ + # Extra PyObject slots when compiled with Py_TRACE_REFS. + ( + "PyObject_HEAD", + ctypes.c_byte * (32 if hasattr(sys, "getobjects") else 16), + ), + # Only care about tb_next as an object, not a traceback. + ("tb_next", ctypes.py_object), ] - class _Traceback(_PyObject): - pass - _Traceback._fields_ = [ - ('tb_next', ctypes.POINTER(_Traceback)), - ('tb_frame', ctypes.POINTER(_PyObject)), - ('tb_lasti', ctypes.c_int), - ('tb_lineno', ctypes.c_int) - ] + def tb_set_next(tb, tb_next): + c_tb = _CTraceback.from_address(id(tb)) - def tb_set_next(tb, next): - """Set the tb_next attribute of a traceback object.""" - if not (isinstance(tb, TracebackType) and - (next is None or isinstance(next, TracebackType))): - raise TypeError('tb_set_next arguments must be traceback objects') - obj = _Traceback.from_address(id(tb)) + # Clear out the old tb_next. if tb.tb_next is not None: - old = _Traceback.from_address(id(tb.tb_next)) - old.ob_refcnt -= 1 - if next is None: - obj.tb_next = ctypes.POINTER(_Traceback)() - else: - next = _Traceback.from_address(id(next)) - next.ob_refcnt += 1 - obj.tb_next = ctypes.pointer(next) + c_tb_next = ctypes.py_object(tb.tb_next) + c_tb.tb_next = ctypes.py_object() + ctypes.pythonapi.Py_DecRef(c_tb_next) - return tb_set_next + # Assign the new tb_next. + if tb_next is not None: + c_tb_next = ctypes.py_object(tb_next) + ctypes.pythonapi.Py_IncRef(c_tb_next) + c_tb.tb_next = c_tb_next - -# try to get a tb_set_next implementation if we don't have transparent -# proxies. -tb_set_next = None -if tproxy is None: - try: - tb_set_next = _init_ugly_crap() - except: - pass - del _init_ugly_crap + return tb diff --git a/pipenv/vendor/jinja2/defaults.py b/pipenv/vendor/jinja2/defaults.py index 7c93dec0..8e0e7d77 100644 --- a/pipenv/vendor/jinja2/defaults.py +++ b/pipenv/vendor/jinja2/defaults.py @@ -1,56 +1,44 @@ # -*- coding: utf-8 -*- -""" - jinja2.defaults - ~~~~~~~~~~~~~~~ - - Jinja default filters and tags. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from jinja2._compat import range_type -from jinja2.utils import generate_lorem_ipsum, Cycler, Joiner, Namespace - +from ._compat import range_type +from .filters import FILTERS as DEFAULT_FILTERS # noqa: F401 +from .tests import TESTS as DEFAULT_TESTS # noqa: F401 +from .utils import Cycler +from .utils import generate_lorem_ipsum +from .utils import Joiner +from .utils import Namespace # defaults for the parser / lexer -BLOCK_START_STRING = '{%' -BLOCK_END_STRING = '%}' -VARIABLE_START_STRING = '{{' -VARIABLE_END_STRING = '}}' -COMMENT_START_STRING = '{#' -COMMENT_END_STRING = '#}' +BLOCK_START_STRING = "{%" +BLOCK_END_STRING = "%}" +VARIABLE_START_STRING = "{{" +VARIABLE_END_STRING = "}}" +COMMENT_START_STRING = "{#" +COMMENT_END_STRING = "#}" LINE_STATEMENT_PREFIX = None LINE_COMMENT_PREFIX = None TRIM_BLOCKS = False LSTRIP_BLOCKS = False -NEWLINE_SEQUENCE = '\n' +NEWLINE_SEQUENCE = "\n" KEEP_TRAILING_NEWLINE = False - # default filters, tests and namespace -from jinja2.filters import FILTERS as DEFAULT_FILTERS -from jinja2.tests import TESTS as DEFAULT_TESTS -DEFAULT_NAMESPACE = { - 'range': range_type, - 'dict': dict, - 'lipsum': generate_lorem_ipsum, - 'cycler': Cycler, - 'joiner': Joiner, - 'namespace': Namespace -} +DEFAULT_NAMESPACE = { + "range": range_type, + "dict": dict, + "lipsum": generate_lorem_ipsum, + "cycler": Cycler, + "joiner": Joiner, + "namespace": Namespace, +} # default policies DEFAULT_POLICIES = { - 'compiler.ascii_str': True, - 'urlize.rel': 'noopener', - 'urlize.target': None, - 'truncate.leeway': 5, - 'json.dumps_function': None, - 'json.dumps_kwargs': {'sort_keys': True}, - 'ext.i18n.trimmed': False, + "compiler.ascii_str": True, + "urlize.rel": "noopener", + "urlize.target": None, + "truncate.leeway": 5, + "json.dumps_function": None, + "json.dumps_kwargs": {"sort_keys": True}, + "ext.i18n.trimmed": False, } - - -# export all constants -__all__ = tuple(x for x in locals().keys() if x.isupper()) diff --git a/pipenv/vendor/jinja2/environment.py b/pipenv/vendor/jinja2/environment.py index 549d9afa..bf44b9de 100644 --- a/pipenv/vendor/jinja2/environment.py +++ b/pipenv/vendor/jinja2/environment.py @@ -1,60 +1,83 @@ # -*- coding: utf-8 -*- -""" - jinja2.environment - ~~~~~~~~~~~~~~~~~~ - - Provides a class that holds runtime and parsing time options. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. +"""Classes for managing templates and their runtime and compile time +options. """ import os import sys import weakref -from functools import reduce, partial -from jinja2 import nodes -from jinja2.defaults import BLOCK_START_STRING, \ - BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ - COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ - LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ - DEFAULT_FILTERS, DEFAULT_TESTS, DEFAULT_NAMESPACE, \ - DEFAULT_POLICIES, KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS -from jinja2.lexer import get_lexer, TokenStream -from jinja2.parser import Parser -from jinja2.nodes import EvalContext -from jinja2.compiler import generate, CodeGenerator -from jinja2.runtime import Undefined, new_context, Context -from jinja2.exceptions import TemplateSyntaxError, TemplateNotFound, \ - TemplatesNotFound, TemplateRuntimeError -from jinja2.utils import import_string, LRUCache, Markup, missing, \ - concat, consume, internalcode, have_async_gen -from jinja2._compat import imap, ifilter, string_types, iteritems, \ - text_type, reraise, implements_iterator, implements_to_string, \ - encode_filename, PY2, PYPY +from functools import partial +from functools import reduce +from markupsafe import Markup + +from . import nodes +from ._compat import encode_filename +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import PYPY +from ._compat import reraise +from ._compat import string_types +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import generate +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import DEFAULT_FILTERS +from .defaults import DEFAULT_NAMESPACE +from .defaults import DEFAULT_POLICIES +from .defaults import DEFAULT_TESTS +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .exceptions import TemplateNotFound +from .exceptions import TemplateRuntimeError +from .exceptions import TemplatesNotFound +from .exceptions import TemplateSyntaxError +from .exceptions import UndefinedError +from .lexer import get_lexer +from .lexer import TokenStream +from .nodes import EvalContext +from .parser import Parser +from .runtime import Context +from .runtime import new_context +from .runtime import Undefined +from .utils import concat +from .utils import consume +from .utils import have_async_gen +from .utils import import_string +from .utils import internalcode +from .utils import LRUCache +from .utils import missing # for direct template usage we have up to ten living environments _spontaneous_environments = LRUCache(10) -# the function to create jinja traceback objects. This is dynamically -# imported on the first exception in the exception handler. -_make_traceback = None +def get_spontaneous_environment(cls, *args): + """Return a new spontaneous environment. A spontaneous environment + is used for templates created directly rather than through an + existing environment. -def get_spontaneous_environment(*args): - """Return a new spontaneous environment. A spontaneous environment is an - unnamed and unaccessible (in theory) environment that is used for - templates generated from a string and not from the file system. + :param cls: Environment class to create. + :param args: Positional arguments passed to environment. """ + key = (cls, args) + try: - env = _spontaneous_environments.get(args) - except TypeError: - return Environment(*args) - if env is not None: + return _spontaneous_environments[key] + except KeyError: + _spontaneous_environments[key] = env = cls(*args) + env.shared = True return env - _spontaneous_environments[args] = env = Environment(*args) - env.shared = True - return env def create_cache(size): @@ -93,20 +116,25 @@ def fail_for_missing_callable(string, name): try: name._fail_with_undefined_error() except Exception as e: - msg = '%s (%s; did you forget to quote the callable name?)' % (msg, e) + msg = "%s (%s; did you forget to quote the callable name?)" % (msg, e) raise TemplateRuntimeError(msg) def _environment_sanity_check(environment): """Perform a sanity check on the environment.""" - assert issubclass(environment.undefined, Undefined), 'undefined must ' \ - 'be a subclass of undefined because filters depend on it.' - assert environment.block_start_string != \ - environment.variable_start_string != \ - environment.comment_start_string, 'block, variable and comment ' \ - 'start strings must be different' - assert environment.newline_sequence in ('\r', '\r\n', '\n'), \ - 'newline_sequence set to unknown line ending string.' + assert issubclass( + environment.undefined, Undefined + ), "undefined must be a subclass of undefined because filters depend on it." + assert ( + environment.block_start_string + != environment.variable_start_string + != environment.comment_start_string + ), "block, variable and comment start strings must be different" + assert environment.newline_sequence in ( + "\r", + "\r\n", + "\n", + ), "newline_sequence set to unknown line ending string." return environment @@ -191,7 +219,7 @@ class Environment(object): `autoescape` If set to ``True`` the XML/HTML autoescaping feature is enabled by default. For more details about autoescaping see - :class:`~jinja2.utils.Markup`. As of Jinja 2.4 this can also + :class:`~markupsafe.Markup`. As of Jinja 2.4 this can also be a callable that is passed the template name and has to return ``True`` or ``False`` depending on autoescape should be enabled by default. @@ -249,10 +277,6 @@ class Environment(object): #: must not be modified shared = False - #: these are currently EXPERIMENTAL undocumented features. - exception_handler = None - exception_formatter = None - #: the class that is used for code generation. See #: :class:`~jinja2.compiler.CodeGenerator` for more information. code_generator_class = CodeGenerator @@ -261,29 +285,31 @@ class Environment(object): #: :class:`~jinja2.runtime.Context` for more information. context_class = Context - def __init__(self, - block_start_string=BLOCK_START_STRING, - block_end_string=BLOCK_END_STRING, - variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING, - comment_start_string=COMMENT_START_STRING, - comment_end_string=COMMENT_END_STRING, - line_statement_prefix=LINE_STATEMENT_PREFIX, - line_comment_prefix=LINE_COMMENT_PREFIX, - trim_blocks=TRIM_BLOCKS, - lstrip_blocks=LSTRIP_BLOCKS, - newline_sequence=NEWLINE_SEQUENCE, - keep_trailing_newline=KEEP_TRAILING_NEWLINE, - extensions=(), - optimized=True, - undefined=Undefined, - finalize=None, - autoescape=False, - loader=None, - cache_size=400, - auto_reload=True, - bytecode_cache=None, - enable_async=False): + def __init__( + self, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + loader=None, + cache_size=400, + auto_reload=True, + bytecode_cache=None, + enable_async=False, + ): # !!Important notice!! # The constructor accepts quite a few arguments that should be # passed by keyword rather than position. However it's important to @@ -334,6 +360,9 @@ class Environment(object): self.enable_async = enable_async self.is_async = self.enable_async and have_async_gen + if self.is_async: + # runs patch_all() to enable async support + from . import asyncsupport # noqa: F401 _environment_sanity_check(self) @@ -353,15 +382,28 @@ class Environment(object): if not hasattr(self, key): setattr(self, key, value) - def overlay(self, block_start_string=missing, block_end_string=missing, - variable_start_string=missing, variable_end_string=missing, - comment_start_string=missing, comment_end_string=missing, - line_statement_prefix=missing, line_comment_prefix=missing, - trim_blocks=missing, lstrip_blocks=missing, - extensions=missing, optimized=missing, - undefined=missing, finalize=missing, autoescape=missing, - loader=missing, cache_size=missing, auto_reload=missing, - bytecode_cache=missing): + def overlay( + self, + block_start_string=missing, + block_end_string=missing, + variable_start_string=missing, + variable_end_string=missing, + comment_start_string=missing, + comment_end_string=missing, + line_statement_prefix=missing, + line_comment_prefix=missing, + trim_blocks=missing, + lstrip_blocks=missing, + extensions=missing, + optimized=missing, + undefined=missing, + finalize=missing, + autoescape=missing, + loader=missing, + cache_size=missing, + auto_reload=missing, + bytecode_cache=missing, + ): """Create a new overlay environment that shares all the data with the current environment except for cache and the overridden attributes. Extensions cannot be removed for an overlayed environment. An overlayed @@ -374,7 +416,7 @@ class Environment(object): through. """ args = dict(locals()) - del args['self'], args['cache_size'], args['extensions'] + del args["self"], args["cache_size"], args["extensions"] rv = object.__new__(self.__class__) rv.__dict__.update(self.__dict__) @@ -402,8 +444,7 @@ class Environment(object): def iter_extensions(self): """Iterates over the extensions by priority.""" - return iter(sorted(self.extensions.values(), - key=lambda x: x.priority)) + return iter(sorted(self.extensions.values(), key=lambda x: x.priority)) def getitem(self, obj, argument): """Get an item or attribute of an object but prefer the item.""" @@ -435,8 +476,9 @@ class Environment(object): except (TypeError, LookupError, AttributeError): return self.undefined(obj=obj, name=attribute) - def call_filter(self, name, value, args=None, kwargs=None, - context=None, eval_ctx=None): + def call_filter( + self, name, value, args=None, kwargs=None, context=None, eval_ctx=None + ): """Invokes a filter on a value the same way the compiler does it. Note that on Python 3 this might return a coroutine in case the @@ -448,21 +490,22 @@ class Environment(object): """ func = self.filters.get(name) if func is None: - fail_for_missing_callable('no filter named %r', name) + fail_for_missing_callable("no filter named %r", name) args = [value] + list(args or ()) - if getattr(func, 'contextfilter', False): + if getattr(func, "contextfilter", False): if context is None: - raise TemplateRuntimeError('Attempted to invoke context ' - 'filter without context') + raise TemplateRuntimeError( + "Attempted to invoke context filter without context" + ) args.insert(0, context) - elif getattr(func, 'evalcontextfilter', False): + elif getattr(func, "evalcontextfilter", False): if eval_ctx is None: if context is not None: eval_ctx = context.eval_ctx else: eval_ctx = EvalContext(self) args.insert(0, eval_ctx) - elif getattr(func, 'environmentfilter', False): + elif getattr(func, "environmentfilter", False): args.insert(0, self) return func(*args, **(kwargs or {})) @@ -473,7 +516,7 @@ class Environment(object): """ func = self.tests.get(name) if func is None: - fail_for_missing_callable('no test named %r', name) + fail_for_missing_callable("no test named %r", name) return func(value, *(args or ()), **(kwargs or {})) @internalcode @@ -483,14 +526,13 @@ class Environment(object): executable source- or bytecode. This is useful for debugging or to extract information from templates. - If you are :ref:`developing Jinja2 extensions <writing-extensions>` + If you are :ref:`developing Jinja extensions <writing-extensions>` this gives you a good overview of the node tree generated. """ try: return self._parse(source, name, filename) except TemplateSyntaxError: - exc_info = sys.exc_info() - self.handle_exception(exc_info, source_hint=source) + self.handle_exception(source=source) def _parse(self, source, name, filename): """Internal parsing function used by `parse` and `compile`.""" @@ -510,16 +552,18 @@ class Environment(object): try: return self.lexer.tokeniter(source, name, filename) except TemplateSyntaxError: - exc_info = sys.exc_info() - self.handle_exception(exc_info, source_hint=source) + self.handle_exception(source=source) def preprocess(self, source, name=None, filename=None): """Preprocesses the source with all extensions. This is automatically called for all parsing and compiling methods but *not* for :meth:`lex` because there you usually only want the actual source tokenized. """ - return reduce(lambda s, e: e.preprocess(s, name, filename), - self.iter_extensions(), text_type(source)) + return reduce( + lambda s, e: e.preprocess(s, name, filename), + self.iter_extensions(), + text_type(source), + ) def _tokenize(self, source, name, filename=None, state=None): """Called by the parser to do the preprocessing and filtering @@ -539,8 +583,14 @@ class Environment(object): .. versionadded:: 2.5 """ - return generate(source, self, name, filename, defer_init=defer_init, - optimized=self.optimized) + return generate( + source, + self, + name, + filename, + defer_init=defer_init, + optimized=self.optimized, + ) def _compile(self, source, filename): """Internal hook that can be overridden to hook a different compile @@ -548,11 +598,10 @@ class Environment(object): .. versionadded:: 2.5 """ - return compile(source, filename, 'exec') + return compile(source, filename, "exec") @internalcode - def compile(self, source, name=None, filename=None, raw=False, - defer_init=False): + def compile(self, source, name=None, filename=None, raw=False, defer_init=False): """Compile a node or template source code. The `name` parameter is the load name of the template after it was joined using :meth:`join_path` if necessary, not the filename on the file system. @@ -577,18 +626,16 @@ class Environment(object): if isinstance(source, string_types): source_hint = source source = self._parse(source, name, filename) - source = self._generate(source, name, filename, - defer_init=defer_init) + source = self._generate(source, name, filename, defer_init=defer_init) if raw: return source if filename is None: - filename = '<template>' + filename = "<template>" else: filename = encode_filename(filename) return self._compile(source, filename) except TemplateSyntaxError: - exc_info = sys.exc_info() - self.handle_exception(exc_info, source_hint=source_hint) + self.handle_exception(source=source_hint) def compile_expression(self, source, undefined_to_none=True): """A handy helper method that returns a callable that accepts keyword @@ -618,26 +665,32 @@ class Environment(object): .. versionadded:: 2.1 """ - parser = Parser(self, source, state='variable') - exc_info = None + parser = Parser(self, source, state="variable") try: expr = parser.parse_expression() if not parser.stream.eos: - raise TemplateSyntaxError('chunk after expression', - parser.stream.current.lineno, - None, None) + raise TemplateSyntaxError( + "chunk after expression", parser.stream.current.lineno, None, None + ) expr.set_environment(self) except TemplateSyntaxError: - exc_info = sys.exc_info() - if exc_info is not None: - self.handle_exception(exc_info, source_hint=source) - body = [nodes.Assign(nodes.Name('result', 'store'), expr, lineno=1)] + if sys.exc_info() is not None: + self.handle_exception(source=source) + + body = [nodes.Assign(nodes.Name("result", "store"), expr, lineno=1)] template = self.from_string(nodes.Template(body, lineno=1)) return TemplateExpression(template, undefined_to_none) - def compile_templates(self, target, extensions=None, filter_func=None, - zip='deflated', log_function=None, - ignore_errors=True, py_compile=False): + def compile_templates( + self, + target, + extensions=None, + filter_func=None, + zip="deflated", + log_function=None, + ignore_errors=True, + py_compile=False, + ): """Finds all the templates the loader can find, compiles them and stores them in `target`. If `zip` is `None`, instead of in a zipfile, the templates will be stored in a directory. @@ -660,42 +713,52 @@ class Environment(object): .. versionadded:: 2.4 """ - from jinja2.loaders import ModuleLoader + from .loaders import ModuleLoader if log_function is None: - log_function = lambda x: None + + def log_function(x): + pass if py_compile: if not PY2 or PYPY: - from warnings import warn - warn(Warning('py_compile has no effect on pypy or Python 3')) + import warnings + + warnings.warn( + "'py_compile=True' has no effect on PyPy or Python" + " 3 and will be removed in version 3.0", + DeprecationWarning, + stacklevel=2, + ) py_compile = False else: import imp import marshal - py_header = imp.get_magic() + \ - u'\xff\xff\xff\xff'.encode('iso-8859-15') + + py_header = imp.get_magic() + u"\xff\xff\xff\xff".encode("iso-8859-15") # Python 3.3 added a source filesize to the header if sys.version_info >= (3, 3): - py_header += u'\x00\x00\x00\x00'.encode('iso-8859-15') + py_header += u"\x00\x00\x00\x00".encode("iso-8859-15") - def write_file(filename, data, mode): + def write_file(filename, data): if zip: info = ZipInfo(filename) info.external_attr = 0o755 << 16 zip_file.writestr(info, data) else: - f = open(os.path.join(target, filename), mode) - try: + if isinstance(data, text_type): + data = data.encode("utf8") + + with open(os.path.join(target, filename), "wb") as f: f.write(data) - finally: - f.close() if zip is not None: from zipfile import ZipFile, ZipInfo, ZIP_DEFLATED, ZIP_STORED - zip_file = ZipFile(target, 'w', dict(deflated=ZIP_DEFLATED, - stored=ZIP_STORED)[zip]) + + zip_file = ZipFile( + target, "w", dict(deflated=ZIP_DEFLATED, stored=ZIP_STORED)[zip] + ) log_function('Compiling into Zip archive "%s"' % target) else: if not os.path.isdir(target): @@ -717,18 +780,16 @@ class Environment(object): if py_compile: c = self._compile(code, encode_filename(filename)) - write_file(filename + 'c', py_header + - marshal.dumps(c), 'wb') - log_function('Byte-compiled "%s" as %s' % - (name, filename + 'c')) + write_file(filename + "c", py_header + marshal.dumps(c)) + log_function('Byte-compiled "%s" as %s' % (name, filename + "c")) else: - write_file(filename, code, 'w') + write_file(filename, code) log_function('Compiled "%s" as %s' % (name, filename)) finally: if zip: zip_file.close() - log_function('Finished compiling templates') + log_function("Finished compiling templates") def list_templates(self, extensions=None, filter_func=None): """Returns a list of templates for this environment. This requires @@ -746,38 +807,29 @@ class Environment(object): .. versionadded:: 2.4 """ - x = self.loader.list_templates() + names = self.loader.list_templates() + if extensions is not None: if filter_func is not None: - raise TypeError('either extensions or filter_func ' - 'can be passed, but not both') - filter_func = lambda x: '.' in x and \ - x.rsplit('.', 1)[1] in extensions - if filter_func is not None: - x = list(ifilter(filter_func, x)) - return x + raise TypeError( + "either extensions or filter_func can be passed, but not both" + ) - def handle_exception(self, exc_info=None, rendered=False, source_hint=None): + def filter_func(x): + return "." in x and x.rsplit(".", 1)[1] in extensions + + if filter_func is not None: + names = [name for name in names if filter_func(name)] + + return names + + def handle_exception(self, source=None): """Exception handling helper. This is used internally to either raise rewritten exceptions or return a rendered traceback for the template. """ - global _make_traceback - if exc_info is None: - exc_info = sys.exc_info() + from .debug import rewrite_traceback_stack - # the debugging module is imported when it's used for the first time. - # we're doing a lot of stuff there and for applications that do not - # get any exceptions in template rendering there is no need to load - # all of that. - if _make_traceback is None: - from jinja2.debug import make_traceback as _make_traceback - traceback = _make_traceback(exc_info, source_hint) - if rendered and self.exception_formatter is not None: - return self.exception_formatter(traceback) - if self.exception_handler is not None: - self.exception_handler(traceback) - exc_type, exc_value, tb = traceback.standard_exc_info - reraise(exc_type, exc_value, tb) + reraise(*rewrite_traceback_stack(source=source)) def join_path(self, template, parent): """Join a template with the parent. By default all the lookups are @@ -794,12 +846,13 @@ class Environment(object): @internalcode def _load_template(self, name, globals): if self.loader is None: - raise TypeError('no loader for this environment specified') + raise TypeError("no loader for this environment specified") cache_key = (weakref.ref(self.loader), name) if self.cache is not None: template = self.cache.get(cache_key) - if template is not None and (not self.auto_reload or - template.is_up_to_date): + if template is not None and ( + not self.auto_reload or template.is_up_to_date + ): return template template = self.loader.load(self, name, globals) if self.cache is not None: @@ -835,15 +888,24 @@ class Environment(object): before it fails. If it cannot find any of the templates, it will raise a :exc:`TemplatesNotFound` exception. - .. versionadded:: 2.3 + .. versionchanged:: 2.11 + If names is :class:`Undefined`, an :exc:`UndefinedError` is + raised instead. If no templates were found and names + contains :class:`Undefined`, the message is more helpful. .. versionchanged:: 2.4 If `names` contains a :class:`Template` object it is returned from the function unchanged. + + .. versionadded:: 2.3 """ + if isinstance(names, Undefined): + names._fail_with_undefined_error() + if not names: - raise TemplatesNotFound(message=u'Tried to select from an empty list ' - u'of templates.') + raise TemplatesNotFound( + message=u"Tried to select from an empty list " u"of templates." + ) globals = self.make_globals(globals) for name in names: if isinstance(name, Template): @@ -852,20 +914,19 @@ class Environment(object): name = self.join_path(name, parent) try: return self._load_template(name, globals) - except TemplateNotFound: + except (TemplateNotFound, UndefinedError): pass raise TemplatesNotFound(names) @internalcode - def get_or_select_template(self, template_name_or_list, - parent=None, globals=None): + def get_or_select_template(self, template_name_or_list, parent=None, globals=None): """Does a typecheck and dispatches to :meth:`select_template` if an iterable of template names is given, otherwise to :meth:`get_template`. .. versionadded:: 2.3 """ - if isinstance(template_name_or_list, string_types): + if isinstance(template_name_or_list, (string_types, Undefined)): return self.get_template(template_name_or_list, parent, globals) elif isinstance(template_name_or_list, Template): return template_name_or_list @@ -916,32 +977,57 @@ class Template(object): StopIteration """ - def __new__(cls, source, - block_start_string=BLOCK_START_STRING, - block_end_string=BLOCK_END_STRING, - variable_start_string=VARIABLE_START_STRING, - variable_end_string=VARIABLE_END_STRING, - comment_start_string=COMMENT_START_STRING, - comment_end_string=COMMENT_END_STRING, - line_statement_prefix=LINE_STATEMENT_PREFIX, - line_comment_prefix=LINE_COMMENT_PREFIX, - trim_blocks=TRIM_BLOCKS, - lstrip_blocks=LSTRIP_BLOCKS, - newline_sequence=NEWLINE_SEQUENCE, - keep_trailing_newline=KEEP_TRAILING_NEWLINE, - extensions=(), - optimized=True, - undefined=Undefined, - finalize=None, - autoescape=False, - enable_async=False): + #: Type of environment to create when creating a template directly + #: rather than through an existing environment. + environment_class = Environment + + def __new__( + cls, + source, + block_start_string=BLOCK_START_STRING, + block_end_string=BLOCK_END_STRING, + variable_start_string=VARIABLE_START_STRING, + variable_end_string=VARIABLE_END_STRING, + comment_start_string=COMMENT_START_STRING, + comment_end_string=COMMENT_END_STRING, + line_statement_prefix=LINE_STATEMENT_PREFIX, + line_comment_prefix=LINE_COMMENT_PREFIX, + trim_blocks=TRIM_BLOCKS, + lstrip_blocks=LSTRIP_BLOCKS, + newline_sequence=NEWLINE_SEQUENCE, + keep_trailing_newline=KEEP_TRAILING_NEWLINE, + extensions=(), + optimized=True, + undefined=Undefined, + finalize=None, + autoescape=False, + enable_async=False, + ): env = get_spontaneous_environment( - block_start_string, block_end_string, variable_start_string, - variable_end_string, comment_start_string, comment_end_string, - line_statement_prefix, line_comment_prefix, trim_blocks, - lstrip_blocks, newline_sequence, keep_trailing_newline, - frozenset(extensions), optimized, undefined, finalize, autoescape, - None, 0, False, None, enable_async) + cls.environment_class, + block_start_string, + block_end_string, + variable_start_string, + variable_end_string, + comment_start_string, + comment_end_string, + line_statement_prefix, + line_comment_prefix, + trim_blocks, + lstrip_blocks, + newline_sequence, + keep_trailing_newline, + frozenset(extensions), + optimized, + undefined, + finalize, + autoescape, + None, + 0, + False, + None, + enable_async, + ) return env.from_string(source, template_class=cls) @classmethod @@ -949,10 +1035,7 @@ class Template(object): """Creates a template object from compiled code and the globals. This is used by the loaders and environment to create a template object. """ - namespace = { - 'environment': environment, - '__file__': code.co_filename - } + namespace = {"environment": environment, "__file__": code.co_filename} exec(code, namespace) rv = cls._from_namespace(environment, namespace, globals) rv._uptodate = uptodate @@ -972,21 +1055,21 @@ class Template(object): t = object.__new__(cls) t.environment = environment t.globals = globals - t.name = namespace['name'] - t.filename = namespace['__file__'] - t.blocks = namespace['blocks'] + t.name = namespace["name"] + t.filename = namespace["__file__"] + t.blocks = namespace["blocks"] # render function and module - t.root_render_func = namespace['root'] + t.root_render_func = namespace["root"] t._module = None # debug and loader helpers - t._debug_info = namespace['debug_info'] + t._debug_info = namespace["debug_info"] t._uptodate = None # store the reference - namespace['environment'] = environment - namespace['__jinja_template__'] = t + namespace["environment"] = environment + namespace["__jinja_template__"] = t return t @@ -1004,8 +1087,7 @@ class Template(object): try: return concat(self.root_render_func(self.new_context(vars))) except Exception: - exc_info = sys.exc_info() - return self.environment.handle_exception(exc_info, True) + self.environment.handle_exception() def render_async(self, *args, **kwargs): """This works similar to :meth:`render` but returns a coroutine @@ -1017,8 +1099,9 @@ class Template(object): await template.render_async(knights='that say nih; asynchronously') """ # see asyncsupport for the actual implementation - raise NotImplementedError('This feature is not available for this ' - 'version of Python') + raise NotImplementedError( + "This feature is not available for this version of Python" + ) def stream(self, *args, **kwargs): """Works exactly like :meth:`generate` but returns a @@ -1039,29 +1122,28 @@ class Template(object): for event in self.root_render_func(self.new_context(vars)): yield event except Exception: - exc_info = sys.exc_info() - else: - return - yield self.environment.handle_exception(exc_info, True) + yield self.environment.handle_exception() def generate_async(self, *args, **kwargs): """An async version of :meth:`generate`. Works very similarly but returns an async iterator instead. """ # see asyncsupport for the actual implementation - raise NotImplementedError('This feature is not available for this ' - 'version of Python') + raise NotImplementedError( + "This feature is not available for this version of Python" + ) def new_context(self, vars=None, shared=False, locals=None): """Create a new :class:`Context` for this template. The vars provided will be passed to the template. Per default the globals are added to the context. If shared is set to `True` the data - is passed as it to the context without adding the globals. + is passed as is to the context without adding the globals. `locals` can be a dict of local variables for internal usage. """ - return new_context(self.environment, self.name, self.blocks, - vars, shared, self.globals, locals) + return new_context( + self.environment, self.name, self.blocks, vars, shared, self.globals, locals + ) def make_module(self, vars=None, shared=False, locals=None): """This method works like the :attr:`module` attribute when called @@ -1074,13 +1156,14 @@ class Template(object): def make_module_async(self, vars=None, shared=False, locals=None): """As template module creation can invoke template code for - asynchronous exections this method must be used instead of the + asynchronous executions this method must be used instead of the normal :meth:`make_module` one. Likewise the module attribute becomes unavailable in async mode. """ # see asyncsupport for the actual implementation - raise NotImplementedError('This feature is not available for this ' - 'version of Python') + raise NotImplementedError( + "This feature is not available for this version of Python" + ) @internalcode def _get_default_module(self): @@ -1124,15 +1207,16 @@ class Template(object): @property def debug_info(self): """The debug info mapping.""" - return [tuple(imap(int, x.split('='))) for x in - self._debug_info.split('&')] + if self._debug_info: + return [tuple(map(int, x.split("="))) for x in self._debug_info.split("&")] + return [] def __repr__(self): if self.name is None: - name = 'memory:%x' % id(self) + name = "memory:%x" % id(self) else: name = repr(self.name) - return '<%s %s>' % (self.__class__.__name__, name) + return "<%s %s>" % (self.__class__.__name__, name) @implements_to_string @@ -1145,10 +1229,12 @@ class TemplateModule(object): def __init__(self, template, context, body_stream=None): if body_stream is None: if context.environment.is_async: - raise RuntimeError('Async mode requires a body stream ' - 'to be passed to a template module. Use ' - 'the async methods of the API you are ' - 'using.') + raise RuntimeError( + "Async mode requires a body stream " + "to be passed to a template module. Use " + "the async methods of the API you are " + "using." + ) body_stream = list(template.root_render_func(context)) self._body_stream = body_stream self.__dict__.update(context.get_exported()) @@ -1162,10 +1248,10 @@ class TemplateModule(object): def __repr__(self): if self.__name__ is None: - name = 'memory:%x' % id(self) + name = "memory:%x" % id(self) else: name = repr(self.__name__) - return '<%s %s>' % (self.__class__.__name__, name) + return "<%s %s>" % (self.__class__.__name__, name) class TemplateExpression(object): @@ -1181,7 +1267,7 @@ class TemplateExpression(object): def __call__(self, *args, **kwargs): context = self._template.new_context(dict(*args, **kwargs)) consume(self._template.root_render_func(context)) - rv = context.vars['result'] + rv = context.vars["result"] if self._undefined_to_none and isinstance(rv, Undefined): rv = None return rv @@ -1203,7 +1289,7 @@ class TemplateStream(object): self._gen = gen self.disable_buffering() - def dump(self, fp, encoding=None, errors='strict'): + def dump(self, fp, encoding=None, errors="strict"): """Dump the complete stream into a file or file-like object. Per default unicode strings are written, if you want to encode before writing specify an `encoding`. @@ -1215,15 +1301,15 @@ class TemplateStream(object): close = False if isinstance(fp, string_types): if encoding is None: - encoding = 'utf-8' - fp = open(fp, 'wb') + encoding = "utf-8" + fp = open(fp, "wb") close = True try: if encoding is not None: iterable = (x.encode(encoding, errors) for x in self) else: iterable = self - if hasattr(fp, 'writelines'): + if hasattr(fp, "writelines"): fp.writelines(iterable) else: for item in iterable: @@ -1259,7 +1345,7 @@ class TemplateStream(object): def enable_buffering(self, size=5): """Enable buffering. Buffer `size` items before yielding them.""" if size <= 1: - raise ValueError('buffer size too small') + raise ValueError("buffer size too small") self.buffered = True self._next = partial(next, self._buffered_generator(size)) diff --git a/pipenv/vendor/jinja2/exceptions.py b/pipenv/vendor/jinja2/exceptions.py index c018a33e..0bf2003e 100644 --- a/pipenv/vendor/jinja2/exceptions.py +++ b/pipenv/vendor/jinja2/exceptions.py @@ -1,23 +1,18 @@ # -*- coding: utf-8 -*- -""" - jinja2.exceptions - ~~~~~~~~~~~~~~~~~ - - Jinja exceptions. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from jinja2._compat import imap, text_type, PY2, implements_to_string +from ._compat import imap +from ._compat import implements_to_string +from ._compat import PY2 +from ._compat import text_type class TemplateError(Exception): """Baseclass for all template errors.""" if PY2: + def __init__(self, message=None): if message is not None: - message = text_type(message).encode('utf-8') + message = text_type(message).encode("utf-8") Exception.__init__(self, message) @property @@ -25,11 +20,13 @@ class TemplateError(Exception): if self.args: message = self.args[0] if message is not None: - return message.decode('utf-8', 'replace') + return message.decode("utf-8", "replace") def __unicode__(self): - return self.message or u'' + return self.message or u"" + else: + def __init__(self, message=None): Exception.__init__(self, message) @@ -43,16 +40,28 @@ class TemplateError(Exception): @implements_to_string class TemplateNotFound(IOError, LookupError, TemplateError): - """Raised if a template does not exist.""" + """Raised if a template does not exist. + + .. versionchanged:: 2.11 + If the given name is :class:`Undefined` and no message was + provided, an :exc:`UndefinedError` is raised. + """ # looks weird, but removes the warning descriptor that just # bogusly warns us about message being deprecated message = None def __init__(self, name, message=None): - IOError.__init__(self) + IOError.__init__(self, name) + if message is None: + from .runtime import Undefined + + if isinstance(name, Undefined): + name._fail_with_undefined_error() + message = name + self.message = message self.name = name self.templates = [name] @@ -66,13 +75,28 @@ class TemplatesNotFound(TemplateNotFound): are selected. This is a subclass of :class:`TemplateNotFound` exception, so just catching the base exception will catch both. + .. versionchanged:: 2.11 + If a name in the list of names is :class:`Undefined`, a message + about it being undefined is shown rather than the empty string. + .. versionadded:: 2.2 """ def __init__(self, names=(), message=None): if message is None: - message = u'none of the templates given were found: ' + \ - u', '.join(imap(text_type, names)) + from .runtime import Undefined + + parts = [] + + for name in names: + if isinstance(name, Undefined): + parts.append(name._undefined_message) + else: + parts.append(name) + + message = u"none of the templates given were found: " + u", ".join( + imap(text_type, parts) + ) TemplateNotFound.__init__(self, names and names[-1] or None, message) self.templates = list(names) @@ -98,11 +122,11 @@ class TemplateSyntaxError(TemplateError): return self.message # otherwise attach some stuff - location = 'line %d' % self.lineno + location = "line %d" % self.lineno name = self.filename or self.name if name: location = 'File "%s", %s' % (name, location) - lines = [self.message, ' ' + location] + lines = [self.message, " " + location] # if the source is set, add the line to the output if self.source is not None: @@ -111,9 +135,16 @@ class TemplateSyntaxError(TemplateError): except IndexError: line = None if line: - lines.append(' ' + line.strip()) + lines.append(" " + line.strip()) - return u'\n'.join(lines) + return u"\n".join(lines) + + def __reduce__(self): + # https://bugs.python.org/issue1692335 Exceptions that take + # multiple required arguments have problems with pickling. + # Without this, raises TypeError: __init__() missing 1 required + # positional argument: 'lineno' + return self.__class__, (self.message, self.lineno, self.name, self.filename) class TemplateAssertionError(TemplateSyntaxError): diff --git a/pipenv/vendor/jinja2/ext.py b/pipenv/vendor/jinja2/ext.py index 0734a84f..9141be4d 100644 --- a/pipenv/vendor/jinja2/ext.py +++ b/pipenv/vendor/jinja2/ext.py @@ -1,42 +1,49 @@ # -*- coding: utf-8 -*- -""" - jinja2.ext - ~~~~~~~~~~ - - Jinja extensions allow to add custom tags similar to the way django custom - tags work. By default two example extensions exist: an i18n and a cache - extension. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" +"""Extension API for adding custom tags and behavior.""" +import pprint import re +from sys import version_info -from jinja2 import nodes -from jinja2.defaults import BLOCK_START_STRING, \ - BLOCK_END_STRING, VARIABLE_START_STRING, VARIABLE_END_STRING, \ - COMMENT_START_STRING, COMMENT_END_STRING, LINE_STATEMENT_PREFIX, \ - LINE_COMMENT_PREFIX, TRIM_BLOCKS, NEWLINE_SEQUENCE, \ - KEEP_TRAILING_NEWLINE, LSTRIP_BLOCKS -from jinja2.environment import Environment -from jinja2.runtime import concat -from jinja2.exceptions import TemplateAssertionError, TemplateSyntaxError -from jinja2.utils import contextfunction, import_string, Markup -from jinja2._compat import with_metaclass, string_types, iteritems +from markupsafe import Markup +from . import nodes +from ._compat import iteritems +from ._compat import string_types +from ._compat import with_metaclass +from .defaults import BLOCK_END_STRING +from .defaults import BLOCK_START_STRING +from .defaults import COMMENT_END_STRING +from .defaults import COMMENT_START_STRING +from .defaults import KEEP_TRAILING_NEWLINE +from .defaults import LINE_COMMENT_PREFIX +from .defaults import LINE_STATEMENT_PREFIX +from .defaults import LSTRIP_BLOCKS +from .defaults import NEWLINE_SEQUENCE +from .defaults import TRIM_BLOCKS +from .defaults import VARIABLE_END_STRING +from .defaults import VARIABLE_START_STRING +from .environment import Environment +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .nodes import ContextReference +from .runtime import concat +from .utils import contextfunction +from .utils import import_string # the only real useful gettext functions for a Jinja template. Note # that ugettext must be assigned to gettext as Jinja doesn't support # non unicode strings. -GETTEXT_FUNCTIONS = ('_', 'gettext', 'ngettext') +GETTEXT_FUNCTIONS = ("_", "gettext", "ngettext") + +_ws_re = re.compile(r"\s*\n\s*") class ExtensionRegistry(type): """Gives the extension an unique identifier.""" - def __new__(cls, name, bases, d): - rv = type.__new__(cls, name, bases, d) - rv.identifier = rv.__module__ + '.' + rv.__name__ + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) + rv.identifier = rv.__module__ + "." + rv.__name__ return rv @@ -91,10 +98,6 @@ class Extension(with_metaclass(ExtensionRegistry, object)): to filter tokens returned. This method has to return an iterable of :class:`~jinja2.lexer.Token`\\s, but it doesn't have to return a :class:`~jinja2.lexer.TokenStream`. - - In the `ext` folder of the Jinja2 source distribution there is a file - called `inlinegettext.py` which implements a filter that utilizes this - method. """ return stream @@ -116,8 +119,9 @@ class Extension(with_metaclass(ExtensionRegistry, object)): """ return nodes.ExtensionAttribute(self.identifier, name, lineno=lineno) - def call_method(self, name, args=None, kwargs=None, dyn_args=None, - dyn_kwargs=None, lineno=None): + def call_method( + self, name, args=None, kwargs=None, dyn_args=None, dyn_kwargs=None, lineno=None + ): """Call a method of the extension. This is a shortcut for :meth:`attr` + :class:`jinja2.nodes.Call`. """ @@ -125,13 +129,19 @@ class Extension(with_metaclass(ExtensionRegistry, object)): args = [] if kwargs is None: kwargs = [] - return nodes.Call(self.attr(name, lineno=lineno), args, kwargs, - dyn_args, dyn_kwargs, lineno=lineno) + return nodes.Call( + self.attr(name, lineno=lineno), + args, + kwargs, + dyn_args, + dyn_kwargs, + lineno=lineno, + ) @contextfunction def _gettext_alias(__context, *args, **kwargs): - return __context.call(__context.resolve('gettext'), *args, **kwargs) + return __context.call(__context.resolve("gettext"), *args, **kwargs) def _make_new_gettext(func): @@ -140,24 +150,31 @@ def _make_new_gettext(func): rv = __context.call(func, __string) if __context.eval_ctx.autoescape: rv = Markup(rv) + # Always treat as a format string, even if there are no + # variables. This makes translation strings more consistent + # and predictable. This requires escaping return rv % variables + return gettext def _make_new_ngettext(func): @contextfunction def ngettext(__context, __singular, __plural, __num, **variables): - variables.setdefault('num', __num) + variables.setdefault("num", __num) rv = __context.call(func, __singular, __plural, __num) if __context.eval_ctx.autoescape: rv = Markup(rv) + # Always treat as a format string, see gettext comment above. return rv % variables + return ngettext class InternationalizationExtension(Extension): - """This extension adds gettext support to Jinja2.""" - tags = set(['trans']) + """This extension adds gettext support to Jinja.""" + + tags = {"trans"} # TODO: the i18n extension is currently reevaluating values in a few # situations. Take this example: @@ -168,30 +185,28 @@ class InternationalizationExtension(Extension): def __init__(self, environment): Extension.__init__(self, environment) - environment.globals['_'] = _gettext_alias + environment.globals["_"] = _gettext_alias environment.extend( install_gettext_translations=self._install, install_null_translations=self._install_null, install_gettext_callables=self._install_callables, uninstall_gettext_translations=self._uninstall, extract_translations=self._extract, - newstyle_gettext=False + newstyle_gettext=False, ) def _install(self, translations, newstyle=None): - gettext = getattr(translations, 'ugettext', None) + gettext = getattr(translations, "ugettext", None) if gettext is None: gettext = translations.gettext - ngettext = getattr(translations, 'ungettext', None) + ngettext = getattr(translations, "ungettext", None) if ngettext is None: ngettext = translations.ngettext self._install_callables(gettext, ngettext, newstyle) def _install_null(self, newstyle=None): self._install_callables( - lambda x: x, - lambda s, p, n: (n != 1 and (p,) or (s,))[0], - newstyle + lambda x: x, lambda s, p, n: (n != 1 and (p,) or (s,))[0], newstyle ) def _install_callables(self, gettext, ngettext, newstyle=None): @@ -200,13 +215,10 @@ class InternationalizationExtension(Extension): if self.environment.newstyle_gettext: gettext = _make_new_gettext(gettext) ngettext = _make_new_ngettext(ngettext) - self.environment.globals.update( - gettext=gettext, - ngettext=ngettext - ) + self.environment.globals.update(gettext=gettext, ngettext=ngettext) def _uninstall(self, translations): - for key in 'gettext', 'ngettext': + for key in "gettext", "ngettext": self.environment.globals.pop(key, None) def _extract(self, source, gettext_functions=GETTEXT_FUNCTIONS): @@ -226,41 +238,44 @@ class InternationalizationExtension(Extension): plural_expr_assignment = None variables = {} trimmed = None - while parser.stream.current.type != 'block_end': + while parser.stream.current.type != "block_end": if variables: - parser.stream.expect('comma') + parser.stream.expect("comma") # skip colon for python compatibility - if parser.stream.skip_if('colon'): + if parser.stream.skip_if("colon"): break - name = parser.stream.expect('name') + name = parser.stream.expect("name") if name.value in variables: - parser.fail('translatable variable %r defined twice.' % - name.value, name.lineno, - exc=TemplateAssertionError) + parser.fail( + "translatable variable %r defined twice." % name.value, + name.lineno, + exc=TemplateAssertionError, + ) # expressions - if parser.stream.current.type == 'assign': + if parser.stream.current.type == "assign": next(parser.stream) variables[name.value] = var = parser.parse_expression() - elif trimmed is None and name.value in ('trimmed', 'notrimmed'): - trimmed = name.value == 'trimmed' + elif trimmed is None and name.value in ("trimmed", "notrimmed"): + trimmed = name.value == "trimmed" continue else: - variables[name.value] = var = nodes.Name(name.value, 'load') + variables[name.value] = var = nodes.Name(name.value, "load") if plural_expr is None: if isinstance(var, nodes.Call): - plural_expr = nodes.Name('_trans', 'load') + plural_expr = nodes.Name("_trans", "load") variables[name.value] = plural_expr plural_expr_assignment = nodes.Assign( - nodes.Name('_trans', 'store'), var) + nodes.Name("_trans", "store"), var + ) else: plural_expr = var - num_called_num = name.value == 'num' + num_called_num = name.value == "num" - parser.stream.expect('block_end') + parser.stream.expect("block_end") plural = None have_plural = False @@ -271,22 +286,24 @@ class InternationalizationExtension(Extension): if singular_names: referenced.update(singular_names) if plural_expr is None: - plural_expr = nodes.Name(singular_names[0], 'load') - num_called_num = singular_names[0] == 'num' + plural_expr = nodes.Name(singular_names[0], "load") + num_called_num = singular_names[0] == "num" # if we have a pluralize block, we parse that too - if parser.stream.current.test('name:pluralize'): + if parser.stream.current.test("name:pluralize"): have_plural = True next(parser.stream) - if parser.stream.current.type != 'block_end': - name = parser.stream.expect('name') + if parser.stream.current.type != "block_end": + name = parser.stream.expect("name") if name.value not in variables: - parser.fail('unknown variable %r for pluralization' % - name.value, name.lineno, - exc=TemplateAssertionError) + parser.fail( + "unknown variable %r for pluralization" % name.value, + name.lineno, + exc=TemplateAssertionError, + ) plural_expr = variables[name.value] - num_called_num = name.value == 'num' - parser.stream.expect('block_end') + num_called_num = name.value == "num" + parser.stream.expect("block_end") plural_names, plural = self._parse_block(parser, False) next(parser.stream) referenced.update(plural_names) @@ -296,88 +313,97 @@ class InternationalizationExtension(Extension): # register free names as simple name expressions for var in referenced: if var not in variables: - variables[var] = nodes.Name(var, 'load') + variables[var] = nodes.Name(var, "load") if not have_plural: plural_expr = None elif plural_expr is None: - parser.fail('pluralize without variables', lineno) + parser.fail("pluralize without variables", lineno) if trimmed is None: - trimmed = self.environment.policies['ext.i18n.trimmed'] + trimmed = self.environment.policies["ext.i18n.trimmed"] if trimmed: singular = self._trim_whitespace(singular) if plural: plural = self._trim_whitespace(plural) - node = self._make_node(singular, plural, variables, plural_expr, - bool(referenced), - num_called_num and have_plural) + node = self._make_node( + singular, + plural, + variables, + plural_expr, + bool(referenced), + num_called_num and have_plural, + ) node.set_lineno(lineno) if plural_expr_assignment is not None: return [plural_expr_assignment, node] else: return node - def _trim_whitespace(self, string, _ws_re=re.compile(r'\s*\n\s*')): - return _ws_re.sub(' ', string.strip()) + def _trim_whitespace(self, string, _ws_re=_ws_re): + return _ws_re.sub(" ", string.strip()) def _parse_block(self, parser, allow_pluralize): """Parse until the next block tag with a given name.""" referenced = [] buf = [] while 1: - if parser.stream.current.type == 'data': - buf.append(parser.stream.current.value.replace('%', '%%')) + if parser.stream.current.type == "data": + buf.append(parser.stream.current.value.replace("%", "%%")) next(parser.stream) - elif parser.stream.current.type == 'variable_begin': + elif parser.stream.current.type == "variable_begin": next(parser.stream) - name = parser.stream.expect('name').value + name = parser.stream.expect("name").value referenced.append(name) - buf.append('%%(%s)s' % name) - parser.stream.expect('variable_end') - elif parser.stream.current.type == 'block_begin': + buf.append("%%(%s)s" % name) + parser.stream.expect("variable_end") + elif parser.stream.current.type == "block_begin": next(parser.stream) - if parser.stream.current.test('name:endtrans'): + if parser.stream.current.test("name:endtrans"): break - elif parser.stream.current.test('name:pluralize'): + elif parser.stream.current.test("name:pluralize"): if allow_pluralize: break - parser.fail('a translatable section can have only one ' - 'pluralize section') - parser.fail('control structures in translatable sections are ' - 'not allowed') + parser.fail( + "a translatable section can have only one pluralize section" + ) + parser.fail( + "control structures in translatable sections are not allowed" + ) elif parser.stream.eos: - parser.fail('unclosed translation block') + parser.fail("unclosed translation block") else: - assert False, 'internal parser error' + raise RuntimeError("internal parser error") return referenced, concat(buf) - def _make_node(self, singular, plural, variables, plural_expr, - vars_referenced, num_called_num): + def _make_node( + self, singular, plural, variables, plural_expr, vars_referenced, num_called_num + ): """Generates a useful node from the data provided.""" # no variables referenced? no need to escape for old style # gettext invocations only if there are vars. if not vars_referenced and not self.environment.newstyle_gettext: - singular = singular.replace('%%', '%') + singular = singular.replace("%%", "%") if plural: - plural = plural.replace('%%', '%') + plural = plural.replace("%%", "%") # singular only: if plural_expr is None: - gettext = nodes.Name('gettext', 'load') - node = nodes.Call(gettext, [nodes.Const(singular)], - [], None, None) + gettext = nodes.Name("gettext", "load") + node = nodes.Call(gettext, [nodes.Const(singular)], [], None, None) # singular and plural else: - ngettext = nodes.Name('ngettext', 'load') - node = nodes.Call(ngettext, [ - nodes.Const(singular), - nodes.Const(plural), - plural_expr - ], [], None, None) + ngettext = nodes.Name("ngettext", "load") + node = nodes.Call( + ngettext, + [nodes.Const(singular), nodes.Const(plural), plural_expr], + [], + None, + None, + ) # in case newstyle gettext is used, the method is powerful # enough to handle the variable expansion and autoescape @@ -386,7 +412,7 @@ class InternationalizationExtension(Extension): for key, value in iteritems(variables): # the function adds that later anyways in case num was # called num, so just skip it. - if num_called_num and key == 'num': + if num_called_num and key == "num": continue node.kwargs.append(nodes.Keyword(key, value)) @@ -396,18 +422,24 @@ class InternationalizationExtension(Extension): # environment with autoescaping turned on node = nodes.MarkSafeIfAutoescape(node) if variables: - node = nodes.Mod(node, nodes.Dict([ - nodes.Pair(nodes.Const(key), value) - for key, value in variables.items() - ])) + node = nodes.Mod( + node, + nodes.Dict( + [ + nodes.Pair(nodes.Const(key), value) + for key, value in variables.items() + ] + ), + ) return nodes.Output([node]) class ExprStmtExtension(Extension): - """Adds a `do` tag to Jinja2 that works like the print statement just + """Adds a `do` tag to Jinja that works like the print statement just that it doesn't print the return value. """ - tags = set(['do']) + + tags = set(["do"]) def parse(self, parser): node = nodes.ExprStmt(lineno=next(parser.stream).lineno) @@ -417,11 +449,12 @@ class ExprStmtExtension(Extension): class LoopControlExtension(Extension): """Adds break and continue to the template engine.""" - tags = set(['break', 'continue']) + + tags = set(["break", "continue"]) def parse(self, parser): token = next(parser.stream) - if token.value == 'break': + if token.value == "break": return nodes.Break(lineno=token.lineno) return nodes.Continue(lineno=token.lineno) @@ -434,8 +467,50 @@ class AutoEscapeExtension(Extension): pass -def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, - babel_style=True): +class DebugExtension(Extension): + """A ``{% debug %}`` tag that dumps the available variables, + filters, and tests. + + .. code-block:: html+jinja + + <pre>{% debug %}</pre> + + .. code-block:: text + + {'context': {'cycler': <class 'jinja2.utils.Cycler'>, + ..., + 'namespace': <class 'jinja2.utils.Namespace'>}, + 'filters': ['abs', 'attr', 'batch', 'capitalize', 'center', 'count', 'd', + ..., 'urlencode', 'urlize', 'wordcount', 'wordwrap', 'xmlattr'], + 'tests': ['!=', '<', '<=', '==', '>', '>=', 'callable', 'defined', + ..., 'odd', 'sameas', 'sequence', 'string', 'undefined', 'upper']} + + .. versionadded:: 2.11.0 + """ + + tags = {"debug"} + + def parse(self, parser): + lineno = parser.stream.expect("name:debug").lineno + context = ContextReference() + result = self.call_method("_render", [context], lineno=lineno) + return nodes.Output([result], lineno=lineno) + + def _render(self, context): + result = { + "context": context.get_all(), + "filters": sorted(self.environment.filters.keys()), + "tests": sorted(self.environment.tests.keys()), + } + + # Set the depth since the intent is to show the top few names. + if version_info[:2] >= (3, 4): + return pprint.pformat(result, depth=3, compact=True) + else: + return pprint.pformat(result, depth=3) + + +def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, babel_style=True): """Extract localizable strings from the given template node. Per default this function returns matches in babel style that means non string parameters as well as keyword arguments are returned as `None`. This @@ -471,19 +546,20 @@ def extract_from_ast(node, gettext_functions=GETTEXT_FUNCTIONS, extraction interface or extract comments yourself. """ for node in node.find_all(nodes.Call): - if not isinstance(node.node, nodes.Name) or \ - node.node.name not in gettext_functions: + if ( + not isinstance(node.node, nodes.Name) + or node.node.name not in gettext_functions + ): continue strings = [] for arg in node.args: - if isinstance(arg, nodes.Const) and \ - isinstance(arg.value, string_types): + if isinstance(arg, nodes.Const) and isinstance(arg.value, string_types): strings.append(arg.value) else: strings.append(None) - for arg in node.kwargs: + for _ in node.kwargs: strings.append(None) if node.dyn_args is not None: strings.append(None) @@ -517,9 +593,10 @@ class _CommentFinder(object): def find_backwards(self, offset): try: - for _, token_type, token_value in \ - reversed(self.tokens[self.offset:offset]): - if token_type in ('comment', 'linecomment'): + for _, token_type, token_value in reversed( + self.tokens[self.offset : offset] + ): + if token_type in ("comment", "linecomment"): try: prefix, comment = token_value.split(None, 1) except ValueError: @@ -533,7 +610,7 @@ class _CommentFinder(object): def find_comments(self, lineno): if not self.comment_tags or self.last_lineno > lineno: return [] - for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset:]): + for idx, (token_lineno, _, _) in enumerate(self.tokens[self.offset :]): if token_lineno > lineno: return self.find_backwards(self.offset + idx) return self.find_backwards(len(self.tokens)) @@ -545,7 +622,7 @@ def babel_extract(fileobj, keywords, comment_tags, options): .. versionchanged:: 2.3 Basic support for translation comments was added. If `comment_tags` is now set to a list of keywords for extraction, the extractor will - try to find the best preceeding comment that begins with one of the + try to find the best preceding comment that begins with one of the keywords. For best results, make sure to not have more than one gettext call in one line of code and the matching comment in the same line or the line before. @@ -568,7 +645,7 @@ def babel_extract(fileobj, keywords, comment_tags, options): (comments will be empty currently) """ extensions = set() - for extension in options.get('extensions', '').split(','): + for extension in options.get("extensions", "").split(","): extension = extension.strip() if not extension: continue @@ -577,38 +654,37 @@ def babel_extract(fileobj, keywords, comment_tags, options): extensions.add(InternationalizationExtension) def getbool(options, key, default=False): - return options.get(key, str(default)).lower() in \ - ('1', 'on', 'yes', 'true') + return options.get(key, str(default)).lower() in ("1", "on", "yes", "true") - silent = getbool(options, 'silent', True) + silent = getbool(options, "silent", True) environment = Environment( - options.get('block_start_string', BLOCK_START_STRING), - options.get('block_end_string', BLOCK_END_STRING), - options.get('variable_start_string', VARIABLE_START_STRING), - options.get('variable_end_string', VARIABLE_END_STRING), - options.get('comment_start_string', COMMENT_START_STRING), - options.get('comment_end_string', COMMENT_END_STRING), - options.get('line_statement_prefix') or LINE_STATEMENT_PREFIX, - options.get('line_comment_prefix') or LINE_COMMENT_PREFIX, - getbool(options, 'trim_blocks', TRIM_BLOCKS), - getbool(options, 'lstrip_blocks', LSTRIP_BLOCKS), + options.get("block_start_string", BLOCK_START_STRING), + options.get("block_end_string", BLOCK_END_STRING), + options.get("variable_start_string", VARIABLE_START_STRING), + options.get("variable_end_string", VARIABLE_END_STRING), + options.get("comment_start_string", COMMENT_START_STRING), + options.get("comment_end_string", COMMENT_END_STRING), + options.get("line_statement_prefix") or LINE_STATEMENT_PREFIX, + options.get("line_comment_prefix") or LINE_COMMENT_PREFIX, + getbool(options, "trim_blocks", TRIM_BLOCKS), + getbool(options, "lstrip_blocks", LSTRIP_BLOCKS), NEWLINE_SEQUENCE, - getbool(options, 'keep_trailing_newline', KEEP_TRAILING_NEWLINE), + getbool(options, "keep_trailing_newline", KEEP_TRAILING_NEWLINE), frozenset(extensions), cache_size=0, - auto_reload=False + auto_reload=False, ) - if getbool(options, 'trimmed'): - environment.policies['ext.i18n.trimmed'] = True - if getbool(options, 'newstyle_gettext'): + if getbool(options, "trimmed"): + environment.policies["ext.i18n.trimmed"] = True + if getbool(options, "newstyle_gettext"): environment.newstyle_gettext = True - source = fileobj.read().decode(options.get('encoding', 'utf-8')) + source = fileobj.read().decode(options.get("encoding", "utf-8")) try: node = environment.parse(source) tokens = list(environment.lex(environment.preprocess(source))) - except TemplateSyntaxError as e: + except TemplateSyntaxError: if not silent: raise # skip templates with syntax errors @@ -625,3 +701,4 @@ do = ExprStmtExtension loopcontrols = LoopControlExtension with_ = WithExtension autoescape = AutoEscapeExtension +debug = DebugExtension diff --git a/pipenv/vendor/jinja2/filters.py b/pipenv/vendor/jinja2/filters.py index 267dddda..1af7ac88 100644 --- a/pipenv/vendor/jinja2/filters.py +++ b/pipenv/vendor/jinja2/filters.py @@ -1,29 +1,31 @@ # -*- coding: utf-8 -*- -""" - jinja2.filters - ~~~~~~~~~~~~~~ - - Bundled jinja filters. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import re +"""Built-in template filters used with the ``|`` operator.""" import math import random +import re import warnings - -from itertools import groupby, chain from collections import namedtuple -from jinja2.utils import Markup, escape, pformat, urlize, soft_unicode, \ - unicode_urlencode, htmlsafe_json_dumps -from jinja2.runtime import Undefined -from jinja2.exceptions import FilterArgumentError -from jinja2._compat import imap, string_types, text_type, iteritems, PY2 +from itertools import chain +from itertools import groupby +from markupsafe import escape +from markupsafe import Markup +from markupsafe import soft_unicode -_word_re = re.compile(r'\w+', re.UNICODE) -_word_beginning_split_re = re.compile(r'([-\s\(\{\[\<]+)', re.UNICODE) +from ._compat import abc +from ._compat import imap +from ._compat import iteritems +from ._compat import string_types +from ._compat import text_type +from .exceptions import FilterArgumentError +from .runtime import Undefined +from .utils import htmlsafe_json_dumps +from .utils import pformat +from .utils import unicode_urlencode +from .utils import urlize + +_word_re = re.compile(r"\w+", re.UNICODE) +_word_beginning_split_re = re.compile(r"([-\s\(\{\[\<]+)", re.UNICODE) def contextfilter(f): @@ -59,23 +61,21 @@ def ignore_case(value): return value.lower() if isinstance(value, string_types) else value -def make_attrgetter(environment, attribute, postprocess=None): +def make_attrgetter(environment, attribute, postprocess=None, default=None): """Returns a callable that looks up the given attribute from a passed object with the rules of the environment. Dots are allowed to access attributes of attributes. Integer parts in paths are looked up as integers. """ - if attribute is None: - attribute = [] - elif isinstance(attribute, string_types): - attribute = [int(x) if x.isdigit() else x for x in attribute.split('.')] - else: - attribute = [attribute] + attribute = _prepare_attribute_parts(attribute) def attrgetter(item): for part in attribute: item = environment.getitem(item, part) + if default and isinstance(item, Undefined): + item = default + if postprocess is not None: item = postprocess(item) @@ -84,32 +84,84 @@ def make_attrgetter(environment, attribute, postprocess=None): return attrgetter +def make_multi_attrgetter(environment, attribute, postprocess=None): + """Returns a callable that looks up the given comma separated + attributes from a passed object with the rules of the environment. + Dots are allowed to access attributes of each attribute. Integer + parts in paths are looked up as integers. + + The value returned by the returned callable is a list of extracted + attribute values. + + Examples of attribute: "attr1,attr2", "attr1.inner1.0,attr2.inner2.0", etc. + """ + attribute_parts = ( + attribute.split(",") if isinstance(attribute, string_types) else [attribute] + ) + attribute = [ + _prepare_attribute_parts(attribute_part) for attribute_part in attribute_parts + ] + + def attrgetter(item): + items = [None] * len(attribute) + for i, attribute_part in enumerate(attribute): + item_i = item + for part in attribute_part: + item_i = environment.getitem(item_i, part) + + if postprocess is not None: + item_i = postprocess(item_i) + + items[i] = item_i + return items + + return attrgetter + + +def _prepare_attribute_parts(attr): + if attr is None: + return [] + elif isinstance(attr, string_types): + return [int(x) if x.isdigit() else x for x in attr.split(".")] + else: + return [attr] + + def do_forceescape(value): """Enforce HTML escaping. This will probably double escape variables.""" - if hasattr(value, '__html__'): + if hasattr(value, "__html__"): value = value.__html__() return escape(text_type(value)) def do_urlencode(value): - """Escape strings for use in URLs (uses UTF-8 encoding). It accepts both - dictionaries and regular strings as well as pairwise iterables. + """Quote data for use in a URL path or query using UTF-8. + + Basic wrapper around :func:`urllib.parse.quote` when given a + string, or :func:`urllib.parse.urlencode` for a dict or iterable. + + :param value: Data to quote. A string will be quoted directly. A + dict or iterable of ``(key, value)`` pairs will be joined as a + query string. + + When given a string, "/" is not quoted. HTTP servers treat "/" and + "%2F" equivalently in paths. If you need quoted slashes, use the + ``|replace("/", "%2F")`` filter. .. versionadded:: 2.7 """ - itemiter = None - if isinstance(value, dict): - itemiter = iteritems(value) - elif not isinstance(value, string_types): - try: - itemiter = iter(value) - except TypeError: - pass - if itemiter is None: + if isinstance(value, string_types) or not isinstance(value, abc.Iterable): return unicode_urlencode(value) - return u'&'.join(unicode_urlencode(k) + '=' + - unicode_urlencode(v, for_qs=True) - for k, v in itemiter) + + if isinstance(value, dict): + items = iteritems(value) + else: + items = iter(value) + + return u"&".join( + "%s=%s" % (unicode_urlencode(k, for_qs=True), unicode_urlencode(v, for_qs=True)) + for k, v in items + ) @evalcontextfilter @@ -132,8 +184,11 @@ def do_replace(eval_ctx, s, old, new, count=None): count = -1 if not eval_ctx.autoescape: return text_type(s).replace(text_type(old), text_type(new), count) - if hasattr(old, '__html__') or hasattr(new, '__html__') and \ - not hasattr(s, '__html__'): + if ( + hasattr(old, "__html__") + or hasattr(new, "__html__") + and not hasattr(s, "__html__") + ): s = escape(s) else: s = soft_unicode(s) @@ -174,13 +229,13 @@ def do_xmlattr(_eval_ctx, d, autospace=True): As you can see it automatically prepends a space in front of the item if the filter returned something unless the second parameter is false. """ - rv = u' '.join( + rv = u" ".join( u'%s="%s"' % (escape(key), escape(value)) for key, value in iteritems(d) if value is not None and not isinstance(value, Undefined) ) if autospace and rv: - rv = u' ' + rv + rv = u" " + rv if _eval_ctx.autoescape: rv = Markup(rv) return rv @@ -197,13 +252,16 @@ def do_title(s): """Return a titlecased version of the value. I.e. words will start with uppercase letters, all remaining characters are lowercase. """ - return ''.join( - [item[0].upper() + item[1:].lower() - for item in _word_beginning_split_re.split(soft_unicode(s)) - if item]) + return "".join( + [ + item[0].upper() + item[1:].lower() + for item in _word_beginning_split_re.split(soft_unicode(s)) + if item + ] + ) -def do_dictsort(value, case_sensitive=False, by='key', reverse=False): +def do_dictsort(value, case_sensitive=False, by="key", reverse=False): """Sort a dict and yield (key, value) pairs. Because python dicts are unsorted you may want to use this function to order them by either key or value: @@ -222,14 +280,12 @@ def do_dictsort(value, case_sensitive=False, by='key', reverse=False): {% for item in mydict|dictsort(false, 'value') %} sort the dict by value, case insensitive """ - if by == 'key': + if by == "key": pos = 0 - elif by == 'value': + elif by == "value": pos = 1 else: - raise FilterArgumentError( - 'You can only sort by either "key" or "value"' - ) + raise FilterArgumentError('You can only sort by either "key" or "value"') def sort_func(item): value = item[pos] @@ -243,48 +299,62 @@ def do_dictsort(value, case_sensitive=False, by='key', reverse=False): @environmentfilter -def do_sort( - environment, value, reverse=False, case_sensitive=False, attribute=None -): - """Sort an iterable. Per default it sorts ascending, if you pass it - true as first argument it will reverse the sorting. - - If the iterable is made of strings the third parameter can be used to - control the case sensitiveness of the comparison which is disabled by - default. +def do_sort(environment, value, reverse=False, case_sensitive=False, attribute=None): + """Sort an iterable using Python's :func:`sorted`. .. sourcecode:: jinja - {% for item in iterable|sort %} + {% for city in cities|sort %} ... {% endfor %} - It is also possible to sort by an attribute (for example to sort - by the date of an object) by specifying the `attribute` parameter: + :param reverse: Sort descending instead of ascending. + :param case_sensitive: When sorting strings, sort upper and lower + case separately. + :param attribute: When sorting objects or dicts, an attribute or + key to sort by. Can use dot notation like ``"address.city"``. + Can be a list of attributes like ``"age,name"``. + + The sort is stable, it does not change the relative order of + elements that compare equal. This makes it is possible to chain + sorts on different attributes and ordering. .. sourcecode:: jinja - {% for item in iterable|sort(attribute='date') %} + {% for user in users|sort(attribute="name") + |sort(reverse=true, attribute="age") %} ... {% endfor %} + As a shortcut to chaining when the direction is the same for all + attributes, pass a comma separate list of attributes. + + .. sourcecode:: jinja + + {% for user users|sort(attribute="age,name") %} + ... + {% endfor %} + + .. versionchanged:: 2.11.0 + The ``attribute`` parameter can be a comma separated list of + attributes, e.g. ``"age,name"``. + .. versionchanged:: 2.6 - The `attribute` parameter was added. + The ``attribute`` parameter was added. """ - key_func = make_attrgetter( - environment, attribute, - postprocess=ignore_case if not case_sensitive else None + key_func = make_multi_attrgetter( + environment, attribute, postprocess=ignore_case if not case_sensitive else None ) return sorted(value, key=key_func, reverse=reverse) @environmentfilter def do_unique(environment, value, case_sensitive=False, attribute=None): - """Returns a list of unique items from the the given iterable. + """Returns a list of unique items from the given iterable. .. sourcecode:: jinja - {{ ['foo', 'bar', 'foobar', 'FooBar']|unique }} + {{ ['foo', 'bar', 'foobar', 'FooBar']|unique|list }} -> ['foo', 'bar', 'foobar'] The unique items are yielded in the same order as their first occurrence in @@ -294,8 +364,7 @@ def do_unique(environment, value, case_sensitive=False, attribute=None): :param attribute: Filter objects with unique values for this attribute. """ getter = make_attrgetter( - environment, attribute, - postprocess=ignore_case if not case_sensitive else None + environment, attribute, postprocess=ignore_case if not case_sensitive else None ) seen = set() @@ -313,11 +382,10 @@ def _min_or_max(environment, value, func, case_sensitive, attribute): try: first = next(it) except StopIteration: - return environment.undefined('No aggregated item, sequence was empty.') + return environment.undefined("No aggregated item, sequence was empty.") key_func = make_attrgetter( - environment, attribute, - ignore_case if not case_sensitive else None + environment, attribute, postprocess=ignore_case if not case_sensitive else None ) return func(chain([first], it), key=key_func) @@ -332,7 +400,7 @@ def do_min(environment, value, case_sensitive=False, attribute=None): -> 1 :param case_sensitive: Treat upper and lower case strings as distinct. - :param attribute: Get the object with the max value of this attribute. + :param attribute: Get the object with the min value of this attribute. """ return _min_or_max(environment, value, min, case_sensitive, attribute) @@ -352,7 +420,7 @@ def do_max(environment, value, case_sensitive=False, attribute=None): return _min_or_max(environment, value, max, case_sensitive, attribute) -def do_default(value, default_value=u'', boolean=False): +def do_default(value, default_value=u"", boolean=False): """If the value is undefined it will return the passed default value, otherwise the value of the variable: @@ -368,6 +436,12 @@ def do_default(value, default_value=u'', boolean=False): .. sourcecode:: jinja {{ ''|default('the string was empty', true) }} + + .. versionchanged:: 2.11 + It's now possible to configure the :class:`~jinja2.Environment` with + :class:`~jinja2.ChainableUndefined` to make the `default` filter work + on nested elements and attributes that may contain undefined values + in the chain without getting an :exc:`~jinja2.UndefinedError`. """ if isinstance(value, Undefined) or (boolean and not value): return default_value @@ -375,7 +449,7 @@ def do_default(value, default_value=u'', boolean=False): @evalcontextfilter -def do_join(eval_ctx, value, d=u'', attribute=None): +def do_join(eval_ctx, value, d=u"", attribute=None): """Return a string which is the concatenation of the strings in the sequence. The separator between elements is an empty string per default, you can define it with the optional parameter: @@ -400,17 +474,17 @@ def do_join(eval_ctx, value, d=u'', attribute=None): if attribute is not None: value = imap(make_attrgetter(eval_ctx.environment, attribute), value) - # no automatic escaping? joining is a lot eaiser then + # no automatic escaping? joining is a lot easier then if not eval_ctx.autoescape: return text_type(d).join(imap(text_type, value)) # if the delimiter doesn't have an html representation we check # if any of the items has. If yes we do a coercion to Markup - if not hasattr(d, '__html__'): + if not hasattr(d, "__html__"): value = list(value) do_escape = False for idx, item in enumerate(value): - if hasattr(item, '__html__'): + if hasattr(item, "__html__"): do_escape = True else: value[idx] = text_type(item) @@ -435,16 +509,25 @@ def do_first(environment, seq): try: return next(iter(seq)) except StopIteration: - return environment.undefined('No first item, sequence was empty.') + return environment.undefined("No first item, sequence was empty.") @environmentfilter def do_last(environment, seq): - """Return the last item of a sequence.""" + """ + Return the last item of a sequence. + + Note: Does not work with generators. You may want to explicitly + convert it to a list: + + .. sourcecode:: jinja + + {{ data | selectattr('name', '==', 'Jinja') | list | last }} + """ try: return next(iter(reversed(seq))) except StopIteration: - return environment.undefined('No last item, sequence was empty.') + return environment.undefined("No last item, sequence was empty.") @contextfilter @@ -453,7 +536,7 @@ def do_random(context, seq): try: return random.choice(seq) except IndexError: - return context.environment.undefined('No random item, sequence was empty.') + return context.environment.undefined("No random item, sequence was empty.") def do_filesizeformat(value, binary=False): @@ -465,25 +548,25 @@ def do_filesizeformat(value, binary=False): bytes = float(value) base = binary and 1024 or 1000 prefixes = [ - (binary and 'KiB' or 'kB'), - (binary and 'MiB' or 'MB'), - (binary and 'GiB' or 'GB'), - (binary and 'TiB' or 'TB'), - (binary and 'PiB' or 'PB'), - (binary and 'EiB' or 'EB'), - (binary and 'ZiB' or 'ZB'), - (binary and 'YiB' or 'YB') + (binary and "KiB" or "kB"), + (binary and "MiB" or "MB"), + (binary and "GiB" or "GB"), + (binary and "TiB" or "TB"), + (binary and "PiB" or "PB"), + (binary and "EiB" or "EB"), + (binary and "ZiB" or "ZB"), + (binary and "YiB" or "YB"), ] if bytes == 1: - return '1 Byte' + return "1 Byte" elif bytes < base: - return '%d Bytes' % bytes + return "%d Bytes" % bytes else: for i, prefix in enumerate(prefixes): unit = base ** (i + 2) if bytes < unit: - return '%.1f %s' % ((base * bytes / unit), prefix) - return '%.1f %s' % ((base * bytes / unit), prefix) + return "%.1f %s" % ((base * bytes / unit), prefix) + return "%.1f %s" % ((base * bytes / unit), prefix) def do_pprint(value, verbose=False): @@ -496,8 +579,9 @@ def do_pprint(value, verbose=False): @evalcontextfilter -def do_urlize(eval_ctx, value, trim_url_limit=None, nofollow=False, - target=None, rel=None): +def do_urlize( + eval_ctx, value, trim_url_limit=None, nofollow=False, target=None, rel=None +): """Converts URLs in plain text into clickable links. If you pass the filter an additional integer it will shorten the urls @@ -520,22 +604,20 @@ def do_urlize(eval_ctx, value, trim_url_limit=None, nofollow=False, The *target* parameter was added. """ policies = eval_ctx.environment.policies - rel = set((rel or '').split() or []) + rel = set((rel or "").split() or []) if nofollow: - rel.add('nofollow') - rel.update((policies['urlize.rel'] or '').split()) + rel.add("nofollow") + rel.update((policies["urlize.rel"] or "").split()) if target is None: - target = policies['urlize.target'] - rel = ' '.join(sorted(rel)) or None + target = policies["urlize.target"] + rel = " ".join(sorted(rel)) or None rv = urlize(value, trim_url_limit, rel=rel, target=target) if eval_ctx.autoescape: rv = Markup(rv) return rv -def do_indent( - s, width=4, first=False, blank=False, indentfirst=None -): +def do_indent(s, width=4, first=False, blank=False, indentfirst=None): """Return a copy of the string with each line indented by 4 spaces. The first line and blank lines are not indented by default. @@ -549,22 +631,31 @@ def do_indent( Rename the ``indentfirst`` argument to ``first``. """ if indentfirst is not None: - warnings.warn(DeprecationWarning( - 'The "indentfirst" argument is renamed to "first".' - ), stacklevel=2) + warnings.warn( + "The 'indentfirst' argument is renamed to 'first' and will" + " be removed in version 3.0.", + DeprecationWarning, + stacklevel=2, + ) first = indentfirst - s += u'\n' # this quirk is necessary for splitlines method - indention = u' ' * width + indention = u" " * width + newline = u"\n" + + if isinstance(s, Markup): + indention = Markup(indention) + newline = Markup(newline) + + s += newline # this quirk is necessary for splitlines method if blank: - rv = (u'\n' + indention).join(s.splitlines()) + rv = (newline + indention).join(s.splitlines()) else: lines = s.splitlines() rv = lines.pop(0) if lines: - rv += u'\n' + u'\n'.join( + rv += newline + newline.join( indention + line if line else line for line in lines ) @@ -575,7 +666,7 @@ def do_indent( @environmentfilter -def do_truncate(env, s, length=255, killwords=False, end='...', leeway=None): +def do_truncate(env, s, length=255, killwords=False, end="...", leeway=None): """Return a truncated copy of the string. The length is specified with the first parameter which defaults to ``255``. If the second parameter is ``true`` the filter will cut the text at length. Otherwise @@ -596,41 +687,76 @@ def do_truncate(env, s, length=255, killwords=False, end='...', leeway=None): {{ "foo bar baz qux"|truncate(11, False, '...', 0) }} -> "foo bar..." - The default leeway on newer Jinja2 versions is 5 and was 0 before but + The default leeway on newer Jinja versions is 5 and was 0 before but can be reconfigured globally. """ if leeway is None: - leeway = env.policies['truncate.leeway'] - assert length >= len(end), 'expected length >= %s, got %s' % (len(end), length) - assert leeway >= 0, 'expected leeway >= 0, got %s' % leeway + leeway = env.policies["truncate.leeway"] + assert length >= len(end), "expected length >= %s, got %s" % (len(end), length) + assert leeway >= 0, "expected leeway >= 0, got %s" % leeway if len(s) <= length + leeway: return s if killwords: - return s[:length - len(end)] + end - result = s[:length - len(end)].rsplit(' ', 1)[0] + return s[: length - len(end)] + end + result = s[: length - len(end)].rsplit(" ", 1)[0] return result + end @environmentfilter -def do_wordwrap(environment, s, width=79, break_long_words=True, - wrapstring=None): - """ - Return a copy of the string passed to the filter wrapped after - ``79`` characters. You can override this default using the first - parameter. If you set the second parameter to `false` Jinja will not - split words apart if they are longer than `width`. By default, the newlines - will be the default newlines for the environment, but this can be changed - using the wrapstring keyword argument. +def do_wordwrap( + environment, + s, + width=79, + break_long_words=True, + wrapstring=None, + break_on_hyphens=True, +): + """Wrap a string to the given width. Existing newlines are treated + as paragraphs to be wrapped separately. - .. versionadded:: 2.7 - Added support for the `wrapstring` parameter. + :param s: Original text to wrap. + :param width: Maximum length of wrapped lines. + :param break_long_words: If a word is longer than ``width``, break + it across lines. + :param break_on_hyphens: If a word contains hyphens, it may be split + across lines. + :param wrapstring: String to join each wrapped line. Defaults to + :attr:`Environment.newline_sequence`. + + .. versionchanged:: 2.11 + Existing newlines are treated as paragraphs wrapped separately. + + .. versionchanged:: 2.11 + Added the ``break_on_hyphens`` parameter. + + .. versionchanged:: 2.7 + Added the ``wrapstring`` parameter. """ + + import textwrap + if not wrapstring: wrapstring = environment.newline_sequence - import textwrap - return wrapstring.join(textwrap.wrap(s, width=width, expand_tabs=False, - replace_whitespace=False, - break_long_words=break_long_words)) + + # textwrap.wrap doesn't consider existing newlines when wrapping. + # If the string has a newline before width, wrap will still insert + # a newline at width, resulting in a short line. Instead, split and + # wrap each paragraph individually. + return wrapstring.join( + [ + wrapstring.join( + textwrap.wrap( + line, + width=width, + expand_tabs=False, + replace_whitespace=False, + break_long_words=break_long_words, + break_on_hyphens=break_on_hyphens, + ) + ) + for line in s.splitlines() + ] + ) def do_wordcount(s): @@ -671,29 +797,40 @@ def do_float(value, default=0.0): def do_format(value, *args, **kwargs): - """ - Apply python string formatting on an object: + """Apply the given values to a `printf-style`_ format string, like + ``string % values``. .. sourcecode:: jinja - {{ "%s - %s"|format("Hello?", "Foo!") }} - -> Hello? - Foo! + {{ "%s, %s!"|format(greeting, name) }} + Hello, World! + + In most cases it should be more convenient and efficient to use the + ``%`` operator or :meth:`str.format`. + + .. code-block:: text + + {{ "%s, %s!" % (greeting, name) }} + {{ "{}, {}!".format(greeting, name) }} + + .. _printf-style: https://docs.python.org/library/stdtypes.html + #printf-style-string-formatting """ if args and kwargs: - raise FilterArgumentError('can\'t handle positional and keyword ' - 'arguments at the same time') + raise FilterArgumentError( + "can't handle positional and keyword arguments at the same time" + ) return soft_unicode(value) % (kwargs or args) -def do_trim(value): - """Strip leading and trailing whitespace.""" - return soft_unicode(value).strip() +def do_trim(value, chars=None): + """Strip leading and trailing characters, by default whitespace.""" + return soft_unicode(value).strip(chars) def do_striptags(value): - """Strip SGML/XML tags and replace adjacent whitespace by one space. - """ - if hasattr(value, '__html__'): + """Strip SGML/XML tags and replace adjacent whitespace by one space.""" + if hasattr(value, "__html__"): value = value.__html__() return Markup(text_type(value)).striptags() @@ -705,7 +842,7 @@ def do_slice(value, slices, fill_with=None): .. sourcecode:: html+jinja - <div class="columwrapper"> + <div class="columnwrapper"> {%- for column in items|slice(3) %} <ul class="column-{{ loop.index }}"> {%- for item in column %} @@ -765,7 +902,7 @@ def do_batch(value, linecount, fill_with=None): yield tmp -def do_round(value, precision=0, method='common'): +def do_round(value, precision=0, method="common"): """Round the number to a given precision. The first parameter specifies the precision (default is ``0``), the second the rounding method: @@ -791,9 +928,9 @@ def do_round(value, precision=0, method='common'): {{ 42.55|round|int }} -> 43 """ - if not method in ('common', 'ceil', 'floor'): - raise FilterArgumentError('method must be common, ceil or floor') - if method == 'common': + if method not in {"common", "ceil", "floor"}: + raise FilterArgumentError("method must be common, ceil or floor") + if method == "common": return round(value, precision) func = getattr(math, method) return func(value * (10 ** precision)) / (10 ** precision) @@ -804,52 +941,51 @@ def do_round(value, precision=0, method='common'): # we do not want to accidentally expose an auto generated repr in case # people start to print this out in comments or something similar for # debugging. -_GroupTuple = namedtuple('_GroupTuple', ['grouper', 'list']) +_GroupTuple = namedtuple("_GroupTuple", ["grouper", "list"]) _GroupTuple.__repr__ = tuple.__repr__ _GroupTuple.__str__ = tuple.__str__ + @environmentfilter def do_groupby(environment, value, attribute): - """Group a sequence of objects by a common attribute. + """Group a sequence of objects by an attribute using Python's + :func:`itertools.groupby`. The attribute can use dot notation for + nested access, like ``"address.city"``. Unlike Python's ``groupby``, + the values are sorted first so only one group is returned for each + unique value. - If you for example have a list of dicts or objects that represent persons - with `gender`, `first_name` and `last_name` attributes and you want to - group all users by genders you can do something like the following - snippet: + For example, a list of ``User`` objects with a ``city`` attribute + can be rendered in groups. In this example, ``grouper`` refers to + the ``city`` value of the group. .. sourcecode:: html+jinja - <ul> - {% for group in persons|groupby('gender') %} - <li>{{ group.grouper }}<ul> - {% for person in group.list %} - <li>{{ person.first_name }} {{ person.last_name }}</li> - {% endfor %}</ul></li> - {% endfor %} - </ul> + <ul>{% for city, items in users|groupby("city") %} + <li>{{ city }} + <ul>{% for user in items %} + <li>{{ user.name }} + {% endfor %}</ul> + </li> + {% endfor %}</ul> - Additionally it's possible to use tuple unpacking for the grouper and - list: + ``groupby`` yields namedtuples of ``(grouper, list)``, which + can be used instead of the tuple unpacking above. ``grouper`` is the + value of the attribute, and ``list`` is the items with that value. .. sourcecode:: html+jinja - <ul> - {% for grouper, list in persons|groupby('gender') %} - ... - {% endfor %} - </ul> - - As you can see the item we're grouping by is stored in the `grouper` - attribute and the `list` contains all the objects that have this grouper - in common. + <ul>{% for group in users|groupby("city") %} + <li>{{ group.grouper }}: {{ group.list|join(", ") }} + {% endfor %}</ul> .. versionchanged:: 2.6 - It's now possible to use dotted notation to group by the child - attribute of another attribute. + The attribute supports dot notation for nested access. """ expr = make_attrgetter(environment, attribute) - return [_GroupTuple(key, list(values)) for key, values - in groupby(sorted(value, key=expr), expr)] + return [ + _GroupTuple(key, list(values)) + for key, values in groupby(sorted(value, key=expr), expr) + ] @environmentfilter @@ -906,7 +1042,7 @@ def do_reverse(value): rv.reverse() return rv except TypeError: - raise FilterArgumentError('argument must be iterable') + raise FilterArgumentError("argument must be iterable") @environmentfilter @@ -927,8 +1063,9 @@ def do_attr(environment, obj, name): except AttributeError: pass else: - if environment.sandboxed and not \ - environment.is_safe_attribute(obj, name, value): + if environment.sandboxed and not environment.is_safe_attribute( + obj, name, value + ): return environment.unsafe_undefined(obj, name) return value return environment.undefined(obj=obj, name=name) @@ -947,6 +1084,13 @@ def do_map(*args, **kwargs): Users on this page: {{ users|map(attribute='username')|join(', ') }} + You can specify a ``default`` value to use if an object in the list + does not have the given attribute. + + .. sourcecode:: jinja + + {{ users|map(attribute="username", default="Anonymous")|join(", ") }} + Alternatively you can let it invoke a filter by passing the name of the filter and the arguments afterwards. A good example would be applying a text conversion filter on a sequence: @@ -955,6 +1099,17 @@ def do_map(*args, **kwargs): Users on this page: {{ titles|map('lower')|join(', ') }} + Similar to a generator comprehension such as: + + .. code-block:: python + + (u.username for u in users) + (u.username or "Anonymous" for u in users) + (do_lower(x) for x in titles) + + .. versionchanged:: 2.11.0 + Added the ``default`` parameter. + .. versionadded:: 2.7 """ seq, func = prepare_map(args, kwargs) @@ -980,6 +1135,13 @@ def do_select(*args, **kwargs): {{ numbers|select("lessthan", 42) }} {{ strings|select("equalto", "mystring") }} + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if test_odd(n)) + (n for n in numbers if test_divisibleby(n, 3)) + .. versionadded:: 2.7 """ return select_or_reject(args, kwargs, lambda x: x, False) @@ -998,6 +1160,12 @@ def do_reject(*args, **kwargs): {{ numbers|reject("odd") }} + Similar to a generator comprehension such as: + + .. code-block:: python + + (n for n in numbers if not test_odd(n)) + .. versionadded:: 2.7 """ return select_or_reject(args, kwargs, lambda x: not x, False) @@ -1019,6 +1187,13 @@ def do_selectattr(*args, **kwargs): {{ users|selectattr("is_active") }} {{ users|selectattr("email", "none") }} + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if user.is_active) + (u for user in users if test_none(user.email)) + .. versionadded:: 2.7 """ return select_or_reject(args, kwargs, lambda x: x, True) @@ -1038,6 +1213,13 @@ def do_rejectattr(*args, **kwargs): {{ users|rejectattr("is_active") }} {{ users|rejectattr("email", "none") }} + Similar to a generator comprehension such as: + + .. code-block:: python + + (u for user in users if not user.is_active) + (u for user in users if not test_none(user.email)) + .. versionadded:: 2.7 """ return select_or_reject(args, kwargs, lambda x: not x, True) @@ -1070,32 +1252,38 @@ def do_tojson(eval_ctx, value, indent=None): .. versionadded:: 2.9 """ policies = eval_ctx.environment.policies - dumper = policies['json.dumps_function'] - options = policies['json.dumps_kwargs'] + dumper = policies["json.dumps_function"] + options = policies["json.dumps_kwargs"] if indent is not None: options = dict(options) - options['indent'] = indent + options["indent"] = indent return htmlsafe_json_dumps(value, dumper=dumper, **options) def prepare_map(args, kwargs): context = args[0] seq = args[1] + default = None - if len(args) == 2 and 'attribute' in kwargs: - attribute = kwargs.pop('attribute') + if len(args) == 2 and "attribute" in kwargs: + attribute = kwargs.pop("attribute") + default = kwargs.pop("default", None) if kwargs: - raise FilterArgumentError('Unexpected keyword argument %r' % - next(iter(kwargs))) - func = make_attrgetter(context.environment, attribute) + raise FilterArgumentError( + "Unexpected keyword argument %r" % next(iter(kwargs)) + ) + func = make_attrgetter(context.environment, attribute, default=default) else: try: name = args[2] args = args[3:] except LookupError: - raise FilterArgumentError('map requires a filter argument') - func = lambda item: context.environment.call_filter( - name, item, args, kwargs, context=context) + raise FilterArgumentError("map requires a filter argument") + + def func(item): + return context.environment.call_filter( + name, item, args, kwargs, context=context + ) return seq, func @@ -1107,18 +1295,22 @@ def prepare_select_or_reject(args, kwargs, modfunc, lookup_attr): try: attr = args[2] except LookupError: - raise FilterArgumentError('Missing parameter for attribute name') + raise FilterArgumentError("Missing parameter for attribute name") transfunc = make_attrgetter(context.environment, attr) off = 1 else: off = 0 - transfunc = lambda x: x + + def transfunc(x): + return x try: name = args[2 + off] - args = args[3 + off:] - func = lambda item: context.environment.call_test( - name, item, args, kwargs) + args = args[3 + off :] + + def func(item): + return context.environment.call_test(name, item, args, kwargs) + except LookupError: func = bool @@ -1134,57 +1326,57 @@ def select_or_reject(args, kwargs, modfunc, lookup_attr): FILTERS = { - 'abs': abs, - 'attr': do_attr, - 'batch': do_batch, - 'capitalize': do_capitalize, - 'center': do_center, - 'count': len, - 'd': do_default, - 'default': do_default, - 'dictsort': do_dictsort, - 'e': escape, - 'escape': escape, - 'filesizeformat': do_filesizeformat, - 'first': do_first, - 'float': do_float, - 'forceescape': do_forceescape, - 'format': do_format, - 'groupby': do_groupby, - 'indent': do_indent, - 'int': do_int, - 'join': do_join, - 'last': do_last, - 'length': len, - 'list': do_list, - 'lower': do_lower, - 'map': do_map, - 'min': do_min, - 'max': do_max, - 'pprint': do_pprint, - 'random': do_random, - 'reject': do_reject, - 'rejectattr': do_rejectattr, - 'replace': do_replace, - 'reverse': do_reverse, - 'round': do_round, - 'safe': do_mark_safe, - 'select': do_select, - 'selectattr': do_selectattr, - 'slice': do_slice, - 'sort': do_sort, - 'string': soft_unicode, - 'striptags': do_striptags, - 'sum': do_sum, - 'title': do_title, - 'trim': do_trim, - 'truncate': do_truncate, - 'unique': do_unique, - 'upper': do_upper, - 'urlencode': do_urlencode, - 'urlize': do_urlize, - 'wordcount': do_wordcount, - 'wordwrap': do_wordwrap, - 'xmlattr': do_xmlattr, - 'tojson': do_tojson, + "abs": abs, + "attr": do_attr, + "batch": do_batch, + "capitalize": do_capitalize, + "center": do_center, + "count": len, + "d": do_default, + "default": do_default, + "dictsort": do_dictsort, + "e": escape, + "escape": escape, + "filesizeformat": do_filesizeformat, + "first": do_first, + "float": do_float, + "forceescape": do_forceescape, + "format": do_format, + "groupby": do_groupby, + "indent": do_indent, + "int": do_int, + "join": do_join, + "last": do_last, + "length": len, + "list": do_list, + "lower": do_lower, + "map": do_map, + "min": do_min, + "max": do_max, + "pprint": do_pprint, + "random": do_random, + "reject": do_reject, + "rejectattr": do_rejectattr, + "replace": do_replace, + "reverse": do_reverse, + "round": do_round, + "safe": do_mark_safe, + "select": do_select, + "selectattr": do_selectattr, + "slice": do_slice, + "sort": do_sort, + "string": soft_unicode, + "striptags": do_striptags, + "sum": do_sum, + "title": do_title, + "trim": do_trim, + "truncate": do_truncate, + "unique": do_unique, + "upper": do_upper, + "urlencode": do_urlencode, + "urlize": do_urlize, + "wordcount": do_wordcount, + "wordwrap": do_wordwrap, + "xmlattr": do_xmlattr, + "tojson": do_tojson, } diff --git a/pipenv/vendor/jinja2/idtracking.py b/pipenv/vendor/jinja2/idtracking.py index 491bfe08..9a0d8380 100644 --- a/pipenv/vendor/jinja2/idtracking.py +++ b/pipenv/vendor/jinja2/idtracking.py @@ -1,11 +1,10 @@ -from jinja2.visitor import NodeVisitor -from jinja2._compat import iteritems +from ._compat import iteritems +from .visitor import NodeVisitor - -VAR_LOAD_PARAMETER = 'param' -VAR_LOAD_RESOLVE = 'resolve' -VAR_LOAD_ALIAS = 'alias' -VAR_LOAD_UNDEFINED = 'undefined' +VAR_LOAD_PARAMETER = "param" +VAR_LOAD_RESOLVE = "resolve" +VAR_LOAD_ALIAS = "alias" +VAR_LOAD_UNDEFINED = "undefined" def find_symbols(nodes, parent_symbols=None): @@ -23,7 +22,6 @@ def symbols_for_node(node, parent_symbols=None): class Symbols(object): - def __init__(self, parent=None, level=None): if level is None: if parent is None: @@ -41,7 +39,7 @@ class Symbols(object): visitor.visit(node, **kwargs) def _define_ref(self, name, load=None): - ident = 'l_%d_%s' % (self.level, name) + ident = "l_%d_%s" % (self.level, name) self.refs[name] = ident if load is not None: self.loads[ident] = load @@ -62,8 +60,10 @@ class Symbols(object): def ref(self, name): rv = self.find_ref(name) if rv is None: - raise AssertionError('Tried to resolve a name to a reference that ' - 'was unknown to the frame (%r)' % name) + raise AssertionError( + "Tried to resolve a name to a reference that " + "was unknown to the frame (%r)" % name + ) return rv def copy(self): @@ -118,7 +118,7 @@ class Symbols(object): if branch_count == len(branch_symbols): continue target = self.find_ref(name) - assert target is not None, 'should not happen' + assert target is not None, "should not happen" if self.parent is not None: outer_target = self.parent.find_ref(name) @@ -149,7 +149,6 @@ class Symbols(object): class RootVisitor(NodeVisitor): - def __init__(self, symbols): self.sym_visitor = FrameSymbolVisitor(symbols) @@ -157,35 +156,39 @@ class RootVisitor(NodeVisitor): for child in node.iter_child_nodes(): self.sym_visitor.visit(child) - visit_Template = visit_Block = visit_Macro = visit_FilterBlock = \ - visit_Scope = visit_If = visit_ScopedEvalContextModifier = \ - _simple_visit + visit_Template = ( + visit_Block + ) = ( + visit_Macro + ) = ( + visit_FilterBlock + ) = visit_Scope = visit_If = visit_ScopedEvalContextModifier = _simple_visit def visit_AssignBlock(self, node, **kwargs): for child in node.body: self.sym_visitor.visit(child) def visit_CallBlock(self, node, **kwargs): - for child in node.iter_child_nodes(exclude=('call',)): + for child in node.iter_child_nodes(exclude=("call",)): self.sym_visitor.visit(child) def visit_OverlayScope(self, node, **kwargs): for child in node.body: self.sym_visitor.visit(child) - def visit_For(self, node, for_branch='body', **kwargs): - if for_branch == 'body': + def visit_For(self, node, for_branch="body", **kwargs): + if for_branch == "body": self.sym_visitor.visit(node.target, store_as_param=True) branch = node.body - elif for_branch == 'else': + elif for_branch == "else": branch = node.else_ - elif for_branch == 'test': + elif for_branch == "test": self.sym_visitor.visit(node.target, store_as_param=True) if node.test is not None: self.sym_visitor.visit(node.test) return else: - raise RuntimeError('Unknown for branch') + raise RuntimeError("Unknown for branch") for item in branch or (): self.sym_visitor.visit(item) @@ -196,8 +199,9 @@ class RootVisitor(NodeVisitor): self.sym_visitor.visit(child) def generic_visit(self, node, *args, **kwargs): - raise NotImplementedError('Cannot find symbols for %r' % - node.__class__.__name__) + raise NotImplementedError( + "Cannot find symbols for %r" % node.__class__.__name__ + ) class FrameSymbolVisitor(NodeVisitor): @@ -208,11 +212,11 @@ class FrameSymbolVisitor(NodeVisitor): def visit_Name(self, node, store_as_param=False, **kwargs): """All assignments to names go through this function.""" - if store_as_param or node.ctx == 'param': + if store_as_param or node.ctx == "param": self.symbols.declare_parameter(node.name) - elif node.ctx == 'store': + elif node.ctx == "store": self.symbols.store(node.name) - elif node.ctx == 'load': + elif node.ctx == "load": self.symbols.load(node.name) def visit_NSRef(self, node, **kwargs): diff --git a/pipenv/vendor/jinja2/lexer.py b/pipenv/vendor/jinja2/lexer.py index 6fd135dd..a2b44e92 100644 --- a/pipenv/vendor/jinja2/lexer.py +++ b/pipenv/vendor/jinja2/lexer.py @@ -1,185 +1,194 @@ # -*- coding: utf-8 -*- -""" - jinja2.lexer - ~~~~~~~~~~~~ - - This module implements a Jinja / Python combination lexer. The - `Lexer` class provided by this module is used to do some preprocessing - for Jinja. - - On the one hand it filters out invalid operators like the bitshift - operators we don't allow in templates. On the other hand it separates - template code and python code in expressions. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. +"""Implements a Jinja / Python combination lexer. The ``Lexer`` class +is used to do some preprocessing. It filters out invalid operators like +the bitshift operators we don't allow in templates. It separates +template code and python code in expressions. """ import re +from ast import literal_eval from collections import deque from operator import itemgetter -from jinja2._compat import implements_iterator, intern, iteritems, text_type -from jinja2.exceptions import TemplateSyntaxError -from jinja2.utils import LRUCache +from ._compat import implements_iterator +from ._compat import intern +from ._compat import iteritems +from ._compat import text_type +from .exceptions import TemplateSyntaxError +from .utils import LRUCache # cache for the lexers. Exists in order to be able to have multiple # environments with the same lexer _lexer_cache = LRUCache(50) # static regular expressions -whitespace_re = re.compile(r'\s+', re.U) -string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'" - r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S) -integer_re = re.compile(r'\d+') +whitespace_re = re.compile(r"\s+", re.U) +newline_re = re.compile(r"(\r\n|\r|\n)") +string_re = re.compile( + r"('([^'\\]*(?:\\.[^'\\]*)*)'" r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S +) +integer_re = re.compile(r"(\d+_)*\d+") +float_re = re.compile( + r""" + (?<!\.) # doesn't start with a . + (\d+_)*\d+ # digits, possibly _ separated + ( + (\.(\d+_)*\d+)? # optional fractional part + e[+\-]?(\d+_)*\d+ # exponent part + | + \.(\d+_)*\d+ # required fractional part + ) + """, + re.IGNORECASE | re.VERBOSE, +) try: # check if this Python supports Unicode identifiers - compile('föö', '<unknown>', 'eval') + compile("föö", "<unknown>", "eval") except SyntaxError: - # no Unicode support, use ASCII identifiers - name_re = re.compile(r'[a-zA-Z_][a-zA-Z0-9_]*') + # Python 2, no Unicode support, use ASCII identifiers + name_re = re.compile(r"[a-zA-Z_][a-zA-Z0-9_]*") check_ident = False else: - # Unicode support, build a pattern to match valid characters, and set flag - # to use str.isidentifier to validate during lexing - from jinja2 import _identifier - name_re = re.compile(r'[\w{0}]+'.format(_identifier.pattern)) - check_ident = True - # remove the pattern from memory after building the regex - import sys - del sys.modules['jinja2._identifier'] - import jinja2 - del jinja2._identifier - del _identifier + # Unicode support, import generated re pattern and set flag to use + # str.isidentifier to validate during lexing. + from ._identifier import pattern as name_re -float_re = re.compile(r'(?<!\.)\d+\.\d+') -newline_re = re.compile(r'(\r\n|\r|\n)') + check_ident = True # internal the tokens and keep references to them -TOKEN_ADD = intern('add') -TOKEN_ASSIGN = intern('assign') -TOKEN_COLON = intern('colon') -TOKEN_COMMA = intern('comma') -TOKEN_DIV = intern('div') -TOKEN_DOT = intern('dot') -TOKEN_EQ = intern('eq') -TOKEN_FLOORDIV = intern('floordiv') -TOKEN_GT = intern('gt') -TOKEN_GTEQ = intern('gteq') -TOKEN_LBRACE = intern('lbrace') -TOKEN_LBRACKET = intern('lbracket') -TOKEN_LPAREN = intern('lparen') -TOKEN_LT = intern('lt') -TOKEN_LTEQ = intern('lteq') -TOKEN_MOD = intern('mod') -TOKEN_MUL = intern('mul') -TOKEN_NE = intern('ne') -TOKEN_PIPE = intern('pipe') -TOKEN_POW = intern('pow') -TOKEN_RBRACE = intern('rbrace') -TOKEN_RBRACKET = intern('rbracket') -TOKEN_RPAREN = intern('rparen') -TOKEN_SEMICOLON = intern('semicolon') -TOKEN_SUB = intern('sub') -TOKEN_TILDE = intern('tilde') -TOKEN_WHITESPACE = intern('whitespace') -TOKEN_FLOAT = intern('float') -TOKEN_INTEGER = intern('integer') -TOKEN_NAME = intern('name') -TOKEN_STRING = intern('string') -TOKEN_OPERATOR = intern('operator') -TOKEN_BLOCK_BEGIN = intern('block_begin') -TOKEN_BLOCK_END = intern('block_end') -TOKEN_VARIABLE_BEGIN = intern('variable_begin') -TOKEN_VARIABLE_END = intern('variable_end') -TOKEN_RAW_BEGIN = intern('raw_begin') -TOKEN_RAW_END = intern('raw_end') -TOKEN_COMMENT_BEGIN = intern('comment_begin') -TOKEN_COMMENT_END = intern('comment_end') -TOKEN_COMMENT = intern('comment') -TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin') -TOKEN_LINESTATEMENT_END = intern('linestatement_end') -TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin') -TOKEN_LINECOMMENT_END = intern('linecomment_end') -TOKEN_LINECOMMENT = intern('linecomment') -TOKEN_DATA = intern('data') -TOKEN_INITIAL = intern('initial') -TOKEN_EOF = intern('eof') +TOKEN_ADD = intern("add") +TOKEN_ASSIGN = intern("assign") +TOKEN_COLON = intern("colon") +TOKEN_COMMA = intern("comma") +TOKEN_DIV = intern("div") +TOKEN_DOT = intern("dot") +TOKEN_EQ = intern("eq") +TOKEN_FLOORDIV = intern("floordiv") +TOKEN_GT = intern("gt") +TOKEN_GTEQ = intern("gteq") +TOKEN_LBRACE = intern("lbrace") +TOKEN_LBRACKET = intern("lbracket") +TOKEN_LPAREN = intern("lparen") +TOKEN_LT = intern("lt") +TOKEN_LTEQ = intern("lteq") +TOKEN_MOD = intern("mod") +TOKEN_MUL = intern("mul") +TOKEN_NE = intern("ne") +TOKEN_PIPE = intern("pipe") +TOKEN_POW = intern("pow") +TOKEN_RBRACE = intern("rbrace") +TOKEN_RBRACKET = intern("rbracket") +TOKEN_RPAREN = intern("rparen") +TOKEN_SEMICOLON = intern("semicolon") +TOKEN_SUB = intern("sub") +TOKEN_TILDE = intern("tilde") +TOKEN_WHITESPACE = intern("whitespace") +TOKEN_FLOAT = intern("float") +TOKEN_INTEGER = intern("integer") +TOKEN_NAME = intern("name") +TOKEN_STRING = intern("string") +TOKEN_OPERATOR = intern("operator") +TOKEN_BLOCK_BEGIN = intern("block_begin") +TOKEN_BLOCK_END = intern("block_end") +TOKEN_VARIABLE_BEGIN = intern("variable_begin") +TOKEN_VARIABLE_END = intern("variable_end") +TOKEN_RAW_BEGIN = intern("raw_begin") +TOKEN_RAW_END = intern("raw_end") +TOKEN_COMMENT_BEGIN = intern("comment_begin") +TOKEN_COMMENT_END = intern("comment_end") +TOKEN_COMMENT = intern("comment") +TOKEN_LINESTATEMENT_BEGIN = intern("linestatement_begin") +TOKEN_LINESTATEMENT_END = intern("linestatement_end") +TOKEN_LINECOMMENT_BEGIN = intern("linecomment_begin") +TOKEN_LINECOMMENT_END = intern("linecomment_end") +TOKEN_LINECOMMENT = intern("linecomment") +TOKEN_DATA = intern("data") +TOKEN_INITIAL = intern("initial") +TOKEN_EOF = intern("eof") # bind operators to token types operators = { - '+': TOKEN_ADD, - '-': TOKEN_SUB, - '/': TOKEN_DIV, - '//': TOKEN_FLOORDIV, - '*': TOKEN_MUL, - '%': TOKEN_MOD, - '**': TOKEN_POW, - '~': TOKEN_TILDE, - '[': TOKEN_LBRACKET, - ']': TOKEN_RBRACKET, - '(': TOKEN_LPAREN, - ')': TOKEN_RPAREN, - '{': TOKEN_LBRACE, - '}': TOKEN_RBRACE, - '==': TOKEN_EQ, - '!=': TOKEN_NE, - '>': TOKEN_GT, - '>=': TOKEN_GTEQ, - '<': TOKEN_LT, - '<=': TOKEN_LTEQ, - '=': TOKEN_ASSIGN, - '.': TOKEN_DOT, - ':': TOKEN_COLON, - '|': TOKEN_PIPE, - ',': TOKEN_COMMA, - ';': TOKEN_SEMICOLON + "+": TOKEN_ADD, + "-": TOKEN_SUB, + "/": TOKEN_DIV, + "//": TOKEN_FLOORDIV, + "*": TOKEN_MUL, + "%": TOKEN_MOD, + "**": TOKEN_POW, + "~": TOKEN_TILDE, + "[": TOKEN_LBRACKET, + "]": TOKEN_RBRACKET, + "(": TOKEN_LPAREN, + ")": TOKEN_RPAREN, + "{": TOKEN_LBRACE, + "}": TOKEN_RBRACE, + "==": TOKEN_EQ, + "!=": TOKEN_NE, + ">": TOKEN_GT, + ">=": TOKEN_GTEQ, + "<": TOKEN_LT, + "<=": TOKEN_LTEQ, + "=": TOKEN_ASSIGN, + ".": TOKEN_DOT, + ":": TOKEN_COLON, + "|": TOKEN_PIPE, + ",": TOKEN_COMMA, + ";": TOKEN_SEMICOLON, } reverse_operators = dict([(v, k) for k, v in iteritems(operators)]) -assert len(operators) == len(reverse_operators), 'operators dropped' -operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in - sorted(operators, key=lambda x: -len(x)))) +assert len(operators) == len(reverse_operators), "operators dropped" +operator_re = re.compile( + "(%s)" % "|".join(re.escape(x) for x in sorted(operators, key=lambda x: -len(x))) +) -ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT, - TOKEN_COMMENT_END, TOKEN_WHITESPACE, - TOKEN_LINECOMMENT_BEGIN, TOKEN_LINECOMMENT_END, - TOKEN_LINECOMMENT]) -ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA, - TOKEN_COMMENT, TOKEN_LINECOMMENT]) +ignored_tokens = frozenset( + [ + TOKEN_COMMENT_BEGIN, + TOKEN_COMMENT, + TOKEN_COMMENT_END, + TOKEN_WHITESPACE, + TOKEN_LINECOMMENT_BEGIN, + TOKEN_LINECOMMENT_END, + TOKEN_LINECOMMENT, + ] +) +ignore_if_empty = frozenset( + [TOKEN_WHITESPACE, TOKEN_DATA, TOKEN_COMMENT, TOKEN_LINECOMMENT] +) def _describe_token_type(token_type): if token_type in reverse_operators: return reverse_operators[token_type] return { - TOKEN_COMMENT_BEGIN: 'begin of comment', - TOKEN_COMMENT_END: 'end of comment', - TOKEN_COMMENT: 'comment', - TOKEN_LINECOMMENT: 'comment', - TOKEN_BLOCK_BEGIN: 'begin of statement block', - TOKEN_BLOCK_END: 'end of statement block', - TOKEN_VARIABLE_BEGIN: 'begin of print statement', - TOKEN_VARIABLE_END: 'end of print statement', - TOKEN_LINESTATEMENT_BEGIN: 'begin of line statement', - TOKEN_LINESTATEMENT_END: 'end of line statement', - TOKEN_DATA: 'template data / text', - TOKEN_EOF: 'end of template' + TOKEN_COMMENT_BEGIN: "begin of comment", + TOKEN_COMMENT_END: "end of comment", + TOKEN_COMMENT: "comment", + TOKEN_LINECOMMENT: "comment", + TOKEN_BLOCK_BEGIN: "begin of statement block", + TOKEN_BLOCK_END: "end of statement block", + TOKEN_VARIABLE_BEGIN: "begin of print statement", + TOKEN_VARIABLE_END: "end of print statement", + TOKEN_LINESTATEMENT_BEGIN: "begin of line statement", + TOKEN_LINESTATEMENT_END: "end of line statement", + TOKEN_DATA: "template data / text", + TOKEN_EOF: "end of template", }.get(token_type, token_type) def describe_token(token): """Returns a description of the token.""" - if token.type == 'name': + if token.type == TOKEN_NAME: return token.value return _describe_token_type(token.type) def describe_token_expr(expr): """Like `describe_token` but for token expressions.""" - if ':' in expr: - type, value = expr.split(':', 1) - if type == 'name': + if ":" in expr: + type, value = expr.split(":", 1) + if type == TOKEN_NAME: return value else: type = expr @@ -197,21 +206,39 @@ def compile_rules(environment): """Compiles all the rules from the environment into a list of rules.""" e = re.escape rules = [ - (len(environment.comment_start_string), 'comment', - e(environment.comment_start_string)), - (len(environment.block_start_string), 'block', - e(environment.block_start_string)), - (len(environment.variable_start_string), 'variable', - e(environment.variable_start_string)) + ( + len(environment.comment_start_string), + TOKEN_COMMENT_BEGIN, + e(environment.comment_start_string), + ), + ( + len(environment.block_start_string), + TOKEN_BLOCK_BEGIN, + e(environment.block_start_string), + ), + ( + len(environment.variable_start_string), + TOKEN_VARIABLE_BEGIN, + e(environment.variable_start_string), + ), ] if environment.line_statement_prefix is not None: - rules.append((len(environment.line_statement_prefix), 'linestatement', - r'^[ \t\v]*' + e(environment.line_statement_prefix))) + rules.append( + ( + len(environment.line_statement_prefix), + TOKEN_LINESTATEMENT_BEGIN, + r"^[ \t\v]*" + e(environment.line_statement_prefix), + ) + ) if environment.line_comment_prefix is not None: - rules.append((len(environment.line_comment_prefix), 'linecomment', - r'(?:^|(?<=\S))[^\S\r\n]*' + - e(environment.line_comment_prefix))) + rules.append( + ( + len(environment.line_comment_prefix), + TOKEN_LINECOMMENT_BEGIN, + r"(?:^|(?<=\S))[^\S\r\n]*" + e(environment.line_comment_prefix), + ) + ) return [x[1:] for x in sorted(rules, reverse=True)] @@ -231,6 +258,7 @@ class Failure(object): class Token(tuple): """Token class.""" + __slots__ = () lineno, type, value = (property(itemgetter(x)) for x in range(3)) @@ -240,7 +268,7 @@ class Token(tuple): def __str__(self): if self.type in reverse_operators: return reverse_operators[self.type] - elif self.type == 'name': + elif self.type == "name": return self.value return self.type @@ -253,8 +281,8 @@ class Token(tuple): # passed an iterable of not interned strings. if self.type == expr: return True - elif ':' in expr: - return expr.split(':', 1) == [self.type, self.value] + elif ":" in expr: + return expr.split(":", 1) == [self.type, self.value] return False def test_any(self, *iterable): @@ -265,11 +293,7 @@ class Token(tuple): return False def __repr__(self): - return 'Token(%r, %r, %r)' % ( - self.lineno, - self.type, - self.value - ) + return "Token(%r, %r, %r)" % (self.lineno, self.type, self.value) @implements_iterator @@ -306,7 +330,7 @@ class TokenStream(object): self.name = name self.filename = filename self.closed = False - self.current = Token(1, TOKEN_INITIAL, '') + self.current = Token(1, TOKEN_INITIAL, "") next(self) def __iter__(self): @@ -314,9 +338,13 @@ class TokenStream(object): def __bool__(self): return bool(self._pushed) or self.current.type is not TOKEN_EOF + __nonzero__ = __bool__ # py2 - eos = property(lambda x: not x, doc="Are we at the end of the stream?") + @property + def eos(self): + """Are we at the end of the stream?""" + return not self def push(self, token): """Push a token back to the stream.""" @@ -332,7 +360,7 @@ class TokenStream(object): def skip(self, n=1): """Got n tokens ahead.""" - for x in range(n): + for _ in range(n): next(self) def next_if(self, expr): @@ -363,7 +391,7 @@ class TokenStream(object): def close(self): """Close the stream.""" - self.current = Token(self.current.lineno, TOKEN_EOF, '') + self.current = Token(self.current.lineno, TOKEN_EOF, "") self._iter = None self.closed = True @@ -374,14 +402,18 @@ class TokenStream(object): if not self.current.test(expr): expr = describe_token_expr(expr) if self.current.type is TOKEN_EOF: - raise TemplateSyntaxError('unexpected end of template, ' - 'expected %r.' % expr, - self.current.lineno, - self.name, self.filename) - raise TemplateSyntaxError("expected token %r, got %r" % - (expr, describe_token(self.current)), - self.current.lineno, - self.name, self.filename) + raise TemplateSyntaxError( + "unexpected end of template, expected %r." % expr, + self.current.lineno, + self.name, + self.filename, + ) + raise TemplateSyntaxError( + "expected token %r, got %r" % (expr, describe_token(self.current)), + self.current.lineno, + self.name, + self.filename, + ) try: return self.current finally: @@ -390,18 +422,20 @@ class TokenStream(object): def get_lexer(environment): """Return a lexer which is probably cached.""" - key = (environment.block_start_string, - environment.block_end_string, - environment.variable_start_string, - environment.variable_end_string, - environment.comment_start_string, - environment.comment_end_string, - environment.line_statement_prefix, - environment.line_comment_prefix, - environment.trim_blocks, - environment.lstrip_blocks, - environment.newline_sequence, - environment.keep_trailing_newline) + key = ( + environment.block_start_string, + environment.block_end_string, + environment.variable_start_string, + environment.variable_end_string, + environment.comment_start_string, + environment.comment_end_string, + environment.line_statement_prefix, + environment.line_comment_prefix, + environment.trim_blocks, + environment.lstrip_blocks, + environment.newline_sequence, + environment.keep_trailing_newline, + ) lexer = _lexer_cache.get(key) if lexer is None: lexer = Lexer(environment) @@ -409,6 +443,19 @@ def get_lexer(environment): return lexer +class OptionalLStrip(tuple): + """A special tuple for marking a point in the state that can have + lstrip applied. + """ + + __slots__ = () + + # Even though it looks like a no-op, creating instances fails + # without this. + def __new__(cls, *members, **kwargs): + return super(OptionalLStrip, cls).__new__(cls, members) + + class Lexer(object): """Class that implements a lexer for a given environment. Automatically created by the environment class, usually you don't have to do that. @@ -419,9 +466,11 @@ class Lexer(object): def __init__(self, environment): # shortcuts - c = lambda x: re.compile(x, re.M | re.S) e = re.escape + def c(x): + return re.compile(x, re.M | re.S) + # lexing rules for tags tag_rules = [ (whitespace_re, TOKEN_WHITESPACE, None), @@ -429,7 +478,7 @@ class Lexer(object): (integer_re, TOKEN_INTEGER, None), (name_re, TOKEN_NAME, None), (string_re, TOKEN_STRING, None), - (operator_re, TOKEN_OPERATOR, None) + (operator_re, TOKEN_OPERATOR, None), ] # assemble the root lexing rule. because "|" is ungreedy @@ -441,108 +490,120 @@ class Lexer(object): root_tag_rules = compile_rules(environment) # block suffix if trimming is enabled - block_suffix_re = environment.trim_blocks and '\\n?' or '' + block_suffix_re = environment.trim_blocks and "\\n?" or "" - # strip leading spaces if lstrip_blocks is enabled - prefix_re = {} - if environment.lstrip_blocks: - # use '{%+' to manually disable lstrip_blocks behavior - no_lstrip_re = e('+') - # detect overlap between block and variable or comment strings - block_diff = c(r'^%s(.*)' % e(environment.block_start_string)) - # make sure we don't mistake a block for a variable or a comment - m = block_diff.match(environment.comment_start_string) - no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' - m = block_diff.match(environment.variable_start_string) - no_lstrip_re += m and r'|%s' % e(m.group(1)) or '' - - # detect overlap between comment and variable strings - comment_diff = c(r'^%s(.*)' % e(environment.comment_start_string)) - m = comment_diff.match(environment.variable_start_string) - no_variable_re = m and r'(?!%s)' % e(m.group(1)) or '' - - lstrip_re = r'^[ \t]*' - block_prefix_re = r'%s%s(?!%s)|%s\+?' % ( - lstrip_re, - e(environment.block_start_string), - no_lstrip_re, - e(environment.block_start_string), - ) - comment_prefix_re = r'%s%s%s|%s\+?' % ( - lstrip_re, - e(environment.comment_start_string), - no_variable_re, - e(environment.comment_start_string), - ) - prefix_re['block'] = block_prefix_re - prefix_re['comment'] = comment_prefix_re - else: - block_prefix_re = '%s' % e(environment.block_start_string) + # If lstrip is enabled, it should not be applied if there is any + # non-whitespace between the newline and block. + self.lstrip_unless_re = c(r"[^ \t]") if environment.lstrip_blocks else None self.newline_sequence = environment.newline_sequence self.keep_trailing_newline = environment.keep_trailing_newline # global lexing rules self.rules = { - 'root': [ + "root": [ # directives - (c('(.*?)(?:%s)' % '|'.join( - [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % ( - e(environment.block_start_string), - block_prefix_re, - e(environment.block_end_string), - e(environment.block_end_string) - )] + [ - r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, prefix_re.get(n,r)) - for n, r in root_tag_rules - ])), (TOKEN_DATA, '#bygroup'), '#bygroup'), + ( + c( + "(.*?)(?:%s)" + % "|".join( + [ + r"(?P<raw_begin>%s(\-|\+|)\s*raw\s*(?:\-%s\s*|%s))" + % ( + e(environment.block_start_string), + e(environment.block_end_string), + e(environment.block_end_string), + ) + ] + + [ + r"(?P<%s>%s(\-|\+|))" % (n, r) + for n, r in root_tag_rules + ] + ) + ), + OptionalLStrip(TOKEN_DATA, "#bygroup"), + "#bygroup", + ), # data - (c('.+'), TOKEN_DATA, None) + (c(".+"), TOKEN_DATA, None), ], # comments TOKEN_COMMENT_BEGIN: [ - (c(r'(.*?)((?:\-%s\s*|%s)%s)' % ( - e(environment.comment_end_string), - e(environment.comment_end_string), - block_suffix_re - )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'), - (c('(.)'), (Failure('Missing end of comment tag'),), None) + ( + c( + r"(.*?)((?:\-%s\s*|%s)%s)" + % ( + e(environment.comment_end_string), + e(environment.comment_end_string), + block_suffix_re, + ) + ), + (TOKEN_COMMENT, TOKEN_COMMENT_END), + "#pop", + ), + (c("(.)"), (Failure("Missing end of comment tag"),), None), ], # blocks TOKEN_BLOCK_BEGIN: [ - (c(r'(?:\-%s\s*|%s)%s' % ( - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re - )), TOKEN_BLOCK_END, '#pop'), - ] + tag_rules, + ( + c( + r"(?:\-%s\s*|%s)%s" + % ( + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re, + ) + ), + TOKEN_BLOCK_END, + "#pop", + ), + ] + + tag_rules, # variables TOKEN_VARIABLE_BEGIN: [ - (c(r'\-%s\s*|%s' % ( - e(environment.variable_end_string), - e(environment.variable_end_string) - )), TOKEN_VARIABLE_END, '#pop') - ] + tag_rules, + ( + c( + r"\-%s\s*|%s" + % ( + e(environment.variable_end_string), + e(environment.variable_end_string), + ) + ), + TOKEN_VARIABLE_END, + "#pop", + ) + ] + + tag_rules, # raw block TOKEN_RAW_BEGIN: [ - (c(r'(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % ( - e(environment.block_start_string), - block_prefix_re, - e(environment.block_end_string), - e(environment.block_end_string), - block_suffix_re - )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'), - (c('(.)'), (Failure('Missing end of raw directive'),), None) + ( + c( + r"(.*?)((?:%s(\-|\+|))\s*endraw\s*(?:\-%s\s*|%s%s))" + % ( + e(environment.block_start_string), + e(environment.block_end_string), + e(environment.block_end_string), + block_suffix_re, + ) + ), + OptionalLStrip(TOKEN_DATA, TOKEN_RAW_END), + "#pop", + ), + (c("(.)"), (Failure("Missing end of raw directive"),), None), ], # line statements TOKEN_LINESTATEMENT_BEGIN: [ - (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop') - ] + tag_rules, + (c(r"\s*(\n|$)"), TOKEN_LINESTATEMENT_END, "#pop") + ] + + tag_rules, # line comments TOKEN_LINECOMMENT_BEGIN: [ - (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT, - TOKEN_LINECOMMENT_END), '#pop') - ] + ( + c(r"(.*?)()(?=\n|$)"), + (TOKEN_LINECOMMENT, TOKEN_LINECOMMENT_END), + "#pop", + ) + ], } def _normalize_newlines(self, value): @@ -550,8 +611,7 @@ class Lexer(object): return newline_re.sub(self.newline_sequence, value) def tokenize(self, source, name=None, filename=None, state=None): - """Calls tokeniter + tokenize and wraps it in a token stream. - """ + """Calls tokeniter + tokenize and wraps it in a token stream.""" stream = self.tokeniter(source, name, filename, state) return TokenStream(self.wrap(stream, name, filename), name, filename) @@ -562,37 +622,40 @@ class Lexer(object): for lineno, token, value in stream: if token in ignored_tokens: continue - elif token == 'linestatement_begin': - token = 'block_begin' - elif token == 'linestatement_end': - token = 'block_end' + elif token == TOKEN_LINESTATEMENT_BEGIN: + token = TOKEN_BLOCK_BEGIN + elif token == TOKEN_LINESTATEMENT_END: + token = TOKEN_BLOCK_END # we are not interested in those tokens in the parser - elif token in ('raw_begin', 'raw_end'): + elif token in (TOKEN_RAW_BEGIN, TOKEN_RAW_END): continue - elif token == 'data': + elif token == TOKEN_DATA: value = self._normalize_newlines(value) - elif token == 'keyword': + elif token == "keyword": token = value - elif token == 'name': + elif token == TOKEN_NAME: value = str(value) if check_ident and not value.isidentifier(): raise TemplateSyntaxError( - 'Invalid character in identifier', - lineno, name, filename) - elif token == 'string': + "Invalid character in identifier", lineno, name, filename + ) + elif token == TOKEN_STRING: # try to unescape string try: - value = self._normalize_newlines(value[1:-1]) \ - .encode('ascii', 'backslashreplace') \ - .decode('unicode-escape') + value = ( + self._normalize_newlines(value[1:-1]) + .encode("ascii", "backslashreplace") + .decode("unicode-escape") + ) except Exception as e: - msg = str(e).split(':')[-1].strip() + msg = str(e).split(":")[-1].strip() raise TemplateSyntaxError(msg, lineno, name, filename) - elif token == 'integer': - value = int(value) - elif token == 'float': - value = float(value) - elif token == 'operator': + elif token == TOKEN_INTEGER: + value = int(value.replace("_", "")) + elif token == TOKEN_FLOAT: + # remove all "_" first to support more Python versions + value = literal_eval(value.replace("_", "")) + elif token == TOKEN_OPERATOR: token = operators[value] yield Token(lineno, token, value) @@ -603,23 +666,21 @@ class Lexer(object): source = text_type(source) lines = source.splitlines() if self.keep_trailing_newline and source: - for newline in ('\r\n', '\r', '\n'): + for newline in ("\r\n", "\r", "\n"): if source.endswith(newline): - lines.append('') + lines.append("") break - source = '\n'.join(lines) + source = "\n".join(lines) pos = 0 lineno = 1 - stack = ['root'] - if state is not None and state != 'root': - assert state in ('variable', 'block'), 'invalid state' - stack.append(state + '_begin') - else: - state = 'root' + stack = ["root"] + if state is not None and state != "root": + assert state in ("variable", "block"), "invalid state" + stack.append(state + "_begin") statetokens = self.rules[stack[-1]] source_length = len(source) - balancing_stack = [] + lstrip_unless_re = self.lstrip_unless_re while 1: # tokenizer loop @@ -633,13 +694,46 @@ class Lexer(object): # are balanced. continue parsing with the lower rule which # is the operator rule. do this only if the end tags look # like operators - if balancing_stack and \ - tokens in ('variable_end', 'block_end', - 'linestatement_end'): + if balancing_stack and tokens in ( + TOKEN_VARIABLE_END, + TOKEN_BLOCK_END, + TOKEN_LINESTATEMENT_END, + ): continue # tuples support more options if isinstance(tokens, tuple): + groups = m.groups() + + if isinstance(tokens, OptionalLStrip): + # Rule supports lstrip. Match will look like + # text, block type, whitespace control, type, control, ... + text = groups[0] + + # Skipping the text and first type, every other group is the + # whitespace control for each type. One of the groups will be + # -, +, or empty string instead of None. + strip_sign = next(g for g in groups[2::2] if g is not None) + + if strip_sign == "-": + # Strip all whitespace between the text and the tag. + groups = (text.rstrip(),) + groups[1:] + elif ( + # Not marked for preserving whitespace. + strip_sign != "+" + # lstrip is enabled. + and lstrip_unless_re is not None + # Not a variable expression. + and not m.groupdict().get(TOKEN_VARIABLE_BEGIN) + ): + # The start of text between the last newline and the tag. + l_pos = text.rfind("\n") + 1 + + # If there's only whitespace between the newline and the + # tag, strip it. + if not lstrip_unless_re.search(text, l_pos): + groups = (text[:l_pos],) + groups[1:] + for idx, token in enumerate(tokens): # failure group if token.__class__ is Failure: @@ -647,51 +741,54 @@ class Lexer(object): # bygroup is a bit more complex, in that case we # yield for the current token the first named # group that matched - elif token == '#bygroup': + elif token == "#bygroup": for key, value in iteritems(m.groupdict()): if value is not None: yield lineno, key, value - lineno += value.count('\n') + lineno += value.count("\n") break else: - raise RuntimeError('%r wanted to resolve ' - 'the token dynamically' - ' but no group matched' - % regex) + raise RuntimeError( + "%r wanted to resolve " + "the token dynamically" + " but no group matched" % regex + ) # normal group else: - data = m.group(idx + 1) + data = groups[idx] if data or token not in ignore_if_empty: yield lineno, token, data - lineno += data.count('\n') + lineno += data.count("\n") # strings as token just are yielded as it. else: data = m.group() # update brace/parentheses balance - if tokens == 'operator': - if data == '{': - balancing_stack.append('}') - elif data == '(': - balancing_stack.append(')') - elif data == '[': - balancing_stack.append(']') - elif data in ('}', ')', ']'): + if tokens == TOKEN_OPERATOR: + if data == "{": + balancing_stack.append("}") + elif data == "(": + balancing_stack.append(")") + elif data == "[": + balancing_stack.append("]") + elif data in ("}", ")", "]"): if not balancing_stack: - raise TemplateSyntaxError('unexpected \'%s\'' % - data, lineno, name, - filename) + raise TemplateSyntaxError( + "unexpected '%s'" % data, lineno, name, filename + ) expected_op = balancing_stack.pop() if expected_op != data: - raise TemplateSyntaxError('unexpected \'%s\', ' - 'expected \'%s\'' % - (data, expected_op), - lineno, name, - filename) + raise TemplateSyntaxError( + "unexpected '%s', " + "expected '%s'" % (data, expected_op), + lineno, + name, + filename, + ) # yield items if data or tokens not in ignore_if_empty: yield lineno, tokens, data - lineno += data.count('\n') + lineno += data.count("\n") # fetch new position into new variable so that we can check # if there is a internal parsing error which would result @@ -701,19 +798,20 @@ class Lexer(object): # handle state changes if new_state is not None: # remove the uppermost state - if new_state == '#pop': + if new_state == "#pop": stack.pop() # resolve the new state by group checking - elif new_state == '#bygroup': + elif new_state == "#bygroup": for key, value in iteritems(m.groupdict()): if value is not None: stack.append(key) break else: - raise RuntimeError('%r wanted to resolve the ' - 'new state dynamically but' - ' no group matched' % - regex) + raise RuntimeError( + "%r wanted to resolve the " + "new state dynamically but" + " no group matched" % regex + ) # direct state name given else: stack.append(new_state) @@ -722,8 +820,9 @@ class Lexer(object): # this means a loop without break condition, avoid that and # raise error elif pos2 == pos: - raise RuntimeError('%r yielded empty string without ' - 'stack change' % regex) + raise RuntimeError( + "%r yielded empty string without stack change" % regex + ) # publish new function and start again pos = pos2 break @@ -734,6 +833,9 @@ class Lexer(object): if pos >= source_length: return # something went wrong - raise TemplateSyntaxError('unexpected char %r at %d' % - (source[pos], pos), lineno, - name, filename) + raise TemplateSyntaxError( + "unexpected char %r at %d" % (source[pos], pos), + lineno, + name, + filename, + ) diff --git a/pipenv/vendor/jinja2/loaders.py b/pipenv/vendor/jinja2/loaders.py index 4c797937..ce5537a0 100644 --- a/pipenv/vendor/jinja2/loaders.py +++ b/pipenv/vendor/jinja2/loaders.py @@ -1,22 +1,23 @@ # -*- coding: utf-8 -*- -""" - jinja2.loaders - ~~~~~~~~~~~~~~ - - Jinja loader classes. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. +"""API and implementations for loading templates from different data +sources. """ import os +import pkgutil import sys import weakref -from types import ModuleType -from os import path from hashlib import sha1 -from jinja2.exceptions import TemplateNotFound -from jinja2.utils import open_if_exists, internalcode -from jinja2._compat import string_types, iteritems +from importlib import import_module +from os import path +from types import ModuleType + +from ._compat import abc +from ._compat import fspath +from ._compat import iteritems +from ._compat import string_types +from .exceptions import TemplateNotFound +from .utils import internalcode +from .utils import open_if_exists def split_template_path(template): @@ -24,12 +25,14 @@ def split_template_path(template): '..' in the path it will raise a `TemplateNotFound` error. """ pieces = [] - for piece in template.split('/'): - if path.sep in piece \ - or (path.altsep and path.altsep in piece) or \ - piece == path.pardir: + for piece in template.split("/"): + if ( + path.sep in piece + or (path.altsep and path.altsep in piece) + or piece == path.pardir + ): raise TemplateNotFound(template) - elif piece and piece != '.': + elif piece and piece != ".": pieces.append(piece) return pieces @@ -86,15 +89,16 @@ class BaseLoader(object): the template will be reloaded. """ if not self.has_source_access: - raise RuntimeError('%s cannot provide access to the source' % - self.__class__.__name__) + raise RuntimeError( + "%s cannot provide access to the source" % self.__class__.__name__ + ) raise TemplateNotFound(template) def list_templates(self): """Iterates over all templates. If the loader does not support that it should raise a :exc:`TypeError` which is the default behavior. """ - raise TypeError('this loader cannot iterate over all templates') + raise TypeError("this loader cannot iterate over all templates") @internalcode def load(self, environment, name, globals=None): @@ -131,8 +135,9 @@ class BaseLoader(object): bucket.code = code bcc.set_bucket(bucket) - return environment.template_class.from_code(environment, code, - globals, uptodate) + return environment.template_class.from_code( + environment, code, globals, uptodate + ) class FileSystemLoader(BaseLoader): @@ -153,14 +158,20 @@ class FileSystemLoader(BaseLoader): >>> loader = FileSystemLoader('/path/to/templates', followlinks=True) - .. versionchanged:: 2.8+ - The *followlinks* parameter was added. + .. versionchanged:: 2.8 + The ``followlinks`` parameter was added. """ - def __init__(self, searchpath, encoding='utf-8', followlinks=False): - if isinstance(searchpath, string_types): + def __init__(self, searchpath, encoding="utf-8", followlinks=False): + if not isinstance(searchpath, abc.Iterable) or isinstance( + searchpath, string_types + ): searchpath = [searchpath] - self.searchpath = list(searchpath) + + # In Python 3.5, os.path.join doesn't support Path. This can be + # simplified to list(searchpath) when Python 3.5 is dropped. + self.searchpath = [fspath(p) for p in searchpath] + self.encoding = encoding self.followlinks = followlinks @@ -183,6 +194,7 @@ class FileSystemLoader(BaseLoader): return path.getmtime(filename) == mtime except OSError: return False + return contents, filename, uptodate raise TemplateNotFound(template) @@ -190,12 +202,14 @@ class FileSystemLoader(BaseLoader): found = set() for searchpath in self.searchpath: walk_dir = os.walk(searchpath, followlinks=self.followlinks) - for dirpath, dirnames, filenames in walk_dir: + for dirpath, _, filenames in walk_dir: for filename in filenames: - template = os.path.join(dirpath, filename) \ - [len(searchpath):].strip(os.path.sep) \ - .replace(os.path.sep, '/') - if template[:2] == './': + template = ( + os.path.join(dirpath, filename)[len(searchpath) :] + .strip(os.path.sep) + .replace(os.path.sep, "/") + ) + if template[:2] == "./": template = template[2:] if template not in found: found.add(template) @@ -203,66 +217,141 @@ class FileSystemLoader(BaseLoader): class PackageLoader(BaseLoader): - """Load templates from python eggs or packages. It is constructed with - the name of the python package and the path to the templates in that - package:: + """Load templates from a directory in a Python package. - loader = PackageLoader('mypackage', 'views') + :param package_name: Import name of the package that contains the + template directory. + :param package_path: Directory within the imported package that + contains the templates. + :param encoding: Encoding of template files. - If the package path is not given, ``'templates'`` is assumed. + The following example looks up templates in the ``pages`` directory + within the ``project.ui`` package. - Per default the template encoding is ``'utf-8'`` which can be changed - by setting the `encoding` parameter to something else. Due to the nature - of eggs it's only possible to reload templates if the package was loaded - from the file system and not a zip file. + .. code-block:: python + + loader = PackageLoader("project.ui", "pages") + + Only packages installed as directories (standard pip behavior) or + zip/egg files (less common) are supported. The Python API for + introspecting data in packages is too limited to support other + installation methods the way this loader requires. + + There is limited support for :pep:`420` namespace packages. The + template directory is assumed to only be in one namespace + contributor. Zip files contributing to a namespace are not + supported. + + .. versionchanged:: 2.11.0 + No longer uses ``setuptools`` as a dependency. + + .. versionchanged:: 2.11.0 + Limited PEP 420 namespace package support. """ - def __init__(self, package_name, package_path='templates', - encoding='utf-8'): - from pkg_resources import DefaultProvider, ResourceManager, \ - get_provider - provider = get_provider(package_name) - self.encoding = encoding - self.manager = ResourceManager() - self.filesystem_bound = isinstance(provider, DefaultProvider) - self.provider = provider + def __init__(self, package_name, package_path="templates", encoding="utf-8"): + if package_path == os.path.curdir: + package_path = "" + elif package_path[:2] == os.path.curdir + os.path.sep: + package_path = package_path[2:] + + package_path = os.path.normpath(package_path).rstrip(os.path.sep) self.package_path = package_path + self.package_name = package_name + self.encoding = encoding + + # Make sure the package exists. This also makes namespace + # packages work, otherwise get_loader returns None. + import_module(package_name) + self._loader = loader = pkgutil.get_loader(package_name) + + # Zip loader's archive attribute points at the zip. + self._archive = getattr(loader, "archive", None) + self._template_root = None + + if hasattr(loader, "get_filename"): + # A standard directory package, or a zip package. + self._template_root = os.path.join( + os.path.dirname(loader.get_filename(package_name)), package_path + ) + elif hasattr(loader, "_path"): + # A namespace package, limited support. Find the first + # contributor with the template directory. + for root in loader._path: + root = os.path.join(root, package_path) + + if os.path.isdir(root): + self._template_root = root + break + + if self._template_root is None: + raise ValueError( + "The %r package was not installed in a way that" + " PackageLoader understands." % package_name + ) def get_source(self, environment, template): - pieces = split_template_path(template) - p = '/'.join((self.package_path,) + tuple(pieces)) - if not self.provider.has_resource(p): - raise TemplateNotFound(template) + p = os.path.join(self._template_root, *split_template_path(template)) - filename = uptodate = None - if self.filesystem_bound: - filename = self.provider.get_resource_filename(self.manager, p) - mtime = path.getmtime(filename) - def uptodate(): - try: - return path.getmtime(filename) == mtime - except OSError: - return False + if self._archive is None: + # Package is a directory. + if not os.path.isfile(p): + raise TemplateNotFound(template) - source = self.provider.get_resource_string(self.manager, p) - return source.decode(self.encoding), filename, uptodate + with open(p, "rb") as f: + source = f.read() + + mtime = os.path.getmtime(p) + + def up_to_date(): + return os.path.isfile(p) and os.path.getmtime(p) == mtime + + else: + # Package is a zip file. + try: + source = self._loader.get_data(p) + except OSError: + raise TemplateNotFound(template) + + # Could use the zip's mtime for all template mtimes, but + # would need to safely reload the module if it's out of + # date, so just report it as always current. + up_to_date = None + + return source.decode(self.encoding), p, up_to_date def list_templates(self): - path = self.package_path - if path[:2] == './': - path = path[2:] - elif path == '.': - path = '' - offset = len(path) results = [] - def _walk(path): - for filename in self.provider.resource_listdir(path): - fullname = path + '/' + filename - if self.provider.resource_isdir(fullname): - _walk(fullname) - else: - results.append(fullname[offset:].lstrip('/')) - _walk(path) + + if self._archive is None: + # Package is a directory. + offset = len(self._template_root) + + for dirpath, _, filenames in os.walk(self._template_root): + dirpath = dirpath[offset:].lstrip(os.path.sep) + results.extend( + os.path.join(dirpath, name).replace(os.path.sep, "/") + for name in filenames + ) + else: + if not hasattr(self._loader, "_files"): + raise TypeError( + "This zip import does not have the required" + " metadata to list templates." + ) + + # Package is a zip file. + prefix = ( + self._template_root[len(self._archive) :].lstrip(os.path.sep) + + os.path.sep + ) + offset = len(prefix) + + for name in self._loader._files.keys(): + # Find names under the templates directory that aren't directories. + if name.startswith(prefix) and name[-1] != os.path.sep: + results.append(name[offset:].replace(os.path.sep, "/")) + results.sort() return results @@ -334,7 +423,7 @@ class PrefixLoader(BaseLoader): by loading ``'app2/index.html'`` the file from the second. """ - def __init__(self, mapping, delimiter='/'): + def __init__(self, mapping, delimiter="/"): self.mapping = mapping self.delimiter = delimiter @@ -434,19 +523,20 @@ class ModuleLoader(BaseLoader): has_source_access = False def __init__(self, path): - package_name = '_jinja2_module_templates_%x' % id(self) + package_name = "_jinja2_module_templates_%x" % id(self) # create a fake module that looks for the templates in the # path given. mod = _TemplateModule(package_name) - if isinstance(path, string_types): - path = [path] - else: - path = list(path) - mod.__path__ = path - sys.modules[package_name] = weakref.proxy(mod, - lambda x: sys.modules.pop(package_name, None)) + if not isinstance(path, abc.Iterable) or isinstance(path, string_types): + path = [path] + + mod.__path__ = [fspath(p) for p in path] + + sys.modules[package_name] = weakref.proxy( + mod, lambda x: sys.modules.pop(package_name, None) + ) # the only strong reference, the sys.modules entry is weak # so that the garbage collector can remove it once the @@ -456,20 +546,20 @@ class ModuleLoader(BaseLoader): @staticmethod def get_template_key(name): - return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest() + return "tmpl_" + sha1(name.encode("utf-8")).hexdigest() @staticmethod def get_module_filename(name): - return ModuleLoader.get_template_key(name) + '.py' + return ModuleLoader.get_template_key(name) + ".py" @internalcode def load(self, environment, name, globals=None): key = self.get_template_key(name) - module = '%s.%s' % (self.package_name, key) + module = "%s.%s" % (self.package_name, key) mod = getattr(self.module, module, None) if mod is None: try: - mod = __import__(module, None, None, ['root']) + mod = __import__(module, None, None, ["root"]) except ImportError: raise TemplateNotFound(name) @@ -478,4 +568,5 @@ class ModuleLoader(BaseLoader): sys.modules.pop(module, None) return environment.template_class.from_module_dict( - environment, mod.__dict__, globals) + environment, mod.__dict__, globals + ) diff --git a/pipenv/vendor/jinja2/meta.py b/pipenv/vendor/jinja2/meta.py index 7421914f..3795aace 100644 --- a/pipenv/vendor/jinja2/meta.py +++ b/pipenv/vendor/jinja2/meta.py @@ -1,25 +1,18 @@ # -*- coding: utf-8 -*- +"""Functions that expose information about templates that might be +interesting for introspection. """ - jinja2.meta - ~~~~~~~~~~~ - - This module implements various functions that exposes information about - templates that might be interesting for various kinds of applications. - - :copyright: (c) 2017 by the Jinja Team, see AUTHORS for more details. - :license: BSD, see LICENSE for more details. -""" -from jinja2 import nodes -from jinja2.compiler import CodeGenerator -from jinja2._compat import string_types, iteritems +from . import nodes +from ._compat import iteritems +from ._compat import string_types +from .compiler import CodeGenerator class TrackingCodeGenerator(CodeGenerator): """We abuse the code generator for introspection.""" def __init__(self, environment): - CodeGenerator.__init__(self, environment, '<introspection>', - '<introspection>') + CodeGenerator.__init__(self, environment, "<introspection>", "<introspection>") self.undeclared_identifiers = set() def write(self, x): @@ -29,7 +22,7 @@ class TrackingCodeGenerator(CodeGenerator): """Remember all undeclared identifiers.""" CodeGenerator.enter_frame(self, frame) for _, (action, param) in iteritems(frame.symbols.loads): - if action == 'resolve': + if action == "resolve" and param not in self.environment.globals: self.undeclared_identifiers.add(param) @@ -72,8 +65,9 @@ def find_referenced_templates(ast): This function is useful for dependency tracking. For example if you want to rebuild parts of the website after a layout template has changed. """ - for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import, - nodes.Include)): + for node in ast.find_all( + (nodes.Extends, nodes.FromImport, nodes.Import, nodes.Include) + ): if not isinstance(node.template, nodes.Const): # a tuple with some non consts in there if isinstance(node.template, (nodes.Tuple, nodes.List)): @@ -96,8 +90,9 @@ def find_referenced_templates(ast): # a tuple or list (latter *should* not happen) made of consts, # yield the consts that are strings. We could warn here for # non string values - elif isinstance(node, nodes.Include) and \ - isinstance(node.template.value, (tuple, list)): + elif isinstance(node, nodes.Include) and isinstance( + node.template.value, (tuple, list) + ): for template_name in node.template.value: if isinstance(template_name, string_types): yield template_name diff --git a/pipenv/vendor/jinja2/nativetypes.py b/pipenv/vendor/jinja2/nativetypes.py index fe17e413..9866c962 100644 --- a/pipenv/vendor/jinja2/nativetypes.py +++ b/pipenv/vendor/jinja2/nativetypes.py @@ -1,19 +1,27 @@ -import sys +import types from ast import literal_eval -from itertools import islice, chain -from jinja2 import nodes -from jinja2._compat import text_type -from jinja2.compiler import CodeGenerator, has_safe_repr -from jinja2.environment import Environment, Template -from jinja2.utils import concat, escape +from itertools import chain +from itertools import islice + +from . import nodes +from ._compat import text_type +from .compiler import CodeGenerator +from .compiler import has_safe_repr +from .environment import Environment +from .environment import Template -def native_concat(nodes): - """Return a native Python type from the list of compiled nodes. If the - result is a single node, its value is returned. Otherwise, the nodes are - concatenated as strings. If the result can be parsed with - :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the - string is returned. +def native_concat(nodes, preserve_quotes=True): + """Return a native Python type from the list of compiled nodes. If + the result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed with + :func:`ast.literal_eval`, the parsed value is returned. Otherwise, + the string is returned. + + :param nodes: Iterable of nodes to concatenate. + :param preserve_quotes: Whether to re-wrap literal strings with + quotes, to preserve quotes around expressions for later parsing. + Should be ``False`` in :meth:`NativeEnvironment.render`. """ head = list(islice(nodes, 2)) @@ -21,200 +29,83 @@ def native_concat(nodes): return None if len(head) == 1: - out = head[0] + raw = head[0] else: - out = u''.join([text_type(v) for v in chain(head, nodes)]) + if isinstance(nodes, types.GeneratorType): + nodes = chain(head, nodes) + raw = u"".join([text_type(v) for v in nodes]) try: - return literal_eval(out) + literal = literal_eval(raw) except (ValueError, SyntaxError, MemoryError): - return out + return raw + + # If literal_eval returned a string, re-wrap with the original + # quote character to avoid dropping quotes between expression nodes. + # Without this, "'{{ a }}', '{{ b }}'" results in "a, b", but should + # be ('a', 'b'). + if preserve_quotes and isinstance(literal, str): + return "{quote}{}{quote}".format(literal, quote=raw[0]) + + return literal class NativeCodeGenerator(CodeGenerator): - """A code generator which avoids injecting ``to_string()`` calls around the - internal code Jinja uses to render templates. + """A code generator which renders Python types by not adding + ``to_string()`` around output nodes, and using :func:`native_concat` + to convert complex strings back to Python types if possible. """ - def visit_Output(self, node, frame): - """Same as :meth:`CodeGenerator.visit_Output`, but do not call - ``to_string`` on output nodes in generated code. - """ - if self.has_known_extends and frame.require_output_check: - return + @staticmethod + def _default_finalize(value): + return value - finalize = self.environment.finalize - finalize_context = getattr(finalize, 'contextfunction', False) - finalize_eval = getattr(finalize, 'evalcontextfunction', False) - finalize_env = getattr(finalize, 'environmentfunction', False) + def _output_const_repr(self, group): + return repr(native_concat(group)) - if finalize is not None: - if finalize_context or finalize_eval: - const_finalize = None - elif finalize_env: - def const_finalize(x): - return finalize(self.environment, x) - else: - const_finalize = finalize - else: - def const_finalize(x): - return x + def _output_child_to_const(self, node, frame, finalize): + const = node.as_const(frame.eval_ctx) - # If we are inside a frame that requires output checking, we do so. - outdent_later = False + if not has_safe_repr(const): + raise nodes.Impossible() - if frame.require_output_check: - self.writeline('if parent_template is None:') - self.indent() - outdent_later = True + if isinstance(node, nodes.TemplateData): + return const - # Try to evaluate as many chunks as possible into a static string at - # compile time. - body = [] + return finalize.const(const) - for child in node.nodes: - try: - if const_finalize is None: - raise nodes.Impossible() + def _output_child_pre(self, node, frame, finalize): + if finalize.src is not None: + self.write(finalize.src) - const = child.as_const(frame.eval_ctx) - if not has_safe_repr(const): - raise nodes.Impossible() - except nodes.Impossible: - body.append(child) - continue - - # the frame can't be volatile here, because otherwise the as_const - # function would raise an Impossible exception at that point - try: - if frame.eval_ctx.autoescape: - if hasattr(const, '__html__'): - const = const.__html__() - else: - const = escape(const) - - const = const_finalize(const) - except Exception: - # if something goes wrong here we evaluate the node at runtime - # for easier debugging - body.append(child) - continue - - if body and isinstance(body[-1], list): - body[-1].append(const) - else: - body.append([const]) - - # if we have less than 3 nodes or a buffer we yield or extend/append - if len(body) < 3 or frame.buffer is not None: - if frame.buffer is not None: - # for one item we append, for more we extend - if len(body) == 1: - self.writeline('%s.append(' % frame.buffer) - else: - self.writeline('%s.extend((' % frame.buffer) - - self.indent() - - for item in body: - if isinstance(item, list): - val = repr(native_concat(item)) - - if frame.buffer is None: - self.writeline('yield ' + val) - else: - self.writeline(val + ',') - else: - if frame.buffer is None: - self.writeline('yield ', item) - else: - self.newline(item) - - close = 0 - - if finalize is not None: - self.write('environment.finalize(') - - if finalize_context: - self.write('context, ') - - close += 1 - - self.visit(item, frame) - - if close > 0: - self.write(')' * close) - - if frame.buffer is not None: - self.write(',') - - if frame.buffer is not None: - # close the open parentheses - self.outdent() - self.writeline(len(body) == 1 and ')' or '))') - - # otherwise we create a format string as this is faster in that case - else: - format = [] - arguments = [] - - for item in body: - if isinstance(item, list): - format.append(native_concat(item).replace('%', '%%')) - else: - format.append('%s') - arguments.append(item) - - self.writeline('yield ') - self.write(repr(concat(format)) + ' % (') - self.indent() - - for argument in arguments: - self.newline(argument) - close = 0 - - if finalize is not None: - self.write('environment.finalize(') - - if finalize_context: - self.write('context, ') - elif finalize_eval: - self.write('context.eval_ctx, ') - elif finalize_env: - self.write('environment, ') - - close += 1 - - self.visit(argument, frame) - self.write(')' * close + ', ') - - self.outdent() - self.writeline(')') - - if outdent_later: - self.outdent() - - -class NativeTemplate(Template): - def render(self, *args, **kwargs): - """Render the template to produce a native Python type. If the result - is a single node, its value is returned. Otherwise, the nodes are - concatenated as strings. If the result can be parsed with - :func:`ast.literal_eval`, the parsed value is returned. Otherwise, the - string is returned. - """ - vars = dict(*args, **kwargs) - - try: - return native_concat(self.root_render_func(self.new_context(vars))) - except Exception: - exc_info = sys.exc_info() - - return self.environment.handle_exception(exc_info, True) + def _output_child_post(self, node, frame, finalize): + if finalize.src is not None: + self.write(")") class NativeEnvironment(Environment): """An environment that renders templates to native Python types.""" code_generator_class = NativeCodeGenerator - template_class = NativeTemplate + + +class NativeTemplate(Template): + environment_class = NativeEnvironment + + def render(self, *args, **kwargs): + """Render the template to produce a native Python type. If the + result is a single node, its value is returned. Otherwise, the + nodes are concatenated as strings. If the result can be parsed + with :func:`ast.literal_eval`, the parsed value is returned. + Otherwise, the string is returned. + """ + vars = dict(*args, **kwargs) + try: + return native_concat( + self.root_render_func(self.new_context(vars)), preserve_quotes=False + ) + except Exception: + return self.environment.handle_exception() + + +NativeEnvironment.template_class = NativeTemplate diff --git a/pipenv/vendor/jinja2/nodes.py b/pipenv/vendor/jinja2/nodes.py index 4d9a01ad..9f3edc05 100644 --- a/pipenv/vendor/jinja2/nodes.py +++ b/pipenv/vendor/jinja2/nodes.py @@ -1,54 +1,39 @@ # -*- coding: utf-8 -*- +"""AST nodes generated by the parser for the compiler. Also provides +some node tree helper functions used by the parser and compiler in order +to normalize nodes. """ - jinja2.nodes - ~~~~~~~~~~~~ - - This module implements additional nodes derived from the ast base node. - - It also provides some node tree helper functions like `in_lineno` and - `get_nodes` used by the parser and translator in order to normalize - python and jinja nodes. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import types import operator - from collections import deque -from jinja2.utils import Markup -from jinja2._compat import izip, with_metaclass, text_type, PY2 +from markupsafe import Markup -#: the types we support for context functions -_context_function_types = (types.FunctionType, types.MethodType) - +from ._compat import izip +from ._compat import PY2 +from ._compat import text_type +from ._compat import with_metaclass _binop_to_func = { - '*': operator.mul, - '/': operator.truediv, - '//': operator.floordiv, - '**': operator.pow, - '%': operator.mod, - '+': operator.add, - '-': operator.sub + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, + "+": operator.add, + "-": operator.sub, } -_uaop_to_func = { - 'not': operator.not_, - '+': operator.pos, - '-': operator.neg -} +_uaop_to_func = {"not": operator.not_, "+": operator.pos, "-": operator.neg} _cmpop_to_func = { - 'eq': operator.eq, - 'ne': operator.ne, - 'gt': operator.gt, - 'gteq': operator.ge, - 'lt': operator.lt, - 'lteq': operator.le, - 'in': lambda a, b: a in b, - 'notin': lambda a, b: a not in b + "eq": operator.eq, + "ne": operator.ne, + "gt": operator.gt, + "gteq": operator.ge, + "lt": operator.lt, + "lteq": operator.le, + "in": lambda a, b: a in b, + "notin": lambda a, b: a not in b, } @@ -61,16 +46,16 @@ class NodeType(type): inheritance. fields and attributes from the parent class are automatically forwarded to the child.""" - def __new__(cls, name, bases, d): - for attr in 'fields', 'attributes': + def __new__(mcs, name, bases, d): + for attr in "fields", "attributes": storage = [] storage.extend(getattr(bases[0], attr, ())) storage.extend(d.get(attr, ())) - assert len(bases) == 1, 'multiple inheritance not allowed' - assert len(storage) == len(set(storage)), 'layout conflict' + assert len(bases) == 1, "multiple inheritance not allowed" + assert len(storage) == len(set(storage)), "layout conflict" d[attr] = tuple(storage) - d.setdefault('abstract', False) - return type.__new__(cls, name, bases, d) + d.setdefault("abstract", False) + return type.__new__(mcs, name, bases, d) class EvalContext(object): @@ -97,15 +82,17 @@ class EvalContext(object): def get_eval_context(node, ctx): if ctx is None: if node.environment is None: - raise RuntimeError('if no eval context is passed, the ' - 'node must have an attached ' - 'environment.') + raise RuntimeError( + "if no eval context is passed, the " + "node must have an attached " + "environment." + ) return EvalContext(node.environment) return ctx class Node(with_metaclass(NodeType, object)): - """Baseclass for all Jinja2 nodes. There are a number of nodes available + """Baseclass for all Jinja nodes. There are a number of nodes available of different types. There are four major types: - :class:`Stmt`: statements @@ -120,30 +107,32 @@ class Node(with_metaclass(NodeType, object)): The `environment` attribute is set at the end of the parsing process for all nodes automatically. """ + fields = () - attributes = ('lineno', 'environment') + attributes = ("lineno", "environment") abstract = True def __init__(self, *fields, **attributes): if self.abstract: - raise TypeError('abstract nodes are not instanciable') + raise TypeError("abstract nodes are not instantiable") if fields: if len(fields) != len(self.fields): if not self.fields: - raise TypeError('%r takes 0 arguments' % - self.__class__.__name__) - raise TypeError('%r takes 0 or %d argument%s' % ( - self.__class__.__name__, - len(self.fields), - len(self.fields) != 1 and 's' or '' - )) + raise TypeError("%r takes 0 arguments" % self.__class__.__name__) + raise TypeError( + "%r takes 0 or %d argument%s" + % ( + self.__class__.__name__, + len(self.fields), + len(self.fields) != 1 and "s" or "", + ) + ) for name, arg in izip(self.fields, fields): setattr(self, name, arg) for attr in self.attributes: setattr(self, attr, attributes.pop(attr, None)) if attributes: - raise TypeError('unknown attribute %r' % - next(iter(attributes))) + raise TypeError("unknown attribute %r" % next(iter(attributes))) def iter_fields(self, exclude=None, only=None): """This method iterates over all fields that are defined and yields @@ -153,9 +142,11 @@ class Node(with_metaclass(NodeType, object)): should be sets or tuples of field names. """ for name in self.fields: - if (exclude is only is None) or \ - (exclude is not None and name not in exclude) or \ - (only is not None and name in only): + if ( + (exclude is only is None) + or (exclude is not None and name not in exclude) + or (only is not None and name in only) + ): try: yield name, getattr(self, name) except AttributeError: @@ -166,7 +157,7 @@ class Node(with_metaclass(NodeType, object)): over all fields and yields the values of they are nodes. If the value of a field is a list all the nodes in that list are returned. """ - for field, item in self.iter_fields(exclude, only): + for _, item in self.iter_fields(exclude, only): if isinstance(item, list): for n in item: if isinstance(n, Node): @@ -200,7 +191,7 @@ class Node(with_metaclass(NodeType, object)): todo = deque([self]) while todo: node = todo.popleft() - if 'ctx' in node.fields: + if "ctx" in node.fields: node.ctx = ctx todo.extend(node.iter_child_nodes()) return self @@ -210,7 +201,7 @@ class Node(with_metaclass(NodeType, object)): todo = deque([self]) while todo: node = todo.popleft() - if 'lineno' in node.attributes: + if "lineno" in node.attributes: if node.lineno is None or override: node.lineno = lineno todo.extend(node.iter_child_nodes()) @@ -226,8 +217,9 @@ class Node(with_metaclass(NodeType, object)): return self def __eq__(self, other): - return type(self) is type(other) and \ - tuple(self.iter_fields()) == tuple(other.iter_fields()) + return type(self) is type(other) and tuple(self.iter_fields()) == tuple( + other.iter_fields() + ) def __ne__(self, other): return not self.__eq__(other) @@ -236,10 +228,9 @@ class Node(with_metaclass(NodeType, object)): __hash__ = object.__hash__ def __repr__(self): - return '%s(%s)' % ( + return "%s(%s)" % ( self.__class__.__name__, - ', '.join('%s=%r' % (arg, getattr(self, arg, None)) for - arg in self.fields) + ", ".join("%s=%r" % (arg, getattr(self, arg, None)) for arg in self.fields), ) def dump(self): @@ -248,37 +239,39 @@ class Node(with_metaclass(NodeType, object)): buf.append(repr(node)) return - buf.append('nodes.%s(' % node.__class__.__name__) + buf.append("nodes.%s(" % node.__class__.__name__) if not node.fields: - buf.append(')') + buf.append(")") return for idx, field in enumerate(node.fields): if idx: - buf.append(', ') + buf.append(", ") value = getattr(node, field) if isinstance(value, list): - buf.append('[') + buf.append("[") for idx, item in enumerate(value): if idx: - buf.append(', ') + buf.append(", ") _dump(item) - buf.append(']') + buf.append("]") else: _dump(value) - buf.append(')') + buf.append(")") + buf = [] _dump(self) - return ''.join(buf) - + return "".join(buf) class Stmt(Node): """Base node for all statements.""" + abstract = True class Helper(Node): """Nodes that exist in a specific context only.""" + abstract = True @@ -286,19 +279,22 @@ class Template(Node): """Node that represents a template. This must be the outermost node that is passed to the compiler. """ - fields = ('body',) + + fields = ("body",) class Output(Stmt): """A node that holds multiple expressions which are then printed out. This is used both for the `print` statement and the regular template data. """ - fields = ('nodes',) + + fields = ("nodes",) class Extends(Stmt): """Represents an extends statement.""" - fields = ('template',) + + fields = ("template",) class For(Stmt): @@ -309,12 +305,14 @@ class For(Stmt): For filtered nodes an expression can be stored as `test`, otherwise `None`. """ - fields = ('target', 'iter', 'body', 'else_', 'test', 'recursive') + + fields = ("target", "iter", "body", "else_", "test", "recursive") class If(Stmt): """If `test` is true, `body` is rendered, else `else_`.""" - fields = ('test', 'body', 'elif_', 'else_') + + fields = ("test", "body", "elif_", "else_") class Macro(Stmt): @@ -322,19 +320,22 @@ class Macro(Stmt): arguments and `defaults` a list of defaults if there are any. `body` is a list of nodes for the macro body. """ - fields = ('name', 'args', 'defaults', 'body') + + fields = ("name", "args", "defaults", "body") class CallBlock(Stmt): """Like a macro without a name but a call instead. `call` is called with the unnamed macro as `caller` argument this node holds. """ - fields = ('call', 'args', 'defaults', 'body') + + fields = ("call", "args", "defaults", "body") class FilterBlock(Stmt): """Node for filter sections.""" - fields = ('body', 'filter') + + fields = ("body", "filter") class With(Stmt): @@ -343,22 +344,26 @@ class With(Stmt): .. versionadded:: 2.9.3 """ - fields = ('targets', 'values', 'body') + + fields = ("targets", "values", "body") class Block(Stmt): """A node that represents a block.""" - fields = ('name', 'body', 'scoped') + + fields = ("name", "body", "scoped") class Include(Stmt): """A node that represents the include tag.""" - fields = ('template', 'with_context', 'ignore_missing') + + fields = ("template", "with_context", "ignore_missing") class Import(Stmt): """A node that represents the import tag.""" - fields = ('template', 'target', 'with_context') + + fields = ("template", "target", "with_context") class FromImport(Stmt): @@ -372,26 +377,31 @@ class FromImport(Stmt): The list of names may contain tuples if aliases are wanted. """ - fields = ('template', 'names', 'with_context') + + fields = ("template", "names", "with_context") class ExprStmt(Stmt): """A statement that evaluates an expression and discards the result.""" - fields = ('node',) + + fields = ("node",) class Assign(Stmt): """Assigns an expression to a target.""" - fields = ('target', 'node') + + fields = ("target", "node") class AssignBlock(Stmt): """Assigns a block to a target.""" - fields = ('target', 'filter', 'body') + + fields = ("target", "filter", "body") class Expr(Node): """Baseclass for all expressions.""" + abstract = True def as_const(self, eval_ctx=None): @@ -414,15 +424,18 @@ class Expr(Node): class BinExpr(Expr): """Baseclass for all binary expressions.""" - fields = ('left', 'right') + + fields = ("left", "right") operator = None abstract = True def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) # intercepted operators cannot be folded at compile time - if self.environment.sandboxed and \ - self.operator in self.environment.intercepted_binops: + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_binops + ): raise Impossible() f = _binop_to_func[self.operator] try: @@ -433,15 +446,18 @@ class BinExpr(Expr): class UnaryExpr(Expr): """Baseclass for all unary expressions.""" - fields = ('node',) + + fields = ("node",) operator = None abstract = True def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) # intercepted operators cannot be folded at compile time - if self.environment.sandboxed and \ - self.operator in self.environment.intercepted_unops: + if ( + self.environment.sandboxed + and self.operator in self.environment.intercepted_unops + ): raise Impossible() f = _uaop_to_func[self.operator] try: @@ -458,16 +474,17 @@ class Name(Expr): - `load`: load that name - `param`: like `store` but if the name was defined as function parameter. """ - fields = ('name', 'ctx') + + fields = ("name", "ctx") def can_assign(self): - return self.name not in ('true', 'false', 'none', - 'True', 'False', 'None') + return self.name not in ("true", "false", "none", "True", "False", "None") class NSRef(Expr): """Reference to a namespace value assignment""" - fields = ('name', 'attr') + + fields = ("name", "attr") def can_assign(self): # We don't need any special checks here; NSRef assignments have a @@ -479,6 +496,7 @@ class NSRef(Expr): class Literal(Expr): """Baseclass for literals.""" + abstract = True @@ -488,14 +506,18 @@ class Const(Literal): complex values such as lists too. Only constants with a safe representation (objects where ``eval(repr(x)) == x`` is true). """ - fields = ('value',) + + fields = ("value",) def as_const(self, eval_ctx=None): rv = self.value - if PY2 and type(rv) is text_type and \ - self.environment.policies['compiler.ascii_str']: + if ( + PY2 + and type(rv) is text_type + and self.environment.policies["compiler.ascii_str"] + ): try: - rv = rv.encode('ascii') + rv = rv.encode("ascii") except UnicodeError: pass return rv @@ -507,6 +529,7 @@ class Const(Literal): an `Impossible` exception. """ from .compiler import has_safe_repr + if not has_safe_repr(value): raise Impossible() return cls(value, lineno=lineno, environment=environment) @@ -514,7 +537,8 @@ class Const(Literal): class TemplateData(Literal): """A constant template string.""" - fields = ('data',) + + fields = ("data",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -530,7 +554,8 @@ class Tuple(Literal): for subscripts. Like for :class:`Name` `ctx` specifies if the tuple is used for loading the names or storing. """ - fields = ('items', 'ctx') + + fields = ("items", "ctx") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -545,7 +570,8 @@ class Tuple(Literal): class List(Literal): """Any list literal such as ``[1, 2, 3]``""" - fields = ('items',) + + fields = ("items",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -556,7 +582,8 @@ class Dict(Literal): """Any dict literal such as ``{1: 2, 3: 4}``. The items must be a list of :class:`Pair` nodes. """ - fields = ('items',) + + fields = ("items",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -565,7 +592,8 @@ class Dict(Literal): class Pair(Helper): """A key, value pair for dicts.""" - fields = ('key', 'value') + + fields = ("key", "value") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -574,7 +602,8 @@ class Pair(Helper): class Keyword(Helper): """A key, value pair for keyword arguments where key is a string.""" - fields = ('key', 'value') + + fields = ("key", "value") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -585,7 +614,8 @@ class CondExpr(Expr): """A conditional expression (inline if expression). (``{{ foo if bar else baz }}``) """ - fields = ('test', 'expr1', 'expr2') + + fields = ("test", "expr1", "expr2") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -626,7 +656,7 @@ class Filter(Expr): filtered. Buffers are created by macros and filter blocks. """ - fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -636,28 +666,27 @@ class Filter(Expr): # we have to be careful here because we call filter_ below. # if this variable would be called filter, 2to3 would wrap the - # call in a list beause it is assuming we are talking about the + # call in a list because it is assuming we are talking about the # builtin filter function here which no longer returns a list in # python 3. because of that, do not rename filter_ to filter! filter_ = self.environment.filters.get(self.name) - if filter_ is None or getattr(filter_, 'contextfilter', False): + if filter_ is None or getattr(filter_, "contextfilter", False): raise Impossible() # We cannot constant handle async filters, so we need to make sure # to not go down this path. - if ( - eval_ctx.environment.is_async - and getattr(filter_, 'asyncfiltervariant', False) + if eval_ctx.environment.is_async and getattr( + filter_, "asyncfiltervariant", False ): raise Impossible() args, kwargs = args_as_const(self, eval_ctx) args.insert(0, self.node.as_const(eval_ctx)) - if getattr(filter_, 'evalcontextfilter', False): + if getattr(filter_, "evalcontextfilter", False): args.insert(0, eval_ctx) - elif getattr(filter_, 'environmentfilter', False): + elif getattr(filter_, "environmentfilter", False): args.insert(0, self.environment) try: @@ -671,7 +700,7 @@ class Test(Expr): rest of the fields are the same as for :class:`Call`. """ - fields = ('node', 'name', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') + fields = ("node", "name", "args", "kwargs", "dyn_args", "dyn_kwargs") def as_const(self, eval_ctx=None): test = self.environment.tests.get(self.name) @@ -696,20 +725,23 @@ class Call(Expr): node for dynamic positional (``*args``) or keyword (``**kwargs``) arguments. """ - fields = ('node', 'args', 'kwargs', 'dyn_args', 'dyn_kwargs') + + fields = ("node", "args", "kwargs", "dyn_args", "dyn_kwargs") class Getitem(Expr): """Get an attribute or item from an expression and prefer the item.""" - fields = ('node', 'arg', 'ctx') + + fields = ("node", "arg", "ctx") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) - if self.ctx != 'load': + if self.ctx != "load": raise Impossible() try: - return self.environment.getitem(self.node.as_const(eval_ctx), - self.arg.as_const(eval_ctx)) + return self.environment.getitem( + self.node.as_const(eval_ctx), self.arg.as_const(eval_ctx) + ) except Exception: raise Impossible() @@ -721,15 +753,15 @@ class Getattr(Expr): """Get an attribute or item from an expression that is a ascii-only bytestring and prefer the attribute. """ - fields = ('node', 'attr', 'ctx') + + fields = ("node", "attr", "ctx") def as_const(self, eval_ctx=None): - if self.ctx != 'load': + if self.ctx != "load": raise Impossible() try: eval_ctx = get_eval_context(self, eval_ctx) - return self.environment.getattr(self.node.as_const(eval_ctx), - self.attr) + return self.environment.getattr(self.node.as_const(eval_ctx), self.attr) except Exception: raise Impossible() @@ -741,14 +773,17 @@ class Slice(Expr): """Represents a slice object. This must only be used as argument for :class:`Subscript`. """ - fields = ('start', 'stop', 'step') + + fields = ("start", "stop", "step") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) + def const(obj): if obj is None: return None return obj.as_const(eval_ctx) + return slice(const(self.start), const(self.stop), const(self.step)) @@ -756,82 +791,103 @@ class Concat(Expr): """Concatenates the list of expressions provided after converting them to unicode. """ - fields = ('nodes',) + + fields = ("nodes",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) - return ''.join(text_type(x.as_const(eval_ctx)) for x in self.nodes) + return "".join(text_type(x.as_const(eval_ctx)) for x in self.nodes) class Compare(Expr): """Compares an expression with some other expressions. `ops` must be a list of :class:`Operand`\\s. """ - fields = ('expr', 'ops') + + fields = ("expr", "ops") def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) result = value = self.expr.as_const(eval_ctx) + try: for op in self.ops: new_value = op.expr.as_const(eval_ctx) result = _cmpop_to_func[op.op](value, new_value) + + if not result: + return False + value = new_value except Exception: raise Impossible() + return result class Operand(Helper): """Holds an operator and an expression.""" - fields = ('op', 'expr') + + fields = ("op", "expr") + if __debug__: - Operand.__doc__ += '\nThe following operators are available: ' + \ - ', '.join(sorted('``%s``' % x for x in set(_binop_to_func) | - set(_uaop_to_func) | set(_cmpop_to_func))) + Operand.__doc__ += "\nThe following operators are available: " + ", ".join( + sorted( + "``%s``" % x + for x in set(_binop_to_func) | set(_uaop_to_func) | set(_cmpop_to_func) + ) + ) class Mul(BinExpr): """Multiplies the left with the right node.""" - operator = '*' + + operator = "*" class Div(BinExpr): """Divides the left by the right node.""" - operator = '/' + + operator = "/" class FloorDiv(BinExpr): """Divides the left by the right node and truncates conver the result into an integer by truncating. """ - operator = '//' + + operator = "//" class Add(BinExpr): """Add the left to the right node.""" - operator = '+' + + operator = "+" class Sub(BinExpr): """Subtract the right from the left node.""" - operator = '-' + + operator = "-" class Mod(BinExpr): """Left modulo right.""" - operator = '%' + + operator = "%" class Pow(BinExpr): """Left to the power of right.""" - operator = '**' + + operator = "**" class And(BinExpr): """Short circuited AND.""" - operator = 'and' + + operator = "and" def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -840,7 +896,8 @@ class And(BinExpr): class Or(BinExpr): """Short circuited OR.""" - operator = 'or' + + operator = "or" def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -849,17 +906,20 @@ class Or(BinExpr): class Not(UnaryExpr): """Negate the expression.""" - operator = 'not' + + operator = "not" class Neg(UnaryExpr): """Make the expression negative.""" - operator = '-' + + operator = "-" class Pos(UnaryExpr): """Make the expression positive (noop for most expressions)""" - operator = '+' + + operator = "+" # Helpers for extensions @@ -869,7 +929,8 @@ class EnvironmentAttribute(Expr): """Loads an attribute from the environment object. This is useful for extensions that want to call a callback stored on the environment. """ - fields = ('name',) + + fields = ("name",) class ExtensionAttribute(Expr): @@ -879,7 +940,8 @@ class ExtensionAttribute(Expr): This node is usually constructed by calling the :meth:`~jinja2.ext.Extension.attr` method on an extension. """ - fields = ('identifier', 'name') + + fields = ("identifier", "name") class ImportedName(Expr): @@ -888,7 +950,8 @@ class ImportedName(Expr): function from the cgi module on evaluation. Imports are optimized by the compiler so there is no need to assign them to local variables. """ - fields = ('importname',) + + fields = ("importname",) class InternalName(Expr): @@ -898,16 +961,20 @@ class InternalName(Expr): a new identifier for you. This identifier is not available from the template and is not threated specially by the compiler. """ - fields = ('name',) + + fields = ("name",) def __init__(self): - raise TypeError('Can\'t create internal names. Use the ' - '`free_identifier` method on a parser.') + raise TypeError( + "Can't create internal names. Use the " + "`free_identifier` method on a parser." + ) class MarkSafe(Expr): """Mark the wrapped expression as safe (wrap it as `Markup`).""" - fields = ('expr',) + + fields = ("expr",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -920,7 +987,8 @@ class MarkSafeIfAutoescape(Expr): .. versionadded:: 2.5 """ - fields = ('expr',) + + fields = ("expr",) def as_const(self, eval_ctx=None): eval_ctx = get_eval_context(self, eval_ctx) @@ -942,6 +1010,20 @@ class ContextReference(Expr): Assign(Name('foo', ctx='store'), Getattr(ContextReference(), 'name')) + + This is basically equivalent to using the + :func:`~jinja2.contextfunction` decorator when using the + high-level API, which causes a reference to the context to be passed + as the first argument to a function. + """ + + +class DerivedContextReference(Expr): + """Return the current template context including locals. Behaves + exactly like :class:`ContextReference`, but includes local + variables, such as from a ``for`` loop. + + .. versionadded:: 2.11 """ @@ -955,7 +1037,8 @@ class Break(Stmt): class Scope(Stmt): """An artificial scope.""" - fields = ('body',) + + fields = ("body",) class OverlayScope(Stmt): @@ -971,7 +1054,8 @@ class OverlayScope(Stmt): .. versionadded:: 2.10 """ - fields = ('context', 'body') + + fields = ("context", "body") class EvalContextModifier(Stmt): @@ -982,7 +1066,8 @@ class EvalContextModifier(Stmt): EvalContextModifier(options=[Keyword('autoescape', Const(True))]) """ - fields = ('options',) + + fields = ("options",) class ScopedEvalContextModifier(EvalContextModifier): @@ -990,10 +1075,14 @@ class ScopedEvalContextModifier(EvalContextModifier): :class:`EvalContextModifier` but will only modify the :class:`~jinja2.nodes.EvalContext` for nodes in the :attr:`body`. """ - fields = ('body',) + + fields = ("body",) # make sure nobody creates custom nodes def _failing_new(*args, **kwargs): - raise TypeError('can\'t create custom node types') -NodeType.__new__ = staticmethod(_failing_new); del _failing_new + raise TypeError("can't create custom node types") + + +NodeType.__new__ = staticmethod(_failing_new) +del _failing_new diff --git a/pipenv/vendor/jinja2/optimizer.py b/pipenv/vendor/jinja2/optimizer.py index 65ab3ceb..7bc78c45 100644 --- a/pipenv/vendor/jinja2/optimizer.py +++ b/pipenv/vendor/jinja2/optimizer.py @@ -1,23 +1,15 @@ # -*- coding: utf-8 -*- +"""The optimizer tries to constant fold expressions and modify the AST +in place so that it should be faster to evaluate. + +Because the AST does not contain all the scoping information and the +compiler has to find that out, we cannot do all the optimizations we +want. For example, loop unrolling doesn't work because unrolled loops +would have a different scope. The solution would be a second syntax tree +that stored the scoping rules. """ - jinja2.optimizer - ~~~~~~~~~~~~~~~~ - - The jinja optimizer is currently trying to constant fold a few expressions - and modify the AST in place so that it should be easier to evaluate it. - - Because the AST does not contain all the scoping information and the - compiler has to find that out, we cannot do all the optimizations we - want. For example loop unrolling doesn't work because unrolled loops would - have a different scoping. - - The solution would be a second syntax tree that has the scoping rules stored. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" -from jinja2 import nodes -from jinja2.visitor import NodeTransformer +from . import nodes +from .visitor import NodeTransformer def optimize(node, environment): @@ -28,22 +20,22 @@ def optimize(node, environment): class Optimizer(NodeTransformer): - def __init__(self, environment): self.environment = environment - def fold(self, node, eval_ctx=None): - """Do constant folding.""" - node = self.generic_visit(node) - try: - return nodes.Const.from_untrusted(node.as_const(eval_ctx), - lineno=node.lineno, - environment=self.environment) - except nodes.Impossible: - return node + def generic_visit(self, node, *args, **kwargs): + node = super(Optimizer, self).generic_visit(node, *args, **kwargs) - visit_Add = visit_Sub = visit_Mul = visit_Div = visit_FloorDiv = \ - visit_Pow = visit_Mod = visit_And = visit_Or = visit_Pos = visit_Neg = \ - visit_Not = visit_Compare = visit_Getitem = visit_Getattr = visit_Call = \ - visit_Filter = visit_Test = visit_CondExpr = fold - del fold + # Do constant folding. Some other nodes besides Expr have + # as_const, but folding them causes errors later on. + if isinstance(node, nodes.Expr): + try: + return nodes.Const.from_untrusted( + node.as_const(args[0] if args else None), + lineno=node.lineno, + environment=self.environment, + ) + except nodes.Impossible: + pass + + return node diff --git a/pipenv/vendor/jinja2/parser.py b/pipenv/vendor/jinja2/parser.py index ed00d970..d5881066 100644 --- a/pipenv/vendor/jinja2/parser.py +++ b/pipenv/vendor/jinja2/parser.py @@ -1,41 +1,46 @@ # -*- coding: utf-8 -*- -""" - jinja2.parser - ~~~~~~~~~~~~~ +"""Parse tokens from the lexer into nodes for the compiler.""" +from . import nodes +from ._compat import imap +from .exceptions import TemplateAssertionError +from .exceptions import TemplateSyntaxError +from .lexer import describe_token +from .lexer import describe_token_expr - Implements the template parser. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -from jinja2 import nodes -from jinja2.exceptions import TemplateSyntaxError, TemplateAssertionError -from jinja2.lexer import describe_token, describe_token_expr -from jinja2._compat import imap - - -_statement_keywords = frozenset(['for', 'if', 'block', 'extends', 'print', - 'macro', 'include', 'from', 'import', - 'set', 'with', 'autoescape']) -_compare_operators = frozenset(['eq', 'ne', 'lt', 'lteq', 'gt', 'gteq']) +_statement_keywords = frozenset( + [ + "for", + "if", + "block", + "extends", + "print", + "macro", + "include", + "from", + "import", + "set", + "with", + "autoescape", + ] +) +_compare_operators = frozenset(["eq", "ne", "lt", "lteq", "gt", "gteq"]) _math_nodes = { - 'add': nodes.Add, - 'sub': nodes.Sub, - 'mul': nodes.Mul, - 'div': nodes.Div, - 'floordiv': nodes.FloorDiv, - 'mod': nodes.Mod, + "add": nodes.Add, + "sub": nodes.Sub, + "mul": nodes.Mul, + "div": nodes.Div, + "floordiv": nodes.FloorDiv, + "mod": nodes.Mod, } class Parser(object): - """This is the central parsing class Jinja2 uses. It's passed to + """This is the central parsing class Jinja uses. It's passed to extensions and can be used to parse expressions or statements. """ - def __init__(self, environment, source, name=None, filename=None, - state=None): + def __init__(self, environment, source, name=None, filename=None, state=None): self.environment = environment self.stream = environment._tokenize(source, name, filename, state) self.name = name @@ -63,31 +68,37 @@ class Parser(object): for exprs in end_token_stack: expected.extend(imap(describe_token_expr, exprs)) if end_token_stack: - currently_looking = ' or '.join( - "'%s'" % describe_token_expr(expr) - for expr in end_token_stack[-1]) + currently_looking = " or ".join( + "'%s'" % describe_token_expr(expr) for expr in end_token_stack[-1] + ) else: currently_looking = None if name is None: - message = ['Unexpected end of template.'] + message = ["Unexpected end of template."] else: - message = ['Encountered unknown tag \'%s\'.' % name] + message = ["Encountered unknown tag '%s'." % name] if currently_looking: if name is not None and name in expected: - message.append('You probably made a nesting mistake. Jinja ' - 'is expecting this tag, but currently looking ' - 'for %s.' % currently_looking) + message.append( + "You probably made a nesting mistake. Jinja " + "is expecting this tag, but currently looking " + "for %s." % currently_looking + ) else: - message.append('Jinja was looking for the following tags: ' - '%s.' % currently_looking) + message.append( + "Jinja was looking for the following tags: " + "%s." % currently_looking + ) if self._tag_stack: - message.append('The innermost block that needs to be ' - 'closed is \'%s\'.' % self._tag_stack[-1]) + message.append( + "The innermost block that needs to be " + "closed is '%s'." % self._tag_stack[-1] + ) - self.fail(' '.join(message), lineno) + self.fail(" ".join(message), lineno) def fail_unknown_tag(self, name, lineno=None): """Called if the parser encounters an unknown tag. Tries to fail @@ -105,7 +116,7 @@ class Parser(object): def is_tuple_end(self, extra_end_rules=None): """Are we at the end of a tuple?""" - if self.stream.current.type in ('variable_end', 'block_end', 'rparen'): + if self.stream.current.type in ("variable_end", "block_end", "rparen"): return True elif extra_end_rules is not None: return self.stream.current.test_any(extra_end_rules) @@ -115,22 +126,22 @@ class Parser(object): """Return a new free identifier as :class:`~jinja2.nodes.InternalName`.""" self._last_identifier += 1 rv = object.__new__(nodes.InternalName) - nodes.Node.__init__(rv, 'fi%d' % self._last_identifier, lineno=lineno) + nodes.Node.__init__(rv, "fi%d" % self._last_identifier, lineno=lineno) return rv def parse_statement(self): """Parse a single statement.""" token = self.stream.current - if token.type != 'name': - self.fail('tag name expected', token.lineno) + if token.type != "name": + self.fail("tag name expected", token.lineno) self._tag_stack.append(token.value) pop_tag = True try: if token.value in _statement_keywords: - return getattr(self, 'parse_' + self.stream.current.value)() - if token.value == 'call': + return getattr(self, "parse_" + self.stream.current.value)() + if token.value == "call": return self.parse_call_block() - if token.value == 'filter': + if token.value == "filter": return self.parse_filter_block() ext = self.extensions.get(token.value) if ext is not None: @@ -157,16 +168,16 @@ class Parser(object): can be set to `True` and the end token is removed. """ # the first token may be a colon for python compatibility - self.stream.skip_if('colon') + self.stream.skip_if("colon") # in the future it would be possible to add whole code sections # by adding some sort of end of statement token and parsing those here. - self.stream.expect('block_end') + self.stream.expect("block_end") result = self.subparse(end_tokens) # we reached the end of the template too early, the subparser # does not check for this, so we do that now - if self.stream.current.type == 'eof': + if self.stream.current.type == "eof": self.fail_eof(end_tokens) if drop_needle: @@ -177,50 +188,47 @@ class Parser(object): """Parse an assign statement.""" lineno = next(self.stream).lineno target = self.parse_assign_target(with_namespace=True) - if self.stream.skip_if('assign'): + if self.stream.skip_if("assign"): expr = self.parse_tuple() return nodes.Assign(target, expr, lineno=lineno) filter_node = self.parse_filter(None) - body = self.parse_statements(('name:endset',), - drop_needle=True) + body = self.parse_statements(("name:endset",), drop_needle=True) return nodes.AssignBlock(target, filter_node, body, lineno=lineno) def parse_for(self): """Parse a for loop.""" - lineno = self.stream.expect('name:for').lineno - target = self.parse_assign_target(extra_end_rules=('name:in',)) - self.stream.expect('name:in') - iter = self.parse_tuple(with_condexpr=False, - extra_end_rules=('name:recursive',)) + lineno = self.stream.expect("name:for").lineno + target = self.parse_assign_target(extra_end_rules=("name:in",)) + self.stream.expect("name:in") + iter = self.parse_tuple( + with_condexpr=False, extra_end_rules=("name:recursive",) + ) test = None - if self.stream.skip_if('name:if'): + if self.stream.skip_if("name:if"): test = self.parse_expression() - recursive = self.stream.skip_if('name:recursive') - body = self.parse_statements(('name:endfor', 'name:else')) - if next(self.stream).value == 'endfor': + recursive = self.stream.skip_if("name:recursive") + body = self.parse_statements(("name:endfor", "name:else")) + if next(self.stream).value == "endfor": else_ = [] else: - else_ = self.parse_statements(('name:endfor',), drop_needle=True) - return nodes.For(target, iter, body, else_, test, - recursive, lineno=lineno) + else_ = self.parse_statements(("name:endfor",), drop_needle=True) + return nodes.For(target, iter, body, else_, test, recursive, lineno=lineno) def parse_if(self): """Parse an if construct.""" - node = result = nodes.If(lineno=self.stream.expect('name:if').lineno) + node = result = nodes.If(lineno=self.stream.expect("name:if").lineno) while 1: node.test = self.parse_tuple(with_condexpr=False) - node.body = self.parse_statements(('name:elif', 'name:else', - 'name:endif')) + node.body = self.parse_statements(("name:elif", "name:else", "name:endif")) node.elif_ = [] node.else_ = [] token = next(self.stream) - if token.test('name:elif'): + if token.test("name:elif"): node = nodes.If(lineno=self.stream.current.lineno) result.elif_.append(node) continue - elif token.test('name:else'): - result.else_ = self.parse_statements(('name:endif',), - drop_needle=True) + elif token.test("name:else"): + result.else_ = self.parse_statements(("name:endif",), drop_needle=True) break return result @@ -228,45 +236,42 @@ class Parser(object): node = nodes.With(lineno=next(self.stream).lineno) targets = [] values = [] - while self.stream.current.type != 'block_end': - lineno = self.stream.current.lineno + while self.stream.current.type != "block_end": if targets: - self.stream.expect('comma') + self.stream.expect("comma") target = self.parse_assign_target() - target.set_ctx('param') + target.set_ctx("param") targets.append(target) - self.stream.expect('assign') + self.stream.expect("assign") values.append(self.parse_expression()) node.targets = targets node.values = values - node.body = self.parse_statements(('name:endwith',), - drop_needle=True) + node.body = self.parse_statements(("name:endwith",), drop_needle=True) return node def parse_autoescape(self): node = nodes.ScopedEvalContextModifier(lineno=next(self.stream).lineno) - node.options = [ - nodes.Keyword('autoescape', self.parse_expression()) - ] - node.body = self.parse_statements(('name:endautoescape',), - drop_needle=True) + node.options = [nodes.Keyword("autoescape", self.parse_expression())] + node.body = self.parse_statements(("name:endautoescape",), drop_needle=True) return nodes.Scope([node]) def parse_block(self): node = nodes.Block(lineno=next(self.stream).lineno) - node.name = self.stream.expect('name').value - node.scoped = self.stream.skip_if('name:scoped') + node.name = self.stream.expect("name").value + node.scoped = self.stream.skip_if("name:scoped") # common problem people encounter when switching from django # to jinja. we do not support hyphens in block names, so let's # raise a nicer error message in that case. - if self.stream.current.type == 'sub': - self.fail('Block names in Jinja have to be valid Python ' - 'identifiers and may not contain hyphens, use an ' - 'underscore instead.') + if self.stream.current.type == "sub": + self.fail( + "Block names in Jinja have to be valid Python " + "identifiers and may not contain hyphens, use an " + "underscore instead." + ) - node.body = self.parse_statements(('name:endblock',), drop_needle=True) - self.stream.skip_if('name:' + node.name) + node.body = self.parse_statements(("name:endblock",), drop_needle=True) + self.stream.skip_if("name:" + node.name) return node def parse_extends(self): @@ -275,9 +280,10 @@ class Parser(object): return node def parse_import_context(self, node, default): - if self.stream.current.test_any('name:with', 'name:without') and \ - self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' + if self.stream.current.test_any( + "name:with", "name:without" + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" self.stream.skip() else: node.with_context = default @@ -286,8 +292,9 @@ class Parser(object): def parse_include(self): node = nodes.Include(lineno=next(self.stream).lineno) node.template = self.parse_expression() - if self.stream.current.test('name:ignore') and \ - self.stream.look().test('name:missing'): + if self.stream.current.test("name:ignore") and self.stream.look().test( + "name:missing" + ): node.ignore_missing = True self.stream.skip(2) else: @@ -297,67 +304,71 @@ class Parser(object): def parse_import(self): node = nodes.Import(lineno=next(self.stream).lineno) node.template = self.parse_expression() - self.stream.expect('name:as') + self.stream.expect("name:as") node.target = self.parse_assign_target(name_only=True).name return self.parse_import_context(node, False) def parse_from(self): node = nodes.FromImport(lineno=next(self.stream).lineno) node.template = self.parse_expression() - self.stream.expect('name:import') + self.stream.expect("name:import") node.names = [] def parse_context(): - if self.stream.current.value in ('with', 'without') and \ - self.stream.look().test('name:context'): - node.with_context = next(self.stream).value == 'with' + if self.stream.current.value in ( + "with", + "without", + ) and self.stream.look().test("name:context"): + node.with_context = next(self.stream).value == "with" self.stream.skip() return True return False while 1: if node.names: - self.stream.expect('comma') - if self.stream.current.type == 'name': + self.stream.expect("comma") + if self.stream.current.type == "name": if parse_context(): break target = self.parse_assign_target(name_only=True) - if target.name.startswith('_'): - self.fail('names starting with an underline can not ' - 'be imported', target.lineno, - exc=TemplateAssertionError) - if self.stream.skip_if('name:as'): + if target.name.startswith("_"): + self.fail( + "names starting with an underline can not be imported", + target.lineno, + exc=TemplateAssertionError, + ) + if self.stream.skip_if("name:as"): alias = self.parse_assign_target(name_only=True) node.names.append((target.name, alias.name)) else: node.names.append(target.name) - if parse_context() or self.stream.current.type != 'comma': + if parse_context() or self.stream.current.type != "comma": break else: - self.stream.expect('name') - if not hasattr(node, 'with_context'): + self.stream.expect("name") + if not hasattr(node, "with_context"): node.with_context = False return node def parse_signature(self, node): node.args = args = [] node.defaults = defaults = [] - self.stream.expect('lparen') - while self.stream.current.type != 'rparen': + self.stream.expect("lparen") + while self.stream.current.type != "rparen": if args: - self.stream.expect('comma') + self.stream.expect("comma") arg = self.parse_assign_target(name_only=True) - arg.set_ctx('param') - if self.stream.skip_if('assign'): + arg.set_ctx("param") + if self.stream.skip_if("assign"): defaults.append(self.parse_expression()) elif defaults: - self.fail('non-default argument follows default argument') + self.fail("non-default argument follows default argument") args.append(arg) - self.stream.expect('rparen') + self.stream.expect("rparen") def parse_call_block(self): node = nodes.CallBlock(lineno=next(self.stream).lineno) - if self.stream.current.type == 'lparen': + if self.stream.current.type == "lparen": self.parse_signature(node) else: node.args = [] @@ -365,37 +376,40 @@ class Parser(object): node.call = self.parse_expression() if not isinstance(node.call, nodes.Call): - self.fail('expected call', node.lineno) - node.body = self.parse_statements(('name:endcall',), drop_needle=True) + self.fail("expected call", node.lineno) + node.body = self.parse_statements(("name:endcall",), drop_needle=True) return node def parse_filter_block(self): node = nodes.FilterBlock(lineno=next(self.stream).lineno) node.filter = self.parse_filter(None, start_inline=True) - node.body = self.parse_statements(('name:endfilter',), - drop_needle=True) + node.body = self.parse_statements(("name:endfilter",), drop_needle=True) return node def parse_macro(self): node = nodes.Macro(lineno=next(self.stream).lineno) node.name = self.parse_assign_target(name_only=True).name self.parse_signature(node) - node.body = self.parse_statements(('name:endmacro',), - drop_needle=True) + node.body = self.parse_statements(("name:endmacro",), drop_needle=True) return node def parse_print(self): node = nodes.Output(lineno=next(self.stream).lineno) node.nodes = [] - while self.stream.current.type != 'block_end': + while self.stream.current.type != "block_end": if node.nodes: - self.stream.expect('comma') + self.stream.expect("comma") node.nodes.append(self.parse_expression()) return node - def parse_assign_target(self, with_tuple=True, name_only=False, - extra_end_rules=None, with_namespace=False): - """Parse an assignment target. As Jinja2 allows assignments to + def parse_assign_target( + self, + with_tuple=True, + name_only=False, + extra_end_rules=None, + with_namespace=False, + ): + """Parse an assignment target. As Jinja allows assignments to tuples, this function can parse all allowed assignment targets. Per default assignments to tuples are parsed, that can be disable however by setting `with_tuple` to `False`. If only assignments to names are @@ -403,24 +417,26 @@ class Parser(object): parameter is forwarded to the tuple parsing function. If `with_namespace` is enabled, a namespace assignment may be parsed. """ - if with_namespace and self.stream.look().type == 'dot': - token = self.stream.expect('name') + if with_namespace and self.stream.look().type == "dot": + token = self.stream.expect("name") next(self.stream) # dot - attr = self.stream.expect('name') + attr = self.stream.expect("name") target = nodes.NSRef(token.value, attr.value, lineno=token.lineno) elif name_only: - token = self.stream.expect('name') - target = nodes.Name(token.value, 'store', lineno=token.lineno) + token = self.stream.expect("name") + target = nodes.Name(token.value, "store", lineno=token.lineno) else: if with_tuple: - target = self.parse_tuple(simplified=True, - extra_end_rules=extra_end_rules) + target = self.parse_tuple( + simplified=True, extra_end_rules=extra_end_rules + ) else: target = self.parse_primary() - target.set_ctx('store') + target.set_ctx("store") if not target.can_assign(): - self.fail('can\'t assign to %r' % target.__class__. - __name__.lower(), target.lineno) + self.fail( + "can't assign to %r" % target.__class__.__name__.lower(), target.lineno + ) return target def parse_expression(self, with_condexpr=True): @@ -435,9 +451,9 @@ class Parser(object): def parse_condexpr(self): lineno = self.stream.current.lineno expr1 = self.parse_or() - while self.stream.skip_if('name:if'): + while self.stream.skip_if("name:if"): expr2 = self.parse_or() - if self.stream.skip_if('name:else'): + if self.stream.skip_if("name:else"): expr3 = self.parse_condexpr() else: expr3 = None @@ -448,7 +464,7 @@ class Parser(object): def parse_or(self): lineno = self.stream.current.lineno left = self.parse_and() - while self.stream.skip_if('name:or'): + while self.stream.skip_if("name:or"): right = self.parse_and() left = nodes.Or(left, right, lineno=lineno) lineno = self.stream.current.lineno @@ -457,14 +473,14 @@ class Parser(object): def parse_and(self): lineno = self.stream.current.lineno left = self.parse_not() - while self.stream.skip_if('name:and'): + while self.stream.skip_if("name:and"): right = self.parse_not() left = nodes.And(left, right, lineno=lineno) lineno = self.stream.current.lineno return left def parse_not(self): - if self.stream.current.test('name:not'): + if self.stream.current.test("name:not"): lineno = next(self.stream).lineno return nodes.Not(self.parse_not(), lineno=lineno) return self.parse_compare() @@ -478,12 +494,13 @@ class Parser(object): if token_type in _compare_operators: next(self.stream) ops.append(nodes.Operand(token_type, self.parse_math1())) - elif self.stream.skip_if('name:in'): - ops.append(nodes.Operand('in', self.parse_math1())) - elif (self.stream.current.test('name:not') and - self.stream.look().test('name:in')): + elif self.stream.skip_if("name:in"): + ops.append(nodes.Operand("in", self.parse_math1())) + elif self.stream.current.test("name:not") and self.stream.look().test( + "name:in" + ): self.stream.skip(2) - ops.append(nodes.Operand('notin', self.parse_math1())) + ops.append(nodes.Operand("notin", self.parse_math1())) else: break lineno = self.stream.current.lineno @@ -494,7 +511,7 @@ class Parser(object): def parse_math1(self): lineno = self.stream.current.lineno left = self.parse_concat() - while self.stream.current.type in ('add', 'sub'): + while self.stream.current.type in ("add", "sub"): cls = _math_nodes[self.stream.current.type] next(self.stream) right = self.parse_concat() @@ -505,7 +522,7 @@ class Parser(object): def parse_concat(self): lineno = self.stream.current.lineno args = [self.parse_math2()] - while self.stream.current.type == 'tilde': + while self.stream.current.type == "tilde": next(self.stream) args.append(self.parse_math2()) if len(args) == 1: @@ -515,7 +532,7 @@ class Parser(object): def parse_math2(self): lineno = self.stream.current.lineno left = self.parse_pow() - while self.stream.current.type in ('mul', 'div', 'floordiv', 'mod'): + while self.stream.current.type in ("mul", "div", "floordiv", "mod"): cls = _math_nodes[self.stream.current.type] next(self.stream) right = self.parse_pow() @@ -526,7 +543,7 @@ class Parser(object): def parse_pow(self): lineno = self.stream.current.lineno left = self.parse_unary() - while self.stream.current.type == 'pow': + while self.stream.current.type == "pow": next(self.stream) right = self.parse_unary() left = nodes.Pow(left, right, lineno=lineno) @@ -536,10 +553,10 @@ class Parser(object): def parse_unary(self, with_filter=True): token_type = self.stream.current.type lineno = self.stream.current.lineno - if token_type == 'sub': + if token_type == "sub": next(self.stream) node = nodes.Neg(self.parse_unary(False), lineno=lineno) - elif token_type == 'add': + elif token_type == "add": next(self.stream) node = nodes.Pos(self.parse_unary(False), lineno=lineno) else: @@ -551,40 +568,44 @@ class Parser(object): def parse_primary(self): token = self.stream.current - if token.type == 'name': - if token.value in ('true', 'false', 'True', 'False'): - node = nodes.Const(token.value in ('true', 'True'), - lineno=token.lineno) - elif token.value in ('none', 'None'): + if token.type == "name": + if token.value in ("true", "false", "True", "False"): + node = nodes.Const(token.value in ("true", "True"), lineno=token.lineno) + elif token.value in ("none", "None"): node = nodes.Const(None, lineno=token.lineno) else: - node = nodes.Name(token.value, 'load', lineno=token.lineno) + node = nodes.Name(token.value, "load", lineno=token.lineno) next(self.stream) - elif token.type == 'string': + elif token.type == "string": next(self.stream) buf = [token.value] lineno = token.lineno - while self.stream.current.type == 'string': + while self.stream.current.type == "string": buf.append(self.stream.current.value) next(self.stream) - node = nodes.Const(''.join(buf), lineno=lineno) - elif token.type in ('integer', 'float'): + node = nodes.Const("".join(buf), lineno=lineno) + elif token.type in ("integer", "float"): next(self.stream) node = nodes.Const(token.value, lineno=token.lineno) - elif token.type == 'lparen': + elif token.type == "lparen": next(self.stream) node = self.parse_tuple(explicit_parentheses=True) - self.stream.expect('rparen') - elif token.type == 'lbracket': + self.stream.expect("rparen") + elif token.type == "lbracket": node = self.parse_list() - elif token.type == 'lbrace': + elif token.type == "lbrace": node = self.parse_dict() else: self.fail("unexpected '%s'" % describe_token(token), token.lineno) return node - def parse_tuple(self, simplified=False, with_condexpr=True, - extra_end_rules=None, explicit_parentheses=False): + def parse_tuple( + self, + simplified=False, + with_condexpr=True, + extra_end_rules=None, + explicit_parentheses=False, + ): """Works like `parse_expression` but if multiple expressions are delimited by a comma a :class:`~jinja2.nodes.Tuple` node is created. This method could also return a regular expression instead of a tuple @@ -609,16 +630,19 @@ class Parser(object): elif with_condexpr: parse = self.parse_expression else: - parse = lambda: self.parse_expression(with_condexpr=False) + + def parse(): + return self.parse_expression(with_condexpr=False) + args = [] is_tuple = False while 1: if args: - self.stream.expect('comma') + self.stream.expect("comma") if self.is_tuple_end(extra_end_rules): break args.append(parse()) - if self.stream.current.type == 'comma': + if self.stream.current.type == "comma": is_tuple = True else: break @@ -633,46 +657,48 @@ class Parser(object): # nothing) in the spot of an expression would be an empty # tuple. if not explicit_parentheses: - self.fail('Expected an expression, got \'%s\'' % - describe_token(self.stream.current)) + self.fail( + "Expected an expression, got '%s'" + % describe_token(self.stream.current) + ) - return nodes.Tuple(args, 'load', lineno=lineno) + return nodes.Tuple(args, "load", lineno=lineno) def parse_list(self): - token = self.stream.expect('lbracket') + token = self.stream.expect("lbracket") items = [] - while self.stream.current.type != 'rbracket': + while self.stream.current.type != "rbracket": if items: - self.stream.expect('comma') - if self.stream.current.type == 'rbracket': + self.stream.expect("comma") + if self.stream.current.type == "rbracket": break items.append(self.parse_expression()) - self.stream.expect('rbracket') + self.stream.expect("rbracket") return nodes.List(items, lineno=token.lineno) def parse_dict(self): - token = self.stream.expect('lbrace') + token = self.stream.expect("lbrace") items = [] - while self.stream.current.type != 'rbrace': + while self.stream.current.type != "rbrace": if items: - self.stream.expect('comma') - if self.stream.current.type == 'rbrace': + self.stream.expect("comma") + if self.stream.current.type == "rbrace": break key = self.parse_expression() - self.stream.expect('colon') + self.stream.expect("colon") value = self.parse_expression() items.append(nodes.Pair(key, value, lineno=key.lineno)) - self.stream.expect('rbrace') + self.stream.expect("rbrace") return nodes.Dict(items, lineno=token.lineno) def parse_postfix(self, node): while 1: token_type = self.stream.current.type - if token_type == 'dot' or token_type == 'lbracket': + if token_type == "dot" or token_type == "lbracket": node = self.parse_subscript(node) # calls are valid both after postfix expressions (getattr # and getitem) as well as filters and tests - elif token_type == 'lparen': + elif token_type == "lparen": node = self.parse_call(node) else: break @@ -681,13 +707,13 @@ class Parser(object): def parse_filter_expr(self, node): while 1: token_type = self.stream.current.type - if token_type == 'pipe': + if token_type == "pipe": node = self.parse_filter(node) - elif token_type == 'name' and self.stream.current.value == 'is': + elif token_type == "name" and self.stream.current.value == "is": node = self.parse_test(node) # calls are valid both after postfix expressions (getattr # and getitem) as well as filters and tests - elif token_type == 'lparen': + elif token_type == "lparen": node = self.parse_call(node) else: break @@ -695,53 +721,54 @@ class Parser(object): def parse_subscript(self, node): token = next(self.stream) - if token.type == 'dot': + if token.type == "dot": attr_token = self.stream.current next(self.stream) - if attr_token.type == 'name': - return nodes.Getattr(node, attr_token.value, 'load', - lineno=token.lineno) - elif attr_token.type != 'integer': - self.fail('expected name or number', attr_token.lineno) + if attr_token.type == "name": + return nodes.Getattr( + node, attr_token.value, "load", lineno=token.lineno + ) + elif attr_token.type != "integer": + self.fail("expected name or number", attr_token.lineno) arg = nodes.Const(attr_token.value, lineno=attr_token.lineno) - return nodes.Getitem(node, arg, 'load', lineno=token.lineno) - if token.type == 'lbracket': + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + if token.type == "lbracket": args = [] - while self.stream.current.type != 'rbracket': + while self.stream.current.type != "rbracket": if args: - self.stream.expect('comma') + self.stream.expect("comma") args.append(self.parse_subscribed()) - self.stream.expect('rbracket') + self.stream.expect("rbracket") if len(args) == 1: arg = args[0] else: - arg = nodes.Tuple(args, 'load', lineno=token.lineno) - return nodes.Getitem(node, arg, 'load', lineno=token.lineno) - self.fail('expected subscript expression', self.lineno) + arg = nodes.Tuple(args, "load", lineno=token.lineno) + return nodes.Getitem(node, arg, "load", lineno=token.lineno) + self.fail("expected subscript expression", token.lineno) def parse_subscribed(self): lineno = self.stream.current.lineno - if self.stream.current.type == 'colon': + if self.stream.current.type == "colon": next(self.stream) args = [None] else: node = self.parse_expression() - if self.stream.current.type != 'colon': + if self.stream.current.type != "colon": return node next(self.stream) args = [node] - if self.stream.current.type == 'colon': + if self.stream.current.type == "colon": args.append(None) - elif self.stream.current.type not in ('rbracket', 'comma'): + elif self.stream.current.type not in ("rbracket", "comma"): args.append(self.parse_expression()) else: args.append(None) - if self.stream.current.type == 'colon': + if self.stream.current.type == "colon": next(self.stream) - if self.stream.current.type not in ('rbracket', 'comma'): + if self.stream.current.type not in ("rbracket", "comma"): args.append(self.parse_expression()) else: args.append(None) @@ -751,7 +778,7 @@ class Parser(object): return nodes.Slice(lineno=lineno, *args) def parse_call(self, node): - token = self.stream.expect('lparen') + token = self.stream.expect("lparen") args = [] kwargs = [] dyn_args = dyn_kwargs = None @@ -759,91 +786,100 @@ class Parser(object): def ensure(expr): if not expr: - self.fail('invalid syntax for function call expression', - token.lineno) + self.fail("invalid syntax for function call expression", token.lineno) - while self.stream.current.type != 'rparen': + while self.stream.current.type != "rparen": if require_comma: - self.stream.expect('comma') + self.stream.expect("comma") # support for trailing comma - if self.stream.current.type == 'rparen': + if self.stream.current.type == "rparen": break - if self.stream.current.type == 'mul': + if self.stream.current.type == "mul": ensure(dyn_args is None and dyn_kwargs is None) next(self.stream) dyn_args = self.parse_expression() - elif self.stream.current.type == 'pow': + elif self.stream.current.type == "pow": ensure(dyn_kwargs is None) next(self.stream) dyn_kwargs = self.parse_expression() else: - ensure(dyn_args is None and dyn_kwargs is None) - if self.stream.current.type == 'name' and \ - self.stream.look().type == 'assign': + if ( + self.stream.current.type == "name" + and self.stream.look().type == "assign" + ): + # Parsing a kwarg + ensure(dyn_kwargs is None) key = self.stream.current.value self.stream.skip(2) value = self.parse_expression() - kwargs.append(nodes.Keyword(key, value, - lineno=value.lineno)) + kwargs.append(nodes.Keyword(key, value, lineno=value.lineno)) else: - ensure(not kwargs) + # Parsing an arg + ensure(dyn_args is None and dyn_kwargs is None and not kwargs) args.append(self.parse_expression()) require_comma = True - self.stream.expect('rparen') + self.stream.expect("rparen") if node is None: return args, kwargs, dyn_args, dyn_kwargs - return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, - lineno=token.lineno) + return nodes.Call(node, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno) def parse_filter(self, node, start_inline=False): - while self.stream.current.type == 'pipe' or start_inline: + while self.stream.current.type == "pipe" or start_inline: if not start_inline: next(self.stream) - token = self.stream.expect('name') + token = self.stream.expect("name") name = token.value - while self.stream.current.type == 'dot': + while self.stream.current.type == "dot": next(self.stream) - name += '.' + self.stream.expect('name').value - if self.stream.current.type == 'lparen': + name += "." + self.stream.expect("name").value + if self.stream.current.type == "lparen": args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) else: args = [] kwargs = [] dyn_args = dyn_kwargs = None - node = nodes.Filter(node, name, args, kwargs, dyn_args, - dyn_kwargs, lineno=token.lineno) + node = nodes.Filter( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) start_inline = False return node def parse_test(self, node): token = next(self.stream) - if self.stream.current.test('name:not'): + if self.stream.current.test("name:not"): next(self.stream) negated = True else: negated = False - name = self.stream.expect('name').value - while self.stream.current.type == 'dot': + name = self.stream.expect("name").value + while self.stream.current.type == "dot": next(self.stream) - name += '.' + self.stream.expect('name').value + name += "." + self.stream.expect("name").value dyn_args = dyn_kwargs = None kwargs = [] - if self.stream.current.type == 'lparen': + if self.stream.current.type == "lparen": args, kwargs, dyn_args, dyn_kwargs = self.parse_call(None) - elif (self.stream.current.type in ('name', 'string', 'integer', - 'float', 'lparen', 'lbracket', - 'lbrace') and not - self.stream.current.test_any('name:else', 'name:or', - 'name:and')): - if self.stream.current.test('name:is'): - self.fail('You cannot chain multiple tests with is') - args = [self.parse_primary()] + elif self.stream.current.type in ( + "name", + "string", + "integer", + "float", + "lparen", + "lbracket", + "lbrace", + ) and not self.stream.current.test_any("name:else", "name:or", "name:and"): + if self.stream.current.test("name:is"): + self.fail("You cannot chain multiple tests with is") + arg_node = self.parse_primary() + arg_node = self.parse_postfix(arg_node) + args = [arg_node] else: args = [] - node = nodes.Test(node, name, args, kwargs, dyn_args, - dyn_kwargs, lineno=token.lineno) + node = nodes.Test( + node, name, args, kwargs, dyn_args, dyn_kwargs, lineno=token.lineno + ) if negated: node = nodes.Not(node, lineno=token.lineno) return node @@ -865,29 +901,29 @@ class Parser(object): try: while self.stream: token = self.stream.current - if token.type == 'data': + if token.type == "data": if token.value: - add_data(nodes.TemplateData(token.value, - lineno=token.lineno)) + add_data(nodes.TemplateData(token.value, lineno=token.lineno)) next(self.stream) - elif token.type == 'variable_begin': + elif token.type == "variable_begin": next(self.stream) add_data(self.parse_tuple(with_condexpr=True)) - self.stream.expect('variable_end') - elif token.type == 'block_begin': + self.stream.expect("variable_end") + elif token.type == "block_begin": flush_data() next(self.stream) - if end_tokens is not None and \ - self.stream.current.test_any(*end_tokens): + if end_tokens is not None and self.stream.current.test_any( + *end_tokens + ): return body rv = self.parse_statement() if isinstance(rv, list): body.extend(rv) else: body.append(rv) - self.stream.expect('block_end') + self.stream.expect("block_end") else: - raise AssertionError('internal parsing error') + raise AssertionError("internal parsing error") flush_data() finally: diff --git a/pipenv/vendor/jinja2/runtime.py b/pipenv/vendor/jinja2/runtime.py index f9d7a680..527d4b5e 100644 --- a/pipenv/vendor/jinja2/runtime.py +++ b/pipenv/vendor/jinja2/runtime.py @@ -1,43 +1,62 @@ # -*- coding: utf-8 -*- -""" - jinja2.runtime - ~~~~~~~~~~~~~~ - - Runtime helpers. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" +"""The runtime functions and state used by compiled templates.""" import sys - from itertools import chain from types import MethodType -from jinja2.nodes import EvalContext, _context_function_types -from jinja2.utils import Markup, soft_unicode, escape, missing, concat, \ - internalcode, object_type_repr, evalcontextfunction, Namespace -from jinja2.exceptions import UndefinedError, TemplateRuntimeError, \ - TemplateNotFound -from jinja2._compat import imap, text_type, iteritems, \ - implements_iterator, implements_to_string, string_types, PY2, \ - with_metaclass +from markupsafe import escape # noqa: F401 +from markupsafe import Markup +from markupsafe import soft_unicode +from ._compat import abc +from ._compat import imap +from ._compat import implements_iterator +from ._compat import implements_to_string +from ._compat import iteritems +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import with_metaclass +from .exceptions import TemplateNotFound # noqa: F401 +from .exceptions import TemplateRuntimeError # noqa: F401 +from .exceptions import UndefinedError +from .nodes import EvalContext +from .utils import concat +from .utils import evalcontextfunction +from .utils import internalcode +from .utils import missing +from .utils import Namespace # noqa: F401 +from .utils import object_type_repr # these variables are exported to the template runtime -__all__ = ['LoopContext', 'TemplateReference', 'Macro', 'Markup', - 'TemplateRuntimeError', 'missing', 'concat', 'escape', - 'markup_join', 'unicode_join', 'to_string', 'identity', - 'TemplateNotFound', 'Namespace'] +exported = [ + "LoopContext", + "TemplateReference", + "Macro", + "Markup", + "TemplateRuntimeError", + "missing", + "concat", + "escape", + "markup_join", + "unicode_join", + "to_string", + "identity", + "TemplateNotFound", + "Namespace", + "Undefined", +] #: the name of the function that is used to convert something into #: a string. We can just use the text type here. to_string = text_type -#: the identity function. Useful for certain things in the environment -identity = lambda x: x -_first_iteration = object() -_last_iteration = object() +def identity(x): + """Returns its argument. Useful for certain things in the + environment. + """ + return x def markup_join(seq): @@ -46,8 +65,8 @@ def markup_join(seq): iterator = imap(soft_unicode, seq) for arg in iterator: buf.append(arg) - if hasattr(arg, '__html__'): - return Markup(u'').join(chain(buf, iterator)) + if hasattr(arg, "__html__"): + return Markup(u"").join(chain(buf, iterator)) return concat(buf) @@ -56,9 +75,16 @@ def unicode_join(seq): return concat(imap(text_type, seq)) -def new_context(environment, template_name, blocks, vars=None, - shared=None, globals=None, locals=None): - """Internal helper to for context creation.""" +def new_context( + environment, + template_name, + blocks, + vars=None, + shared=None, + globals=None, + locals=None, +): + """Internal helper for context creation.""" if vars is None: vars = {} if shared: @@ -73,8 +99,7 @@ def new_context(environment, template_name, blocks, vars=None, for key, value in iteritems(locals): if value is not missing: parent[key] = value - return environment.context_class(environment, parent, template_name, - blocks) + return environment.context_class(environment, parent, template_name, blocks) class TemplateReference(object): @@ -88,20 +113,16 @@ class TemplateReference(object): return BlockReference(name, self.__context, blocks, 0) def __repr__(self): - return '<%s %r>' % ( - self.__class__.__name__, - self.__context.name - ) + return "<%s %r>" % (self.__class__.__name__, self.__context.name) def _get_func(x): - return getattr(x, '__func__', x) + return getattr(x, "__func__", x) class ContextMeta(type): - - def __new__(cls, name, bases, d): - rv = type.__new__(cls, name, bases, d) + def __new__(mcs, name, bases, d): + rv = type.__new__(mcs, name, bases, d) if bases == (): return rv @@ -112,11 +133,15 @@ class ContextMeta(type): # If we have a changed resolve but no changed default or missing # resolve we invert the call logic. - if resolve is not default_resolve and \ - resolve_or_missing is default_resolve_or_missing: + if ( + resolve is not default_resolve + and resolve_or_missing is default_resolve_or_missing + ): rv._legacy_resolve_mode = True - elif resolve is default_resolve and \ - resolve_or_missing is default_resolve_or_missing: + elif ( + resolve is default_resolve + and resolve_or_missing is default_resolve_or_missing + ): rv._fast_resolve_mode = True return rv @@ -149,6 +174,7 @@ class Context(with_metaclass(ContextMeta)): method that doesn't fail with a `KeyError` but returns an :class:`Undefined` object for missing variables. """ + # XXX: we want to eventually make this be a deprecation warning and # remove it. _legacy_resolve_mode = False @@ -179,9 +205,9 @@ class Context(with_metaclass(ContextMeta)): index = blocks.index(current) + 1 blocks[index] except LookupError: - return self.environment.undefined('there is no parent block ' - 'called %r.' % name, - name='super') + return self.environment.undefined( + "there is no parent block called %r." % name, name="super" + ) return BlockReference(name, self, blocks, index) def get(self, key, default=None): @@ -232,7 +258,7 @@ class Context(with_metaclass(ContextMeta)): return dict(self.parent, **self.vars) @internalcode - def call(__self, __obj, *args, **kwargs): + def call(__self, __obj, *args, **kwargs): # noqa: B902 """Call the callable with the arguments and keyword arguments provided but inject the active context or environment as first argument if the callable is a :func:`contextfunction` or @@ -242,55 +268,62 @@ class Context(with_metaclass(ContextMeta)): __traceback_hide__ = True # noqa # Allow callable classes to take a context - if hasattr(__obj, '__call__'): + if hasattr(__obj, "__call__"): # noqa: B004 fn = __obj.__call__ - for fn_type in ('contextfunction', - 'evalcontextfunction', - 'environmentfunction'): + for fn_type in ( + "contextfunction", + "evalcontextfunction", + "environmentfunction", + ): if hasattr(fn, fn_type): __obj = fn break - if isinstance(__obj, _context_function_types): - if getattr(__obj, 'contextfunction', 0): + if callable(__obj): + if getattr(__obj, "contextfunction", 0): args = (__self,) + args - elif getattr(__obj, 'evalcontextfunction', 0): + elif getattr(__obj, "evalcontextfunction", 0): args = (__self.eval_ctx,) + args - elif getattr(__obj, 'environmentfunction', 0): + elif getattr(__obj, "environmentfunction", 0): args = (__self.environment,) + args try: return __obj(*args, **kwargs) except StopIteration: - return __self.environment.undefined('value was undefined because ' - 'a callable raised a ' - 'StopIteration exception') + return __self.environment.undefined( + "value was undefined because " + "a callable raised a " + "StopIteration exception" + ) def derived(self, locals=None): """Internal helper function to create a derived context. This is used in situations where the system needs a new context in the same template that is independent. """ - context = new_context(self.environment, self.name, {}, - self.get_all(), True, None, locals) + context = new_context( + self.environment, self.name, {}, self.get_all(), True, None, locals + ) context.eval_ctx = self.eval_ctx context.blocks.update((k, list(v)) for k, v in iteritems(self.blocks)) return context - def _all(meth): - proxy = lambda self: getattr(self.get_all(), meth)() + def _all(meth): # noqa: B902 + def proxy(self): + return getattr(self.get_all(), meth)() + proxy.__doc__ = getattr(dict, meth).__doc__ proxy.__name__ = meth return proxy - keys = _all('keys') - values = _all('values') - items = _all('items') + keys = _all("keys") + values = _all("values") + items = _all("items") # not available on python 3 if PY2: - iterkeys = _all('iterkeys') - itervalues = _all('itervalues') - iteritems = _all('iteritems') + iterkeys = _all("iterkeys") + itervalues = _all("itervalues") + iteritems = _all("iteritems") del _all def __contains__(self, name): @@ -306,19 +339,14 @@ class Context(with_metaclass(ContextMeta)): return item def __repr__(self): - return '<%s %s of %r>' % ( + return "<%s %s of %r>" % ( self.__class__.__name__, repr(self.get_all()), - self.name + self.name, ) -# register the context as mapping if possible -try: - from collections import Mapping - Mapping.register(Context) -except ImportError: - pass +abc.Mapping.register(Context) class BlockReference(object): @@ -334,11 +362,10 @@ class BlockReference(object): def super(self): """Super the block.""" if self._depth + 1 >= len(self._stack): - return self._context.environment. \ - undefined('there is no parent block called %r.' % - self.name, name='super') - return BlockReference(self.name, self._context, self._stack, - self._depth + 1) + return self._context.environment.undefined( + "there is no parent block called %r." % self.name, name="super" + ) + return BlockReference(self.name, self._context, self._stack, self._depth + 1) @internalcode def __call__(self): @@ -348,143 +375,212 @@ class BlockReference(object): return rv -class LoopContextBase(object): - """A loop context for dynamic iteration.""" +@implements_iterator +class LoopContext: + """A wrapper iterable for dynamic ``for`` loops, with information + about the loop and iteration. + """ + + #: Current iteration of the loop, starting at 0. + index0 = -1 - _before = _first_iteration - _current = _first_iteration - _after = _last_iteration _length = None + _after = missing + _current = missing + _before = missing + _last_changed_value = missing - def __init__(self, undefined, recurse=None, depth0=0): + def __init__(self, iterable, undefined, recurse=None, depth0=0): + """ + :param iterable: Iterable to wrap. + :param undefined: :class:`Undefined` class to use for next and + previous items. + :param recurse: The function to render the loop body when the + loop is marked recursive. + :param depth0: Incremented when looping recursively. + """ + self._iterable = iterable + self._iterator = self._to_iterator(iterable) self._undefined = undefined self._recurse = recurse - self.index0 = -1 + #: How many levels deep a recursive loop currently is, starting at 0. self.depth0 = depth0 - self._last_checked_value = missing - def cycle(self, *args): - """Cycles among the arguments with the current loop index.""" - if not args: - raise TypeError('no items for cycling given') - return args[self.index0 % len(args)] - - def changed(self, *value): - """Checks whether the value has changed since the last call.""" - if self._last_checked_value != value: - self._last_checked_value = value - return True - return False - - first = property(lambda x: x.index0 == 0) - last = property(lambda x: x._after is _last_iteration) - index = property(lambda x: x.index0 + 1) - revindex = property(lambda x: x.length - x.index0) - revindex0 = property(lambda x: x.length - x.index) - depth = property(lambda x: x.depth0 + 1) + @staticmethod + def _to_iterator(iterable): + return iter(iterable) @property - def previtem(self): - if self._before is _first_iteration: - return self._undefined('there is no previous item') - return self._before + def length(self): + """Length of the iterable. - @property - def nextitem(self): - if self._after is _last_iteration: - return self._undefined('there is no next item') - return self._after + If the iterable is a generator or otherwise does not have a + size, it is eagerly evaluated to get a size. + """ + if self._length is not None: + return self._length + + try: + self._length = len(self._iterable) + except TypeError: + iterable = list(self._iterator) + self._iterator = self._to_iterator(iterable) + self._length = len(iterable) + self.index + (self._after is not missing) + + return self._length def __len__(self): return self.length - @internalcode - def loop(self, iterable): - if self._recurse is None: - raise TypeError('Tried to call non recursive loop. Maybe you ' - "forgot the 'recursive' modifier.") - return self._recurse(iterable, self._recurse, self.depth0 + 1) - - # a nifty trick to enhance the error message if someone tried to call - # the the loop without or with too many arguments. - __call__ = loop - del loop - - def __repr__(self): - return '<%s %r/%r>' % ( - self.__class__.__name__, - self.index, - self.length - ) - - -class LoopContext(LoopContextBase): - - def __init__(self, iterable, undefined, recurse=None, depth0=0): - LoopContextBase.__init__(self, undefined, recurse, depth0) - self._iterator = iter(iterable) - - # try to get the length of the iterable early. This must be done - # here because there are some broken iterators around where there - # __len__ is the number of iterations left (i'm looking at your - # listreverseiterator!). - try: - self._length = len(iterable) - except (TypeError, AttributeError): - self._length = None - self._after = self._safe_next() + @property + def depth(self): + """How many levels deep a recursive loop currently is, starting at 1.""" + return self.depth0 + 1 @property - def length(self): - if self._length is None: - # if was not possible to get the length of the iterator when - # the loop context was created (ie: iterating over a generator) - # we have to convert the iterable into a sequence and use the - # length of that + the number of iterations so far. - iterable = tuple(self._iterator) - self._iterator = iter(iterable) - iterations_done = self.index0 + 2 - self._length = len(iterable) + iterations_done - return self._length + def index(self): + """Current iteration of the loop, starting at 1.""" + return self.index0 + 1 - def __iter__(self): - return LoopContextIterator(self) + @property + def revindex0(self): + """Number of iterations from the end of the loop, ending at 0. - def _safe_next(self): - try: - return next(self._iterator) - except StopIteration: - return _last_iteration + Requires calculating :attr:`length`. + """ + return self.length - self.index + @property + def revindex(self): + """Number of iterations from the end of the loop, ending at 1. -@implements_iterator -class LoopContextIterator(object): - """The iterator for a loop context.""" - __slots__ = ('context',) + Requires calculating :attr:`length`. + """ + return self.length - self.index0 - def __init__(self, context): - self.context = context + @property + def first(self): + """Whether this is the first iteration of the loop.""" + return self.index0 == 0 + + def _peek_next(self): + """Return the next element in the iterable, or :data:`missing` + if the iterable is exhausted. Only peeks one item ahead, caching + the result in :attr:`_last` for use in subsequent checks. The + cache is reset when :meth:`__next__` is called. + """ + if self._after is not missing: + return self._after + + self._after = next(self._iterator, missing) + return self._after + + @property + def last(self): + """Whether this is the last iteration of the loop. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + return self._peek_next() is missing + + @property + def previtem(self): + """The item in the previous iteration. Undefined during the + first iteration. + """ + if self.first: + return self._undefined("there is no previous item") + + return self._before + + @property + def nextitem(self): + """The item in the next iteration. Undefined during the last + iteration. + + Causes the iterable to advance early. See + :func:`itertools.groupby` for issues this can cause. + The :func:`groupby` filter avoids that issue. + """ + rv = self._peek_next() + + if rv is missing: + return self._undefined("there is no next item") + + return rv + + def cycle(self, *args): + """Return a value from the given args, cycling through based on + the current :attr:`index0`. + + :param args: One or more values to cycle through. + """ + if not args: + raise TypeError("no items for cycling given") + + return args[self.index0 % len(args)] + + def changed(self, *value): + """Return ``True`` if previously called with a different value + (including when called for the first time). + + :param value: One or more values to compare to the last call. + """ + if self._last_changed_value != value: + self._last_changed_value = value + return True + + return False def __iter__(self): return self def __next__(self): - ctx = self.context - ctx.index0 += 1 - if ctx._after is _last_iteration: - raise StopIteration() - ctx._before = ctx._current - ctx._current = ctx._after - ctx._after = ctx._safe_next() - return ctx._current, ctx + if self._after is not missing: + rv = self._after + self._after = missing + else: + rv = next(self._iterator) + + self.index0 += 1 + self._before = self._current + self._current = rv + return rv, self + + @internalcode + def __call__(self, iterable): + """When iterating over nested data, render the body of the loop + recursively with the given inner iterable data. + + The loop must have the ``recursive`` marker for this to work. + """ + if self._recurse is None: + raise TypeError( + "The loop must have the 'recursive' marker to be called recursively." + ) + + return self._recurse(iterable, self._recurse, depth=self.depth) + + def __repr__(self): + return "<%s %d/%d>" % (self.__class__.__name__, self.index, self.length) class Macro(object): """Wraps a macro function.""" - def __init__(self, environment, func, name, arguments, - catch_kwargs, catch_varargs, caller, - default_autoescape=None): + def __init__( + self, + environment, + func, + name, + arguments, + catch_kwargs, + catch_varargs, + caller, + default_autoescape=None, + ): self._environment = environment self._func = func self._argument_count = len(arguments) @@ -493,7 +589,7 @@ class Macro(object): self.catch_kwargs = catch_kwargs self.catch_varargs = catch_varargs self.caller = caller - self.explicit_caller = 'caller' in arguments + self.explicit_caller = "caller" in arguments if default_autoescape is None: default_autoescape = environment.autoescape self._default_autoescape = default_autoescape @@ -505,9 +601,8 @@ class Macro(object): # decide largely based on compile-time information if a macro is # safe or unsafe. While there was a volatile mode it was largely # unused for deciding on escaping. This turns out to be - # problemtic for macros because if a macro is safe or not not so - # much depends on the escape mode when it was defined but when it - # was used. + # problematic for macros because whether a macro is safe depends not + # on the escape mode when it was defined, but rather when it was used. # # Because however we export macros from the module system and # there are historic callers that do not pass an eval context (and @@ -515,7 +610,7 @@ class Macro(object): # check here. # # This is considered safe because an eval context is not a valid - # argument to callables otherwise anwyays. Worst case here is + # argument to callables otherwise anyway. Worst case here is # that if no eval context is passed we fall back to the compile # time autoescape flag. if args and isinstance(args[0], EvalContext): @@ -525,7 +620,7 @@ class Macro(object): autoescape = self._default_autoescape # try to consume the positional arguments - arguments = list(args[:self._argument_count]) + arguments = list(args[: self._argument_count]) off = len(arguments) # For information why this is necessary refer to the handling @@ -536,12 +631,12 @@ class Macro(object): # arguments expected we start filling in keyword arguments # and defaults. if off != self._argument_count: - for idx, name in enumerate(self.arguments[len(arguments):]): + for name in self.arguments[len(arguments) :]: try: value = kwargs.pop(name) except KeyError: value = missing - if name == 'caller': + if name == "caller": found_caller = True arguments.append(value) else: @@ -551,26 +646,31 @@ class Macro(object): # if not also changed in the compiler's `function_scoping` method. # the order is caller, keyword arguments, positional arguments! if self.caller and not found_caller: - caller = kwargs.pop('caller', None) + caller = kwargs.pop("caller", None) if caller is None: - caller = self._environment.undefined('No caller defined', - name='caller') + caller = self._environment.undefined("No caller defined", name="caller") arguments.append(caller) if self.catch_kwargs: arguments.append(kwargs) elif kwargs: - if 'caller' in kwargs: - raise TypeError('macro %r was invoked with two values for ' - 'the special caller argument. This is ' - 'most likely a bug.' % self.name) - raise TypeError('macro %r takes no keyword argument %r' % - (self.name, next(iter(kwargs)))) + if "caller" in kwargs: + raise TypeError( + "macro %r was invoked with two values for " + "the special caller argument. This is " + "most likely a bug." % self.name + ) + raise TypeError( + "macro %r takes no keyword argument %r" + % (self.name, next(iter(kwargs))) + ) if self.catch_varargs: - arguments.append(args[self._argument_count:]) + arguments.append(args[self._argument_count :]) elif len(args) > self._argument_count: - raise TypeError('macro %r takes not more than %d argument(s)' % - (self.name, len(self.arguments))) + raise TypeError( + "macro %r takes not more than %d argument(s)" + % (self.name, len(self.arguments)) + ) return self._invoke(arguments, autoescape) @@ -582,16 +682,16 @@ class Macro(object): return rv def __repr__(self): - return '<%s %s>' % ( + return "<%s %s>" % ( self.__class__.__name__, - self.name is None and 'anonymous' or repr(self.name) + self.name is None and "anonymous" or repr(self.name), ) @implements_to_string class Undefined(object): """The default undefined type. This undefined type can be printed and - iterated over, but every other access will raise an :exc:`jinja2.exceptions.UndefinedError`: + iterated over, but every other access will raise an :exc:`UndefinedError`: >>> foo = Undefined(name='foo') >>> str(foo) @@ -603,8 +703,13 @@ class Undefined(object): ... jinja2.exceptions.UndefinedError: 'foo' is undefined """ - __slots__ = ('_undefined_hint', '_undefined_obj', '_undefined_name', - '_undefined_exception') + + __slots__ = ( + "_undefined_hint", + "_undefined_obj", + "_undefined_name", + "_undefined_exception", + ) def __init__(self, hint=None, obj=missing, name=None, exc=UndefinedError): self._undefined_hint = hint @@ -612,40 +717,86 @@ class Undefined(object): self._undefined_name = name self._undefined_exception = exc + @property + def _undefined_message(self): + """Build a message about the undefined value based on how it was + accessed. + """ + if self._undefined_hint: + return self._undefined_hint + + if self._undefined_obj is missing: + return "%r is undefined" % self._undefined_name + + if not isinstance(self._undefined_name, string_types): + return "%s has no element %r" % ( + object_type_repr(self._undefined_obj), + self._undefined_name, + ) + + return "%r has no attribute %r" % ( + object_type_repr(self._undefined_obj), + self._undefined_name, + ) + @internalcode def _fail_with_undefined_error(self, *args, **kwargs): - """Regular callback function for undefined objects that raises an - `jinja2.exceptions.UndefinedError` on call. + """Raise an :exc:`UndefinedError` when operations are performed + on the undefined value. """ - if self._undefined_hint is None: - if self._undefined_obj is missing: - hint = '%r is undefined' % self._undefined_name - elif not isinstance(self._undefined_name, string_types): - hint = '%s has no element %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = '%r has no attribute %r' % ( - object_type_repr(self._undefined_obj), - self._undefined_name - ) - else: - hint = self._undefined_hint - raise self._undefined_exception(hint) + raise self._undefined_exception(self._undefined_message) @internalcode def __getattr__(self, name): - if name[:2] == '__': + if name[:2] == "__": raise AttributeError(name) return self._fail_with_undefined_error() - __add__ = __radd__ = __mul__ = __rmul__ = __div__ = __rdiv__ = \ - __truediv__ = __rtruediv__ = __floordiv__ = __rfloordiv__ = \ - __mod__ = __rmod__ = __pos__ = __neg__ = __call__ = \ - __getitem__ = __lt__ = __le__ = __gt__ = __ge__ = __int__ = \ - __float__ = __complex__ = __pow__ = __rpow__ = __sub__ = \ - __rsub__ = _fail_with_undefined_error + __add__ = ( + __radd__ + ) = ( + __mul__ + ) = ( + __rmul__ + ) = ( + __div__ + ) = ( + __rdiv__ + ) = ( + __truediv__ + ) = ( + __rtruediv__ + ) = ( + __floordiv__ + ) = ( + __rfloordiv__ + ) = ( + __mod__ + ) = ( + __rmod__ + ) = ( + __pos__ + ) = ( + __neg__ + ) = ( + __call__ + ) = ( + __getitem__ + ) = ( + __lt__ + ) = ( + __le__ + ) = ( + __gt__ + ) = ( + __ge__ + ) = ( + __int__ + ) = ( + __float__ + ) = ( + __complex__ + ) = __pow__ = __rpow__ = __sub__ = __rsub__ = _fail_with_undefined_error def __eq__(self, other): return type(self) is type(other) @@ -657,7 +808,7 @@ class Undefined(object): return id(type(self)) def __str__(self): - return u'' + return u"" def __len__(self): return 0 @@ -668,10 +819,11 @@ class Undefined(object): def __nonzero__(self): return False + __bool__ = __nonzero__ def __repr__(self): - return 'Undefined' + return "Undefined" def make_logging_undefined(logger=None, base=None): @@ -696,6 +848,7 @@ def make_logging_undefined(logger=None, base=None): """ if logger is None: import logging + logger = logging.getLogger(__name__) logger.addHandler(logging.StreamHandler(sys.stderr)) if base is None: @@ -704,26 +857,27 @@ def make_logging_undefined(logger=None, base=None): def _log_message(undef): if undef._undefined_hint is None: if undef._undefined_obj is missing: - hint = '%s is undefined' % undef._undefined_name + hint = "%s is undefined" % undef._undefined_name elif not isinstance(undef._undefined_name, string_types): - hint = '%s has no element %s' % ( + hint = "%s has no element %s" % ( object_type_repr(undef._undefined_obj), - undef._undefined_name) + undef._undefined_name, + ) else: - hint = '%s has no attribute %s' % ( + hint = "%s has no attribute %s" % ( object_type_repr(undef._undefined_obj), - undef._undefined_name) + undef._undefined_name, + ) else: hint = undef._undefined_hint - logger.warning('Template variable warning: %s', hint) + logger.warning("Template variable warning: %s", hint) class LoggingUndefined(base): - def _fail_with_undefined_error(self, *args, **kwargs): try: return base._fail_with_undefined_error(self, *args, **kwargs) except self._undefined_exception as e: - logger.error('Template variable error: %s', str(e)) + logger.error("Template variable error: %s", str(e)) raise e def __str__(self): @@ -737,6 +891,7 @@ def make_logging_undefined(logger=None, base=None): return rv if PY2: + def __nonzero__(self): rv = base.__nonzero__(self) _log_message(self) @@ -746,7 +901,9 @@ def make_logging_undefined(logger=None, base=None): rv = base.__unicode__(self) _log_message(self) return rv + else: + def __bool__(self): rv = base.__bool__(self) _log_message(self) @@ -755,6 +912,36 @@ def make_logging_undefined(logger=None, base=None): return LoggingUndefined +# No @implements_to_string decorator here because __str__ +# is not overwritten from Undefined in this class. +# This would cause a recursion error in Python 2. +class ChainableUndefined(Undefined): + """An undefined that is chainable, where both ``__getattr__`` and + ``__getitem__`` return itself rather than raising an + :exc:`UndefinedError`. + + >>> foo = ChainableUndefined(name='foo') + >>> str(foo.bar['baz']) + '' + >>> foo.bar['baz'] + 42 + Traceback (most recent call last): + ... + jinja2.exceptions.UndefinedError: 'foo' is undefined + + .. versionadded:: 2.11.0 + """ + + __slots__ = () + + def __html__(self): + return self.__str__() + + def __getattr__(self, _): + return self + + __getitem__ = __getattr__ + + @implements_to_string class DebugUndefined(Undefined): """An undefined that returns the debug info when printed. @@ -769,17 +956,18 @@ class DebugUndefined(Undefined): ... jinja2.exceptions.UndefinedError: 'foo' is undefined """ + __slots__ = () def __str__(self): if self._undefined_hint is None: if self._undefined_obj is missing: - return u'{{ %s }}' % self._undefined_name - return '{{ no such element: %s[%r] }}' % ( + return u"{{ %s }}" % self._undefined_name + return "{{ no such element: %s[%r] }}" % ( object_type_repr(self._undefined_obj), - self._undefined_name + self._undefined_name, ) - return u'{{ undefined value printed: %s }}' % self._undefined_hint + return u"{{ undefined value printed: %s }}" % self._undefined_hint @implements_to_string @@ -802,12 +990,22 @@ class StrictUndefined(Undefined): ... jinja2.exceptions.UndefinedError: 'foo' is undefined """ + __slots__ = () - __iter__ = __str__ = __len__ = __nonzero__ = __eq__ = \ - __ne__ = __bool__ = __hash__ = \ - Undefined._fail_with_undefined_error + __iter__ = ( + __str__ + ) = ( + __len__ + ) = ( + __nonzero__ + ) = __eq__ = __ne__ = __bool__ = __hash__ = Undefined._fail_with_undefined_error # remove remaining slots attributes, after the metaclass did the magic they # are unneeded and irritating as they contain wrong data for the subclasses. -del Undefined.__slots__, DebugUndefined.__slots__, StrictUndefined.__slots__ +del ( + Undefined.__slots__, + ChainableUndefined.__slots__, + DebugUndefined.__slots__, + StrictUndefined.__slots__, +) diff --git a/pipenv/vendor/jinja2/sandbox.py b/pipenv/vendor/jinja2/sandbox.py index 93fb9d45..cfd7993a 100644 --- a/pipenv/vendor/jinja2/sandbox.py +++ b/pipenv/vendor/jinja2/sandbox.py @@ -1,71 +1,66 @@ # -*- coding: utf-8 -*- +"""A sandbox layer that ensures unsafe operations cannot be performed. +Useful when the template itself comes from an untrusted source. """ - jinja2.sandbox - ~~~~~~~~~~~~~~ - - Adds a sandbox layer to Jinja as it was the default behavior in the old - Jinja 1 releases. This sandbox is slightly different from Jinja 1 as the - default behavior is easier to use. - - The behavior can be changed by subclassing the environment. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" -import types import operator -from collections import Mapping -from jinja2.environment import Environment -from jinja2.exceptions import SecurityError -from jinja2._compat import string_types, PY2 -from jinja2.utils import Markup - -from markupsafe import EscapeFormatter +import types +import warnings +from collections import deque from string import Formatter +from markupsafe import EscapeFormatter +from markupsafe import Markup + +from ._compat import abc +from ._compat import PY2 +from ._compat import range_type +from ._compat import string_types +from .environment import Environment +from .exceptions import SecurityError #: maximum number of items a range may produce MAX_RANGE = 100000 #: attributes of function objects that are considered unsafe. if PY2: - UNSAFE_FUNCTION_ATTRIBUTES = set(['func_closure', 'func_code', 'func_dict', - 'func_defaults', 'func_globals']) + UNSAFE_FUNCTION_ATTRIBUTES = { + "func_closure", + "func_code", + "func_dict", + "func_defaults", + "func_globals", + } else: # On versions > python 2 the special attributes on functions are gone, # but they remain on methods and generators for whatever reason. UNSAFE_FUNCTION_ATTRIBUTES = set() - #: unsafe method attributes. function attributes are unsafe for methods too -UNSAFE_METHOD_ATTRIBUTES = set(['im_class', 'im_func', 'im_self']) +UNSAFE_METHOD_ATTRIBUTES = {"im_class", "im_func", "im_self"} -#: unsafe generator attirbutes. -UNSAFE_GENERATOR_ATTRIBUTES = set(['gi_frame', 'gi_code']) +#: unsafe generator attributes. +UNSAFE_GENERATOR_ATTRIBUTES = {"gi_frame", "gi_code"} #: unsafe attributes on coroutines -UNSAFE_COROUTINE_ATTRIBUTES = set(['cr_frame', 'cr_code']) +UNSAFE_COROUTINE_ATTRIBUTES = {"cr_frame", "cr_code"} #: unsafe attributes on async generators -UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = set(['ag_code', 'ag_frame']) - -import warnings +UNSAFE_ASYNC_GENERATOR_ATTRIBUTES = {"ag_code", "ag_frame"} # make sure we don't warn in python 2.6 about stuff we don't care about -warnings.filterwarnings('ignore', 'the sets module', DeprecationWarning, - module='jinja2.sandbox') - -from collections import deque +warnings.filterwarnings( + "ignore", "the sets module", DeprecationWarning, module=__name__ +) _mutable_set_types = (set,) _mutable_mapping_types = (dict,) _mutable_sequence_types = (list,) - # on python 2.x we can register the user collection types try: from UserDict import UserDict, DictMixin from UserList import UserList + _mutable_mapping_types += (UserDict, DictMixin) _mutable_set_types += (UserList,) except ImportError: @@ -74,36 +69,60 @@ except ImportError: # if sets is still available, register the mutable set from there as well try: from sets import Set + _mutable_set_types += (Set,) except ImportError: pass #: register Python 2.6 abstract base classes -from collections import MutableSet, MutableMapping, MutableSequence -_mutable_set_types += (MutableSet,) -_mutable_mapping_types += (MutableMapping,) -_mutable_sequence_types += (MutableSequence,) - +_mutable_set_types += (abc.MutableSet,) +_mutable_mapping_types += (abc.MutableMapping,) +_mutable_sequence_types += (abc.MutableSequence,) _mutable_spec = ( - (_mutable_set_types, frozenset([ - 'add', 'clear', 'difference_update', 'discard', 'pop', 'remove', - 'symmetric_difference_update', 'update' - ])), - (_mutable_mapping_types, frozenset([ - 'clear', 'pop', 'popitem', 'setdefault', 'update' - ])), - (_mutable_sequence_types, frozenset([ - 'append', 'reverse', 'insert', 'sort', 'extend', 'remove' - ])), - (deque, frozenset([ - 'append', 'appendleft', 'clear', 'extend', 'extendleft', 'pop', - 'popleft', 'remove', 'rotate' - ])) + ( + _mutable_set_types, + frozenset( + [ + "add", + "clear", + "difference_update", + "discard", + "pop", + "remove", + "symmetric_difference_update", + "update", + ] + ), + ), + ( + _mutable_mapping_types, + frozenset(["clear", "pop", "popitem", "setdefault", "update"]), + ), + ( + _mutable_sequence_types, + frozenset(["append", "reverse", "insert", "sort", "extend", "remove"]), + ), + ( + deque, + frozenset( + [ + "append", + "appendleft", + "clear", + "extend", + "extendleft", + "pop", + "popleft", + "remove", + "rotate", + ] + ), + ), ) -class _MagicFormatMapping(Mapping): +class _MagicFormatMapping(abc.Mapping): """This class implements a dummy wrapper to fix a bug in the Python standard library for string formatting. @@ -117,7 +136,7 @@ class _MagicFormatMapping(Mapping): self._last_index = 0 def __getitem__(self, key): - if key == '': + if key == "": idx = self._last_index self._last_index += 1 try: @@ -135,9 +154,9 @@ class _MagicFormatMapping(Mapping): def inspect_format_method(callable): - if not isinstance(callable, (types.MethodType, - types.BuiltinMethodType)) or \ - callable.__name__ != 'format': + if not isinstance( + callable, (types.MethodType, types.BuiltinMethodType) + ) or callable.__name__ not in ("format", "format_map"): return None obj = callable.__self__ if isinstance(obj, string_types): @@ -148,10 +167,14 @@ def safe_range(*args): """A range that can't generate ranges with a length of more than MAX_RANGE items. """ - rng = range(*args) + rng = range_type(*args) + if len(rng) > MAX_RANGE: - raise OverflowError('range too big, maximum size for range is %d' % - MAX_RANGE) + raise OverflowError( + "Range too big. The sandbox blocks ranges larger than" + " MAX_RANGE (%d)." % MAX_RANGE + ) + return rng @@ -184,24 +207,25 @@ def is_internal_attribute(obj, attr): if attr in UNSAFE_FUNCTION_ATTRIBUTES: return True elif isinstance(obj, types.MethodType): - if attr in UNSAFE_FUNCTION_ATTRIBUTES or \ - attr in UNSAFE_METHOD_ATTRIBUTES: + if attr in UNSAFE_FUNCTION_ATTRIBUTES or attr in UNSAFE_METHOD_ATTRIBUTES: return True elif isinstance(obj, type): - if attr == 'mro': + if attr == "mro": return True elif isinstance(obj, (types.CodeType, types.TracebackType, types.FrameType)): return True elif isinstance(obj, types.GeneratorType): if attr in UNSAFE_GENERATOR_ATTRIBUTES: return True - elif hasattr(types, 'CoroutineType') and isinstance(obj, types.CoroutineType): + elif hasattr(types, "CoroutineType") and isinstance(obj, types.CoroutineType): if attr in UNSAFE_COROUTINE_ATTRIBUTES: return True - elif hasattr(types, 'AsyncGeneratorType') and isinstance(obj, types.AsyncGeneratorType): + elif hasattr(types, "AsyncGeneratorType") and isinstance( + obj, types.AsyncGeneratorType + ): if attr in UNSAFE_ASYNC_GENERATOR_ATTRIBUTES: return True - return attr.startswith('__') + return attr.startswith("__") def modifies_known_mutable(obj, attr): @@ -242,28 +266,26 @@ class SandboxedEnvironment(Environment): raised. However also other exceptions may occur during the rendering so the caller has to ensure that all exceptions are caught. """ + sandboxed = True #: default callback table for the binary operators. A copy of this is #: available on each instance of a sandboxed environment as #: :attr:`binop_table` default_binop_table = { - '+': operator.add, - '-': operator.sub, - '*': operator.mul, - '/': operator.truediv, - '//': operator.floordiv, - '**': operator.pow, - '%': operator.mod + "+": operator.add, + "-": operator.sub, + "*": operator.mul, + "/": operator.truediv, + "//": operator.floordiv, + "**": operator.pow, + "%": operator.mod, } #: default callback table for the unary operators. A copy of this is #: available on each instance of a sandboxed environment as #: :attr:`unop_table` - default_unop_table = { - '+': operator.pos, - '-': operator.neg - } + default_unop_table = {"+": operator.pos, "-": operator.neg} #: a set of binary operators that should be intercepted. Each operator #: that is added to this set (empty by default) is delegated to the @@ -299,7 +321,7 @@ class SandboxedEnvironment(Environment): def intercept_unop(self, operator): """Called during template compilation with the name of a unary operator to check if it should be intercepted at runtime. If this - method returns `True`, :meth:`call_unop` is excuted for this unary + method returns `True`, :meth:`call_unop` is executed for this unary operator. The default implementation of :meth:`call_unop` will use the :attr:`unop_table` dictionary to perform the operator with the same logic as the builtin one. @@ -313,10 +335,9 @@ class SandboxedEnvironment(Environment): """ return False - def __init__(self, *args, **kwargs): Environment.__init__(self, *args, **kwargs) - self.globals['range'] = safe_range + self.globals["range"] = safe_range self.binop_table = self.default_binop_table.copy() self.unop_table = self.default_unop_table.copy() @@ -327,7 +348,7 @@ class SandboxedEnvironment(Environment): special attributes of internal python objects as returned by the :func:`is_internal_attribute` function. """ - return not (attr.startswith('_') or is_internal_attribute(obj, attr)) + return not (attr.startswith("_") or is_internal_attribute(obj, attr)) def is_safe_callable(self, obj): """Check if an object is safely callable. Per default a function is @@ -335,8 +356,9 @@ class SandboxedEnvironment(Environment): True. Override this method to alter the behavior, but this won't affect the `unsafe` decorator from this module. """ - return not (getattr(obj, 'unsafe_callable', False) or - getattr(obj, 'alters_data', False)) + return not ( + getattr(obj, "unsafe_callable", False) or getattr(obj, "alters_data", False) + ) def call_binop(self, context, operator, left, right): """For intercepted binary operator calls (:meth:`intercepted_binops`) @@ -396,13 +418,15 @@ class SandboxedEnvironment(Environment): def unsafe_undefined(self, obj, attribute): """Return an undefined object for unsafe attributes.""" - return self.undefined('access to attribute %r of %r ' - 'object is unsafe.' % ( - attribute, - obj.__class__.__name__ - ), name=attribute, obj=obj, exc=SecurityError) + return self.undefined( + "access to attribute %r of %r " + "object is unsafe." % (attribute, obj.__class__.__name__), + name=attribute, + obj=obj, + exc=SecurityError, + ) - def format_string(self, s, args, kwargs): + def format_string(self, s, args, kwargs, format_func=None): """If a format call is detected, then this is routed through this method so that our safety sandbox can be used for it. """ @@ -410,20 +434,31 @@ class SandboxedEnvironment(Environment): formatter = SandboxedEscapeFormatter(self, s.escape) else: formatter = SandboxedFormatter(self) + + if format_func is not None and format_func.__name__ == "format_map": + if len(args) != 1 or kwargs: + raise TypeError( + "format_map() takes exactly one argument %d given" + % (len(args) + (kwargs is not None)) + ) + + kwargs = args[0] + args = None + kwargs = _MagicFormatMapping(args, kwargs) rv = formatter.vformat(s, args, kwargs) return type(s)(rv) - def call(__self, __context, __obj, *args, **kwargs): + def call(__self, __context, __obj, *args, **kwargs): # noqa: B902 """Call an object from sandboxed code.""" fmt = inspect_format_method(__obj) if fmt is not None: - return __self.format_string(fmt, args, kwargs) + return __self.format_string(fmt, args, kwargs, __obj) # the double prefixes are to avoid double keyword argument # errors when proxying the call. if not __self.is_safe_callable(__obj): - raise SecurityError('%r is not safely callable' % (__obj,)) + raise SecurityError("%r is not safely callable" % (__obj,)) return __context.call(__obj, *args, **kwargs) @@ -439,16 +474,16 @@ class ImmutableSandboxedEnvironment(SandboxedEnvironment): return not modifies_known_mutable(obj, attr) -# This really is not a public API apparenlty. +# This really is not a public API apparently. try: from _string import formatter_field_name_split except ImportError: + def formatter_field_name_split(field_name): return field_name._formatter_field_name_split() class SandboxedFormatterMixin(object): - def __init__(self, env): self._env = env @@ -462,14 +497,14 @@ class SandboxedFormatterMixin(object): obj = self._env.getitem(obj, i) return obj, first -class SandboxedFormatter(SandboxedFormatterMixin, Formatter): +class SandboxedFormatter(SandboxedFormatterMixin, Formatter): def __init__(self, env): SandboxedFormatterMixin.__init__(self, env) Formatter.__init__(self) -class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter): +class SandboxedEscapeFormatter(SandboxedFormatterMixin, EscapeFormatter): def __init__(self, env, escape): SandboxedFormatterMixin.__init__(self, env) EscapeFormatter.__init__(self, escape) diff --git a/pipenv/vendor/jinja2/tests.py b/pipenv/vendor/jinja2/tests.py index 0adc3d4d..fabd4ce5 100644 --- a/pipenv/vendor/jinja2/tests.py +++ b/pipenv/vendor/jinja2/tests.py @@ -1,24 +1,17 @@ # -*- coding: utf-8 -*- -""" - jinja2.tests - ~~~~~~~~~~~~ - - Jinja test functions. Used with the "is" operator. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" +"""Built-in template tests used with the ``is`` operator.""" +import decimal import operator import re -from collections import Mapping -from jinja2.runtime import Undefined -from jinja2._compat import text_type, string_types, integer_types -import decimal -number_re = re.compile(r'^-?\d+(\.\d+)?$') +from ._compat import abc +from ._compat import integer_types +from ._compat import string_types +from ._compat import text_type +from .runtime import Undefined + +number_re = re.compile(r"^-?\d+(\.\d+)?$") regex_type = type(number_re) - - test_callable = callable @@ -64,6 +57,48 @@ def test_none(value): return value is None +def test_boolean(value): + """Return true if the object is a boolean value. + + .. versionadded:: 2.11 + """ + return value is True or value is False + + +def test_false(value): + """Return true if the object is False. + + .. versionadded:: 2.11 + """ + return value is False + + +def test_true(value): + """Return true if the object is True. + + .. versionadded:: 2.11 + """ + return value is True + + +# NOTE: The existing 'number' test matches booleans and floats +def test_integer(value): + """Return true if the object is an integer. + + .. versionadded:: 2.11 + """ + return isinstance(value, integer_types) and value is not True and value is not False + + +# NOTE: The existing 'number' test matches booleans and integers +def test_float(value): + """Return true if the object is a float. + + .. versionadded:: 2.11 + """ + return isinstance(value, float) + + def test_lower(value): """Return true if the variable is lowercased.""" return text_type(value).islower() @@ -84,7 +119,7 @@ def test_mapping(value): .. versionadded:: 2.6 """ - return isinstance(value, Mapping) + return isinstance(value, abc.Mapping) def test_number(value): @@ -99,7 +134,7 @@ def test_sequence(value): try: len(value) value.__getitem__ - except: + except Exception: return False return True @@ -128,7 +163,7 @@ def test_iterable(value): def test_escaped(value): """Check if the value is escaped.""" - return hasattr(value, '__html__') + return hasattr(value, "__html__") def test_in(value, seq): @@ -140,36 +175,41 @@ def test_in(value, seq): TESTS = { - 'odd': test_odd, - 'even': test_even, - 'divisibleby': test_divisibleby, - 'defined': test_defined, - 'undefined': test_undefined, - 'none': test_none, - 'lower': test_lower, - 'upper': test_upper, - 'string': test_string, - 'mapping': test_mapping, - 'number': test_number, - 'sequence': test_sequence, - 'iterable': test_iterable, - 'callable': test_callable, - 'sameas': test_sameas, - 'escaped': test_escaped, - 'in': test_in, - '==': operator.eq, - 'eq': operator.eq, - 'equalto': operator.eq, - '!=': operator.ne, - 'ne': operator.ne, - '>': operator.gt, - 'gt': operator.gt, - 'greaterthan': operator.gt, - 'ge': operator.ge, - '>=': operator.ge, - '<': operator.lt, - 'lt': operator.lt, - 'lessthan': operator.lt, - '<=': operator.le, - 'le': operator.le, + "odd": test_odd, + "even": test_even, + "divisibleby": test_divisibleby, + "defined": test_defined, + "undefined": test_undefined, + "none": test_none, + "boolean": test_boolean, + "false": test_false, + "true": test_true, + "integer": test_integer, + "float": test_float, + "lower": test_lower, + "upper": test_upper, + "string": test_string, + "mapping": test_mapping, + "number": test_number, + "sequence": test_sequence, + "iterable": test_iterable, + "callable": test_callable, + "sameas": test_sameas, + "escaped": test_escaped, + "in": test_in, + "==": operator.eq, + "eq": operator.eq, + "equalto": operator.eq, + "!=": operator.ne, + "ne": operator.ne, + ">": operator.gt, + "gt": operator.gt, + "greaterthan": operator.gt, + "ge": operator.ge, + ">=": operator.ge, + "<": operator.lt, + "lt": operator.lt, + "lessthan": operator.lt, + "<=": operator.le, + "le": operator.le, } diff --git a/pipenv/vendor/jinja2/utils.py b/pipenv/vendor/jinja2/utils.py index 502a311c..e3285e8e 100644 --- a/pipenv/vendor/jinja2/utils.py +++ b/pipenv/vendor/jinja2/utils.py @@ -1,44 +1,44 @@ # -*- coding: utf-8 -*- -""" - jinja2.utils - ~~~~~~~~~~~~ - - Utility functions. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD, see LICENSE for more details. -""" -import re import json -import errno +import os +import re +import warnings from collections import deque +from random import choice +from random import randrange from threading import Lock -from jinja2._compat import text_type, string_types, implements_iterator, \ - url_quote +from markupsafe import escape +from markupsafe import Markup -_word_split_re = re.compile(r'(\s+)') +from ._compat import abc +from ._compat import string_types +from ._compat import text_type +from ._compat import url_quote + +_word_split_re = re.compile(r"(\s+)") _punctuation_re = re.compile( - '^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$' % ( - '|'.join(map(re.escape, ('(', '<', '<'))), - '|'.join(map(re.escape, ('.', ',', ')', '>', '\n', '>'))) + "^(?P<lead>(?:%s)*)(?P<middle>.*?)(?P<trail>(?:%s)*)$" + % ( + "|".join(map(re.escape, ("(", "<", "<"))), + "|".join(map(re.escape, (".", ",", ")", ">", "\n", ">"))), ) ) -_simple_email_re = re.compile(r'^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$') -_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)') -_entity_re = re.compile(r'&([^;]+);') -_letters = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' -_digits = '0123456789' +_simple_email_re = re.compile(r"^\S+@[a-zA-Z0-9._-]+\.[a-zA-Z0-9._-]+$") +_striptags_re = re.compile(r"(<!--.*?-->|<[^>]*>)") +_entity_re = re.compile(r"&([^;]+);") +_letters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" +_digits = "0123456789" # special singleton representing missing values for the runtime -missing = type('MissingType', (), {'__repr__': lambda x: 'missing'})() +missing = type("MissingType", (), {"__repr__": lambda x: "missing"})() # internal code internal_code = set() -concat = u''.join +concat = u"".join -_slash_escape = '\\/' not in json.dumps('/') +_slash_escape = "\\/" not in json.dumps("/") def contextfunction(f): @@ -98,24 +98,26 @@ def is_undefined(obj): return default return var """ - from jinja2.runtime import Undefined + from .runtime import Undefined + return isinstance(obj, Undefined) def consume(iterable): """Consumes an iterable without doing anything with it.""" - for event in iterable: + for _ in iterable: pass def clear_caches(): - """Jinja2 keeps internal caches for environments and lexers. These are - used so that Jinja2 doesn't have to recreate environments and lexers all + """Jinja keeps internal caches for environments and lexers. These are + used so that Jinja doesn't have to recreate environments and lexers all the time. Normally you don't have to care about that but if you are measuring memory consumption you may want to clean the caches. """ - from jinja2.environment import _spontaneous_environments - from jinja2.lexer import _lexer_cache + from .environment import _spontaneous_environments + from .lexer import _lexer_cache + _spontaneous_environments.clear() _lexer_cache.clear() @@ -132,12 +134,10 @@ def import_string(import_name, silent=False): :return: imported object """ try: - if ':' in import_name: - module, obj = import_name.split(':', 1) - elif '.' in import_name: - items = import_name.split('.') - module = '.'.join(items[:-1]) - obj = items[-1] + if ":" in import_name: + module, obj = import_name.split(":", 1) + elif "." in import_name: + module, _, obj = import_name.rpartition(".") else: return __import__(import_name) return getattr(__import__(module, None, None, [obj]), obj) @@ -146,15 +146,14 @@ def import_string(import_name, silent=False): raise -def open_if_exists(filename, mode='rb'): +def open_if_exists(filename, mode="rb"): """Returns a file descriptor for the filename if that file exists, - otherwise `None`. + otherwise ``None``. """ - try: - return open(filename, mode) - except IOError as e: - if e.errno not in (errno.ENOENT, errno.EISDIR, errno.EINVAL): - raise + if not os.path.isfile(filename): + return None + + return open(filename, mode) def object_type_repr(obj): @@ -163,15 +162,15 @@ def object_type_repr(obj): example for `None` and `Ellipsis`). """ if obj is None: - return 'None' + return "None" elif obj is Ellipsis: - return 'Ellipsis' + return "Ellipsis" # __builtin__ in 2.x, builtins in 3.x - if obj.__class__.__module__ in ('__builtin__', 'builtins'): + if obj.__class__.__module__ in ("__builtin__", "builtins"): name = obj.__class__.__name__ else: - name = obj.__class__.__module__ + '.' + obj.__class__.__name__ - return '%s object' % name + name = obj.__class__.__module__ + "." + obj.__class__.__name__ + return "%s object" % name def pformat(obj, verbose=False): @@ -180,9 +179,11 @@ def pformat(obj, verbose=False): """ try: from pretty import pretty + return pretty(obj, verbose=verbose) except ImportError: from pprint import pformat + return pformat(obj) @@ -200,45 +201,60 @@ def urlize(text, trim_url_limit=None, rel=None, target=None): If target is not None, a target attribute will be added to the link. """ - trim_url = lambda x, limit=trim_url_limit: limit is not None \ - and (x[:limit] + (len(x) >=limit and '...' - or '')) or x + trim_url = ( + lambda x, limit=trim_url_limit: limit is not None + and (x[:limit] + (len(x) >= limit and "..." or "")) + or x + ) words = _word_split_re.split(text_type(escape(text))) - rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or '' - target_attr = target and ' target="%s"' % escape(target) or '' + rel_attr = rel and ' rel="%s"' % text_type(escape(rel)) or "" + target_attr = target and ' target="%s"' % escape(target) or "" for i, word in enumerate(words): match = _punctuation_re.match(word) if match: lead, middle, trail = match.groups() - if middle.startswith('www.') or ( - '@' not in middle and - not middle.startswith('http://') and - not middle.startswith('https://') and - len(middle) > 0 and - middle[0] in _letters + _digits and ( - middle.endswith('.org') or - middle.endswith('.net') or - middle.endswith('.com') - )): - middle = '<a href="http://%s"%s%s>%s</a>' % (middle, - rel_attr, target_attr, trim_url(middle)) - if middle.startswith('http://') or \ - middle.startswith('https://'): - middle = '<a href="%s"%s%s>%s</a>' % (middle, - rel_attr, target_attr, trim_url(middle)) - if '@' in middle and not middle.startswith('www.') and \ - not ':' in middle and _simple_email_re.match(middle): + if middle.startswith("www.") or ( + "@" not in middle + and not middle.startswith("http://") + and not middle.startswith("https://") + and len(middle) > 0 + and middle[0] in _letters + _digits + and ( + middle.endswith(".org") + or middle.endswith(".net") + or middle.endswith(".com") + ) + ): + middle = '<a href="http://%s"%s%s>%s</a>' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + if middle.startswith("http://") or middle.startswith("https://"): + middle = '<a href="%s"%s%s>%s</a>' % ( + middle, + rel_attr, + target_attr, + trim_url(middle), + ) + if ( + "@" in middle + and not middle.startswith("www.") + and ":" not in middle + and _simple_email_re.match(middle) + ): middle = '<a href="mailto:%s">%s</a>' % (middle, middle) if lead + middle + trail != word: words[i] = lead + middle + trail - return u''.join(words) + return u"".join(words) def generate_lorem_ipsum(n=5, html=True, min=20, max=100): """Generate some lorem ipsum for the template.""" - from jinja2.constants import LOREM_IPSUM_WORDS - from random import choice, randrange + from .constants import LOREM_IPSUM_WORDS + words = LOREM_IPSUM_WORDS.split() result = [] @@ -263,43 +279,53 @@ def generate_lorem_ipsum(n=5, html=True, min=20, max=100): if idx - randrange(3, 8) > last_comma: last_comma = idx last_fullstop += 2 - word += ',' + word += "," # add end of sentences if idx - randrange(10, 20) > last_fullstop: last_comma = last_fullstop = idx - word += '.' + word += "." next_capitalized = True p.append(word) # ensure that the paragraph ends with a dot. - p = u' '.join(p) - if p.endswith(','): - p = p[:-1] + '.' - elif not p.endswith('.'): - p += '.' + p = u" ".join(p) + if p.endswith(","): + p = p[:-1] + "." + elif not p.endswith("."): + p += "." result.append(p) if not html: - return u'\n\n'.join(result) - return Markup(u'\n'.join(u'<p>%s</p>' % escape(x) for x in result)) + return u"\n\n".join(result) + return Markup(u"\n".join(u"<p>%s</p>" % escape(x) for x in result)) -def unicode_urlencode(obj, charset='utf-8', for_qs=False): - """URL escapes a single bytestring or unicode string with the - given charset if applicable to URL safe quoting under all rules - that need to be considered under all supported Python versions. +def unicode_urlencode(obj, charset="utf-8", for_qs=False): + """Quote a string for use in a URL using the given charset. - If non strings are provided they are converted to their unicode - representation first. + This function is misnamed, it is a wrapper around + :func:`urllib.parse.quote`. + + :param obj: String or bytes to quote. Other types are converted to + string then encoded to bytes using the given charset. + :param charset: Encode text to bytes using this charset. + :param for_qs: Quote "/" and use "+" for spaces. """ if not isinstance(obj, string_types): obj = text_type(obj) + if isinstance(obj, text_type): obj = obj.encode(charset) - safe = not for_qs and b'/' or b'' - rv = text_type(url_quote(obj, safe)) + + safe = b"" if for_qs else b"/" + rv = url_quote(obj, safe) + + if not isinstance(rv, text_type): + rv = rv.decode("utf-8") + if for_qs: - rv = rv.replace('%20', '+') + rv = rv.replace("%20", "+") + return rv @@ -326,9 +352,9 @@ class LRUCache(object): def __getstate__(self): return { - 'capacity': self.capacity, - '_mapping': self._mapping, - '_queue': self._queue + "capacity": self.capacity, + "_mapping": self._mapping, + "_queue": self._queue, } def __setstate__(self, d): @@ -342,7 +368,7 @@ class LRUCache(object): """Return a shallow copy of the instance.""" rv = self.__class__(self.capacity) rv._mapping.update(self._mapping) - rv._queue = deque(self._queue) + rv._queue.extend(self._queue) return rv def get(self, key, default=None): @@ -356,15 +382,11 @@ class LRUCache(object): """Set `default` if the key is not in the cache otherwise leave unchanged. Return the value of this key. """ - self._wlock.acquire() try: - try: - return self[key] - except KeyError: - self[key] = default - return default - finally: - self._wlock.release() + return self[key] + except KeyError: + self[key] = default + return default def clear(self): """Clear the cache.""" @@ -384,10 +406,7 @@ class LRUCache(object): return len(self._mapping) def __repr__(self): - return '<%s %r>' % ( - self.__class__.__name__, - self._mapping - ) + return "<%s %r>" % (self.__class__.__name__, self._mapping) def __getitem__(self, key): """Get an item from the cache. Moves the item up so that it has the @@ -436,7 +455,6 @@ class LRUCache(object): try: self._remove(key) except ValueError: - # __getitem__ is not locked, it might happen pass finally: self._wlock.release() @@ -449,6 +467,12 @@ class LRUCache(object): def iteritems(self): """Iterate over all items.""" + warnings.warn( + "'iteritems()' will be removed in version 3.0. Use" + " 'iter(cache.items())' instead.", + DeprecationWarning, + stacklevel=2, + ) return iter(self.items()) def values(self): @@ -457,6 +481,22 @@ class LRUCache(object): def itervalue(self): """Iterate over all values.""" + warnings.warn( + "'itervalue()' will be removed in version 3.0. Use" + " 'iter(cache.values())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self.values()) + + def itervalues(self): + """Iterate over all values.""" + warnings.warn( + "'itervalues()' will be removed in version 3.0. Use" + " 'iter(cache.values())' instead.", + DeprecationWarning, + stacklevel=2, + ) return iter(self.values()) def keys(self): @@ -467,12 +507,19 @@ class LRUCache(object): """Iterate over all keys in the cache dict, ordered by the most recent usage. """ + warnings.warn( + "'iterkeys()' will be removed in version 3.0. Use" + " 'iter(cache.keys())' instead.", + DeprecationWarning, + stacklevel=2, + ) + return iter(self) + + def __iter__(self): return reversed(tuple(self._queue)) - __iter__ = iterkeys - def __reversed__(self): - """Iterate over the values in the cache dict, oldest items + """Iterate over the keys in the cache dict, oldest items coming first. """ return iter(tuple(self._queue)) @@ -480,18 +527,15 @@ class LRUCache(object): __copy__ = copy -# register the LRU cache as mutable mapping if possible -try: - from collections import MutableMapping - MutableMapping.register(LRUCache) -except ImportError: - pass +abc.MutableMapping.register(LRUCache) -def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), - disabled_extensions=(), - default_for_string=True, - default=False): +def select_autoescape( + enabled_extensions=("html", "htm", "xml"), + disabled_extensions=(), + default_for_string=True, + default=False, +): """Intelligently sets the initial value of autoescaping based on the filename of the template. This is the recommended way to configure autoescaping if you do not want to write a custom function yourself. @@ -526,10 +570,9 @@ def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), .. versionadded:: 2.9 """ - enabled_patterns = tuple('.' + x.lstrip('.').lower() - for x in enabled_extensions) - disabled_patterns = tuple('.' + x.lstrip('.').lower() - for x in disabled_extensions) + enabled_patterns = tuple("." + x.lstrip(".").lower() for x in enabled_extensions) + disabled_patterns = tuple("." + x.lstrip(".").lower() for x in disabled_extensions) + def autoescape(template_name): if template_name is None: return default_for_string @@ -539,6 +582,7 @@ def select_autoescape(enabled_extensions=('html', 'htm', 'xml'), if template_name.endswith(disabled_patterns): return False return default + return autoescape @@ -562,35 +606,63 @@ def htmlsafe_json_dumps(obj, dumper=None, **kwargs): """ if dumper is None: dumper = json.dumps - rv = dumper(obj, **kwargs) \ - .replace(u'<', u'\\u003c') \ - .replace(u'>', u'\\u003e') \ - .replace(u'&', u'\\u0026') \ - .replace(u"'", u'\\u0027') + rv = ( + dumper(obj, **kwargs) + .replace(u"<", u"\\u003c") + .replace(u">", u"\\u003e") + .replace(u"&", u"\\u0026") + .replace(u"'", u"\\u0027") + ) return Markup(rv) -@implements_iterator class Cycler(object): - """A cycle helper for templates.""" + """Cycle through values by yield them one at a time, then restarting + once the end is reached. Available as ``cycler`` in templates. + + Similar to ``loop.cycle``, but can be used outside loops or across + multiple loops. For example, render a list of folders and files in a + list, alternating giving them "odd" and "even" classes. + + .. code-block:: html+jinja + + {% set row_class = cycler("odd", "even") %} + <ul class="browser"> + {% for folder in folders %} + <li class="folder {{ row_class.next() }}">{{ folder }} + {% endfor %} + {% for file in files %} + <li class="file {{ row_class.next() }}">{{ file }} + {% endfor %} + </ul> + + :param items: Each positional argument will be yielded in the order + given for each cycle. + + .. versionadded:: 2.1 + """ def __init__(self, *items): if not items: - raise RuntimeError('at least one item has to be provided') + raise RuntimeError("at least one item has to be provided") self.items = items - self.reset() + self.pos = 0 def reset(self): - """Resets the cycle.""" + """Resets the current item to the first item.""" self.pos = 0 @property def current(self): - """Returns the current item.""" + """Return the current item. Equivalent to the item that will be + returned next time :meth:`next` is called. + """ return self.items[self.pos] def next(self): - """Goes one item ahead and returns it.""" + """Return the current item, then advance :attr:`current` to the + next item. + """ rv = self.current self.pos = (self.pos + 1) % len(self.items) return rv @@ -601,27 +673,27 @@ class Cycler(object): class Joiner(object): """A joining helper for templates.""" - def __init__(self, sep=u', '): + def __init__(self, sep=u", "): self.sep = sep self.used = False def __call__(self): if not self.used: self.used = True - return u'' + return u"" return self.sep class Namespace(object): """A namespace object that can hold arbitrary attributes. It may be - initialized from a dictionary or with keyword argments.""" + initialized from a dictionary or with keyword arguments.""" - def __init__(*args, **kwargs): + def __init__(*args, **kwargs): # noqa: B902 self, args = args[0], args[1:] self.__attrs = dict(*args, **kwargs) def __getattribute__(self, name): - if name == '_Namespace__attrs': + if name == "_Namespace__attrs": return object.__getattribute__(self, name) try: return self.__attrs[name] @@ -632,16 +704,24 @@ class Namespace(object): self.__attrs[name] = value def __repr__(self): - return '<Namespace %r>' % self.__attrs + return "<Namespace %r>" % self.__attrs # does this python version support async for in and async generators? try: - exec('async def _():\n async for _ in ():\n yield _') + exec("async def _():\n async for _ in ():\n yield _") have_async_gen = True except SyntaxError: have_async_gen = False -# Imported here because that's where it was in the past -from markupsafe import Markup, escape, soft_unicode +def soft_unicode(s): + from markupsafe import soft_unicode + + warnings.warn( + "'jinja2.utils.soft_unicode' will be removed in version 3.0." + " Use 'markupsafe.soft_unicode' instead.", + DeprecationWarning, + stacklevel=2, + ) + return soft_unicode(s) diff --git a/pipenv/vendor/jinja2/visitor.py b/pipenv/vendor/jinja2/visitor.py index ba526dfa..d1365bf1 100644 --- a/pipenv/vendor/jinja2/visitor.py +++ b/pipenv/vendor/jinja2/visitor.py @@ -1,14 +1,8 @@ # -*- coding: utf-8 -*- +"""API for traversing the AST nodes. Implemented by the compiler and +meta introspection. """ - jinja2.visitor - ~~~~~~~~~~~~~~ - - This module implements a visitor for the nodes. - - :copyright: (c) 2017 by the Jinja Team. - :license: BSD. -""" -from jinja2.nodes import Node +from .nodes import Node class NodeVisitor(object): @@ -28,7 +22,7 @@ class NodeVisitor(object): exists for this node. In that case the generic visit function is used instead. """ - method = 'visit_' + node.__class__.__name__ + method = "visit_" + node.__class__.__name__ return getattr(self, method, None) def visit(self, node, *args, **kwargs): diff --git a/pipenv/vendor/markupsafe/LICENSE b/pipenv/vendor/markupsafe/LICENSE deleted file mode 100644 index 5d269389..00000000 --- a/pipenv/vendor/markupsafe/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/pipenv/vendor/markupsafe/LICENSE.rst b/pipenv/vendor/markupsafe/LICENSE.rst new file mode 100644 index 00000000..9d227a0c --- /dev/null +++ b/pipenv/vendor/markupsafe/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/markupsafe/__init__.py b/pipenv/vendor/markupsafe/__init__.py index 68dc85f6..da05ed32 100644 --- a/pipenv/vendor/markupsafe/__init__.py +++ b/pipenv/vendor/markupsafe/__init__.py @@ -1,75 +1,74 @@ # -*- coding: utf-8 -*- """ - markupsafe - ~~~~~~~~~~ +markupsafe +~~~~~~~~~~ - Implements a Markup string. +Implements an escape function and a Markup string to replace HTML +special characters with safe representations. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import re import string -from collections import Mapping -from markupsafe._compat import text_type, string_types, int_types, \ - unichr, iteritems, PY2 -__version__ = "1.0" +from ._compat import int_types +from ._compat import iteritems +from ._compat import Mapping +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import unichr -__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] +__version__ = "1.1.1" +__all__ = ["Markup", "soft_unicode", "escape", "escape_silent"] -_striptags_re = re.compile(r'(<!--.*?-->|<[^>]*>)') -_entity_re = re.compile(r'&([^& ;]+);') +_striptags_re = re.compile(r"(<!--.*?-->|<[^>]*>)") +_entity_re = re.compile(r"&([^& ;]+);") class Markup(text_type): - r"""Marks a string as being safe for inclusion in HTML/XML output without - needing to be escaped. This implements the `__html__` interface a couple - of frameworks and web applications use. :class:`Markup` is a direct - subclass of `unicode` and provides all the methods of `unicode` just that - it escapes arguments passed and always returns `Markup`. + """A string that is ready to be safely inserted into an HTML or XML + document, either because it was escaped or because it was marked + safe. - The `escape` function returns markup objects so that double escaping can't - happen. + Passing an object to the constructor converts it to text and wraps + it to mark it safe without escaping. To escape the text, use the + :meth:`escape` class method instead. - The constructor of the :class:`Markup` class can be used for three - different things: When passed an unicode object it's assumed to be safe, - when passed an object with an HTML representation (has an `__html__` - method) that representation is used, otherwise the object passed is - converted into a unicode string and then assumed to be safe: + >>> Markup('Hello, <em>World</em>!') + Markup('Hello, <em>World</em>!') + >>> Markup(42) + Markup('42') + >>> Markup.escape('Hello, <em>World</em>!') + Markup('Hello <em>World</em>!') - >>> Markup("Hello <em>World</em>!") - Markup(u'Hello <em>World</em>!') - >>> class Foo(object): - ... def __html__(self): - ... return '<a href="#">foo</a>' + This implements the ``__html__()`` interface that some frameworks + use. Passing an object that implements ``__html__()`` will wrap the + output of that method, marking it safe. + + >>> class Foo: + ... def __html__(self): + ... return '<a href="/foo">foo</a>' ... >>> Markup(Foo()) - Markup(u'<a href="#">foo</a>') + Markup('<a href="/foo">foo</a>') - If you want object passed being always treated as unsafe you can use the - :meth:`escape` classmethod to create a :class:`Markup` object: + This is a subclass of the text type (``str`` in Python 3, + ``unicode`` in Python 2). It has the same methods as that type, but + all methods escape their arguments and return a ``Markup`` instance. - >>> Markup.escape("Hello <em>World</em>!") - Markup(u'Hello <em>World</em>!') - - Operations on a markup string are markup aware which means that all - arguments are passed through the :func:`escape` function: - - >>> em = Markup("<em>%s</em>") - >>> em % "foo & bar" - Markup(u'<em>foo & bar</em>') - >>> strong = Markup("<strong>%(text)s</strong>") - >>> strong % {'text': '<blink>hacker here</blink>'} - Markup(u'<strong><blink>hacker here</blink></strong>') - >>> Markup("<em>Hello</em> ") + "<foo>" - Markup(u'<em>Hello</em> <foo>') + >>> Markup('<em>%s</em>') % 'foo & bar' + Markup('<em>foo & bar</em>') + >>> Markup('<em>Hello</em> ') + '<foo>' + Markup('<em>Hello</em> <foo>') """ + __slots__ = () - def __new__(cls, base=u'', encoding=None, errors='strict'): - if hasattr(base, '__html__'): + def __new__(cls, base=u"", encoding=None, errors="strict"): + if hasattr(base, "__html__"): base = base.__html__() if encoding is None: return text_type.__new__(cls, base) @@ -79,12 +78,12 @@ class Markup(text_type): return self def __add__(self, other): - if isinstance(other, string_types) or hasattr(other, '__html__'): + if isinstance(other, string_types) or hasattr(other, "__html__"): return self.__class__(super(Markup, self).__add__(self.escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, string_types): + if hasattr(other, "__html__") or isinstance(other, string_types): return self.escape(other).__add__(self) return NotImplemented @@ -92,6 +91,7 @@ class Markup(text_type): if isinstance(num, int_types): return self.__class__(text_type.__mul__(self, num)) return NotImplemented + __rmul__ = __mul__ def __mod__(self, arg): @@ -102,115 +102,124 @@ class Markup(text_type): return self.__class__(text_type.__mod__(self, arg)) def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - text_type.__repr__(self) - ) + return "%s(%s)" % (self.__class__.__name__, text_type.__repr__(self)) def join(self, seq): return self.__class__(text_type.join(self, map(self.escape, seq))) + join.__doc__ = text_type.join.__doc__ def split(self, *args, **kwargs): return list(map(self.__class__, text_type.split(self, *args, **kwargs))) + split.__doc__ = text_type.split.__doc__ def rsplit(self, *args, **kwargs): return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) + rsplit.__doc__ = text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return list(map(self.__class__, text_type.splitlines( - self, *args, **kwargs))) + return list(map(self.__class__, text_type.splitlines(self, *args, **kwargs))) + splitlines.__doc__ = text_type.splitlines.__doc__ def unescape(self): - r"""Unescape markup again into an text_type string. This also resolves - known HTML4 and XHTML entities: + """Convert escaped markup back into a text string. This replaces + HTML entities with the characters they represent. - >>> Markup("Main » <em>About</em>").unescape() - u'Main \xbb <em>About</em>' + >>> Markup('Main » <em>About</em>').unescape() + 'Main » <em>About</em>' """ - from markupsafe._constants import HTML_ENTITIES + from ._constants import HTML_ENTITIES + def handle_match(m): name = m.group(1) if name in HTML_ENTITIES: return unichr(HTML_ENTITIES[name]) try: - if name[:2] in ('#x', '#X'): + if name[:2] in ("#x", "#X"): return unichr(int(name[2:], 16)) - elif name.startswith('#'): + elif name.startswith("#"): return unichr(int(name[1:])) except ValueError: pass # Don't modify unexpected input. return m.group() + return _entity_re.sub(handle_match, text_type(self)) def striptags(self): - r"""Unescape markup into an text_type string and strip all tags. This - also resolves known HTML4 and XHTML entities. Whitespace is - normalized to one: + """:meth:`unescape` the markup, remove tags, and normalize + whitespace to single spaces. - >>> Markup("Main » <em>About</em>").striptags() - u'Main \xbb About' + >>> Markup('Main »\t<em>About</em>').striptags() + 'Main » About' """ - stripped = u' '.join(_striptags_re.sub('', self).split()) + stripped = u" ".join(_striptags_re.sub("", self).split()) return Markup(stripped).unescape() @classmethod def escape(cls, s): - """Escape the string. Works like :func:`escape` with the difference - that for subclasses of :class:`Markup` this function would return the - correct subclass. + """Escape a string. Calls :func:`escape` and ensures that for + subclasses the correct type is returned. """ rv = escape(s) if rv.__class__ is not cls: return cls(rv) return rv - def make_simple_escaping_wrapper(name): + def make_simple_escaping_wrapper(name): # noqa: B902 orig = getattr(text_type, name) + def func(self, *args, **kwargs): args = _escape_argspec(list(args), enumerate(args), self.escape) _escape_argspec(kwargs, iteritems(kwargs), self.escape) return self.__class__(orig(self, *args, **kwargs)) + func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ return func - for method in '__getitem__', 'capitalize', \ - 'title', 'lower', 'upper', 'replace', 'ljust', \ - 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ - 'translate', 'expandtabs', 'swapcase', 'zfill': + for method in ( + "__getitem__", + "capitalize", + "title", + "lower", + "upper", + "replace", + "ljust", + "rjust", + "lstrip", + "rstrip", + "center", + "strip", + "translate", + "expandtabs", + "swapcase", + "zfill", + ): locals()[method] = make_simple_escaping_wrapper(method) - # new in python 2.5 - if hasattr(text_type, 'partition'): - def partition(self, sep): - return tuple(map(self.__class__, - text_type.partition(self, self.escape(sep)))) - def rpartition(self, sep): - return tuple(map(self.__class__, - text_type.rpartition(self, self.escape(sep)))) + def partition(self, sep): + return tuple(map(self.__class__, text_type.partition(self, self.escape(sep)))) - # new in python 2.6 - if hasattr(text_type, 'format'): - def format(*args, **kwargs): - self, args = args[0], args[1:] - formatter = EscapeFormatter(self.escape) - kwargs = _MagicFormatMapping(args, kwargs) - return self.__class__(formatter.vformat(self, args, kwargs)) + def rpartition(self, sep): + return tuple(map(self.__class__, text_type.rpartition(self, self.escape(sep)))) - def __html_format__(self, format_spec): - if format_spec: - raise ValueError('Unsupported format specification ' - 'for Markup.') - return self + def format(self, *args, **kwargs): + formatter = EscapeFormatter(self.escape) + kwargs = _MagicFormatMapping(args, kwargs) + return self.__class__(formatter.vformat(self, args, kwargs)) + + def __html_format__(self, format_spec): + if format_spec: + raise ValueError("Unsupported format specification " "for Markup.") + return self # not in python 3 - if hasattr(text_type, '__getslice__'): - __getslice__ = make_simple_escaping_wrapper('__getslice__') + if hasattr(text_type, "__getslice__"): + __getslice__ = make_simple_escaping_wrapper("__getslice__") del method, make_simple_escaping_wrapper @@ -229,7 +238,7 @@ class _MagicFormatMapping(Mapping): self._last_index = 0 def __getitem__(self, key): - if key == '': + if key == "": idx = self._last_index self._last_index += 1 try: @@ -246,35 +255,37 @@ class _MagicFormatMapping(Mapping): return len(self._kwargs) -if hasattr(text_type, 'format'): - class EscapeFormatter(string.Formatter): +if hasattr(text_type, "format"): + class EscapeFormatter(string.Formatter): def __init__(self, escape): self.escape = escape def format_field(self, value, format_spec): - if hasattr(value, '__html_format__'): + if hasattr(value, "__html_format__"): rv = value.__html_format__(format_spec) - elif hasattr(value, '__html__'): + elif hasattr(value, "__html__"): if format_spec: - raise ValueError('No format specification allowed ' - 'when formatting an object with ' - 'its __html__ method.') + raise ValueError( + "Format specifier {0} given, but {1} does not" + " define __html_format__. A class that defines" + " __html__ must define __html_format__ to work" + " with format specifiers.".format(format_spec, type(value)) + ) rv = value.__html__() else: # We need to make sure the format spec is unicode here as # otherwise the wrong callback methods are invoked. For # instance a byte string there would invoke __str__ and # not __unicode__. - rv = string.Formatter.format_field( - self, value, text_type(format_spec)) + rv = string.Formatter.format_field(self, value, text_type(format_spec)) return text_type(self.escape(rv)) def _escape_argspec(obj, iterable, escape): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, string_types): + if hasattr(value, "__html__") or isinstance(value, string_types): obj[key] = escape(value) return obj @@ -286,20 +297,31 @@ class _MarkupEscapeHelper(object): self.obj = obj self.escape = escape - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) - __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) - __repr__ = lambda s: str(s.escape(repr(s.obj))) - __int__ = lambda s: int(s.obj) - __float__ = lambda s: float(s.obj) + def __getitem__(self, item): + return _MarkupEscapeHelper(self.obj[item], self.escape) + + def __str__(self): + return text_type(self.escape(self.obj)) + + __unicode__ = __str__ + + def __repr__(self): + return str(self.escape(repr(self.obj))) + + def __int__(self): + return int(self.obj) + + def __float__(self): + return float(self.obj) # we have to import it down here as the speedups and native # modules imports the markup type which is define above. try: - from markupsafe._speedups import escape, escape_silent, soft_unicode + from ._speedups import escape, escape_silent, soft_unicode except ImportError: - from markupsafe._native import escape, escape_silent, soft_unicode + from ._native import escape, escape_silent, soft_unicode if not PY2: soft_str = soft_unicode - __all__.append('soft_str') + __all__.append("soft_str") diff --git a/pipenv/vendor/markupsafe/_compat.py b/pipenv/vendor/markupsafe/_compat.py index 62e5632a..bc05090f 100644 --- a/pipenv/vendor/markupsafe/_compat.py +++ b/pipenv/vendor/markupsafe/_compat.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- """ - markupsafe._compat - ~~~~~~~~~~~~~~~~~~ +markupsafe._compat +~~~~~~~~~~~~~~~~~~ - Compatibility module for different Python versions. - - :copyright: (c) 2013 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import sys @@ -17,10 +15,19 @@ if not PY2: string_types = (str,) unichr = chr int_types = (int,) - iteritems = lambda x: iter(x.items()) + + def iteritems(x): + return iter(x.items()) + + from collections.abc import Mapping + else: text_type = unicode string_types = (str, unicode) unichr = unichr int_types = (int, long) - iteritems = lambda x: x.iteritems() + + def iteritems(x): + return x.iteritems() + + from collections import Mapping diff --git a/pipenv/vendor/markupsafe/_constants.py b/pipenv/vendor/markupsafe/_constants.py index 919bf03c..7c57c2d2 100644 --- a/pipenv/vendor/markupsafe/_constants.py +++ b/pipenv/vendor/markupsafe/_constants.py @@ -1,267 +1,264 @@ # -*- coding: utf-8 -*- """ - markupsafe._constants - ~~~~~~~~~~~~~~~~~~~~~ +markupsafe._constants +~~~~~~~~~~~~~~~~~~~~~ - Highlevel implementation of the Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ - HTML_ENTITIES = { - 'AElig': 198, - 'Aacute': 193, - 'Acirc': 194, - 'Agrave': 192, - 'Alpha': 913, - 'Aring': 197, - 'Atilde': 195, - 'Auml': 196, - 'Beta': 914, - 'Ccedil': 199, - 'Chi': 935, - 'Dagger': 8225, - 'Delta': 916, - 'ETH': 208, - 'Eacute': 201, - 'Ecirc': 202, - 'Egrave': 200, - 'Epsilon': 917, - 'Eta': 919, - 'Euml': 203, - 'Gamma': 915, - 'Iacute': 205, - 'Icirc': 206, - 'Igrave': 204, - 'Iota': 921, - 'Iuml': 207, - 'Kappa': 922, - 'Lambda': 923, - 'Mu': 924, - 'Ntilde': 209, - 'Nu': 925, - 'OElig': 338, - 'Oacute': 211, - 'Ocirc': 212, - 'Ograve': 210, - 'Omega': 937, - 'Omicron': 927, - 'Oslash': 216, - 'Otilde': 213, - 'Ouml': 214, - 'Phi': 934, - 'Pi': 928, - 'Prime': 8243, - 'Psi': 936, - 'Rho': 929, - 'Scaron': 352, - 'Sigma': 931, - 'THORN': 222, - 'Tau': 932, - 'Theta': 920, - 'Uacute': 218, - 'Ucirc': 219, - 'Ugrave': 217, - 'Upsilon': 933, - 'Uuml': 220, - 'Xi': 926, - 'Yacute': 221, - 'Yuml': 376, - 'Zeta': 918, - 'aacute': 225, - 'acirc': 226, - 'acute': 180, - 'aelig': 230, - 'agrave': 224, - 'alefsym': 8501, - 'alpha': 945, - 'amp': 38, - 'and': 8743, - 'ang': 8736, - 'apos': 39, - 'aring': 229, - 'asymp': 8776, - 'atilde': 227, - 'auml': 228, - 'bdquo': 8222, - 'beta': 946, - 'brvbar': 166, - 'bull': 8226, - 'cap': 8745, - 'ccedil': 231, - 'cedil': 184, - 'cent': 162, - 'chi': 967, - 'circ': 710, - 'clubs': 9827, - 'cong': 8773, - 'copy': 169, - 'crarr': 8629, - 'cup': 8746, - 'curren': 164, - 'dArr': 8659, - 'dagger': 8224, - 'darr': 8595, - 'deg': 176, - 'delta': 948, - 'diams': 9830, - 'divide': 247, - 'eacute': 233, - 'ecirc': 234, - 'egrave': 232, - 'empty': 8709, - 'emsp': 8195, - 'ensp': 8194, - 'epsilon': 949, - 'equiv': 8801, - 'eta': 951, - 'eth': 240, - 'euml': 235, - 'euro': 8364, - 'exist': 8707, - 'fnof': 402, - 'forall': 8704, - 'frac12': 189, - 'frac14': 188, - 'frac34': 190, - 'frasl': 8260, - 'gamma': 947, - 'ge': 8805, - 'gt': 62, - 'hArr': 8660, - 'harr': 8596, - 'hearts': 9829, - 'hellip': 8230, - 'iacute': 237, - 'icirc': 238, - 'iexcl': 161, - 'igrave': 236, - 'image': 8465, - 'infin': 8734, - 'int': 8747, - 'iota': 953, - 'iquest': 191, - 'isin': 8712, - 'iuml': 239, - 'kappa': 954, - 'lArr': 8656, - 'lambda': 955, - 'lang': 9001, - 'laquo': 171, - 'larr': 8592, - 'lceil': 8968, - 'ldquo': 8220, - 'le': 8804, - 'lfloor': 8970, - 'lowast': 8727, - 'loz': 9674, - 'lrm': 8206, - 'lsaquo': 8249, - 'lsquo': 8216, - 'lt': 60, - 'macr': 175, - 'mdash': 8212, - 'micro': 181, - 'middot': 183, - 'minus': 8722, - 'mu': 956, - 'nabla': 8711, - 'nbsp': 160, - 'ndash': 8211, - 'ne': 8800, - 'ni': 8715, - 'not': 172, - 'notin': 8713, - 'nsub': 8836, - 'ntilde': 241, - 'nu': 957, - 'oacute': 243, - 'ocirc': 244, - 'oelig': 339, - 'ograve': 242, - 'oline': 8254, - 'omega': 969, - 'omicron': 959, - 'oplus': 8853, - 'or': 8744, - 'ordf': 170, - 'ordm': 186, - 'oslash': 248, - 'otilde': 245, - 'otimes': 8855, - 'ouml': 246, - 'para': 182, - 'part': 8706, - 'permil': 8240, - 'perp': 8869, - 'phi': 966, - 'pi': 960, - 'piv': 982, - 'plusmn': 177, - 'pound': 163, - 'prime': 8242, - 'prod': 8719, - 'prop': 8733, - 'psi': 968, - 'quot': 34, - 'rArr': 8658, - 'radic': 8730, - 'rang': 9002, - 'raquo': 187, - 'rarr': 8594, - 'rceil': 8969, - 'rdquo': 8221, - 'real': 8476, - 'reg': 174, - 'rfloor': 8971, - 'rho': 961, - 'rlm': 8207, - 'rsaquo': 8250, - 'rsquo': 8217, - 'sbquo': 8218, - 'scaron': 353, - 'sdot': 8901, - 'sect': 167, - 'shy': 173, - 'sigma': 963, - 'sigmaf': 962, - 'sim': 8764, - 'spades': 9824, - 'sub': 8834, - 'sube': 8838, - 'sum': 8721, - 'sup': 8835, - 'sup1': 185, - 'sup2': 178, - 'sup3': 179, - 'supe': 8839, - 'szlig': 223, - 'tau': 964, - 'there4': 8756, - 'theta': 952, - 'thetasym': 977, - 'thinsp': 8201, - 'thorn': 254, - 'tilde': 732, - 'times': 215, - 'trade': 8482, - 'uArr': 8657, - 'uacute': 250, - 'uarr': 8593, - 'ucirc': 251, - 'ugrave': 249, - 'uml': 168, - 'upsih': 978, - 'upsilon': 965, - 'uuml': 252, - 'weierp': 8472, - 'xi': 958, - 'yacute': 253, - 'yen': 165, - 'yuml': 255, - 'zeta': 950, - 'zwj': 8205, - 'zwnj': 8204 + "AElig": 198, + "Aacute": 193, + "Acirc": 194, + "Agrave": 192, + "Alpha": 913, + "Aring": 197, + "Atilde": 195, + "Auml": 196, + "Beta": 914, + "Ccedil": 199, + "Chi": 935, + "Dagger": 8225, + "Delta": 916, + "ETH": 208, + "Eacute": 201, + "Ecirc": 202, + "Egrave": 200, + "Epsilon": 917, + "Eta": 919, + "Euml": 203, + "Gamma": 915, + "Iacute": 205, + "Icirc": 206, + "Igrave": 204, + "Iota": 921, + "Iuml": 207, + "Kappa": 922, + "Lambda": 923, + "Mu": 924, + "Ntilde": 209, + "Nu": 925, + "OElig": 338, + "Oacute": 211, + "Ocirc": 212, + "Ograve": 210, + "Omega": 937, + "Omicron": 927, + "Oslash": 216, + "Otilde": 213, + "Ouml": 214, + "Phi": 934, + "Pi": 928, + "Prime": 8243, + "Psi": 936, + "Rho": 929, + "Scaron": 352, + "Sigma": 931, + "THORN": 222, + "Tau": 932, + "Theta": 920, + "Uacute": 218, + "Ucirc": 219, + "Ugrave": 217, + "Upsilon": 933, + "Uuml": 220, + "Xi": 926, + "Yacute": 221, + "Yuml": 376, + "Zeta": 918, + "aacute": 225, + "acirc": 226, + "acute": 180, + "aelig": 230, + "agrave": 224, + "alefsym": 8501, + "alpha": 945, + "amp": 38, + "and": 8743, + "ang": 8736, + "apos": 39, + "aring": 229, + "asymp": 8776, + "atilde": 227, + "auml": 228, + "bdquo": 8222, + "beta": 946, + "brvbar": 166, + "bull": 8226, + "cap": 8745, + "ccedil": 231, + "cedil": 184, + "cent": 162, + "chi": 967, + "circ": 710, + "clubs": 9827, + "cong": 8773, + "copy": 169, + "crarr": 8629, + "cup": 8746, + "curren": 164, + "dArr": 8659, + "dagger": 8224, + "darr": 8595, + "deg": 176, + "delta": 948, + "diams": 9830, + "divide": 247, + "eacute": 233, + "ecirc": 234, + "egrave": 232, + "empty": 8709, + "emsp": 8195, + "ensp": 8194, + "epsilon": 949, + "equiv": 8801, + "eta": 951, + "eth": 240, + "euml": 235, + "euro": 8364, + "exist": 8707, + "fnof": 402, + "forall": 8704, + "frac12": 189, + "frac14": 188, + "frac34": 190, + "frasl": 8260, + "gamma": 947, + "ge": 8805, + "gt": 62, + "hArr": 8660, + "harr": 8596, + "hearts": 9829, + "hellip": 8230, + "iacute": 237, + "icirc": 238, + "iexcl": 161, + "igrave": 236, + "image": 8465, + "infin": 8734, + "int": 8747, + "iota": 953, + "iquest": 191, + "isin": 8712, + "iuml": 239, + "kappa": 954, + "lArr": 8656, + "lambda": 955, + "lang": 9001, + "laquo": 171, + "larr": 8592, + "lceil": 8968, + "ldquo": 8220, + "le": 8804, + "lfloor": 8970, + "lowast": 8727, + "loz": 9674, + "lrm": 8206, + "lsaquo": 8249, + "lsquo": 8216, + "lt": 60, + "macr": 175, + "mdash": 8212, + "micro": 181, + "middot": 183, + "minus": 8722, + "mu": 956, + "nabla": 8711, + "nbsp": 160, + "ndash": 8211, + "ne": 8800, + "ni": 8715, + "not": 172, + "notin": 8713, + "nsub": 8836, + "ntilde": 241, + "nu": 957, + "oacute": 243, + "ocirc": 244, + "oelig": 339, + "ograve": 242, + "oline": 8254, + "omega": 969, + "omicron": 959, + "oplus": 8853, + "or": 8744, + "ordf": 170, + "ordm": 186, + "oslash": 248, + "otilde": 245, + "otimes": 8855, + "ouml": 246, + "para": 182, + "part": 8706, + "permil": 8240, + "perp": 8869, + "phi": 966, + "pi": 960, + "piv": 982, + "plusmn": 177, + "pound": 163, + "prime": 8242, + "prod": 8719, + "prop": 8733, + "psi": 968, + "quot": 34, + "rArr": 8658, + "radic": 8730, + "rang": 9002, + "raquo": 187, + "rarr": 8594, + "rceil": 8969, + "rdquo": 8221, + "real": 8476, + "reg": 174, + "rfloor": 8971, + "rho": 961, + "rlm": 8207, + "rsaquo": 8250, + "rsquo": 8217, + "sbquo": 8218, + "scaron": 353, + "sdot": 8901, + "sect": 167, + "shy": 173, + "sigma": 963, + "sigmaf": 962, + "sim": 8764, + "spades": 9824, + "sub": 8834, + "sube": 8838, + "sum": 8721, + "sup": 8835, + "sup1": 185, + "sup2": 178, + "sup3": 179, + "supe": 8839, + "szlig": 223, + "tau": 964, + "there4": 8756, + "theta": 952, + "thetasym": 977, + "thinsp": 8201, + "thorn": 254, + "tilde": 732, + "times": 215, + "trade": 8482, + "uArr": 8657, + "uacute": 250, + "uarr": 8593, + "ucirc": 251, + "ugrave": 249, + "uml": 168, + "upsih": 978, + "upsilon": 965, + "uuml": 252, + "weierp": 8472, + "xi": 958, + "yacute": 253, + "yen": 165, + "yuml": 255, + "zeta": 950, + "zwj": 8205, + "zwnj": 8204, } diff --git a/pipenv/vendor/markupsafe/_native.py b/pipenv/vendor/markupsafe/_native.py index 5e83f10a..cd08752c 100644 --- a/pipenv/vendor/markupsafe/_native.py +++ b/pipenv/vendor/markupsafe/_native.py @@ -1,36 +1,49 @@ # -*- coding: utf-8 -*- """ - markupsafe._native - ~~~~~~~~~~~~~~~~~~ +markupsafe._native +~~~~~~~~~~~~~~~~~~ - Native Python implementation the C module is not compiled. +Native Python implementation used when the C module is not compiled. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ -from markupsafe import Markup -from markupsafe._compat import text_type +from . import Markup +from ._compat import text_type def escape(s): - """Convert the characters &, <, >, ' and " in string s to HTML-safe - sequences. Use this if you need to display text that might contain - such characters in HTML. Marks return value as markup string. + """Replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in + the string with HTML-safe sequences. Use this if you need to display + text that might contain such characters in HTML. + + If the object has an ``__html__`` method, it is called and the + return value is assumed to already be safe for HTML. + + :param s: An object to be converted to a string and escaped. + :return: A :class:`Markup` string with the escaped text. """ - if hasattr(s, '__html__'): - return s.__html__() - return Markup(text_type(s) - .replace('&', '&') - .replace('>', '>') - .replace('<', '<') - .replace("'", ''') - .replace('"', '"') + if hasattr(s, "__html__"): + return Markup(s.__html__()) + return Markup( + text_type(s) + .replace("&", "&") + .replace(">", ">") + .replace("<", "<") + .replace("'", "'") + .replace('"', """) ) def escape_silent(s): - """Like :func:`escape` but converts `None` into an empty - markup string. + """Like :func:`escape` but treats ``None`` as the empty string. + Useful with optional values, as otherwise you get the string + ``'None'`` when the value is ``None``. + + >>> escape(None) + Markup('None') + >>> escape_silent(None) + Markup('') """ if s is None: return Markup() @@ -38,8 +51,18 @@ def escape_silent(s): def soft_unicode(s): - """Make a string unicode if it isn't already. That way a markup - string is not converted back to unicode. + """Convert an object to a string if it isn't already. This preserves + a :class:`Markup` string rather than converting it back to a basic + string, so it will still be marked as safe and won't be escaped + again. + + >>> value = escape('<User 1>') + >>> value + Markup('<User 1>') + >>> escape(str(value)) + Markup('&lt;User 1&gt;') + >>> escape(soft_unicode(value)) + Markup('<User 1>') """ if not isinstance(s, text_type): s = text_type(s) diff --git a/pipenv/vendor/markupsafe/_speedups.c b/pipenv/vendor/markupsafe/_speedups.c index d779a68c..12d2c4a7 100644 --- a/pipenv/vendor/markupsafe/_speedups.c +++ b/pipenv/vendor/markupsafe/_speedups.c @@ -2,33 +2,30 @@ * markupsafe._speedups * ~~~~~~~~~~~~~~~~~~~~ * - * This module implements functions for automatic escaping in C for better - * performance. + * C implementation of escaping for better performance. Used instead of + * the native Python implementation when compiled. * - * :copyright: (c) 2010 by Armin Ronacher. - * :license: BSD. + * :copyright: 2010 Pallets + * :license: BSD-3-Clause */ - #include <Python.h> +#if PY_MAJOR_VERSION < 3 #define ESCAPED_CHARS_TABLE_SIZE 63 #define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - - -static PyObject* markup; static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; +#endif + +static PyObject* markup; static int init_constants(void) { PyObject *module; + +#if PY_MAJOR_VERSION < 3 /* mapping of characters to replace */ escaped_chars_repl['"'] = UNICHR("""); escaped_chars_repl['\''] = UNICHR("'"); @@ -41,6 +38,7 @@ init_constants(void) escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ escaped_chars_delta_len['&'] = 4; escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; +#endif /* import markup type so that we can mark the return value */ module = PyImport_ImportModule("markupsafe"); @@ -52,6 +50,7 @@ init_constants(void) return 1; } +#if PY_MAJOR_VERSION < 3 static PyObject* escape_unicode(PyUnicodeObject *in) { @@ -112,13 +111,192 @@ escape_unicode(PyUnicodeObject *in) return (PyObject*)out; } +#else /* PY_MAJOR_VERSION < 3 */ +#define GET_DELTA(inp, inp_end, delta) \ + while (inp < inp_end) { \ + switch (*inp++) { \ + case '"': \ + case '\'': \ + case '&': \ + delta += 4; \ + break; \ + case '<': \ + case '>': \ + delta += 3; \ + break; \ + } \ + } + +#define DO_ESCAPE(inp, inp_end, outp) \ + { \ + Py_ssize_t ncopy = 0; \ + while (inp < inp_end) { \ + switch (*inp) { \ + case '"': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '4'; \ + *outp++ = ';'; \ + break; \ + case '\'': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '9'; \ + *outp++ = ';'; \ + break; \ + case '&': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'a'; \ + *outp++ = 'm'; \ + *outp++ = 'p'; \ + *outp++ = ';'; \ + break; \ + case '<': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'l'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + case '>': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'g'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + default: \ + ncopy++; \ + } \ + inp++; \ + } \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + } + +static PyObject* +escape_unicode_kind1(PyUnicodeObject *in) +{ + Py_UCS1 *inp = PyUnicode_1BYTE_DATA(in); + Py_UCS1 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS1 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, + PyUnicode_IS_ASCII(in) ? 127 : 255); + if (!out) + return NULL; + + inp = PyUnicode_1BYTE_DATA(in); + outp = PyUnicode_1BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode_kind2(PyUnicodeObject *in) +{ + Py_UCS2 *inp = PyUnicode_2BYTE_DATA(in); + Py_UCS2 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS2 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 65535); + if (!out) + return NULL; + + inp = PyUnicode_2BYTE_DATA(in); + outp = PyUnicode_2BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + + +static PyObject* +escape_unicode_kind4(PyUnicodeObject *in) +{ + Py_UCS4 *inp = PyUnicode_4BYTE_DATA(in); + Py_UCS4 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS4 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 1114111); + if (!out) + return NULL; + + inp = PyUnicode_4BYTE_DATA(in); + outp = PyUnicode_4BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + if (PyUnicode_READY(in)) + return NULL; + + switch (PyUnicode_KIND(in)) { + case PyUnicode_1BYTE_KIND: + return escape_unicode_kind1(in); + case PyUnicode_2BYTE_KIND: + return escape_unicode_kind2(in); + case PyUnicode_4BYTE_KIND: + return escape_unicode_kind4(in); + } + assert(0); /* shouldn't happen */ + return NULL; +} +#endif /* PY_MAJOR_VERSION < 3 */ static PyObject* escape(PyObject *self, PyObject *text) { + static PyObject *id_html; PyObject *s = NULL, *rv = NULL, *html; + if (id_html == NULL) { +#if PY_MAJOR_VERSION < 3 + id_html = PyString_InternFromString("__html__"); +#else + id_html = PyUnicode_InternFromString("__html__"); +#endif + if (id_html == NULL) { + return NULL; + } + } + /* we don't have to escape integers, bools or floats */ if (PyLong_CheckExact(text) || #if PY_MAJOR_VERSION < 3 @@ -129,10 +307,16 @@ escape(PyObject *self, PyObject *text) return PyObject_CallFunctionObjArgs(markup, text, NULL); /* if the object has an __html__ method that performs the escaping */ - html = PyObject_GetAttrString(text, "__html__"); + html = PyObject_GetAttr(text ,id_html); if (html) { - rv = PyObject_CallObject(html, NULL); + s = PyObject_CallObject(html, NULL); Py_DECREF(html); + if (s == NULL) { + return NULL; + } + /* Convert to Markup object */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); return rv; } diff --git a/pipenv/vendor/more_itertools/LICENSE b/pipenv/vendor/more_itertools/LICENSE new file mode 100644 index 00000000..0a523bec --- /dev/null +++ b/pipenv/vendor/more_itertools/LICENSE @@ -0,0 +1,19 @@ +Copyright (c) 2012 Erik Rose + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/more_itertools/__init__.py b/pipenv/vendor/more_itertools/__init__.py new file mode 100644 index 00000000..bba462c3 --- /dev/null +++ b/pipenv/vendor/more_itertools/__init__.py @@ -0,0 +1,2 @@ +from more_itertools.more import * # noqa +from more_itertools.recipes import * # noqa diff --git a/pipenv/vendor/more_itertools/more.py b/pipenv/vendor/more_itertools/more.py new file mode 100644 index 00000000..bd32a261 --- /dev/null +++ b/pipenv/vendor/more_itertools/more.py @@ -0,0 +1,2333 @@ +from __future__ import print_function + +from collections import Counter, defaultdict, deque +from functools import partial, wraps +from heapq import merge +from itertools import ( + chain, + compress, + count, + cycle, + dropwhile, + groupby, + islice, + repeat, + starmap, + takewhile, + tee +) +from operator import itemgetter, lt, gt, sub +from sys import maxsize, version_info +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + +from six import binary_type, string_types, text_type +from six.moves import filter, map, range, zip, zip_longest + +from .recipes import consume, flatten, take + +__all__ = [ + 'adjacent', + 'always_iterable', + 'always_reversible', + 'bucket', + 'chunked', + 'circular_shifts', + 'collapse', + 'collate', + 'consecutive_groups', + 'consumer', + 'count_cycle', + 'difference', + 'distinct_permutations', + 'distribute', + 'divide', + 'exactly_n', + 'first', + 'groupby_transform', + 'ilen', + 'interleave_longest', + 'interleave', + 'intersperse', + 'islice_extended', + 'iterate', + 'last', + 'locate', + 'lstrip', + 'make_decorator', + 'map_reduce', + 'numeric_range', + 'one', + 'padded', + 'peekable', + 'replace', + 'rlocate', + 'rstrip', + 'run_length', + 'seekable', + 'SequenceView', + 'side_effect', + 'sliced', + 'sort_together', + 'split_at', + 'split_after', + 'split_before', + 'split_into', + 'spy', + 'stagger', + 'strip', + 'substrings', + 'unique_to_each', + 'unzip', + 'windowed', + 'with_iter', + 'zip_offset', +] + +_marker = object() + + +def chunked(iterable, n): + """Break *iterable* into lists of length *n*: + + >>> list(chunked([1, 2, 3, 4, 5, 6], 3)) + [[1, 2, 3], [4, 5, 6]] + + If the length of *iterable* is not evenly divisible by *n*, the last + returned list will be shorter: + + >>> list(chunked([1, 2, 3, 4, 5, 6, 7, 8], 3)) + [[1, 2, 3], [4, 5, 6], [7, 8]] + + To use a fill-in value instead, see the :func:`grouper` recipe. + + :func:`chunked` is useful for splitting up a computation on a large number + of keys into batches, to be pickled and sent off to worker processes. One + example is operations on rows in MySQL, which does not implement + server-side cursors properly and would otherwise load the entire dataset + into RAM on the client. + + """ + return iter(partial(take, n, iter(iterable)), []) + + +def first(iterable, default=_marker): + """Return the first item of *iterable*, or *default* if *iterable* is + empty. + + >>> first([0, 1, 2, 3]) + 0 + >>> first([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + + :func:`first` is useful when you have a generator of expensive-to-retrieve + values and want any arbitrary one. It is marginally shorter than + ``next(iter(iterable), default)``. + + """ + try: + return next(iter(iterable)) + except StopIteration: + # I'm on the edge about raising ValueError instead of StopIteration. At + # the moment, ValueError wins, because the caller could conceivably + # want to do something different with flow control when I raise the + # exception, and it's weird to explicitly catch StopIteration. + if default is _marker: + raise ValueError('first() was called on an empty iterable, and no ' + 'default value was provided.') + return default + + +def last(iterable, default=_marker): + """Return the last item of *iterable*, or *default* if *iterable* is + empty. + + >>> last([0, 1, 2, 3]) + 3 + >>> last([], 'some default') + 'some default' + + If *default* is not provided and there are no items in the iterable, + raise ``ValueError``. + """ + try: + try: + # Try to access the last item directly + return iterable[-1] + except (TypeError, AttributeError, KeyError): + # If not slice-able, iterate entirely using length-1 deque + return deque(iterable, maxlen=1)[0] + except IndexError: # If the iterable was empty + if default is _marker: + raise ValueError('last() was called on an empty iterable, and no ' + 'default value was provided.') + return default + + +class peekable(object): + """Wrap an iterator to allow lookahead and prepending elements. + + Call :meth:`peek` on the result to get the value that will be returned + by :func:`next`. This won't advance the iterator: + + >>> p = peekable(['a', 'b']) + >>> p.peek() + 'a' + >>> next(p) + 'a' + + Pass :meth:`peek` a default value to return that instead of raising + ``StopIteration`` when the iterator is exhausted. + + >>> p = peekable([]) + >>> p.peek('hi') + 'hi' + + peekables also offer a :meth:`prepend` method, which "inserts" items + at the head of the iterable: + + >>> p = peekable([1, 2, 3]) + >>> p.prepend(10, 11, 12) + >>> next(p) + 10 + >>> p.peek() + 11 + >>> list(p) + [11, 12, 1, 2, 3] + + peekables can be indexed. Index 0 is the item that will be returned by + :func:`next`, index 1 is the item after that, and so on: + The values up to the given index will be cached. + + >>> p = peekable(['a', 'b', 'c', 'd']) + >>> p[0] + 'a' + >>> p[1] + 'b' + >>> next(p) + 'a' + + Negative indexes are supported, but be aware that they will cache the + remaining items in the source iterator, which may require significant + storage. + + To check whether a peekable is exhausted, check its truth value: + + >>> p = peekable(['a', 'b']) + >>> if p: # peekable has items + ... list(p) + ['a', 'b'] + >>> if not p: # peekable is exhaused + ... list(p) + [] + + """ + def __init__(self, iterable): + self._it = iter(iterable) + self._cache = deque() + + def __iter__(self): + return self + + def __bool__(self): + try: + self.peek() + except StopIteration: + return False + return True + + def __nonzero__(self): + # For Python 2 compatibility + return self.__bool__() + + def peek(self, default=_marker): + """Return the item that will be next returned from ``next()``. + + Return ``default`` if there are no items left. If ``default`` is not + provided, raise ``StopIteration``. + + """ + if not self._cache: + try: + self._cache.append(next(self._it)) + except StopIteration: + if default is _marker: + raise + return default + return self._cache[0] + + def prepend(self, *items): + """Stack up items to be the next ones returned from ``next()`` or + ``self.peek()``. The items will be returned in + first in, first out order:: + + >>> p = peekable([1, 2, 3]) + >>> p.prepend(10, 11, 12) + >>> next(p) + 10 + >>> list(p) + [11, 12, 1, 2, 3] + + It is possible, by prepending items, to "resurrect" a peekable that + previously raised ``StopIteration``. + + >>> p = peekable([]) + >>> next(p) + Traceback (most recent call last): + ... + StopIteration + >>> p.prepend(1) + >>> next(p) + 1 + >>> next(p) + Traceback (most recent call last): + ... + StopIteration + + """ + self._cache.extendleft(reversed(items)) + + def __next__(self): + if self._cache: + return self._cache.popleft() + + return next(self._it) + + next = __next__ # For Python 2 compatibility + + def _get_slice(self, index): + # Normalize the slice's arguments + step = 1 if (index.step is None) else index.step + if step > 0: + start = 0 if (index.start is None) else index.start + stop = maxsize if (index.stop is None) else index.stop + elif step < 0: + start = -1 if (index.start is None) else index.start + stop = (-maxsize - 1) if (index.stop is None) else index.stop + else: + raise ValueError('slice step cannot be zero') + + # If either the start or stop index is negative, we'll need to cache + # the rest of the iterable in order to slice from the right side. + if (start < 0) or (stop < 0): + self._cache.extend(self._it) + # Otherwise we'll need to find the rightmost index and cache to that + # point. + else: + n = min(max(start, stop) + 1, maxsize) + cache_len = len(self._cache) + if n >= cache_len: + self._cache.extend(islice(self._it, n - cache_len)) + + return list(self._cache)[index] + + def __getitem__(self, index): + if isinstance(index, slice): + return self._get_slice(index) + + cache_len = len(self._cache) + if index < 0: + self._cache.extend(self._it) + elif index >= cache_len: + self._cache.extend(islice(self._it, index + 1 - cache_len)) + + return self._cache[index] + + +def _collate(*iterables, **kwargs): + """Helper for ``collate()``, called when the user is using the ``reverse`` + or ``key`` keyword arguments on Python versions below 3.5. + + """ + key = kwargs.pop('key', lambda a: a) + reverse = kwargs.pop('reverse', False) + + min_or_max = partial(max if reverse else min, key=itemgetter(0)) + peekables = [peekable(it) for it in iterables] + peekables = [p for p in peekables if p] # Kill empties. + while peekables: + _, p = min_or_max((key(p.peek()), p) for p in peekables) + yield next(p) + peekables = [x for x in peekables if x] + + +def collate(*iterables, **kwargs): + """Return a sorted merge of the items from each of several already-sorted + *iterables*. + + >>> list(collate('ACDZ', 'AZ', 'JKL')) + ['A', 'A', 'C', 'D', 'J', 'K', 'L', 'Z', 'Z'] + + Works lazily, keeping only the next value from each iterable in memory. Use + :func:`collate` to, for example, perform a n-way mergesort of items that + don't fit in memory. + + If a *key* function is specified, the iterables will be sorted according + to its result: + + >>> key = lambda s: int(s) # Sort by numeric value, not by string + >>> list(collate(['1', '10'], ['2', '11'], key=key)) + ['1', '2', '10', '11'] + + + If the *iterables* are sorted in descending order, set *reverse* to + ``True``: + + >>> list(collate([5, 3, 1], [4, 2, 0], reverse=True)) + [5, 4, 3, 2, 1, 0] + + If the elements of the passed-in iterables are out of order, you might get + unexpected results. + + On Python 2.7, this function delegates to :func:`heapq.merge` if neither + of the keyword arguments are specified. On Python 3.5+, this function + is an alias for :func:`heapq.merge`. + + """ + if not kwargs: + return merge(*iterables) + + return _collate(*iterables, **kwargs) + + +# If using Python version 3.5 or greater, heapq.merge() will be faster than +# collate - use that instead. +if version_info >= (3, 5, 0): + _collate_docstring = collate.__doc__ + collate = partial(merge) + collate.__doc__ = _collate_docstring + + +def consumer(func): + """Decorator that automatically advances a PEP-342-style "reverse iterator" + to its first yield point so you don't have to call ``next()`` on it + manually. + + >>> @consumer + ... def tally(): + ... i = 0 + ... while True: + ... print('Thing number %s is %s.' % (i, (yield))) + ... i += 1 + ... + >>> t = tally() + >>> t.send('red') + Thing number 0 is red. + >>> t.send('fish') + Thing number 1 is fish. + + Without the decorator, you would have to call ``next(t)`` before + ``t.send()`` could be used. + + """ + @wraps(func) + def wrapper(*args, **kwargs): + gen = func(*args, **kwargs) + next(gen) + return gen + return wrapper + + +def ilen(iterable): + """Return the number of items in *iterable*. + + >>> ilen(x for x in range(1000000) if x % 3 == 0) + 333334 + + This consumes the iterable, so handle with care. + + """ + # This approach was selected because benchmarks showed it's likely the + # fastest of the known implementations at the time of writing. + # See GitHub tracker: #236, #230. + counter = count() + deque(zip(iterable, counter), maxlen=0) + return next(counter) + + +def iterate(func, start): + """Return ``start``, ``func(start)``, ``func(func(start))``, ... + + >>> from itertools import islice + >>> list(islice(iterate(lambda x: 2*x, 1), 10)) + [1, 2, 4, 8, 16, 32, 64, 128, 256, 512] + + """ + while True: + yield start + start = func(start) + + +def with_iter(context_manager): + """Wrap an iterable in a ``with`` statement, so it closes once exhausted. + + For example, this will close the file when the iterator is exhausted:: + + upper_lines = (line.upper() for line in with_iter(open('foo'))) + + Any context manager which returns an iterable is a candidate for + ``with_iter``. + + """ + with context_manager as iterable: + for item in iterable: + yield item + + +def one(iterable, too_short=None, too_long=None): + """Return the first item from *iterable*, which is expected to contain only + that item. Raise an exception if *iterable* is empty or has more than one + item. + + :func:`one` is useful for ensuring that an iterable contains only one item. + For example, it can be used to retrieve the result of a database query + that is expected to return a single row. + + If *iterable* is empty, ``ValueError`` will be raised. You may specify a + different exception with the *too_short* keyword: + + >>> it = [] + >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: too many items in iterable (expected 1)' + >>> too_short = IndexError('too few items') + >>> one(it, too_short=too_short) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + IndexError: too few items + + Similarly, if *iterable* contains more than one item, ``ValueError`` will + be raised. You may specify a different exception with the *too_long* + keyword: + + >>> it = ['too', 'many'] + >>> one(it) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + ValueError: too many items in iterable (expected 1)' + >>> too_long = RuntimeError + >>> one(it, too_long=too_long) # doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + ... + RuntimeError + + Note that :func:`one` attempts to advance *iterable* twice to ensure there + is only one item. If there is more than one, both items will be discarded. + See :func:`spy` or :func:`peekable` to check iterable contents less + destructively. + + """ + it = iter(iterable) + + try: + value = next(it) + except StopIteration: + raise too_short or ValueError('too few items in iterable (expected 1)') + + try: + next(it) + except StopIteration: + pass + else: + raise too_long or ValueError('too many items in iterable (expected 1)') + + return value + + +def distinct_permutations(iterable): + """Yield successive distinct permutations of the elements in *iterable*. + + >>> sorted(distinct_permutations([1, 0, 1])) + [(0, 1, 1), (1, 0, 1), (1, 1, 0)] + + Equivalent to ``set(permutations(iterable))``, except duplicates are not + generated and thrown away. For larger input sequences this is much more + efficient. + + Duplicate permutations arise when there are duplicated elements in the + input iterable. The number of items returned is + `n! / (x_1! * x_2! * ... * x_n!)`, where `n` is the total number of + items input, and each `x_i` is the count of a distinct item in the input + sequence. + + """ + def perm_unique_helper(item_counts, perm, i): + """Internal helper function + + :arg item_counts: Stores the unique items in ``iterable`` and how many + times they are repeated + :arg perm: The permutation that is being built for output + :arg i: The index of the permutation being modified + + The output permutations are built up recursively; the distinct items + are placed until their repetitions are exhausted. + """ + if i < 0: + yield tuple(perm) + else: + for item in item_counts: + if item_counts[item] <= 0: + continue + perm[i] = item + item_counts[item] -= 1 + for x in perm_unique_helper(item_counts, perm, i - 1): + yield x + item_counts[item] += 1 + + item_counts = Counter(iterable) + length = sum(item_counts.values()) + + return perm_unique_helper(item_counts, [None] * length, length - 1) + + +def intersperse(e, iterable, n=1): + """Intersperse filler element *e* among the items in *iterable*, leaving + *n* items between each filler element. + + >>> list(intersperse('!', [1, 2, 3, 4, 5])) + [1, '!', 2, '!', 3, '!', 4, '!', 5] + + >>> list(intersperse(None, [1, 2, 3, 4, 5], n=2)) + [1, 2, None, 3, 4, None, 5] + + """ + if n == 0: + raise ValueError('n must be > 0') + elif n == 1: + # interleave(repeat(e), iterable) -> e, x_0, e, e, x_1, e, x_2... + # islice(..., 1, None) -> x_0, e, e, x_1, e, x_2... + return islice(interleave(repeat(e), iterable), 1, None) + else: + # interleave(filler, chunks) -> [e], [x_0, x_1], [e], [x_2, x_3]... + # islice(..., 1, None) -> [x_0, x_1], [e], [x_2, x_3]... + # flatten(...) -> x_0, x_1, e, x_2, x_3... + filler = repeat([e]) + chunks = chunked(iterable, n) + return flatten(islice(interleave(filler, chunks), 1, None)) + + +def unique_to_each(*iterables): + """Return the elements from each of the input iterables that aren't in the + other input iterables. + + For example, suppose you have a set of packages, each with a set of + dependencies:: + + {'pkg_1': {'A', 'B'}, 'pkg_2': {'B', 'C'}, 'pkg_3': {'B', 'D'}} + + If you remove one package, which dependencies can also be removed? + + If ``pkg_1`` is removed, then ``A`` is no longer necessary - it is not + associated with ``pkg_2`` or ``pkg_3``. Similarly, ``C`` is only needed for + ``pkg_2``, and ``D`` is only needed for ``pkg_3``:: + + >>> unique_to_each({'A', 'B'}, {'B', 'C'}, {'B', 'D'}) + [['A'], ['C'], ['D']] + + If there are duplicates in one input iterable that aren't in the others + they will be duplicated in the output. Input order is preserved:: + + >>> unique_to_each("mississippi", "missouri") + [['p', 'p'], ['o', 'u', 'r']] + + It is assumed that the elements of each iterable are hashable. + + """ + pool = [list(it) for it in iterables] + counts = Counter(chain.from_iterable(map(set, pool))) + uniques = {element for element in counts if counts[element] == 1} + return [list(filter(uniques.__contains__, it)) for it in pool] + + +def windowed(seq, n, fillvalue=None, step=1): + """Return a sliding window of width *n* over the given iterable. + + >>> all_windows = windowed([1, 2, 3, 4, 5], 3) + >>> list(all_windows) + [(1, 2, 3), (2, 3, 4), (3, 4, 5)] + + When the window is larger than the iterable, *fillvalue* is used in place + of missing values:: + + >>> list(windowed([1, 2, 3], 4)) + [(1, 2, 3, None)] + + Each window will advance in increments of *step*: + + >>> list(windowed([1, 2, 3, 4, 5, 6], 3, fillvalue='!', step=2)) + [(1, 2, 3), (3, 4, 5), (5, 6, '!')] + + """ + if n < 0: + raise ValueError('n must be >= 0') + if n == 0: + yield tuple() + return + if step < 1: + raise ValueError('step must be >= 1') + + it = iter(seq) + window = deque([], n) + append = window.append + + # Initial deque fill + for _ in range(n): + append(next(it, fillvalue)) + yield tuple(window) + + # Appending new items to the right causes old items to fall off the left + i = 0 + for item in it: + append(item) + i = (i + 1) % step + if i % step == 0: + yield tuple(window) + + # If there are items from the iterable in the window, pad with the given + # value and emit them. + if (i % step) and (step - i < n): + for _ in range(step - i): + append(fillvalue) + yield tuple(window) + + +def substrings(iterable, join_func=None): + """Yield all of the substrings of *iterable*. + + >>> [''.join(s) for s in substrings('more')] + ['m', 'o', 'r', 'e', 'mo', 'or', 're', 'mor', 'ore', 'more'] + + Note that non-string iterables can also be subdivided. + + >>> list(substrings([0, 1, 2])) + [(0,), (1,), (2,), (0, 1), (1, 2), (0, 1, 2)] + + """ + # The length-1 substrings + seq = [] + for item in iter(iterable): + seq.append(item) + yield (item,) + seq = tuple(seq) + item_count = len(seq) + + # And the rest + for n in range(2, item_count + 1): + for i in range(item_count - n + 1): + yield seq[i:i + n] + + +class bucket(object): + """Wrap *iterable* and return an object that buckets it iterable into + child iterables based on a *key* function. + + >>> iterable = ['a1', 'b1', 'c1', 'a2', 'b2', 'c2', 'b3'] + >>> s = bucket(iterable, key=lambda x: x[0]) + >>> a_iterable = s['a'] + >>> next(a_iterable) + 'a1' + >>> next(a_iterable) + 'a2' + >>> list(s['b']) + ['b1', 'b2', 'b3'] + + The original iterable will be advanced and its items will be cached until + they are used by the child iterables. This may require significant storage. + + By default, attempting to select a bucket to which no items belong will + exhaust the iterable and cache all values. + If you specify a *validator* function, selected buckets will instead be + checked against it. + + >>> from itertools import count + >>> it = count(1, 2) # Infinite sequence of odd numbers + >>> key = lambda x: x % 10 # Bucket by last digit + >>> validator = lambda x: x in {1, 3, 5, 7, 9} # Odd digits only + >>> s = bucket(it, key=key, validator=validator) + >>> 2 in s + False + >>> list(s[2]) + [] + + """ + def __init__(self, iterable, key, validator=None): + self._it = iter(iterable) + self._key = key + self._cache = defaultdict(deque) + self._validator = validator or (lambda x: True) + + def __contains__(self, value): + if not self._validator(value): + return False + + try: + item = next(self[value]) + except StopIteration: + return False + else: + self._cache[value].appendleft(item) + + return True + + def _get_values(self, value): + """ + Helper to yield items from the parent iterator that match *value*. + Items that don't match are stored in the local cache as they + are encountered. + """ + while True: + # If we've cached some items that match the target value, emit + # the first one and evict it from the cache. + if self._cache[value]: + yield self._cache[value].popleft() + # Otherwise we need to advance the parent iterator to search for + # a matching item, caching the rest. + else: + while True: + try: + item = next(self._it) + except StopIteration: + return + item_value = self._key(item) + if item_value == value: + yield item + break + elif self._validator(item_value): + self._cache[item_value].append(item) + + def __getitem__(self, value): + if not self._validator(value): + return iter(()) + + return self._get_values(value) + + +def spy(iterable, n=1): + """Return a 2-tuple with a list containing the first *n* elements of + *iterable*, and an iterator with the same items as *iterable*. + This allows you to "look ahead" at the items in the iterable without + advancing it. + + There is one item in the list by default: + + >>> iterable = 'abcdefg' + >>> head, iterable = spy(iterable) + >>> head + ['a'] + >>> list(iterable) + ['a', 'b', 'c', 'd', 'e', 'f', 'g'] + + You may use unpacking to retrieve items instead of lists: + + >>> (head,), iterable = spy('abcdefg') + >>> head + 'a' + >>> (first, second), iterable = spy('abcdefg', 2) + >>> first + 'a' + >>> second + 'b' + + The number of items requested can be larger than the number of items in + the iterable: + + >>> iterable = [1, 2, 3, 4, 5] + >>> head, iterable = spy(iterable, 10) + >>> head + [1, 2, 3, 4, 5] + >>> list(iterable) + [1, 2, 3, 4, 5] + + """ + it = iter(iterable) + head = take(n, it) + + return head, chain(head, it) + + +def interleave(*iterables): + """Return a new iterable yielding from each iterable in turn, + until the shortest is exhausted. + + >>> list(interleave([1, 2, 3], [4, 5], [6, 7, 8])) + [1, 4, 6, 2, 5, 7] + + For a version that doesn't terminate after the shortest iterable is + exhausted, see :func:`interleave_longest`. + + """ + return chain.from_iterable(zip(*iterables)) + + +def interleave_longest(*iterables): + """Return a new iterable yielding from each iterable in turn, + skipping any that are exhausted. + + >>> list(interleave_longest([1, 2, 3], [4, 5], [6, 7, 8])) + [1, 4, 6, 2, 5, 7, 3, 8] + + This function produces the same output as :func:`roundrobin`, but may + perform better for some inputs (in particular when the number of iterables + is large). + + """ + i = chain.from_iterable(zip_longest(*iterables, fillvalue=_marker)) + return (x for x in i if x is not _marker) + + +def collapse(iterable, base_type=None, levels=None): + """Flatten an iterable with multiple levels of nesting (e.g., a list of + lists of tuples) into non-iterable types. + + >>> iterable = [(1, 2), ([3, 4], [[5], [6]])] + >>> list(collapse(iterable)) + [1, 2, 3, 4, 5, 6] + + String types are not considered iterable and will not be collapsed. + To avoid collapsing other types, specify *base_type*: + + >>> iterable = ['ab', ('cd', 'ef'), ['gh', 'ij']] + >>> list(collapse(iterable, base_type=tuple)) + ['ab', ('cd', 'ef'), 'gh', 'ij'] + + Specify *levels* to stop flattening after a certain level: + + >>> iterable = [('a', ['b']), ('c', ['d'])] + >>> list(collapse(iterable)) # Fully flattened + ['a', 'b', 'c', 'd'] + >>> list(collapse(iterable, levels=1)) # Only one level flattened + ['a', ['b'], 'c', ['d']] + + """ + def walk(node, level): + if ( + ((levels is not None) and (level > levels)) or + isinstance(node, string_types) or + ((base_type is not None) and isinstance(node, base_type)) + ): + yield node + return + + try: + tree = iter(node) + except TypeError: + yield node + return + else: + for child in tree: + for x in walk(child, level + 1): + yield x + + for x in walk(iterable, 0): + yield x + + +def side_effect(func, iterable, chunk_size=None, before=None, after=None): + """Invoke *func* on each item in *iterable* (or on each *chunk_size* group + of items) before yielding the item. + + `func` must be a function that takes a single argument. Its return value + will be discarded. + + *before* and *after* are optional functions that take no arguments. They + will be executed before iteration starts and after it ends, respectively. + + `side_effect` can be used for logging, updating progress bars, or anything + that is not functionally "pure." + + Emitting a status message: + + >>> from more_itertools import consume + >>> func = lambda item: print('Received {}'.format(item)) + >>> consume(side_effect(func, range(2))) + Received 0 + Received 1 + + Operating on chunks of items: + + >>> pair_sums = [] + >>> func = lambda chunk: pair_sums.append(sum(chunk)) + >>> list(side_effect(func, [0, 1, 2, 3, 4, 5], 2)) + [0, 1, 2, 3, 4, 5] + >>> list(pair_sums) + [1, 5, 9] + + Writing to a file-like object: + + >>> from io import StringIO + >>> from more_itertools import consume + >>> f = StringIO() + >>> func = lambda x: print(x, file=f) + >>> before = lambda: print(u'HEADER', file=f) + >>> after = f.close + >>> it = [u'a', u'b', u'c'] + >>> consume(side_effect(func, it, before=before, after=after)) + >>> f.closed + True + + """ + try: + if before is not None: + before() + + if chunk_size is None: + for item in iterable: + func(item) + yield item + else: + for chunk in chunked(iterable, chunk_size): + func(chunk) + for item in chunk: + yield item + finally: + if after is not None: + after() + + +def sliced(seq, n): + """Yield slices of length *n* from the sequence *seq*. + + >>> list(sliced((1, 2, 3, 4, 5, 6), 3)) + [(1, 2, 3), (4, 5, 6)] + + If the length of the sequence is not divisible by the requested slice + length, the last slice will be shorter. + + >>> list(sliced((1, 2, 3, 4, 5, 6, 7, 8), 3)) + [(1, 2, 3), (4, 5, 6), (7, 8)] + + This function will only work for iterables that support slicing. + For non-sliceable iterables, see :func:`chunked`. + + """ + return takewhile(bool, (seq[i: i + n] for i in count(0, n))) + + +def split_at(iterable, pred): + """Yield lists of items from *iterable*, where each list is delimited by + an item where callable *pred* returns ``True``. The lists do not include + the delimiting items. + + >>> list(split_at('abcdcba', lambda x: x == 'b')) + [['a'], ['c', 'd', 'c'], ['a']] + + >>> list(split_at(range(10), lambda n: n % 2 == 1)) + [[0], [2], [4], [6], [8], []] + """ + buf = [] + for item in iterable: + if pred(item): + yield buf + buf = [] + else: + buf.append(item) + yield buf + + +def split_before(iterable, pred): + """Yield lists of items from *iterable*, where each list starts with an + item where callable *pred* returns ``True``: + + >>> list(split_before('OneTwo', lambda s: s.isupper())) + [['O', 'n', 'e'], ['T', 'w', 'o']] + + >>> list(split_before(range(10), lambda n: n % 3 == 0)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9]] + + """ + buf = [] + for item in iterable: + if pred(item) and buf: + yield buf + buf = [] + buf.append(item) + yield buf + + +def split_after(iterable, pred): + """Yield lists of items from *iterable*, where each list ends with an + item where callable *pred* returns ``True``: + + >>> list(split_after('one1two2', lambda s: s.isdigit())) + [['o', 'n', 'e', '1'], ['t', 'w', 'o', '2']] + + >>> list(split_after(range(10), lambda n: n % 3 == 0)) + [[0], [1, 2, 3], [4, 5, 6], [7, 8, 9]] + + """ + buf = [] + for item in iterable: + buf.append(item) + if pred(item) and buf: + yield buf + buf = [] + if buf: + yield buf + + +def split_into(iterable, sizes): + """Yield a list of sequential items from *iterable* of length 'n' for each + integer 'n' in *sizes*. + + >>> list(split_into([1,2,3,4,5,6], [1,2,3])) + [[1], [2, 3], [4, 5, 6]] + + If the sum of *sizes* is smaller than the length of *iterable*, then the + remaining items of *iterable* will not be returned. + + >>> list(split_into([1,2,3,4,5,6], [2,3])) + [[1, 2], [3, 4, 5]] + + If the sum of *sizes* is larger than the length of *iterable*, fewer items + will be returned in the iteration that overruns *iterable* and further + lists will be empty: + + >>> list(split_into([1,2,3,4], [1,2,3,4])) + [[1], [2, 3], [4], []] + + When a ``None`` object is encountered in *sizes*, the returned list will + contain items up to the end of *iterable* the same way that itertools.slice + does: + + >>> list(split_into([1,2,3,4,5,6,7,8,9,0], [2,3,None])) + [[1, 2], [3, 4, 5], [6, 7, 8, 9, 0]] + + :func:`split_into` can be useful for grouping a series of items where the + sizes of the groups are not uniform. An example would be where in a row + from a table, multiple columns represent elements of the same feature + (e.g. a point represented by x,y,z) but, the format is not the same for + all columns. + """ + # convert the iterable argument into an iterator so its contents can + # be consumed by islice in case it is a generator + it = iter(iterable) + + for size in sizes: + if size is None: + yield list(it) + return + else: + yield list(islice(it, size)) + + +def padded(iterable, fillvalue=None, n=None, next_multiple=False): + """Yield the elements from *iterable*, followed by *fillvalue*, such that + at least *n* items are emitted. + + >>> list(padded([1, 2, 3], '?', 5)) + [1, 2, 3, '?', '?'] + + If *next_multiple* is ``True``, *fillvalue* will be emitted until the + number of items emitted is a multiple of *n*:: + + >>> list(padded([1, 2, 3, 4], n=3, next_multiple=True)) + [1, 2, 3, 4, None, None] + + If *n* is ``None``, *fillvalue* will be emitted indefinitely. + + """ + it = iter(iterable) + if n is None: + for item in chain(it, repeat(fillvalue)): + yield item + elif n < 1: + raise ValueError('n must be at least 1') + else: + item_count = 0 + for item in it: + yield item + item_count += 1 + + remaining = (n - item_count) % n if next_multiple else n - item_count + for _ in range(remaining): + yield fillvalue + + +def distribute(n, iterable): + """Distribute the items from *iterable* among *n* smaller iterables. + + >>> group_1, group_2 = distribute(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 3, 5] + >>> list(group_2) + [2, 4, 6] + + If the length of *iterable* is not evenly divisible by *n*, then the + length of the returned iterables will not be identical: + + >>> children = distribute(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 4, 7], [2, 5], [3, 6]] + + If the length of *iterable* is smaller than *n*, then the last returned + iterables will be empty: + + >>> children = distribute(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + + This function uses :func:`itertools.tee` and may require significant + storage. If you need the order items in the smaller iterables to match the + original iterable, see :func:`divide`. + + """ + if n < 1: + raise ValueError('n must be at least 1') + + children = tee(iterable, n) + return [islice(it, index, None, n) for index, it in enumerate(children)] + + +def stagger(iterable, offsets=(-1, 0, 1), longest=False, fillvalue=None): + """Yield tuples whose elements are offset from *iterable*. + The amount by which the `i`-th item in each tuple is offset is given by + the `i`-th item in *offsets*. + + >>> list(stagger([0, 1, 2, 3])) + [(None, 0, 1), (0, 1, 2), (1, 2, 3)] + >>> list(stagger(range(8), offsets=(0, 2, 4))) + [(0, 2, 4), (1, 3, 5), (2, 4, 6), (3, 5, 7)] + + By default, the sequence will end when the final element of a tuple is the + last item in the iterable. To continue until the first element of a tuple + is the last item in the iterable, set *longest* to ``True``:: + + >>> list(stagger([0, 1, 2, 3], longest=True)) + [(None, 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, None), (3, None, None)] + + By default, ``None`` will be used to replace offsets beyond the end of the + sequence. Specify *fillvalue* to use some other value. + + """ + children = tee(iterable, len(offsets)) + + return zip_offset( + *children, offsets=offsets, longest=longest, fillvalue=fillvalue + ) + + +def zip_offset(*iterables, **kwargs): + """``zip`` the input *iterables* together, but offset the `i`-th iterable + by the `i`-th item in *offsets*. + + >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1))) + [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e')] + + This can be used as a lightweight alternative to SciPy or pandas to analyze + data sets in which some series have a lead or lag relationship. + + By default, the sequence will end when the shortest iterable is exhausted. + To continue until the longest iterable is exhausted, set *longest* to + ``True``. + + >>> list(zip_offset('0123', 'abcdef', offsets=(0, 1), longest=True)) + [('0', 'b'), ('1', 'c'), ('2', 'd'), ('3', 'e'), (None, 'f')] + + By default, ``None`` will be used to replace offsets beyond the end of the + sequence. Specify *fillvalue* to use some other value. + + """ + offsets = kwargs['offsets'] + longest = kwargs.get('longest', False) + fillvalue = kwargs.get('fillvalue', None) + + if len(iterables) != len(offsets): + raise ValueError("Number of iterables and offsets didn't match") + + staggered = [] + for it, n in zip(iterables, offsets): + if n < 0: + staggered.append(chain(repeat(fillvalue, -n), it)) + elif n > 0: + staggered.append(islice(it, n, None)) + else: + staggered.append(it) + + if longest: + return zip_longest(*staggered, fillvalue=fillvalue) + + return zip(*staggered) + + +def sort_together(iterables, key_list=(0,), reverse=False): + """Return the input iterables sorted together, with *key_list* as the + priority for sorting. All iterables are trimmed to the length of the + shortest one. + + This can be used like the sorting function in a spreadsheet. If each + iterable represents a column of data, the key list determines which + columns are used for sorting. + + By default, all iterables are sorted using the ``0``-th iterable:: + + >>> iterables = [(4, 3, 2, 1), ('a', 'b', 'c', 'd')] + >>> sort_together(iterables) + [(1, 2, 3, 4), ('d', 'c', 'b', 'a')] + + Set a different key list to sort according to another iterable. + Specifying multiple keys dictates how ties are broken:: + + >>> iterables = [(3, 1, 2), (0, 1, 0), ('c', 'b', 'a')] + >>> sort_together(iterables, key_list=(1, 2)) + [(2, 3, 1), (0, 0, 1), ('a', 'c', 'b')] + + Set *reverse* to ``True`` to sort in descending order. + + >>> sort_together([(1, 2, 3), ('c', 'b', 'a')], reverse=True) + [(3, 2, 1), ('a', 'b', 'c')] + + """ + return list(zip(*sorted(zip(*iterables), + key=itemgetter(*key_list), + reverse=reverse))) + + +def unzip(iterable): + """The inverse of :func:`zip`, this function disaggregates the elements + of the zipped *iterable*. + + The ``i``-th iterable contains the ``i``-th element from each element + of the zipped iterable. The first element is used to to determine the + length of the remaining elements. + + >>> iterable = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + >>> letters, numbers = unzip(iterable) + >>> list(letters) + ['a', 'b', 'c', 'd'] + >>> list(numbers) + [1, 2, 3, 4] + + This is similar to using ``zip(*iterable)``, but it avoids reading + *iterable* into memory. Note, however, that this function uses + :func:`itertools.tee` and thus may require significant storage. + + """ + head, iterable = spy(iter(iterable)) + if not head: + # empty iterable, e.g. zip([], [], []) + return () + # spy returns a one-length iterable as head + head = head[0] + iterables = tee(iterable, len(head)) + + def itemgetter(i): + def getter(obj): + try: + return obj[i] + except IndexError: + # basically if we have an iterable like + # iter([(1, 2, 3), (4, 5), (6,)]) + # the second unzipped iterable would fail at the third tuple + # since it would try to access tup[1] + # same with the third unzipped iterable and the second tuple + # to support these "improperly zipped" iterables, + # we create a custom itemgetter + # which just stops the unzipped iterables + # at first length mismatch + raise StopIteration + return getter + + return tuple(map(itemgetter(i), it) for i, it in enumerate(iterables)) + + +def divide(n, iterable): + """Divide the elements from *iterable* into *n* parts, maintaining + order. + + >>> group_1, group_2 = divide(2, [1, 2, 3, 4, 5, 6]) + >>> list(group_1) + [1, 2, 3] + >>> list(group_2) + [4, 5, 6] + + If the length of *iterable* is not evenly divisible by *n*, then the + length of the returned iterables will not be identical: + + >>> children = divide(3, [1, 2, 3, 4, 5, 6, 7]) + >>> [list(c) for c in children] + [[1, 2, 3], [4, 5], [6, 7]] + + If the length of the iterable is smaller than n, then the last returned + iterables will be empty: + + >>> children = divide(5, [1, 2, 3]) + >>> [list(c) for c in children] + [[1], [2], [3], [], []] + + This function will exhaust the iterable before returning and may require + significant storage. If order is not important, see :func:`distribute`, + which does not first pull the iterable into memory. + + """ + if n < 1: + raise ValueError('n must be at least 1') + + seq = tuple(iterable) + q, r = divmod(len(seq), n) + + ret = [] + for i in range(n): + start = (i * q) + (i if i < r else r) + stop = ((i + 1) * q) + (i + 1 if i + 1 < r else r) + ret.append(iter(seq[start:stop])) + + return ret + + +def always_iterable(obj, base_type=(text_type, binary_type)): + """If *obj* is iterable, return an iterator over its items:: + + >>> obj = (1, 2, 3) + >>> list(always_iterable(obj)) + [1, 2, 3] + + If *obj* is not iterable, return a one-item iterable containing *obj*:: + + >>> obj = 1 + >>> list(always_iterable(obj)) + [1] + + If *obj* is ``None``, return an empty iterable: + + >>> obj = None + >>> list(always_iterable(None)) + [] + + By default, binary and text strings are not considered iterable:: + + >>> obj = 'foo' + >>> list(always_iterable(obj)) + ['foo'] + + If *base_type* is set, objects for which ``isinstance(obj, base_type)`` + returns ``True`` won't be considered iterable. + + >>> obj = {'a': 1} + >>> list(always_iterable(obj)) # Iterate over the dict's keys + ['a'] + >>> list(always_iterable(obj, base_type=dict)) # Treat dicts as a unit + [{'a': 1}] + + Set *base_type* to ``None`` to avoid any special handling and treat objects + Python considers iterable as iterable: + + >>> obj = 'foo' + >>> list(always_iterable(obj, base_type=None)) + ['f', 'o', 'o'] + """ + if obj is None: + return iter(()) + + if (base_type is not None) and isinstance(obj, base_type): + return iter((obj,)) + + try: + return iter(obj) + except TypeError: + return iter((obj,)) + + +def adjacent(predicate, iterable, distance=1): + """Return an iterable over `(bool, item)` tuples where the `item` is + drawn from *iterable* and the `bool` indicates whether + that item satisfies the *predicate* or is adjacent to an item that does. + + For example, to find whether items are adjacent to a ``3``:: + + >>> list(adjacent(lambda x: x == 3, range(6))) + [(False, 0), (False, 1), (True, 2), (True, 3), (True, 4), (False, 5)] + + Set *distance* to change what counts as adjacent. For example, to find + whether items are two places away from a ``3``: + + >>> list(adjacent(lambda x: x == 3, range(6), distance=2)) + [(False, 0), (True, 1), (True, 2), (True, 3), (True, 4), (True, 5)] + + This is useful for contextualizing the results of a search function. + For example, a code comparison tool might want to identify lines that + have changed, but also surrounding lines to give the viewer of the diff + context. + + The predicate function will only be called once for each item in the + iterable. + + See also :func:`groupby_transform`, which can be used with this function + to group ranges of items with the same `bool` value. + + """ + # Allow distance=0 mainly for testing that it reproduces results with map() + if distance < 0: + raise ValueError('distance must be at least 0') + + i1, i2 = tee(iterable) + padding = [False] * distance + selected = chain(padding, map(predicate, i1), padding) + adjacent_to_selected = map(any, windowed(selected, 2 * distance + 1)) + return zip(adjacent_to_selected, i2) + + +def groupby_transform(iterable, keyfunc=None, valuefunc=None): + """An extension of :func:`itertools.groupby` that transforms the values of + *iterable* after grouping them. + *keyfunc* is a function used to compute a grouping key for each item. + *valuefunc* is a function for transforming the items after grouping. + + >>> iterable = 'AaaABbBCcA' + >>> keyfunc = lambda x: x.upper() + >>> valuefunc = lambda x: x.lower() + >>> grouper = groupby_transform(iterable, keyfunc, valuefunc) + >>> [(k, ''.join(g)) for k, g in grouper] + [('A', 'aaaa'), ('B', 'bbb'), ('C', 'cc'), ('A', 'a')] + + *keyfunc* and *valuefunc* default to identity functions if they are not + specified. + + :func:`groupby_transform` is useful when grouping elements of an iterable + using a separate iterable as the key. To do this, :func:`zip` the iterables + and pass a *keyfunc* that extracts the first element and a *valuefunc* + that extracts the second element:: + + >>> from operator import itemgetter + >>> keys = [0, 0, 1, 1, 1, 2, 2, 2, 3] + >>> values = 'abcdefghi' + >>> iterable = zip(keys, values) + >>> grouper = groupby_transform(iterable, itemgetter(0), itemgetter(1)) + >>> [(k, ''.join(g)) for k, g in grouper] + [(0, 'ab'), (1, 'cde'), (2, 'fgh'), (3, 'i')] + + Note that the order of items in the iterable is significant. + Only adjacent items are grouped together, so if you don't want any + duplicate groups, you should sort the iterable by the key function. + + """ + valuefunc = (lambda x: x) if valuefunc is None else valuefunc + return ((k, map(valuefunc, g)) for k, g in groupby(iterable, keyfunc)) + + +def numeric_range(*args): + """An extension of the built-in ``range()`` function whose arguments can + be any orderable numeric type. + + With only *stop* specified, *start* defaults to ``0`` and *step* + defaults to ``1``. The output items will match the type of *stop*: + + >>> list(numeric_range(3.5)) + [0.0, 1.0, 2.0, 3.0] + + With only *start* and *stop* specified, *step* defaults to ``1``. The + output items will match the type of *start*: + + >>> from decimal import Decimal + >>> start = Decimal('2.1') + >>> stop = Decimal('5.1') + >>> list(numeric_range(start, stop)) + [Decimal('2.1'), Decimal('3.1'), Decimal('4.1')] + + With *start*, *stop*, and *step* specified the output items will match + the type of ``start + step``: + + >>> from fractions import Fraction + >>> start = Fraction(1, 2) # Start at 1/2 + >>> stop = Fraction(5, 2) # End at 5/2 + >>> step = Fraction(1, 2) # Count by 1/2 + >>> list(numeric_range(start, stop, step)) + [Fraction(1, 2), Fraction(1, 1), Fraction(3, 2), Fraction(2, 1)] + + If *step* is zero, ``ValueError`` is raised. Negative steps are supported: + + >>> list(numeric_range(3, -1, -1.0)) + [3.0, 2.0, 1.0, 0.0] + + Be aware of the limitations of floating point numbers; the representation + of the yielded numbers may be surprising. + + """ + argc = len(args) + if argc == 1: + stop, = args + start = type(stop)(0) + step = 1 + elif argc == 2: + start, stop = args + step = 1 + elif argc == 3: + start, stop, step = args + else: + err_msg = 'numeric_range takes at most 3 arguments, got {}' + raise TypeError(err_msg.format(argc)) + + values = (start + (step * n) for n in count()) + if step > 0: + return takewhile(partial(gt, stop), values) + elif step < 0: + return takewhile(partial(lt, stop), values) + else: + raise ValueError('numeric_range arg 3 must not be zero') + + +def count_cycle(iterable, n=None): + """Cycle through the items from *iterable* up to *n* times, yielding + the number of completed cycles along with each item. If *n* is omitted the + process repeats indefinitely. + + >>> list(count_cycle('AB', 3)) + [(0, 'A'), (0, 'B'), (1, 'A'), (1, 'B'), (2, 'A'), (2, 'B')] + + """ + iterable = tuple(iterable) + if not iterable: + return iter(()) + counter = count() if n is None else range(n) + return ((i, item) for i in counter for item in iterable) + + +def locate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(locate([0, 1, 1, 0, 1, 0, 0])) + [1, 2, 4] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item. + + >>> list(locate(['a', 'b', 'c', 'b'], lambda x: x == 'b')) + [1, 3] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(locate(iterable, pred=pred, window_size=3)) + [1, 5, 9] + + Use with :func:`seekable` to find indexes and then retrieve the associated + items: + + >>> from itertools import count + >>> from more_itertools import seekable + >>> source = (3 * n + 1 if (n % 2) else n // 2 for n in count()) + >>> it = seekable(source) + >>> pred = lambda x: x > 100 + >>> indexes = locate(it, pred=pred) + >>> i = next(indexes) + >>> it.seek(i) + >>> next(it) + 106 + + """ + if window_size is None: + return compress(count(), map(pred, iterable)) + + if window_size < 1: + raise ValueError('window size must be at least 1') + + it = windowed(iterable, window_size, fillvalue=_marker) + return compress(count(), starmap(pred, it)) + + +def lstrip(iterable, pred): + """Yield the items from *iterable*, but strip any from the beginning + for which *pred* returns ``True``. + + For example, to remove a set of items from the start of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(lstrip(iterable, pred)) + [1, 2, None, 3, False, None] + + This function is analogous to to :func:`str.lstrip`, and is essentially + an wrapper for :func:`itertools.dropwhile`. + + """ + return dropwhile(pred, iterable) + + +def rstrip(iterable, pred): + """Yield the items from *iterable*, but strip any from the end + for which *pred* returns ``True``. + + For example, to remove a set of items from the end of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(rstrip(iterable, pred)) + [None, False, None, 1, 2, None, 3] + + This function is analogous to :func:`str.rstrip`. + + """ + cache = [] + cache_append = cache.append + for x in iterable: + if pred(x): + cache_append(x) + else: + for y in cache: + yield y + del cache[:] + yield x + + +def strip(iterable, pred): + """Yield the items from *iterable*, but strip any from the + beginning and end for which *pred* returns ``True``. + + For example, to remove a set of items from both ends of an iterable: + + >>> iterable = (None, False, None, 1, 2, None, 3, False, None) + >>> pred = lambda x: x in {None, False, ''} + >>> list(strip(iterable, pred)) + [1, 2, None, 3] + + This function is analogous to :func:`str.strip`. + + """ + return rstrip(lstrip(iterable, pred), pred) + + +def islice_extended(iterable, *args): + """An extension of :func:`itertools.islice` that supports negative values + for *stop*, *start*, and *step*. + + >>> iterable = iter('abcdefgh') + >>> list(islice_extended(iterable, -4, -1)) + ['e', 'f', 'g'] + + Slices with negative values require some caching of *iterable*, but this + function takes care to minimize the amount of memory required. + + For example, you can use a negative step with an infinite iterator: + + >>> from itertools import count + >>> list(islice_extended(count(), 110, 99, -2)) + [110, 108, 106, 104, 102, 100] + + """ + s = slice(*args) + start = s.start + stop = s.stop + if s.step == 0: + raise ValueError('step argument must be a non-zero integer or None.') + step = s.step or 1 + + it = iter(iterable) + + if step > 0: + start = 0 if (start is None) else start + + if (start < 0): + # Consume all but the last -start items + cache = deque(enumerate(it, 1), maxlen=-start) + len_iter = cache[-1][0] if cache else 0 + + # Adjust start to be positive + i = max(len_iter + start, 0) + + # Adjust stop to be positive + if stop is None: + j = len_iter + elif stop >= 0: + j = min(stop, len_iter) + else: + j = max(len_iter + stop, 0) + + # Slice the cache + n = j - i + if n <= 0: + return + + for index, item in islice(cache, 0, n, step): + yield item + elif (stop is not None) and (stop < 0): + # Advance to the start position + next(islice(it, start, start), None) + + # When stop is negative, we have to carry -stop items while + # iterating + cache = deque(islice(it, -stop), maxlen=-stop) + + for index, item in enumerate(it): + cached_item = cache.popleft() + if index % step == 0: + yield cached_item + cache.append(item) + else: + # When both start and stop are positive we have the normal case + for item in islice(it, start, stop, step): + yield item + else: + start = -1 if (start is None) else start + + if (stop is not None) and (stop < 0): + # Consume all but the last items + n = -stop - 1 + cache = deque(enumerate(it, 1), maxlen=n) + len_iter = cache[-1][0] if cache else 0 + + # If start and stop are both negative they are comparable and + # we can just slice. Otherwise we can adjust start to be negative + # and then slice. + if start < 0: + i, j = start, stop + else: + i, j = min(start - len_iter, -1), None + + for index, item in list(cache)[i:j:step]: + yield item + else: + # Advance to the stop position + if stop is not None: + m = stop + 1 + next(islice(it, m, m), None) + + # stop is positive, so if start is negative they are not comparable + # and we need the rest of the items. + if start < 0: + i = start + n = None + # stop is None and start is positive, so we just need items up to + # the start index. + elif stop is None: + i = None + n = start + 1 + # Both stop and start are positive, so they are comparable. + else: + i = None + n = start - stop + if n <= 0: + return + + cache = list(islice(it, n)) + + for item in cache[i::step]: + yield item + + +def always_reversible(iterable): + """An extension of :func:`reversed` that supports all iterables, not + just those which implement the ``Reversible`` or ``Sequence`` protocols. + + >>> print(*always_reversible(x for x in range(3))) + 2 1 0 + + If the iterable is already reversible, this function returns the + result of :func:`reversed()`. If the iterable is not reversible, + this function will cache the remaining items in the iterable and + yield them in reverse order, which may require significant storage. + """ + try: + return reversed(iterable) + except TypeError: + return reversed(list(iterable)) + + +def consecutive_groups(iterable, ordering=lambda x: x): + """Yield groups of consecutive items using :func:`itertools.groupby`. + The *ordering* function determines whether two items are adjacent by + returning their position. + + By default, the ordering function is the identity function. This is + suitable for finding runs of numbers: + + >>> iterable = [1, 10, 11, 12, 20, 30, 31, 32, 33, 40] + >>> for group in consecutive_groups(iterable): + ... print(list(group)) + [1] + [10, 11, 12] + [20] + [30, 31, 32, 33] + [40] + + For finding runs of adjacent letters, try using the :meth:`index` method + of a string of letters: + + >>> from string import ascii_lowercase + >>> iterable = 'abcdfgilmnop' + >>> ordering = ascii_lowercase.index + >>> for group in consecutive_groups(iterable, ordering): + ... print(list(group)) + ['a', 'b', 'c', 'd'] + ['f', 'g'] + ['i'] + ['l', 'm', 'n', 'o', 'p'] + + """ + for k, g in groupby( + enumerate(iterable), key=lambda x: x[0] - ordering(x[1]) + ): + yield map(itemgetter(1), g) + + +def difference(iterable, func=sub): + """By default, compute the first difference of *iterable* using + :func:`operator.sub`. + + >>> iterable = [0, 1, 3, 6, 10] + >>> list(difference(iterable)) + [0, 1, 2, 3, 4] + + This is the opposite of :func:`accumulate`'s default behavior: + + >>> from more_itertools import accumulate + >>> iterable = [0, 1, 2, 3, 4] + >>> list(accumulate(iterable)) + [0, 1, 3, 6, 10] + >>> list(difference(accumulate(iterable))) + [0, 1, 2, 3, 4] + + By default *func* is :func:`operator.sub`, but other functions can be + specified. They will be applied as follows:: + + A, B, C, D, ... --> A, func(B, A), func(C, B), func(D, C), ... + + For example, to do progressive division: + + >>> iterable = [1, 2, 6, 24, 120] # Factorial sequence + >>> func = lambda x, y: x // y + >>> list(difference(iterable, func)) + [1, 2, 3, 4, 5] + + """ + a, b = tee(iterable) + try: + item = next(b) + except StopIteration: + return iter([]) + return chain([item], map(lambda x: func(x[1], x[0]), zip(a, b))) + + +class SequenceView(Sequence): + """Return a read-only view of the sequence object *target*. + + :class:`SequenceView` objects are analogous to Python's built-in + "dictionary view" types. They provide a dynamic view of a sequence's items, + meaning that when the sequence updates, so does the view. + + >>> seq = ['0', '1', '2'] + >>> view = SequenceView(seq) + >>> view + SequenceView(['0', '1', '2']) + >>> seq.append('3') + >>> view + SequenceView(['0', '1', '2', '3']) + + Sequence views support indexing, slicing, and length queries. They act + like the underlying sequence, except they don't allow assignment: + + >>> view[1] + '1' + >>> view[1:-1] + ['1', '2'] + >>> len(view) + 4 + + Sequence views are useful as an alternative to copying, as they don't + require (much) extra storage. + + """ + def __init__(self, target): + if not isinstance(target, Sequence): + raise TypeError + self._target = target + + def __getitem__(self, index): + return self._target[index] + + def __len__(self): + return len(self._target) + + def __repr__(self): + return '{}({})'.format(self.__class__.__name__, repr(self._target)) + + +class seekable(object): + """Wrap an iterator to allow for seeking backward and forward. This + progressively caches the items in the source iterable so they can be + re-visited. + + Call :meth:`seek` with an index to seek to that position in the source + iterable. + + To "reset" an iterator, seek to ``0``: + + >>> from itertools import count + >>> it = seekable((str(n) for n in count())) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> it.seek(0) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> next(it) + '3' + + You can also seek forward: + + >>> it = seekable((str(n) for n in range(20))) + >>> it.seek(10) + >>> next(it) + '10' + >>> it.seek(20) # Seeking past the end of the source isn't a problem + >>> list(it) + [] + >>> it.seek(0) # Resetting works even after hitting the end + >>> next(it), next(it), next(it) + ('0', '1', '2') + + The cache grows as the source iterable progresses, so beware of wrapping + very large or infinite iterables. + + You may view the contents of the cache with the :meth:`elements` method. + That returns a :class:`SequenceView`, a view that updates automatically: + + >>> it = seekable((str(n) for n in range(10))) + >>> next(it), next(it), next(it) + ('0', '1', '2') + >>> elements = it.elements() + >>> elements + SequenceView(['0', '1', '2']) + >>> next(it) + '3' + >>> elements + SequenceView(['0', '1', '2', '3']) + + """ + + def __init__(self, iterable): + self._source = iter(iterable) + self._cache = [] + self._index = None + + def __iter__(self): + return self + + def __next__(self): + if self._index is not None: + try: + item = self._cache[self._index] + except IndexError: + self._index = None + else: + self._index += 1 + return item + + item = next(self._source) + self._cache.append(item) + return item + + next = __next__ + + def elements(self): + return SequenceView(self._cache) + + def seek(self, index): + self._index = index + remainder = index - len(self._cache) + if remainder > 0: + consume(self, remainder) + + +class run_length(object): + """ + :func:`run_length.encode` compresses an iterable with run-length encoding. + It yields groups of repeated items with the count of how many times they + were repeated: + + >>> uncompressed = 'abbcccdddd' + >>> list(run_length.encode(uncompressed)) + [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + + :func:`run_length.decode` decompresses an iterable that was previously + compressed with run-length encoding. It yields the items of the + decompressed iterable: + + >>> compressed = [('a', 1), ('b', 2), ('c', 3), ('d', 4)] + >>> list(run_length.decode(compressed)) + ['a', 'b', 'b', 'c', 'c', 'c', 'd', 'd', 'd', 'd'] + + """ + + @staticmethod + def encode(iterable): + return ((k, ilen(g)) for k, g in groupby(iterable)) + + @staticmethod + def decode(iterable): + return chain.from_iterable(repeat(k, n) for k, n in iterable) + + +def exactly_n(iterable, n, predicate=bool): + """Return ``True`` if exactly ``n`` items in the iterable are ``True`` + according to the *predicate* function. + + >>> exactly_n([True, True, False], 2) + True + >>> exactly_n([True, True, False], 1) + False + >>> exactly_n([0, 1, 2, 3, 4, 5], 3, lambda x: x < 3) + True + + The iterable will be advanced until ``n + 1`` truthy items are encountered, + so avoid calling it on infinite iterables. + + """ + return len(take(n + 1, filter(predicate, iterable))) == n + + +def circular_shifts(iterable): + """Return a list of circular shifts of *iterable*. + + >>> circular_shifts(range(4)) + [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] + """ + lst = list(iterable) + return take(len(lst), windowed(cycle(lst), len(lst))) + + +def make_decorator(wrapping_func, result_index=0): + """Return a decorator version of *wrapping_func*, which is a function that + modifies an iterable. *result_index* is the position in that function's + signature where the iterable goes. + + This lets you use itertools on the "production end," i.e. at function + definition. This can augment what the function returns without changing the + function's code. + + For example, to produce a decorator version of :func:`chunked`: + + >>> from more_itertools import chunked + >>> chunker = make_decorator(chunked, result_index=0) + >>> @chunker(3) + ... def iter_range(n): + ... return iter(range(n)) + ... + >>> list(iter_range(9)) + [[0, 1, 2], [3, 4, 5], [6, 7, 8]] + + To only allow truthy items to be returned: + + >>> truth_serum = make_decorator(filter, result_index=1) + >>> @truth_serum(bool) + ... def boolean_test(): + ... return [0, 1, '', ' ', False, True] + ... + >>> list(boolean_test()) + [1, ' ', True] + + The :func:`peekable` and :func:`seekable` wrappers make for practical + decorators: + + >>> from more_itertools import peekable + >>> peekable_function = make_decorator(peekable) + >>> @peekable_function() + ... def str_range(*args): + ... return (str(x) for x in range(*args)) + ... + >>> it = str_range(1, 20, 2) + >>> next(it), next(it), next(it) + ('1', '3', '5') + >>> it.peek() + '7' + >>> next(it) + '7' + + """ + # See https://sites.google.com/site/bbayles/index/decorator_factory for + # notes on how this works. + def decorator(*wrapping_args, **wrapping_kwargs): + def outer_wrapper(f): + def inner_wrapper(*args, **kwargs): + result = f(*args, **kwargs) + wrapping_args_ = list(wrapping_args) + wrapping_args_.insert(result_index, result) + return wrapping_func(*wrapping_args_, **wrapping_kwargs) + + return inner_wrapper + + return outer_wrapper + + return decorator + + +def map_reduce(iterable, keyfunc, valuefunc=None, reducefunc=None): + """Return a dictionary that maps the items in *iterable* to categories + defined by *keyfunc*, transforms them with *valuefunc*, and + then summarizes them by category with *reducefunc*. + + *valuefunc* defaults to the identity function if it is unspecified. + If *reducefunc* is unspecified, no summarization takes place: + + >>> keyfunc = lambda x: x.upper() + >>> result = map_reduce('abbccc', keyfunc) + >>> sorted(result.items()) + [('A', ['a']), ('B', ['b', 'b']), ('C', ['c', 'c', 'c'])] + + Specifying *valuefunc* transforms the categorized items: + + >>> keyfunc = lambda x: x.upper() + >>> valuefunc = lambda x: 1 + >>> result = map_reduce('abbccc', keyfunc, valuefunc) + >>> sorted(result.items()) + [('A', [1]), ('B', [1, 1]), ('C', [1, 1, 1])] + + Specifying *reducefunc* summarizes the categorized items: + + >>> keyfunc = lambda x: x.upper() + >>> valuefunc = lambda x: 1 + >>> reducefunc = sum + >>> result = map_reduce('abbccc', keyfunc, valuefunc, reducefunc) + >>> sorted(result.items()) + [('A', 1), ('B', 2), ('C', 3)] + + You may want to filter the input iterable before applying the map/reduce + procedure: + + >>> all_items = range(30) + >>> items = [x for x in all_items if 10 <= x <= 20] # Filter + >>> keyfunc = lambda x: x % 2 # Evens map to 0; odds to 1 + >>> categories = map_reduce(items, keyfunc=keyfunc) + >>> sorted(categories.items()) + [(0, [10, 12, 14, 16, 18, 20]), (1, [11, 13, 15, 17, 19])] + >>> summaries = map_reduce(items, keyfunc=keyfunc, reducefunc=sum) + >>> sorted(summaries.items()) + [(0, 90), (1, 75)] + + Note that all items in the iterable are gathered into a list before the + summarization step, which may require significant storage. + + The returned object is a :obj:`collections.defaultdict` with the + ``default_factory`` set to ``None``, such that it behaves like a normal + dictionary. + + """ + valuefunc = (lambda x: x) if (valuefunc is None) else valuefunc + + ret = defaultdict(list) + for item in iterable: + key = keyfunc(item) + value = valuefunc(item) + ret[key].append(value) + + if reducefunc is not None: + for key, value_list in ret.items(): + ret[key] = reducefunc(value_list) + + ret.default_factory = None + return ret + + +def rlocate(iterable, pred=bool, window_size=None): + """Yield the index of each item in *iterable* for which *pred* returns + ``True``, starting from the right and moving left. + + *pred* defaults to :func:`bool`, which will select truthy items: + + >>> list(rlocate([0, 1, 1, 0, 1, 0, 0])) # Truthy at 1, 2, and 4 + [4, 2, 1] + + Set *pred* to a custom function to, e.g., find the indexes for a particular + item: + + >>> iterable = iter('abcb') + >>> pred = lambda x: x == 'b' + >>> list(rlocate(iterable, pred)) + [3, 1] + + If *window_size* is given, then the *pred* function will be called with + that many items. This enables searching for sub-sequences: + + >>> iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] + >>> pred = lambda *args: args == (1, 2, 3) + >>> list(rlocate(iterable, pred=pred, window_size=3)) + [9, 5, 1] + + Beware, this function won't return anything for infinite iterables. + If *iterable* is reversible, ``rlocate`` will reverse it and search from + the right. Otherwise, it will search from the left and return the results + in reverse order. + + See :func:`locate` to for other example applications. + + """ + if window_size is None: + try: + len_iter = len(iterable) + return ( + len_iter - i - 1 for i in locate(reversed(iterable), pred) + ) + except TypeError: + pass + + return reversed(list(locate(iterable, pred, window_size))) + + +def replace(iterable, pred, substitutes, count=None, window_size=1): + """Yield the items from *iterable*, replacing the items for which *pred* + returns ``True`` with the items from the iterable *substitutes*. + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1] + >>> pred = lambda x: x == 0 + >>> substitutes = (2, 3) + >>> list(replace(iterable, pred, substitutes)) + [1, 1, 2, 3, 1, 1, 2, 3, 1, 1] + + If *count* is given, the number of replacements will be limited: + + >>> iterable = [1, 1, 0, 1, 1, 0, 1, 1, 0] + >>> pred = lambda x: x == 0 + >>> substitutes = [None] + >>> list(replace(iterable, pred, substitutes, count=2)) + [1, 1, None, 1, 1, None, 1, 1, 0] + + Use *window_size* to control the number of items passed as arguments to + *pred*. This allows for locating and replacing subsequences. + + >>> iterable = [0, 1, 2, 5, 0, 1, 2, 5] + >>> window_size = 3 + >>> pred = lambda *args: args == (0, 1, 2) # 3 items passed to pred + >>> substitutes = [3, 4] # Splice in these items + >>> list(replace(iterable, pred, substitutes, window_size=window_size)) + [3, 4, 5, 3, 4, 5] + + """ + if window_size < 1: + raise ValueError('window_size must be at least 1') + + # Save the substitutes iterable, since it's used more than once + substitutes = tuple(substitutes) + + # Add padding such that the number of windows matches the length of the + # iterable + it = chain(iterable, [_marker] * (window_size - 1)) + windows = windowed(it, window_size) + + n = 0 + for w in windows: + # If the current window matches our predicate (and we haven't hit + # our maximum number of replacements), splice in the substitutes + # and then consume the following windows that overlap with this one. + # For example, if the iterable is (0, 1, 2, 3, 4...) + # and the window size is 2, we have (0, 1), (1, 2), (2, 3)... + # If the predicate matches on (0, 1), we need to zap (0, 1) and (1, 2) + if pred(*w): + if (count is None) or (n < count): + n += 1 + for s in substitutes: + yield s + consume(windows, window_size - 1) + continue + + # If there was no match (or we've reached the replacement limit), + # yield the first item from the window. + if w and (w[0] is not _marker): + yield w[0] diff --git a/pipenv/vendor/more_itertools/recipes.py b/pipenv/vendor/more_itertools/recipes.py new file mode 100644 index 00000000..3b455d4e --- /dev/null +++ b/pipenv/vendor/more_itertools/recipes.py @@ -0,0 +1,577 @@ +"""Imported from the recipes section of the itertools documentation. + +All functions taken from the recipes section of the itertools library docs +[1]_. +Some backward-compatible usability improvements have been made. + +.. [1] http://docs.python.org/library/itertools.html#recipes + +""" +from collections import deque +from itertools import ( + chain, combinations, count, cycle, groupby, islice, repeat, starmap, tee +) +import operator +from random import randrange, sample, choice + +from six import PY2 +from six.moves import filter, filterfalse, map, range, zip, zip_longest + +__all__ = [ + 'accumulate', + 'all_equal', + 'consume', + 'dotproduct', + 'first_true', + 'flatten', + 'grouper', + 'iter_except', + 'ncycles', + 'nth', + 'nth_combination', + 'padnone', + 'pairwise', + 'partition', + 'powerset', + 'prepend', + 'quantify', + 'random_combination_with_replacement', + 'random_combination', + 'random_permutation', + 'random_product', + 'repeatfunc', + 'roundrobin', + 'tabulate', + 'tail', + 'take', + 'unique_everseen', + 'unique_justseen', +] + + +def accumulate(iterable, func=operator.add): + """ + Return an iterator whose items are the accumulated results of a function + (specified by the optional *func* argument) that takes two arguments. + By default, returns accumulated sums with :func:`operator.add`. + + >>> list(accumulate([1, 2, 3, 4, 5])) # Running sum + [1, 3, 6, 10, 15] + >>> list(accumulate([1, 2, 3], func=operator.mul)) # Running product + [1, 2, 6] + >>> list(accumulate([0, 1, -1, 2, 3, 2], func=max)) # Running maximum + [0, 1, 1, 2, 3, 3] + + This function is available in the ``itertools`` module for Python 3.2 and + greater. + + """ + it = iter(iterable) + try: + total = next(it) + except StopIteration: + return + else: + yield total + + for element in it: + total = func(total, element) + yield total + + +def take(n, iterable): + """Return first *n* items of the iterable as a list. + + >>> take(3, range(10)) + [0, 1, 2] + >>> take(5, range(3)) + [0, 1, 2] + + Effectively a short replacement for ``next`` based iterator consumption + when you want more than one item, but less than the whole iterator. + + """ + return list(islice(iterable, n)) + + +def tabulate(function, start=0): + """Return an iterator over the results of ``func(start)``, + ``func(start + 1)``, ``func(start + 2)``... + + *func* should be a function that accepts one integer argument. + + If *start* is not specified it defaults to 0. It will be incremented each + time the iterator is advanced. + + >>> square = lambda x: x ** 2 + >>> iterator = tabulate(square, -3) + >>> take(4, iterator) + [9, 4, 1, 0] + + """ + return map(function, count(start)) + + +def tail(n, iterable): + """Return an iterator over the last *n* items of *iterable*. + + >>> t = tail(3, 'ABCDEFG') + >>> list(t) + ['E', 'F', 'G'] + + """ + return iter(deque(iterable, maxlen=n)) + + +def consume(iterator, n=None): + """Advance *iterable* by *n* steps. If *n* is ``None``, consume it + entirely. + + Efficiently exhausts an iterator without returning values. Defaults to + consuming the whole iterator, but an optional second argument may be + provided to limit consumption. + + >>> i = (x for x in range(10)) + >>> next(i) + 0 + >>> consume(i, 3) + >>> next(i) + 4 + >>> consume(i) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + + If the iterator has fewer items remaining than the provided limit, the + whole iterator will be consumed. + + >>> i = (x for x in range(3)) + >>> consume(i, 5) + >>> next(i) + Traceback (most recent call last): + File "<stdin>", line 1, in <module> + StopIteration + + """ + # Use functions that consume iterators at C speed. + if n is None: + # feed the entire iterator into a zero-length deque + deque(iterator, maxlen=0) + else: + # advance to the empty slice starting at position n + next(islice(iterator, n, n), None) + + +def nth(iterable, n, default=None): + """Returns the nth item or a default value. + + >>> l = range(10) + >>> nth(l, 3) + 3 + >>> nth(l, 20, "zebra") + 'zebra' + + """ + return next(islice(iterable, n, None), default) + + +def all_equal(iterable): + """ + Returns ``True`` if all the elements are equal to each other. + + >>> all_equal('aaaa') + True + >>> all_equal('aaab') + False + + """ + g = groupby(iterable) + return next(g, True) and not next(g, False) + + +def quantify(iterable, pred=bool): + """Return the how many times the predicate is true. + + >>> quantify([True, False, True]) + 2 + + """ + return sum(map(pred, iterable)) + + +def padnone(iterable): + """Returns the sequence of elements and then returns ``None`` indefinitely. + + >>> take(5, padnone(range(3))) + [0, 1, 2, None, None] + + Useful for emulating the behavior of the built-in :func:`map` function. + + See also :func:`padded`. + + """ + return chain(iterable, repeat(None)) + + +def ncycles(iterable, n): + """Returns the sequence elements *n* times + + >>> list(ncycles(["a", "b"], 3)) + ['a', 'b', 'a', 'b', 'a', 'b'] + + """ + return chain.from_iterable(repeat(tuple(iterable), n)) + + +def dotproduct(vec1, vec2): + """Returns the dot product of the two iterables. + + >>> dotproduct([10, 10], [20, 20]) + 400 + + """ + return sum(map(operator.mul, vec1, vec2)) + + +def flatten(listOfLists): + """Return an iterator flattening one level of nesting in a list of lists. + + >>> list(flatten([[0, 1], [2, 3]])) + [0, 1, 2, 3] + + See also :func:`collapse`, which can flatten multiple levels of nesting. + + """ + return chain.from_iterable(listOfLists) + + +def repeatfunc(func, times=None, *args): + """Call *func* with *args* repeatedly, returning an iterable over the + results. + + If *times* is specified, the iterable will terminate after that many + repetitions: + + >>> from operator import add + >>> times = 4 + >>> args = 3, 5 + >>> list(repeatfunc(add, times, *args)) + [8, 8, 8, 8] + + If *times* is ``None`` the iterable will not terminate: + + >>> from random import randrange + >>> times = None + >>> args = 1, 11 + >>> take(6, repeatfunc(randrange, times, *args)) # doctest:+SKIP + [2, 4, 8, 1, 8, 4] + + """ + if times is None: + return starmap(func, repeat(args)) + return starmap(func, repeat(args, times)) + + +def pairwise(iterable): + """Returns an iterator of paired items, overlapping, from the original + + >>> take(4, pairwise(count())) + [(0, 1), (1, 2), (2, 3), (3, 4)] + + """ + a, b = tee(iterable) + next(b, None) + return zip(a, b) + + +def grouper(n, iterable, fillvalue=None): + """Collect data into fixed-length chunks or blocks. + + >>> list(grouper(3, 'ABCDEFG', 'x')) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G', 'x', 'x')] + + """ + args = [iter(iterable)] * n + return zip_longest(fillvalue=fillvalue, *args) + + +def roundrobin(*iterables): + """Yields an item from each iterable, alternating between them. + + >>> list(roundrobin('ABC', 'D', 'EF')) + ['A', 'D', 'E', 'B', 'F', 'C'] + + This function produces the same output as :func:`interleave_longest`, but + may perform better for some inputs (in particular when the number of + iterables is small). + + """ + # Recipe credited to George Sakkis + pending = len(iterables) + if PY2: + nexts = cycle(iter(it).next for it in iterables) + else: + nexts = cycle(iter(it).__next__ for it in iterables) + while pending: + try: + for next in nexts: + yield next() + except StopIteration: + pending -= 1 + nexts = cycle(islice(nexts, pending)) + + +def partition(pred, iterable): + """ + Returns a 2-tuple of iterables derived from the input iterable. + The first yields the items that have ``pred(item) == False``. + The second yields the items that have ``pred(item) == True``. + + >>> is_odd = lambda x: x % 2 != 0 + >>> iterable = range(10) + >>> even_items, odd_items = partition(is_odd, iterable) + >>> list(even_items), list(odd_items) + ([0, 2, 4, 6, 8], [1, 3, 5, 7, 9]) + + """ + # partition(is_odd, range(10)) --> 0 2 4 6 8 and 1 3 5 7 9 + t1, t2 = tee(iterable) + return filterfalse(pred, t1), filter(pred, t2) + + +def powerset(iterable): + """Yields all possible subsets of the iterable. + + >>> list(powerset([1, 2, 3])) + [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] + + :func:`powerset` will operate on iterables that aren't :class:`set` + instances, so repeated elements in the input will produce repeated elements + in the output. Use :func:`unique_everseen` on the input to avoid generating + duplicates: + + >>> seq = [1, 1, 0] + >>> list(powerset(seq)) + [(), (1,), (1,), (0,), (1, 1), (1, 0), (1, 0), (1, 1, 0)] + >>> from more_itertools import unique_everseen + >>> list(powerset(unique_everseen(seq))) + [(), (1,), (0,), (1, 0)] + + """ + s = list(iterable) + return chain.from_iterable(combinations(s, r) for r in range(len(s) + 1)) + + +def unique_everseen(iterable, key=None): + """ + Yield unique elements, preserving order. + + >>> list(unique_everseen('AAAABBBCCDAABBB')) + ['A', 'B', 'C', 'D'] + >>> list(unique_everseen('ABBCcAD', str.lower)) + ['A', 'B', 'C', 'D'] + + Sequences with a mix of hashable and unhashable items can be used. + The function will be slower (i.e., `O(n^2)`) for unhashable items. + + """ + seenset = set() + seenset_add = seenset.add + seenlist = [] + seenlist_add = seenlist.append + if key is None: + for element in iterable: + try: + if element not in seenset: + seenset_add(element) + yield element + except TypeError: + if element not in seenlist: + seenlist_add(element) + yield element + else: + for element in iterable: + k = key(element) + try: + if k not in seenset: + seenset_add(k) + yield element + except TypeError: + if k not in seenlist: + seenlist_add(k) + yield element + + +def unique_justseen(iterable, key=None): + """Yields elements in order, ignoring serial duplicates + + >>> list(unique_justseen('AAAABBBCCDAABBB')) + ['A', 'B', 'C', 'D', 'A', 'B'] + >>> list(unique_justseen('ABBCcAD', str.lower)) + ['A', 'B', 'C', 'A', 'D'] + + """ + return map(next, map(operator.itemgetter(1), groupby(iterable, key))) + + +def iter_except(func, exception, first=None): + """Yields results from a function repeatedly until an exception is raised. + + Converts a call-until-exception interface to an iterator interface. + Like ``iter(func, sentinel)``, but uses an exception instead of a sentinel + to end the loop. + + >>> l = [0, 1, 2] + >>> list(iter_except(l.pop, IndexError)) + [2, 1, 0] + + """ + try: + if first is not None: + yield first() + while 1: + yield func() + except exception: + pass + + +def first_true(iterable, default=None, pred=None): + """ + Returns the first true value in the iterable. + + If no true value is found, returns *default* + + If *pred* is not None, returns the first item for which + ``pred(item) == True`` . + + >>> first_true(range(10)) + 1 + >>> first_true(range(10), pred=lambda x: x > 5) + 6 + >>> first_true(range(10), default='missing', pred=lambda x: x > 9) + 'missing' + + """ + return next(filter(pred, iterable), default) + + +def random_product(*args, **kwds): + """Draw an item at random from each of the input iterables. + + >>> random_product('abc', range(4), 'XYZ') # doctest:+SKIP + ('c', 3, 'Z') + + If *repeat* is provided as a keyword argument, that many items will be + drawn from each iterable. + + >>> random_product('abcd', range(4), repeat=2) # doctest:+SKIP + ('a', 2, 'd', 3) + + This equivalent to taking a random selection from + ``itertools.product(*args, **kwarg)``. + + """ + pools = [tuple(pool) for pool in args] * kwds.get('repeat', 1) + return tuple(choice(pool) for pool in pools) + + +def random_permutation(iterable, r=None): + """Return a random *r* length permutation of the elements in *iterable*. + + If *r* is not specified or is ``None``, then *r* defaults to the length of + *iterable*. + + >>> random_permutation(range(5)) # doctest:+SKIP + (3, 4, 0, 1, 2) + + This equivalent to taking a random selection from + ``itertools.permutations(iterable, r)``. + + """ + pool = tuple(iterable) + r = len(pool) if r is None else r + return tuple(sample(pool, r)) + + +def random_combination(iterable, r): + """Return a random *r* length subsequence of the elements in *iterable*. + + >>> random_combination(range(5), 3) # doctest:+SKIP + (2, 3, 4) + + This equivalent to taking a random selection from + ``itertools.combinations(iterable, r)``. + + """ + pool = tuple(iterable) + n = len(pool) + indices = sorted(sample(range(n), r)) + return tuple(pool[i] for i in indices) + + +def random_combination_with_replacement(iterable, r): + """Return a random *r* length subsequence of elements in *iterable*, + allowing individual elements to be repeated. + + >>> random_combination_with_replacement(range(3), 5) # doctest:+SKIP + (0, 0, 1, 2, 2) + + This equivalent to taking a random selection from + ``itertools.combinations_with_replacement(iterable, r)``. + + """ + pool = tuple(iterable) + n = len(pool) + indices = sorted(randrange(n) for i in range(r)) + return tuple(pool[i] for i in indices) + + +def nth_combination(iterable, r, index): + """Equivalent to ``list(combinations(iterable, r))[index]``. + + The subsequences of *iterable* that are of length *r* can be ordered + lexicographically. :func:`nth_combination` computes the subsequence at + sort position *index* directly, without computing the previous + subsequences. + + """ + pool = tuple(iterable) + n = len(pool) + if (r < 0) or (r > n): + raise ValueError + + c = 1 + k = min(r, n - r) + for i in range(1, k + 1): + c = c * (n - k + i) // i + + if index < 0: + index += c + + if (index < 0) or (index >= c): + raise IndexError + + result = [] + while r: + c, n, r = c * r // n, n - 1, r - 1 + while index >= c: + index -= c + c, n = c * (n - r) // n, n - 1 + result.append(pool[-1 - n]) + + return tuple(result) + + +def prepend(value, iterator): + """Yield *value*, followed by the elements in *iterator*. + + >>> value = '0' + >>> iterator = ['1', '2', '3'] + >>> list(prepend(value, iterator)) + ['0', '1', '2', '3'] + + To prepend multiple values, see :func:`itertools.chain`. + + """ + return chain([value], iterator) diff --git a/pipenv/vendor/more_itertools/tests/__init__.py b/pipenv/vendor/more_itertools/tests/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/more_itertools/tests/test_more.py b/pipenv/vendor/more_itertools/tests/test_more.py new file mode 100644 index 00000000..eacf8a8a --- /dev/null +++ b/pipenv/vendor/more_itertools/tests/test_more.py @@ -0,0 +1,2313 @@ +from __future__ import division, print_function, unicode_literals + +from collections import OrderedDict +from decimal import Decimal +from doctest import DocTestSuite +from fractions import Fraction +from functools import partial, reduce +from heapq import merge +from io import StringIO +from itertools import ( + chain, + count, + groupby, + islice, + permutations, + product, + repeat, +) +from operator import add, mul, itemgetter +from unittest import TestCase + +from six.moves import filter, map, range, zip + +import more_itertools as mi + + +def load_tests(loader, tests, ignore): + # Add the doctests + tests.addTests(DocTestSuite('more_itertools.more')) + return tests + + +class CollateTests(TestCase): + """Unit tests for ``collate()``""" + # Also accidentally tests peekable, though that could use its own tests + + def test_default(self): + """Test with the default `key` function.""" + iterables = [range(4), range(7), range(3, 6)] + self.assertEqual( + sorted(reduce(list.__add__, [list(it) for it in iterables])), + list(mi.collate(*iterables)) + ) + + def test_key(self): + """Test using a custom `key` function.""" + iterables = [range(5, 0, -1), range(4, 0, -1)] + actual = sorted( + reduce(list.__add__, [list(it) for it in iterables]), reverse=True + ) + expected = list(mi.collate(*iterables, key=lambda x: -x)) + self.assertEqual(actual, expected) + + def test_empty(self): + """Be nice if passed an empty list of iterables.""" + self.assertEqual([], list(mi.collate())) + + def test_one(self): + """Work when only 1 iterable is passed.""" + self.assertEqual([0, 1], list(mi.collate(range(2)))) + + def test_reverse(self): + """Test the `reverse` kwarg.""" + iterables = [range(4, 0, -1), range(7, 0, -1), range(3, 6, -1)] + + actual = sorted( + reduce(list.__add__, [list(it) for it in iterables]), reverse=True + ) + expected = list(mi.collate(*iterables, reverse=True)) + self.assertEqual(actual, expected) + + def test_alias(self): + self.assertNotEqual(merge.__doc__, mi.collate.__doc__) + self.assertNotEqual(partial.__doc__, mi.collate.__doc__) + + +class ChunkedTests(TestCase): + """Tests for ``chunked()``""" + + def test_even(self): + """Test when ``n`` divides evenly into the length of the iterable.""" + self.assertEqual( + list(mi.chunked('ABCDEF', 3)), [['A', 'B', 'C'], ['D', 'E', 'F']] + ) + + def test_odd(self): + """Test when ``n`` does not divide evenly into the length of the + iterable. + + """ + self.assertEqual( + list(mi.chunked('ABCDE', 3)), [['A', 'B', 'C'], ['D', 'E']] + ) + + +class FirstTests(TestCase): + """Tests for ``first()``""" + + def test_many(self): + """Test that it works on many-item iterables.""" + # Also try it on a generator expression to make sure it works on + # whatever those return, across Python versions. + self.assertEqual(mi.first(x for x in range(4)), 0) + + def test_one(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.first([3]), 3) + + def test_empty_stop_iteration(self): + """It should raise StopIteration for empty iterables.""" + self.assertRaises(ValueError, lambda: mi.first([])) + + def test_default(self): + """It should return the provided default arg for empty iterables.""" + self.assertEqual(mi.first([], 'boo'), 'boo') + + +class IterOnlyRange: + """User-defined iterable class which only support __iter__. + + It is not specified to inherit ``object``, so indexing on a instance will + raise an ``AttributeError`` rather than ``TypeError`` in Python 2. + + >>> r = IterOnlyRange(5) + >>> r[0] + AttributeError: IterOnlyRange instance has no attribute '__getitem__' + + Note: In Python 3, ``TypeError`` will be raised because ``object`` is + inherited implicitly by default. + + >>> r[0] + TypeError: 'IterOnlyRange' object does not support indexing + """ + def __init__(self, n): + """Set the length of the range.""" + self.n = n + + def __iter__(self): + """Works same as range().""" + return iter(range(self.n)) + + +class LastTests(TestCase): + """Tests for ``last()``""" + + def test_many_nonsliceable(self): + """Test that it works on many-item non-slice-able iterables.""" + # Also try it on a generator expression to make sure it works on + # whatever those return, across Python versions. + self.assertEqual(mi.last(x for x in range(4)), 3) + + def test_one_nonsliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last(x for x in range(1)), 0) + + def test_empty_stop_iteration_nonsliceable(self): + """It should raise ValueError for empty non-slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last(x for x in range(0))) + + def test_default_nonsliceable(self): + """It should return the provided default arg for empty non-slice-able + iterables. + """ + self.assertEqual(mi.last((x for x in range(0)), 'boo'), 'boo') + + def test_many_sliceable(self): + """Test that it works on many-item slice-able iterables.""" + self.assertEqual(mi.last([0, 1, 2, 3]), 3) + + def test_one_sliceable(self): + """Test that it doesn't raise StopIteration prematurely.""" + self.assertEqual(mi.last([3]), 3) + + def test_empty_stop_iteration_sliceable(self): + """It should raise ValueError for empty slice-able iterables.""" + self.assertRaises(ValueError, lambda: mi.last([])) + + def test_default_sliceable(self): + """It should return the provided default arg for empty slice-able + iterables. + """ + self.assertEqual(mi.last([], 'boo'), 'boo') + + def test_dict(self): + """last(dic) and last(dic.keys()) should return same result.""" + dic = {'a': 1, 'b': 2, 'c': 3} + self.assertEqual(mi.last(dic), mi.last(dic.keys())) + + def test_ordereddict(self): + """last(dic) should return the last key.""" + od = OrderedDict() + od['a'] = 1 + od['b'] = 2 + od['c'] = 3 + self.assertEqual(mi.last(od), 'c') + + def test_customrange(self): + """It should work on custom class where [] raises AttributeError.""" + self.assertEqual(mi.last(IterOnlyRange(5)), 4) + + +class PeekableTests(TestCase): + """Tests for ``peekable()`` behavor not incidentally covered by testing + ``collate()`` + + """ + def test_peek_default(self): + """Make sure passing a default into ``peek()`` works.""" + p = mi.peekable([]) + self.assertEqual(p.peek(7), 7) + + def test_truthiness(self): + """Make sure a ``peekable`` tests true iff there are items remaining in + the iterable. + + """ + p = mi.peekable([]) + self.assertFalse(p) + + p = mi.peekable(range(3)) + self.assertTrue(p) + + def test_simple_peeking(self): + """Make sure ``next`` and ``peek`` advance and don't advance the + iterator, respectively. + + """ + p = mi.peekable(range(10)) + self.assertEqual(next(p), 0) + self.assertEqual(p.peek(), 1) + self.assertEqual(next(p), 1) + + def test_indexing(self): + """ + Indexing into the peekable shouldn't advance the iterator. + """ + p = mi.peekable('abcdefghijkl') + + # The 0th index is what ``next()`` will return + self.assertEqual(p[0], 'a') + self.assertEqual(next(p), 'a') + + # Indexing further into the peekable shouldn't advance the itertor + self.assertEqual(p[2], 'd') + self.assertEqual(next(p), 'b') + + # The 0th index moves up with the iterator; the last index follows + self.assertEqual(p[0], 'c') + self.assertEqual(p[9], 'l') + + self.assertEqual(next(p), 'c') + self.assertEqual(p[8], 'l') + + # Negative indexing should work too + self.assertEqual(p[-2], 'k') + self.assertEqual(p[-9], 'd') + self.assertRaises(IndexError, lambda: p[-10]) + + def test_slicing(self): + """Slicing the peekable shouldn't advance the iterator.""" + seq = list('abcdefghijkl') + p = mi.peekable(seq) + + # Slicing the peekable should just be like slicing a re-iterable + self.assertEqual(p[1:4], seq[1:4]) + + # Advancing the iterator moves the slices up also + self.assertEqual(next(p), 'a') + self.assertEqual(p[1:4], seq[1:][1:4]) + + # Implicit starts and stop should work + self.assertEqual(p[:5], seq[1:][:5]) + self.assertEqual(p[:], seq[1:][:]) + + # Indexing past the end should work + self.assertEqual(p[:100], seq[1:][:100]) + + # Steps should work, including negative + self.assertEqual(p[::2], seq[1:][::2]) + self.assertEqual(p[::-1], seq[1:][::-1]) + + def test_slicing_reset(self): + """Test slicing on a fresh iterable each time""" + iterable = ['0', '1', '2', '3', '4', '5'] + indexes = list(range(-4, len(iterable) + 4)) + [None] + steps = [1, 2, 3, 4, -1, -2, -3, 4] + for slice_args in product(indexes, indexes, steps): + it = iter(iterable) + p = mi.peekable(it) + next(p) + index = slice(*slice_args) + actual = p[index] + expected = iterable[1:][index] + self.assertEqual(actual, expected, slice_args) + + def test_slicing_error(self): + iterable = '01234567' + p = mi.peekable(iter(iterable)) + + # Prime the cache + p.peek() + old_cache = list(p._cache) + + # Illegal slice + with self.assertRaises(ValueError): + p[1:-1:0] + + # Neither the cache nor the iteration should be affected + self.assertEqual(old_cache, list(p._cache)) + self.assertEqual(list(p), list(iterable)) + + def test_passthrough(self): + """Iterating a peekable without using ``peek()`` or ``prepend()`` + should just give the underlying iterable's elements (a trivial test but + useful to set a baseline in case something goes wrong)""" + expected = [1, 2, 3, 4, 5] + actual = list(mi.peekable(expected)) + self.assertEqual(actual, expected) + + # prepend() behavior tests + + def test_prepend(self): + """Tests intersperesed ``prepend()`` and ``next()`` calls""" + it = mi.peekable(range(2)) + actual = [] + + # Test prepend() before next() + it.prepend(10) + actual += [next(it), next(it)] + + # Test prepend() between next()s + it.prepend(11) + actual += [next(it), next(it)] + + # Test prepend() after source iterable is consumed + it.prepend(12) + actual += [next(it)] + + expected = [10, 0, 11, 1, 12] + self.assertEqual(actual, expected) + + def test_multi_prepend(self): + """Tests prepending multiple items and getting them in proper order""" + it = mi.peekable(range(5)) + actual = [next(it), next(it)] + it.prepend(10, 11, 12) + it.prepend(20, 21) + actual += list(it) + expected = [0, 1, 20, 21, 10, 11, 12, 2, 3, 4] + self.assertEqual(actual, expected) + + def test_empty(self): + """Tests prepending in front of an empty iterable""" + it = mi.peekable([]) + it.prepend(10) + actual = list(it) + expected = [10] + self.assertEqual(actual, expected) + + def test_prepend_truthiness(self): + """Tests that ``__bool__()`` or ``__nonzero__()`` works properly + with ``prepend()``""" + it = mi.peekable(range(5)) + self.assertTrue(it) + actual = list(it) + self.assertFalse(it) + it.prepend(10) + self.assertTrue(it) + actual += [next(it)] + self.assertFalse(it) + expected = [0, 1, 2, 3, 4, 10] + self.assertEqual(actual, expected) + + def test_multi_prepend_peek(self): + """Tests prepending multiple elements and getting them in reverse order + while peeking""" + it = mi.peekable(range(5)) + actual = [next(it), next(it)] + self.assertEqual(it.peek(), 2) + it.prepend(10, 11, 12) + self.assertEqual(it.peek(), 10) + it.prepend(20, 21) + self.assertEqual(it.peek(), 20) + actual += list(it) + self.assertFalse(it) + expected = [0, 1, 20, 21, 10, 11, 12, 2, 3, 4] + self.assertEqual(actual, expected) + + def test_prepend_after_stop(self): + """Test resuming iteration after a previous exhaustion""" + it = mi.peekable(range(3)) + self.assertEqual(list(it), [0, 1, 2]) + self.assertRaises(StopIteration, lambda: next(it)) + it.prepend(10) + self.assertEqual(next(it), 10) + self.assertRaises(StopIteration, lambda: next(it)) + + def test_prepend_slicing(self): + """Tests interaction between prepending and slicing""" + seq = list(range(20)) + p = mi.peekable(seq) + + p.prepend(30, 40, 50) + pseq = [30, 40, 50] + seq # pseq for prepended_seq + + # adapt the specific tests from test_slicing + self.assertEqual(p[0], 30) + self.assertEqual(p[1:8], pseq[1:8]) + self.assertEqual(p[1:], pseq[1:]) + self.assertEqual(p[:5], pseq[:5]) + self.assertEqual(p[:], pseq[:]) + self.assertEqual(p[:100], pseq[:100]) + self.assertEqual(p[::2], pseq[::2]) + self.assertEqual(p[::-1], pseq[::-1]) + + def test_prepend_indexing(self): + """Tests interaction between prepending and indexing""" + seq = list(range(20)) + p = mi.peekable(seq) + + p.prepend(30, 40, 50) + + self.assertEqual(p[0], 30) + self.assertEqual(next(p), 30) + self.assertEqual(p[2], 0) + self.assertEqual(next(p), 40) + self.assertEqual(p[0], 50) + self.assertEqual(p[9], 8) + self.assertEqual(next(p), 50) + self.assertEqual(p[8], 8) + self.assertEqual(p[-2], 18) + self.assertEqual(p[-9], 11) + self.assertRaises(IndexError, lambda: p[-21]) + + def test_prepend_iterable(self): + """Tests prepending from an iterable""" + it = mi.peekable(range(5)) + # Don't directly use the range() object to avoid any range-specific + # optimizations + it.prepend(*(x for x in range(5))) + actual = list(it) + expected = list(chain(range(5), range(5))) + self.assertEqual(actual, expected) + + def test_prepend_many(self): + """Tests that prepending a huge number of elements works""" + it = mi.peekable(range(5)) + # Don't directly use the range() object to avoid any range-specific + # optimizations + it.prepend(*(x for x in range(20000))) + actual = list(it) + expected = list(chain(range(20000), range(5))) + self.assertEqual(actual, expected) + + def test_prepend_reversed(self): + """Tests prepending from a reversed iterable""" + it = mi.peekable(range(3)) + it.prepend(*reversed((10, 11, 12))) + actual = list(it) + expected = [12, 11, 10, 0, 1, 2] + self.assertEqual(actual, expected) + + +class ConsumerTests(TestCase): + """Tests for ``consumer()``""" + + def test_consumer(self): + @mi.consumer + def eater(): + while True: + x = yield # noqa + + e = eater() + e.send('hi') # without @consumer, would raise TypeError + + +class DistinctPermutationsTests(TestCase): + def test_distinct_permutations(self): + """Make sure the output for ``distinct_permutations()`` is the same as + set(permutations(it)). + + """ + iterable = ['z', 'a', 'a', 'q', 'q', 'q', 'y'] + test_output = sorted(mi.distinct_permutations(iterable)) + ref_output = sorted(set(permutations(iterable))) + self.assertEqual(test_output, ref_output) + + def test_other_iterables(self): + """Make sure ``distinct_permutations()`` accepts a different type of + iterables. + + """ + # a generator + iterable = (c for c in ['z', 'a', 'a', 'q', 'q', 'q', 'y']) + test_output = sorted(mi.distinct_permutations(iterable)) + # "reload" it + iterable = (c for c in ['z', 'a', 'a', 'q', 'q', 'q', 'y']) + ref_output = sorted(set(permutations(iterable))) + self.assertEqual(test_output, ref_output) + + # an iterator + iterable = iter(['z', 'a', 'a', 'q', 'q', 'q', 'y']) + test_output = sorted(mi.distinct_permutations(iterable)) + # "reload" it + iterable = iter(['z', 'a', 'a', 'q', 'q', 'q', 'y']) + ref_output = sorted(set(permutations(iterable))) + self.assertEqual(test_output, ref_output) + + +class IlenTests(TestCase): + def test_ilen(self): + """Sanity-checks for ``ilen()``.""" + # Non-empty + self.assertEqual( + mi.ilen(filter(lambda x: x % 10 == 0, range(101))), 11 + ) + + # Empty + self.assertEqual(mi.ilen((x for x in range(0))), 0) + + # Iterable with __len__ + self.assertEqual(mi.ilen(list(range(6))), 6) + + +class WithIterTests(TestCase): + def test_with_iter(self): + s = StringIO('One fish\nTwo fish') + initial_words = [line.split()[0] for line in mi.with_iter(s)] + + # Iterable's items should be faithfully represented + self.assertEqual(initial_words, ['One', 'Two']) + # The file object should be closed + self.assertTrue(s.closed) + + +class OneTests(TestCase): + def test_basic(self): + it = iter(['item']) + self.assertEqual(mi.one(it), 'item') + + def test_too_short(self): + it = iter([]) + self.assertRaises(ValueError, lambda: mi.one(it)) + self.assertRaises(IndexError, lambda: mi.one(it, too_short=IndexError)) + + def test_too_long(self): + it = count() + self.assertRaises(ValueError, lambda: mi.one(it)) # burn 0 and 1 + self.assertEqual(next(it), 2) + self.assertRaises( + OverflowError, lambda: mi.one(it, too_long=OverflowError) + ) + + +class IntersperseTest(TestCase): + """ Tests for intersperse() """ + + def test_even(self): + iterable = (x for x in '01') + self.assertEqual( + list(mi.intersperse(None, iterable)), ['0', None, '1'] + ) + + def test_odd(self): + iterable = (x for x in '012') + self.assertEqual( + list(mi.intersperse(None, iterable)), ['0', None, '1', None, '2'] + ) + + def test_nested(self): + element = ('a', 'b') + iterable = (x for x in '012') + actual = list(mi.intersperse(element, iterable)) + expected = ['0', ('a', 'b'), '1', ('a', 'b'), '2'] + self.assertEqual(actual, expected) + + def test_not_iterable(self): + self.assertRaises(TypeError, lambda: mi.intersperse('x', 1)) + + def test_n(self): + for n, element, expected in [ + (1, '_', ['0', '_', '1', '_', '2', '_', '3', '_', '4', '_', '5']), + (2, '_', ['0', '1', '_', '2', '3', '_', '4', '5']), + (3, '_', ['0', '1', '2', '_', '3', '4', '5']), + (4, '_', ['0', '1', '2', '3', '_', '4', '5']), + (5, '_', ['0', '1', '2', '3', '4', '_', '5']), + (6, '_', ['0', '1', '2', '3', '4', '5']), + (7, '_', ['0', '1', '2', '3', '4', '5']), + (3, ['a', 'b'], ['0', '1', '2', ['a', 'b'], '3', '4', '5']), + ]: + iterable = (x for x in '012345') + actual = list(mi.intersperse(element, iterable, n=n)) + self.assertEqual(actual, expected) + + def test_n_zero(self): + self.assertRaises( + ValueError, lambda: list(mi.intersperse('x', '012', n=0)) + ) + + +class UniqueToEachTests(TestCase): + """Tests for ``unique_to_each()``""" + + def test_all_unique(self): + """When all the input iterables are unique the output should match + the input.""" + iterables = [[1, 2], [3, 4, 5], [6, 7, 8]] + self.assertEqual(mi.unique_to_each(*iterables), iterables) + + def test_duplicates(self): + """When there are duplicates in any of the input iterables that aren't + in the rest, those duplicates should be emitted.""" + iterables = ["mississippi", "missouri"] + self.assertEqual( + mi.unique_to_each(*iterables), [['p', 'p'], ['o', 'u', 'r']] + ) + + def test_mixed(self): + """When the input iterables contain different types the function should + still behave properly""" + iterables = ['x', (i for i in range(3)), [1, 2, 3], tuple()] + self.assertEqual(mi.unique_to_each(*iterables), [['x'], [0], [3], []]) + + +class WindowedTests(TestCase): + """Tests for ``windowed()``""" + + def test_basic(self): + actual = list(mi.windowed([1, 2, 3, 4, 5], 3)) + expected = [(1, 2, 3), (2, 3, 4), (3, 4, 5)] + self.assertEqual(actual, expected) + + def test_large_size(self): + """ + When the window size is larger than the iterable, and no fill value is + given,``None`` should be filled in. + """ + actual = list(mi.windowed([1, 2, 3, 4, 5], 6)) + expected = [(1, 2, 3, 4, 5, None)] + self.assertEqual(actual, expected) + + def test_fillvalue(self): + """ + When sizes don't match evenly, the given fill value should be used. + """ + iterable = [1, 2, 3, 4, 5] + + for n, kwargs, expected in [ + (6, {}, [(1, 2, 3, 4, 5, '!')]), # n > len(iterable) + (3, {'step': 3}, [(1, 2, 3), (4, 5, '!')]), # using ``step`` + ]: + actual = list(mi.windowed(iterable, n, fillvalue='!', **kwargs)) + self.assertEqual(actual, expected) + + def test_zero(self): + """When the window size is zero, an empty tuple should be emitted.""" + actual = list(mi.windowed([1, 2, 3, 4, 5], 0)) + expected = [tuple()] + self.assertEqual(actual, expected) + + def test_negative(self): + """When the window size is negative, ValueError should be raised.""" + with self.assertRaises(ValueError): + list(mi.windowed([1, 2, 3, 4, 5], -1)) + + def test_step(self): + """The window should advance by the number of steps provided""" + iterable = [1, 2, 3, 4, 5, 6, 7] + for n, step, expected in [ + (3, 2, [(1, 2, 3), (3, 4, 5), (5, 6, 7)]), # n > step + (3, 3, [(1, 2, 3), (4, 5, 6), (7, None, None)]), # n == step + (3, 4, [(1, 2, 3), (5, 6, 7)]), # line up nicely + (3, 5, [(1, 2, 3), (6, 7, None)]), # off by one + (3, 6, [(1, 2, 3), (7, None, None)]), # off by two + (3, 7, [(1, 2, 3)]), # step past the end + (7, 8, [(1, 2, 3, 4, 5, 6, 7)]), # step > len(iterable) + ]: + actual = list(mi.windowed(iterable, n, step=step)) + self.assertEqual(actual, expected) + + # Step must be greater than or equal to 1 + with self.assertRaises(ValueError): + list(mi.windowed(iterable, 3, step=0)) + + +class SubstringsTests(TestCase): + def test_basic(self): + iterable = (x for x in range(4)) + actual = list(mi.substrings(iterable)) + expected = [ + (0,), + (1,), + (2,), + (3,), + (0, 1), + (1, 2), + (2, 3), + (0, 1, 2), + (1, 2, 3), + (0, 1, 2, 3), + ] + self.assertEqual(actual, expected) + + def test_strings(self): + iterable = 'abc' + actual = list(mi.substrings(iterable)) + expected = [ + ('a',), ('b',), ('c',), ('a', 'b'), ('b', 'c'), ('a', 'b', 'c') + ] + self.assertEqual(actual, expected) + + def test_empty(self): + iterable = iter([]) + actual = list(mi.substrings(iterable)) + expected = [] + self.assertEqual(actual, expected) + + def test_order(self): + iterable = [2, 0, 1] + actual = list(mi.substrings(iterable)) + expected = [(2,), (0,), (1,), (2, 0), (0, 1), (2, 0, 1)] + self.assertEqual(actual, expected) + + +class BucketTests(TestCase): + """Tests for ``bucket()``""" + + def test_basic(self): + iterable = [10, 20, 30, 11, 21, 31, 12, 22, 23, 33] + D = mi.bucket(iterable, key=lambda x: 10 * (x // 10)) + + # In-order access + self.assertEqual(list(D[10]), [10, 11, 12]) + + # Out of order access + self.assertEqual(list(D[30]), [30, 31, 33]) + self.assertEqual(list(D[20]), [20, 21, 22, 23]) + + self.assertEqual(list(D[40]), []) # Nothing in here! + + def test_in(self): + iterable = [10, 20, 30, 11, 21, 31, 12, 22, 23, 33] + D = mi.bucket(iterable, key=lambda x: 10 * (x // 10)) + + self.assertIn(10, D) + self.assertNotIn(40, D) + self.assertIn(20, D) + self.assertNotIn(21, D) + + # Checking in-ness shouldn't advance the iterator + self.assertEqual(next(D[10]), 10) + + def test_validator(self): + iterable = count(0) + key = lambda x: int(str(x)[0]) # First digit of each number + validator = lambda x: 0 < x < 10 # No leading zeros + D = mi.bucket(iterable, key, validator=validator) + self.assertEqual(mi.take(3, D[1]), [1, 10, 11]) + self.assertNotIn(0, D) # Non-valid entries don't return True + self.assertNotIn(0, D._cache) # Don't store non-valid entries + self.assertEqual(list(D[0]), []) + + +class SpyTests(TestCase): + """Tests for ``spy()``""" + + def test_basic(self): + original_iterable = iter('abcdefg') + head, new_iterable = mi.spy(original_iterable) + self.assertEqual(head, ['a']) + self.assertEqual( + list(new_iterable), ['a', 'b', 'c', 'd', 'e', 'f', 'g'] + ) + + def test_unpacking(self): + original_iterable = iter('abcdefg') + (first, second, third), new_iterable = mi.spy(original_iterable, 3) + self.assertEqual(first, 'a') + self.assertEqual(second, 'b') + self.assertEqual(third, 'c') + self.assertEqual( + list(new_iterable), ['a', 'b', 'c', 'd', 'e', 'f', 'g'] + ) + + def test_too_many(self): + original_iterable = iter('abc') + head, new_iterable = mi.spy(original_iterable, 4) + self.assertEqual(head, ['a', 'b', 'c']) + self.assertEqual(list(new_iterable), ['a', 'b', 'c']) + + def test_zero(self): + original_iterable = iter('abc') + head, new_iterable = mi.spy(original_iterable, 0) + self.assertEqual(head, []) + self.assertEqual(list(new_iterable), ['a', 'b', 'c']) + + +class InterleaveTests(TestCase): + def test_even(self): + actual = list(mi.interleave([1, 4, 7], [2, 5, 8], [3, 6, 9])) + expected = [1, 2, 3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_short(self): + actual = list(mi.interleave([1, 4], [2, 5, 7], [3, 6, 8])) + expected = [1, 2, 3, 4, 5, 6] + self.assertEqual(actual, expected) + + def test_mixed_types(self): + it_list = ['a', 'b', 'c', 'd'] + it_str = '12345' + it_inf = count() + actual = list(mi.interleave(it_list, it_str, it_inf)) + expected = ['a', '1', 0, 'b', '2', 1, 'c', '3', 2, 'd', '4', 3] + self.assertEqual(actual, expected) + + +class InterleaveLongestTests(TestCase): + def test_even(self): + actual = list(mi.interleave_longest([1, 4, 7], [2, 5, 8], [3, 6, 9])) + expected = [1, 2, 3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_short(self): + actual = list(mi.interleave_longest([1, 4], [2, 5, 7], [3, 6, 8])) + expected = [1, 2, 3, 4, 5, 6, 7, 8] + self.assertEqual(actual, expected) + + def test_mixed_types(self): + it_list = ['a', 'b', 'c', 'd'] + it_str = '12345' + it_gen = (x for x in range(3)) + actual = list(mi.interleave_longest(it_list, it_str, it_gen)) + expected = ['a', '1', 0, 'b', '2', 1, 'c', '3', 2, 'd', '4', '5'] + self.assertEqual(actual, expected) + + +class TestCollapse(TestCase): + """Tests for ``collapse()``""" + + def test_collapse(self): + l = [[1], 2, [[3], 4], [[[5]]]] + self.assertEqual(list(mi.collapse(l)), [1, 2, 3, 4, 5]) + + def test_collapse_to_string(self): + l = [["s1"], "s2", [["s3"], "s4"], [[["s5"]]]] + self.assertEqual(list(mi.collapse(l)), ["s1", "s2", "s3", "s4", "s5"]) + + def test_collapse_flatten(self): + l = [[1], [2], [[3], 4], [[[5]]]] + self.assertEqual(list(mi.collapse(l, levels=1)), list(mi.flatten(l))) + + def test_collapse_to_level(self): + l = [[1], 2, [[3], 4], [[[5]]]] + self.assertEqual(list(mi.collapse(l, levels=2)), [1, 2, 3, 4, [5]]) + self.assertEqual( + list(mi.collapse(mi.collapse(l, levels=1), levels=1)), + list(mi.collapse(l, levels=2)) + ) + + def test_collapse_to_list(self): + l = (1, [2], (3, [4, (5,)], 'ab')) + actual = list(mi.collapse(l, base_type=list)) + expected = [1, [2], 3, [4, (5,)], 'ab'] + self.assertEqual(actual, expected) + + +class SideEffectTests(TestCase): + """Tests for ``side_effect()``""" + + def test_individual(self): + # The function increments the counter for each call + counter = [0] + + def func(arg): + counter[0] += 1 + + result = list(mi.side_effect(func, range(10))) + self.assertEqual(result, list(range(10))) + self.assertEqual(counter[0], 10) + + def test_chunked(self): + # The function increments the counter for each call + counter = [0] + + def func(arg): + counter[0] += 1 + + result = list(mi.side_effect(func, range(10), 2)) + self.assertEqual(result, list(range(10))) + self.assertEqual(counter[0], 5) + + def test_before_after(self): + f = StringIO() + collector = [] + + def func(item): + print(item, file=f) + collector.append(f.getvalue()) + + def it(): + yield 'a' + yield 'b' + raise RuntimeError('kaboom') + + before = lambda: print('HEADER', file=f) + after = f.close + + try: + mi.consume(mi.side_effect(func, it(), before=before, after=after)) + except RuntimeError: + pass + + # The iterable should have been written to the file + self.assertEqual(collector, ['HEADER\na\n', 'HEADER\na\nb\n']) + + # The file should be closed even though something bad happened + self.assertTrue(f.closed) + + def test_before_fails(self): + f = StringIO() + func = lambda x: print(x, file=f) + + def before(): + raise RuntimeError('ouch') + + try: + mi.consume( + mi.side_effect(func, 'abc', before=before, after=f.close) + ) + except RuntimeError: + pass + + # The file should be closed even though something bad happened in the + # before function + self.assertTrue(f.closed) + + +class SlicedTests(TestCase): + """Tests for ``sliced()``""" + + def test_even(self): + """Test when the length of the sequence is divisible by *n*""" + seq = 'ABCDEFGHI' + self.assertEqual(list(mi.sliced(seq, 3)), ['ABC', 'DEF', 'GHI']) + + def test_odd(self): + """Test when the length of the sequence is not divisible by *n*""" + seq = 'ABCDEFGHI' + self.assertEqual(list(mi.sliced(seq, 4)), ['ABCD', 'EFGH', 'I']) + + def test_not_sliceable(self): + seq = (x for x in 'ABCDEFGHI') + + with self.assertRaises(TypeError): + list(mi.sliced(seq, 3)) + + +class SplitAtTests(TestCase): + """Tests for ``split()``""" + + def comp_with_str_split(self, str_to_split, delim): + pred = lambda c: c == delim + actual = list(map(''.join, mi.split_at(str_to_split, pred))) + expected = str_to_split.split(delim) + self.assertEqual(actual, expected) + + def test_seperators(self): + test_strs = ['', 'abcba', 'aaabbbcccddd', 'e'] + for s, delim in product(test_strs, 'abcd'): + self.comp_with_str_split(s, delim) + + +class SplitBeforeTest(TestCase): + """Tests for ``split_before()``""" + + def test_starts_with_sep(self): + actual = list(mi.split_before('xooxoo', lambda c: c == 'x')) + expected = [['x', 'o', 'o'], ['x', 'o', 'o']] + self.assertEqual(actual, expected) + + def test_ends_with_sep(self): + actual = list(mi.split_before('ooxoox', lambda c: c == 'x')) + expected = [['o', 'o'], ['x', 'o', 'o'], ['x']] + self.assertEqual(actual, expected) + + def test_no_sep(self): + actual = list(mi.split_before('ooo', lambda c: c == 'x')) + expected = [['o', 'o', 'o']] + self.assertEqual(actual, expected) + + +class SplitAfterTest(TestCase): + """Tests for ``split_after()``""" + + def test_starts_with_sep(self): + actual = list(mi.split_after('xooxoo', lambda c: c == 'x')) + expected = [['x'], ['o', 'o', 'x'], ['o', 'o']] + self.assertEqual(actual, expected) + + def test_ends_with_sep(self): + actual = list(mi.split_after('ooxoox', lambda c: c == 'x')) + expected = [['o', 'o', 'x'], ['o', 'o', 'x']] + self.assertEqual(actual, expected) + + def test_no_sep(self): + actual = list(mi.split_after('ooo', lambda c: c == 'x')) + expected = [['o', 'o', 'o']] + self.assertEqual(actual, expected) + + +class SplitIntoTests(TestCase): + """Tests for ``split_into()``""" + + def test_iterable_just_right(self): + """Size of ``iterable`` equals the sum of ``sizes``.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [2, 3, 4] + expected = [[1, 2], [3, 4, 5], [6, 7, 8, 9]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_iterable_too_small(self): + """Size of ``iterable`` is smaller than sum of ``sizes``. Last return + list is shorter as a result.""" + iterable = [1, 2, 3, 4, 5, 6, 7] + sizes = [2, 3, 4] + expected = [[1, 2], [3, 4, 5], [6, 7]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_iterable_too_small_extra(self): + """Size of ``iterable`` is smaller than sum of ``sizes``. Second last + return list is shorter and last return list is empty as a result.""" + iterable = [1, 2, 3, 4, 5, 6, 7] + sizes = [2, 3, 4, 5] + expected = [[1, 2], [3, 4, 5], [6, 7], []] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_iterable_too_large(self): + """Size of ``iterable`` is larger than sum of ``sizes``. Not all + items of iterable are returned.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [2, 3, 2] + expected = [[1, 2], [3, 4, 5], [6, 7]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_using_none_with_leftover(self): + """Last item of ``sizes`` is None when items still remain in + ``iterable``. Last list returned stretches to fit all remaining items + of ``iterable``.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [2, 3, None] + expected = [[1, 2], [3, 4, 5], [6, 7, 8, 9]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_using_none_without_leftover(self): + """Last item of ``sizes`` is None when no items remain in + ``iterable``. Last list returned is empty.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [2, 3, 4, None] + expected = [[1, 2], [3, 4, 5], [6, 7, 8, 9], []] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_using_none_mid_sizes(self): + """None is present in ``sizes`` but is not the last item. Last list + returned stretches to fit all remaining items of ``iterable`` but + all items in ``sizes`` after None are ignored.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [2, 3, None, 4] + expected = [[1, 2], [3, 4, 5], [6, 7, 8, 9]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_iterable_empty(self): + """``iterable`` argument is empty but ``sizes`` is not. An empty + list is returned for each item in ``sizes``.""" + iterable = [] + sizes = [2, 4, 2] + expected = [[], [], []] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_iterable_empty_using_none(self): + """``iterable`` argument is empty but ``sizes`` is not. An empty + list is returned for each item in ``sizes`` that is not after a + None item.""" + iterable = [] + sizes = [2, 4, None, 2] + expected = [[], [], []] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_sizes_empty(self): + """``sizes`` argument is empty but ``iterable`` is not. An empty + generator is returned.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [] + expected = [] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_both_empty(self): + """Both ``sizes`` and ``iterable`` arguments are empty. An empty + generator is returned.""" + iterable = [] + sizes = [] + expected = [] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_bool_in_sizes(self): + """A bool object is present in ``sizes`` is treated as a 1 or 0 for + ``True`` or ``False`` due to bool being an instance of int.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [3, True, 2, False] + expected = [[1, 2, 3], [4], [5, 6], []] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_invalid_in_sizes(self): + """A ValueError is raised if an object in ``sizes`` is neither ``None`` + or an integer.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [1, [], 3] + with self.assertRaises(ValueError): + list(mi.split_into(iterable, sizes)) + + def test_invalid_in_sizes_after_none(self): + """A item in ``sizes`` that is invalid will not raise a TypeError if it + comes after a ``None`` item.""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = [3, 4, None, []] + expected = [[1, 2, 3], [4, 5, 6, 7], [8, 9]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + def test_generator_iterable_integrity(self): + """Check that if ``iterable`` is an iterator, it is consumed only by as + many items as the sum of ``sizes``.""" + iterable = (i for i in range(10)) + sizes = [2, 3] + + expected = [[0, 1], [2, 3, 4]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + iterable_expected = [5, 6, 7, 8, 9] + iterable_actual = list(iterable) + self.assertEqual(iterable_actual, iterable_expected) + + def test_generator_sizes_integrity(self): + """Check that if ``sizes`` is an iterator, it is consumed only until a + ``None`` item is reached""" + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9] + sizes = (i for i in [1, 2, None, 3, 4]) + + expected = [[1], [2, 3], [4, 5, 6, 7, 8, 9]] + actual = list(mi.split_into(iterable, sizes)) + self.assertEqual(actual, expected) + + sizes_expected = [3, 4] + sizes_actual = list(sizes) + self.assertEqual(sizes_actual, sizes_expected) + + +class PaddedTest(TestCase): + """Tests for ``padded()``""" + + def test_no_n(self): + seq = [1, 2, 3] + + # No fillvalue + self.assertEqual(mi.take(5, mi.padded(seq)), [1, 2, 3, None, None]) + + # With fillvalue + self.assertEqual( + mi.take(5, mi.padded(seq, fillvalue='')), [1, 2, 3, '', ''] + ) + + def test_invalid_n(self): + self.assertRaises(ValueError, lambda: list(mi.padded([1, 2, 3], n=-1))) + self.assertRaises(ValueError, lambda: list(mi.padded([1, 2, 3], n=0))) + + def test_valid_n(self): + seq = [1, 2, 3, 4, 5] + + # No need for padding: len(seq) <= n + self.assertEqual(list(mi.padded(seq, n=4)), [1, 2, 3, 4, 5]) + self.assertEqual(list(mi.padded(seq, n=5)), [1, 2, 3, 4, 5]) + + # No fillvalue + self.assertEqual( + list(mi.padded(seq, n=7)), [1, 2, 3, 4, 5, None, None] + ) + + # With fillvalue + self.assertEqual( + list(mi.padded(seq, fillvalue='', n=7)), [1, 2, 3, 4, 5, '', ''] + ) + + def test_next_multiple(self): + seq = [1, 2, 3, 4, 5, 6] + + # No need for padding: len(seq) % n == 0 + self.assertEqual( + list(mi.padded(seq, n=3, next_multiple=True)), [1, 2, 3, 4, 5, 6] + ) + + # Padding needed: len(seq) < n + self.assertEqual( + list(mi.padded(seq, n=8, next_multiple=True)), + [1, 2, 3, 4, 5, 6, None, None] + ) + + # No padding needed: len(seq) == n + self.assertEqual( + list(mi.padded(seq, n=6, next_multiple=True)), [1, 2, 3, 4, 5, 6] + ) + + # Padding needed: len(seq) > n + self.assertEqual( + list(mi.padded(seq, n=4, next_multiple=True)), + [1, 2, 3, 4, 5, 6, None, None] + ) + + # With fillvalue + self.assertEqual( + list(mi.padded(seq, fillvalue='', n=4, next_multiple=True)), + [1, 2, 3, 4, 5, 6, '', ''] + ) + + +class DistributeTest(TestCase): + """Tests for distribute()""" + + def test_invalid_n(self): + self.assertRaises(ValueError, lambda: mi.distribute(-1, [1, 2, 3])) + self.assertRaises(ValueError, lambda: mi.distribute(0, [1, 2, 3])) + + def test_basic(self): + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + for n, expected in [ + (1, [iterable]), + (2, [[1, 3, 5, 7, 9], [2, 4, 6, 8, 10]]), + (3, [[1, 4, 7, 10], [2, 5, 8], [3, 6, 9]]), + (10, [[n] for n in range(1, 10 + 1)]), + ]: + self.assertEqual( + [list(x) for x in mi.distribute(n, iterable)], expected + ) + + def test_large_n(self): + iterable = [1, 2, 3, 4] + self.assertEqual( + [list(x) for x in mi.distribute(6, iterable)], + [[1], [2], [3], [4], [], []] + ) + + +class StaggerTest(TestCase): + """Tests for ``stagger()``""" + + def test_default(self): + iterable = [0, 1, 2, 3] + actual = list(mi.stagger(iterable)) + expected = [(None, 0, 1), (0, 1, 2), (1, 2, 3)] + self.assertEqual(actual, expected) + + def test_offsets(self): + iterable = [0, 1, 2, 3] + for offsets, expected in [ + ((-2, 0, 2), [('', 0, 2), ('', 1, 3)]), + ((-2, -1), [('', ''), ('', 0), (0, 1), (1, 2), (2, 3)]), + ((1, 2), [(1, 2), (2, 3)]), + ]: + all_groups = mi.stagger(iterable, offsets=offsets, fillvalue='') + self.assertEqual(list(all_groups), expected) + + def test_longest(self): + iterable = [0, 1, 2, 3] + for offsets, expected in [ + ( + (-1, 0, 1), + [('', 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, ''), (3, '', '')] + ), + ((-2, -1), [('', ''), ('', 0), (0, 1), (1, 2), (2, 3), (3, '')]), + ((1, 2), [(1, 2), (2, 3), (3, '')]), + ]: + all_groups = mi.stagger( + iterable, offsets=offsets, fillvalue='', longest=True + ) + self.assertEqual(list(all_groups), expected) + + +class ZipOffsetTest(TestCase): + """Tests for ``zip_offset()``""" + + def test_shortest(self): + a_1 = [0, 1, 2, 3] + a_2 = [0, 1, 2, 3, 4, 5] + a_3 = [0, 1, 2, 3, 4, 5, 6, 7] + actual = list( + mi.zip_offset(a_1, a_2, a_3, offsets=(-1, 0, 1), fillvalue='') + ) + expected = [('', 0, 1), (0, 1, 2), (1, 2, 3), (2, 3, 4), (3, 4, 5)] + self.assertEqual(actual, expected) + + def test_longest(self): + a_1 = [0, 1, 2, 3] + a_2 = [0, 1, 2, 3, 4, 5] + a_3 = [0, 1, 2, 3, 4, 5, 6, 7] + actual = list( + mi.zip_offset(a_1, a_2, a_3, offsets=(-1, 0, 1), longest=True) + ) + expected = [ + (None, 0, 1), + (0, 1, 2), + (1, 2, 3), + (2, 3, 4), + (3, 4, 5), + (None, 5, 6), + (None, None, 7), + ] + self.assertEqual(actual, expected) + + def test_mismatch(self): + iterables = [0, 1, 2], [2, 3, 4] + offsets = (-1, 0, 1) + self.assertRaises( + ValueError, + lambda: list(mi.zip_offset(*iterables, offsets=offsets)) + ) + + +class UnzipTests(TestCase): + """Tests for unzip()""" + + def test_empty_iterable(self): + self.assertEqual(list(mi.unzip([])), []) + # in reality zip([], [], []) is equivalent to iter([]) + # but it doesn't hurt to test both + self.assertEqual(list(mi.unzip(zip([], [], []))), []) + + def test_length_one_iterable(self): + xs, ys, zs = mi.unzip(zip([1], [2], [3])) + self.assertEqual(list(xs), [1]) + self.assertEqual(list(ys), [2]) + self.assertEqual(list(zs), [3]) + + def test_normal_case(self): + xs, ys, zs = range(10), range(1, 11), range(2, 12) + zipped = zip(xs, ys, zs) + xs, ys, zs = mi.unzip(zipped) + self.assertEqual(list(xs), list(range(10))) + self.assertEqual(list(ys), list(range(1, 11))) + self.assertEqual(list(zs), list(range(2, 12))) + + def test_improperly_zipped(self): + zipped = iter([(1, 2, 3), (4, 5), (6,)]) + xs, ys, zs = mi.unzip(zipped) + self.assertEqual(list(xs), [1, 4, 6]) + self.assertEqual(list(ys), [2, 5]) + self.assertEqual(list(zs), [3]) + + def test_increasingly_zipped(self): + zipped = iter([(1, 2), (3, 4, 5), (6, 7, 8, 9)]) + unzipped = mi.unzip(zipped) + # from the docstring: + # len(first tuple) is the number of iterables zipped + self.assertEqual(len(unzipped), 2) + xs, ys = unzipped + self.assertEqual(list(xs), [1, 3, 6]) + self.assertEqual(list(ys), [2, 4, 7]) + + +class SortTogetherTest(TestCase): + """Tests for sort_together()""" + + def test_key_list(self): + """tests `key_list` including default, iterables include duplicates""" + iterables = [ + ['GA', 'GA', 'GA', 'CT', 'CT', 'CT'], + ['May', 'Aug.', 'May', 'June', 'July', 'July'], + [97, 20, 100, 70, 100, 20] + ] + + self.assertEqual( + mi.sort_together(iterables), + [ + ('CT', 'CT', 'CT', 'GA', 'GA', 'GA'), + ('June', 'July', 'July', 'May', 'Aug.', 'May'), + (70, 100, 20, 97, 20, 100) + ] + ) + + self.assertEqual( + mi.sort_together(iterables, key_list=(0, 1)), + [ + ('CT', 'CT', 'CT', 'GA', 'GA', 'GA'), + ('July', 'July', 'June', 'Aug.', 'May', 'May'), + (100, 20, 70, 20, 97, 100) + ] + ) + + self.assertEqual( + mi.sort_together(iterables, key_list=(0, 1, 2)), + [ + ('CT', 'CT', 'CT', 'GA', 'GA', 'GA'), + ('July', 'July', 'June', 'Aug.', 'May', 'May'), + (20, 100, 70, 20, 97, 100) + ] + ) + + self.assertEqual( + mi.sort_together(iterables, key_list=(2,)), + [ + ('GA', 'CT', 'CT', 'GA', 'GA', 'CT'), + ('Aug.', 'July', 'June', 'May', 'May', 'July'), + (20, 20, 70, 97, 100, 100) + ] + ) + + def test_invalid_key_list(self): + """tests `key_list` for indexes not available in `iterables`""" + iterables = [ + ['GA', 'GA', 'GA', 'CT', 'CT', 'CT'], + ['May', 'Aug.', 'May', 'June', 'July', 'July'], + [97, 20, 100, 70, 100, 20] + ] + + self.assertRaises( + IndexError, lambda: mi.sort_together(iterables, key_list=(5,)) + ) + + def test_reverse(self): + """tests `reverse` to ensure a reverse sort for `key_list` iterables""" + iterables = [ + ['GA', 'GA', 'GA', 'CT', 'CT', 'CT'], + ['May', 'Aug.', 'May', 'June', 'July', 'July'], + [97, 20, 100, 70, 100, 20] + ] + + self.assertEqual( + mi.sort_together(iterables, key_list=(0, 1, 2), reverse=True), + [('GA', 'GA', 'GA', 'CT', 'CT', 'CT'), + ('May', 'May', 'Aug.', 'June', 'July', 'July'), + (100, 97, 20, 70, 100, 20)] + ) + + def test_uneven_iterables(self): + """tests trimming of iterables to the shortest length before sorting""" + iterables = [['GA', 'GA', 'GA', 'CT', 'CT', 'CT', 'MA'], + ['May', 'Aug.', 'May', 'June', 'July', 'July'], + [97, 20, 100, 70, 100, 20, 0]] + + self.assertEqual( + mi.sort_together(iterables), + [ + ('CT', 'CT', 'CT', 'GA', 'GA', 'GA'), + ('June', 'July', 'July', 'May', 'Aug.', 'May'), + (70, 100, 20, 97, 20, 100) + ] + ) + + +class DivideTest(TestCase): + """Tests for divide()""" + + def test_invalid_n(self): + self.assertRaises(ValueError, lambda: mi.divide(-1, [1, 2, 3])) + self.assertRaises(ValueError, lambda: mi.divide(0, [1, 2, 3])) + + def test_basic(self): + iterable = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + for n, expected in [ + (1, [iterable]), + (2, [[1, 2, 3, 4, 5], [6, 7, 8, 9, 10]]), + (3, [[1, 2, 3, 4], [5, 6, 7], [8, 9, 10]]), + (10, [[n] for n in range(1, 10 + 1)]), + ]: + self.assertEqual( + [list(x) for x in mi.divide(n, iterable)], expected + ) + + def test_large_n(self): + iterable = [1, 2, 3, 4] + self.assertEqual( + [list(x) for x in mi.divide(6, iterable)], + [[1], [2], [3], [4], [], []] + ) + + +class TestAlwaysIterable(TestCase): + """Tests for always_iterable()""" + def test_single(self): + self.assertEqual(list(mi.always_iterable(1)), [1]) + + def test_strings(self): + for obj in ['foo', b'bar', 'baz']: + actual = list(mi.always_iterable(obj)) + expected = [obj] + self.assertEqual(actual, expected) + + def test_base_type(self): + dict_obj = {'a': 1, 'b': 2} + str_obj = '123' + + # Default: dicts are iterable like they normally are + default_actual = list(mi.always_iterable(dict_obj)) + default_expected = list(dict_obj) + self.assertEqual(default_actual, default_expected) + + # Unitary types set: dicts are not iterable + custom_actual = list(mi.always_iterable(dict_obj, base_type=dict)) + custom_expected = [dict_obj] + self.assertEqual(custom_actual, custom_expected) + + # With unitary types set, strings are iterable + str_actual = list(mi.always_iterable(str_obj, base_type=None)) + str_expected = list(str_obj) + self.assertEqual(str_actual, str_expected) + + def test_iterables(self): + self.assertEqual(list(mi.always_iterable([0, 1])), [0, 1]) + self.assertEqual( + list(mi.always_iterable([0, 1], base_type=list)), [[0, 1]] + ) + self.assertEqual( + list(mi.always_iterable(iter('foo'))), ['f', 'o', 'o'] + ) + self.assertEqual(list(mi.always_iterable([])), []) + + def test_none(self): + self.assertEqual(list(mi.always_iterable(None)), []) + + def test_generator(self): + def _gen(): + yield 0 + yield 1 + + self.assertEqual(list(mi.always_iterable(_gen())), [0, 1]) + + +class AdjacentTests(TestCase): + def test_typical(self): + actual = list(mi.adjacent(lambda x: x % 5 == 0, range(10))) + expected = [(True, 0), (True, 1), (False, 2), (False, 3), (True, 4), + (True, 5), (True, 6), (False, 7), (False, 8), (False, 9)] + self.assertEqual(actual, expected) + + def test_empty_iterable(self): + actual = list(mi.adjacent(lambda x: x % 5 == 0, [])) + expected = [] + self.assertEqual(actual, expected) + + def test_length_one(self): + actual = list(mi.adjacent(lambda x: x % 5 == 0, [0])) + expected = [(True, 0)] + self.assertEqual(actual, expected) + + actual = list(mi.adjacent(lambda x: x % 5 == 0, [1])) + expected = [(False, 1)] + self.assertEqual(actual, expected) + + def test_consecutive_true(self): + """Test that when the predicate matches multiple consecutive elements + it doesn't repeat elements in the output""" + actual = list(mi.adjacent(lambda x: x % 5 < 2, range(10))) + expected = [(True, 0), (True, 1), (True, 2), (False, 3), (True, 4), + (True, 5), (True, 6), (True, 7), (False, 8), (False, 9)] + self.assertEqual(actual, expected) + + def test_distance(self): + actual = list(mi.adjacent(lambda x: x % 5 == 0, range(10), distance=2)) + expected = [(True, 0), (True, 1), (True, 2), (True, 3), (True, 4), + (True, 5), (True, 6), (True, 7), (False, 8), (False, 9)] + self.assertEqual(actual, expected) + + actual = list(mi.adjacent(lambda x: x % 5 == 0, range(10), distance=3)) + expected = [(True, 0), (True, 1), (True, 2), (True, 3), (True, 4), + (True, 5), (True, 6), (True, 7), (True, 8), (False, 9)] + self.assertEqual(actual, expected) + + def test_large_distance(self): + """Test distance larger than the length of the iterable""" + iterable = range(10) + actual = list(mi.adjacent(lambda x: x % 5 == 4, iterable, distance=20)) + expected = list(zip(repeat(True), iterable)) + self.assertEqual(actual, expected) + + actual = list(mi.adjacent(lambda x: False, iterable, distance=20)) + expected = list(zip(repeat(False), iterable)) + self.assertEqual(actual, expected) + + def test_zero_distance(self): + """Test that adjacent() reduces to zip+map when distance is 0""" + iterable = range(1000) + predicate = lambda x: x % 4 == 2 + actual = mi.adjacent(predicate, iterable, 0) + expected = zip(map(predicate, iterable), iterable) + self.assertTrue(all(a == e for a, e in zip(actual, expected))) + + def test_negative_distance(self): + """Test that adjacent() raises an error with negative distance""" + pred = lambda x: x + self.assertRaises( + ValueError, lambda: mi.adjacent(pred, range(1000), -1) + ) + self.assertRaises( + ValueError, lambda: mi.adjacent(pred, range(10), -10) + ) + + def test_grouping(self): + """Test interaction of adjacent() with groupby_transform()""" + iterable = mi.adjacent(lambda x: x % 5 == 0, range(10)) + grouper = mi.groupby_transform(iterable, itemgetter(0), itemgetter(1)) + actual = [(k, list(g)) for k, g in grouper] + expected = [ + (True, [0, 1]), + (False, [2, 3]), + (True, [4, 5, 6]), + (False, [7, 8, 9]), + ] + self.assertEqual(actual, expected) + + def test_call_once(self): + """Test that the predicate is only called once per item.""" + already_seen = set() + iterable = range(10) + + def predicate(item): + self.assertNotIn(item, already_seen) + already_seen.add(item) + return True + + actual = list(mi.adjacent(predicate, iterable)) + expected = [(True, x) for x in iterable] + self.assertEqual(actual, expected) + + +class GroupByTransformTests(TestCase): + def assertAllGroupsEqual(self, groupby1, groupby2): + """Compare two groupby objects for equality, both keys and groups.""" + for a, b in zip(groupby1, groupby2): + key1, group1 = a + key2, group2 = b + self.assertEqual(key1, key2) + self.assertListEqual(list(group1), list(group2)) + self.assertRaises(StopIteration, lambda: next(groupby1)) + self.assertRaises(StopIteration, lambda: next(groupby2)) + + def test_default_funcs(self): + """Test that groupby_transform() with default args mimics groupby()""" + iterable = [(x // 5, x) for x in range(1000)] + actual = mi.groupby_transform(iterable) + expected = groupby(iterable) + self.assertAllGroupsEqual(actual, expected) + + def test_valuefunc(self): + iterable = [(int(x / 5), int(x / 3), x) for x in range(10)] + + # Test the standard usage of grouping one iterable using another's keys + grouper = mi.groupby_transform( + iterable, keyfunc=itemgetter(0), valuefunc=itemgetter(-1) + ) + actual = [(k, list(g)) for k, g in grouper] + expected = [(0, [0, 1, 2, 3, 4]), (1, [5, 6, 7, 8, 9])] + self.assertEqual(actual, expected) + + grouper = mi.groupby_transform( + iterable, keyfunc=itemgetter(1), valuefunc=itemgetter(-1) + ) + actual = [(k, list(g)) for k, g in grouper] + expected = [(0, [0, 1, 2]), (1, [3, 4, 5]), (2, [6, 7, 8]), (3, [9])] + self.assertEqual(actual, expected) + + # and now for something a little different + d = dict(zip(range(10), 'abcdefghij')) + grouper = mi.groupby_transform( + range(10), keyfunc=lambda x: x // 5, valuefunc=d.get + ) + actual = [(k, ''.join(g)) for k, g in grouper] + expected = [(0, 'abcde'), (1, 'fghij')] + self.assertEqual(actual, expected) + + def test_no_valuefunc(self): + iterable = range(1000) + + def key(x): + return x // 5 + + actual = mi.groupby_transform(iterable, key, valuefunc=None) + expected = groupby(iterable, key) + self.assertAllGroupsEqual(actual, expected) + + actual = mi.groupby_transform(iterable, key) # default valuefunc + expected = groupby(iterable, key) + self.assertAllGroupsEqual(actual, expected) + + +class NumericRangeTests(TestCase): + def test_basic(self): + for args, expected in [ + ((4,), [0, 1, 2, 3]), + ((4.0,), [0.0, 1.0, 2.0, 3.0]), + ((1.0, 4), [1.0, 2.0, 3.0]), + ((1, 4.0), [1, 2, 3]), + ((1.0, 5), [1.0, 2.0, 3.0, 4.0]), + ((0, 20, 5), [0, 5, 10, 15]), + ((0, 20, 5.0), [0.0, 5.0, 10.0, 15.0]), + ((0, 10, 3), [0, 3, 6, 9]), + ((0, 10, 3.0), [0.0, 3.0, 6.0, 9.0]), + ((0, -5, -1), [0, -1, -2, -3, -4]), + ((0.0, -5, -1), [0.0, -1.0, -2.0, -3.0, -4.0]), + ((1, 2, Fraction(1, 2)), [Fraction(1, 1), Fraction(3, 2)]), + ((0,), []), + ((0.0,), []), + ((1, 0), []), + ((1.0, 0.0), []), + ((Fraction(2, 1),), [Fraction(0, 1), Fraction(1, 1)]), + ((Decimal('2.0'),), [Decimal('0.0'), Decimal('1.0')]), + ]: + actual = list(mi.numeric_range(*args)) + self.assertEqual(actual, expected) + self.assertTrue( + all(type(a) == type(e) for a, e in zip(actual, expected)) + ) + + def test_arg_count(self): + self.assertRaises(TypeError, lambda: list(mi.numeric_range())) + self.assertRaises( + TypeError, lambda: list(mi.numeric_range(0, 1, 2, 3)) + ) + + def test_zero_step(self): + self.assertRaises( + ValueError, lambda: list(mi.numeric_range(1, 2, 0)) + ) + + +class CountCycleTests(TestCase): + def test_basic(self): + expected = [ + (0, 'a'), (0, 'b'), (0, 'c'), + (1, 'a'), (1, 'b'), (1, 'c'), + (2, 'a'), (2, 'b'), (2, 'c'), + ] + for actual in [ + mi.take(9, mi.count_cycle('abc')), # n=None + list(mi.count_cycle('abc', 3)), # n=3 + ]: + self.assertEqual(actual, expected) + + def test_empty(self): + self.assertEqual(list(mi.count_cycle('')), []) + self.assertEqual(list(mi.count_cycle('', 2)), []) + + def test_negative(self): + self.assertEqual(list(mi.count_cycle('abc', -3)), []) + + +class LocateTests(TestCase): + def test_default_pred(self): + iterable = [0, 1, 1, 0, 1, 0, 0] + actual = list(mi.locate(iterable)) + expected = [1, 2, 4] + self.assertEqual(actual, expected) + + def test_no_matches(self): + iterable = [0, 0, 0] + actual = list(mi.locate(iterable)) + expected = [] + self.assertEqual(actual, expected) + + def test_custom_pred(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda x: x == '0' + actual = list(mi.locate(iterable, pred)) + expected = [0, 3, 5, 6] + self.assertEqual(actual, expected) + + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + actual = list(mi.locate(iterable, pred, window_size=2)) + expected = [0, 3] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + actual = list(mi.locate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + + +class StripFunctionTests(TestCase): + def test_hashable(self): + iterable = list('www.example.com') + pred = lambda x: x in set('cmowz.') + + self.assertEqual(list(mi.lstrip(iterable, pred)), list('example.com')) + self.assertEqual(list(mi.rstrip(iterable, pred)), list('www.example')) + self.assertEqual(list(mi.strip(iterable, pred)), list('example')) + + def test_not_hashable(self): + iterable = [ + list('http://'), list('www'), list('.example'), list('.com') + ] + pred = lambda x: x in [list('http://'), list('www'), list('.com')] + + self.assertEqual(list(mi.lstrip(iterable, pred)), iterable[2:]) + self.assertEqual(list(mi.rstrip(iterable, pred)), iterable[:3]) + self.assertEqual(list(mi.strip(iterable, pred)), iterable[2: 3]) + + def test_math(self): + iterable = [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2] + pred = lambda x: x <= 2 + + self.assertEqual(list(mi.lstrip(iterable, pred)), iterable[3:]) + self.assertEqual(list(mi.rstrip(iterable, pred)), iterable[:-3]) + self.assertEqual(list(mi.strip(iterable, pred)), iterable[3:-3]) + + +class IsliceExtendedTests(TestCase): + def test_all(self): + iterable = ['0', '1', '2', '3', '4', '5'] + indexes = list(range(-4, len(iterable) + 4)) + [None] + steps = [1, 2, 3, 4, -1, -2, -3, 4] + for slice_args in product(indexes, indexes, steps): + try: + actual = list(mi.islice_extended(iterable, *slice_args)) + except Exception as e: + self.fail((slice_args, e)) + + expected = iterable[slice(*slice_args)] + self.assertEqual(actual, expected, slice_args) + + def test_zero_step(self): + with self.assertRaises(ValueError): + list(mi.islice_extended([1, 2, 3], 0, 1, 0)) + + +class ConsecutiveGroupsTest(TestCase): + def test_numbers(self): + iterable = [-10, -8, -7, -6, 1, 2, 4, 5, -1, 7] + actual = [list(g) for g in mi.consecutive_groups(iterable)] + expected = [[-10], [-8, -7, -6], [1, 2], [4, 5], [-1], [7]] + self.assertEqual(actual, expected) + + def test_custom_ordering(self): + iterable = ['1', '10', '11', '20', '21', '22', '30', '31'] + ordering = lambda x: int(x) + actual = [list(g) for g in mi.consecutive_groups(iterable, ordering)] + expected = [['1'], ['10', '11'], ['20', '21', '22'], ['30', '31']] + self.assertEqual(actual, expected) + + def test_exotic_ordering(self): + iterable = [ + ('a', 'b', 'c', 'd'), + ('a', 'c', 'b', 'd'), + ('a', 'c', 'd', 'b'), + ('a', 'd', 'b', 'c'), + ('d', 'b', 'c', 'a'), + ('d', 'c', 'a', 'b'), + ] + ordering = list(permutations('abcd')).index + actual = [list(g) for g in mi.consecutive_groups(iterable, ordering)] + expected = [ + [('a', 'b', 'c', 'd')], + [('a', 'c', 'b', 'd'), ('a', 'c', 'd', 'b'), ('a', 'd', 'b', 'c')], + [('d', 'b', 'c', 'a'), ('d', 'c', 'a', 'b')], + ] + self.assertEqual(actual, expected) + + +class DifferenceTest(TestCase): + def test_normal(self): + iterable = [10, 20, 30, 40, 50] + actual = list(mi.difference(iterable)) + expected = [10, 10, 10, 10, 10] + self.assertEqual(actual, expected) + + def test_custom(self): + iterable = [10, 20, 30, 40, 50] + actual = list(mi.difference(iterable, add)) + expected = [10, 30, 50, 70, 90] + self.assertEqual(actual, expected) + + def test_roundtrip(self): + original = list(range(100)) + accumulated = mi.accumulate(original) + actual = list(mi.difference(accumulated)) + self.assertEqual(actual, original) + + def test_one(self): + self.assertEqual(list(mi.difference([0])), [0]) + + def test_empty(self): + self.assertEqual(list(mi.difference([])), []) + + +class SeekableTest(TestCase): + def test_exhaustion_reset(self): + iterable = [str(n) for n in range(10)] + + s = mi.seekable(iterable) + self.assertEqual(list(s), iterable) # Normal iteration + self.assertEqual(list(s), []) # Iterable is exhausted + + s.seek(0) + self.assertEqual(list(s), iterable) # Back in action + + def test_partial_reset(self): + iterable = [str(n) for n in range(10)] + + s = mi.seekable(iterable) + self.assertEqual(mi.take(5, s), iterable[:5]) # Normal iteration + + s.seek(1) + self.assertEqual(list(s), iterable[1:]) # Get the rest of the iterable + + def test_forward(self): + iterable = [str(n) for n in range(10)] + + s = mi.seekable(iterable) + self.assertEqual(mi.take(1, s), iterable[:1]) # Normal iteration + + s.seek(3) # Skip over index 2 + self.assertEqual(list(s), iterable[3:]) # Result is similar to slicing + + s.seek(0) # Back to 0 + self.assertEqual(list(s), iterable) # No difference in result + + def test_past_end(self): + iterable = [str(n) for n in range(10)] + + s = mi.seekable(iterable) + self.assertEqual(mi.take(1, s), iterable[:1]) # Normal iteration + + s.seek(20) + self.assertEqual(list(s), []) # Iterable is exhausted + + s.seek(0) # Back to 0 + self.assertEqual(list(s), iterable) # No difference in result + + def test_elements(self): + iterable = map(str, count()) + + s = mi.seekable(iterable) + mi.take(10, s) + + elements = s.elements() + self.assertEqual( + [elements[i] for i in range(10)], [str(n) for n in range(10)] + ) + self.assertEqual(len(elements), 10) + + mi.take(10, s) + self.assertEqual(list(elements), [str(n) for n in range(20)]) + + +class SequenceViewTests(TestCase): + def test_init(self): + view = mi.SequenceView((1, 2, 3)) + self.assertEqual(repr(view), "SequenceView((1, 2, 3))") + self.assertRaises(TypeError, lambda: mi.SequenceView({})) + + def test_update(self): + seq = [1, 2, 3] + view = mi.SequenceView(seq) + self.assertEqual(len(view), 3) + self.assertEqual(repr(view), "SequenceView([1, 2, 3])") + + seq.pop() + self.assertEqual(len(view), 2) + self.assertEqual(repr(view), "SequenceView([1, 2])") + + def test_indexing(self): + seq = ('a', 'b', 'c', 'd', 'e', 'f') + view = mi.SequenceView(seq) + for i in range(-len(seq), len(seq)): + self.assertEqual(view[i], seq[i]) + + def test_slicing(self): + seq = ('a', 'b', 'c', 'd', 'e', 'f') + view = mi.SequenceView(seq) + n = len(seq) + indexes = list(range(-n - 1, n + 1)) + [None] + steps = list(range(-n, n + 1)) + steps.remove(0) + for slice_args in product(indexes, indexes, steps): + i = slice(*slice_args) + self.assertEqual(view[i], seq[i]) + + def test_abc_methods(self): + # collections.Sequence should provide all of this functionality + seq = ('a', 'b', 'c', 'd', 'e', 'f', 'f') + view = mi.SequenceView(seq) + + # __contains__ + self.assertIn('b', view) + self.assertNotIn('g', view) + + # __iter__ + self.assertEqual(list(iter(view)), list(seq)) + + # __reversed__ + self.assertEqual(list(reversed(view)), list(reversed(seq))) + + # index + self.assertEqual(view.index('b'), 1) + + # count + self.assertEqual(seq.count('f'), 2) + + +class RunLengthTest(TestCase): + def test_encode(self): + iterable = (int(str(n)[0]) for n in count(800)) + actual = mi.take(4, mi.run_length.encode(iterable)) + expected = [(8, 100), (9, 100), (1, 1000), (2, 1000)] + self.assertEqual(actual, expected) + + def test_decode(self): + iterable = [('d', 4), ('c', 3), ('b', 2), ('a', 1)] + actual = ''.join(mi.run_length.decode(iterable)) + expected = 'ddddcccbba' + self.assertEqual(actual, expected) + + +class ExactlyNTests(TestCase): + """Tests for ``exactly_n()``""" + + def test_true(self): + """Iterable has ``n`` ``True`` elements""" + self.assertTrue(mi.exactly_n([True, False, True], 2)) + self.assertTrue(mi.exactly_n([1, 1, 1, 0], 3)) + self.assertTrue(mi.exactly_n([False, False], 0)) + self.assertTrue(mi.exactly_n(range(100), 10, lambda x: x < 10)) + + def test_false(self): + """Iterable does not have ``n`` ``True`` elements""" + self.assertFalse(mi.exactly_n([True, False, False], 2)) + self.assertFalse(mi.exactly_n([True, True, False], 1)) + self.assertFalse(mi.exactly_n([False], 1)) + self.assertFalse(mi.exactly_n([True], -1)) + self.assertFalse(mi.exactly_n(repeat(True), 100)) + + def test_empty(self): + """Return ``True`` if the iterable is empty and ``n`` is 0""" + self.assertTrue(mi.exactly_n([], 0)) + self.assertFalse(mi.exactly_n([], 1)) + + +class AlwaysReversibleTests(TestCase): + """Tests for ``always_reversible()``""" + + def test_regular_reversed(self): + self.assertEqual(list(reversed(range(10))), + list(mi.always_reversible(range(10)))) + self.assertEqual(list(reversed([1, 2, 3])), + list(mi.always_reversible([1, 2, 3]))) + self.assertEqual(reversed([1, 2, 3]).__class__, + mi.always_reversible([1, 2, 3]).__class__) + + def test_nonseq_reversed(self): + # Create a non-reversible generator from a sequence + with self.assertRaises(TypeError): + reversed(x for x in range(10)) + + self.assertEqual(list(reversed(range(10))), + list(mi.always_reversible(x for x in range(10)))) + self.assertEqual(list(reversed([1, 2, 3])), + list(mi.always_reversible(x for x in [1, 2, 3]))) + self.assertNotEqual(reversed((1, 2)).__class__, + mi.always_reversible(x for x in (1, 2)).__class__) + + +class CircularShiftsTests(TestCase): + def test_empty(self): + # empty iterable -> empty list + self.assertEqual(list(mi.circular_shifts([])), []) + + def test_simple_circular_shifts(self): + # test the a simple iterator case + self.assertEqual( + mi.circular_shifts(range(4)), + [(0, 1, 2, 3), (1, 2, 3, 0), (2, 3, 0, 1), (3, 0, 1, 2)] + ) + + def test_duplicates(self): + # test non-distinct entries + self.assertEqual( + mi.circular_shifts([0, 1, 0, 1]), + [(0, 1, 0, 1), (1, 0, 1, 0), (0, 1, 0, 1), (1, 0, 1, 0)] + ) + + +class MakeDecoratorTests(TestCase): + def test_basic(self): + slicer = mi.make_decorator(islice) + + @slicer(1, 10, 2) + def user_function(arg_1, arg_2, kwarg_1=None): + self.assertEqual(arg_1, 'arg_1') + self.assertEqual(arg_2, 'arg_2') + self.assertEqual(kwarg_1, 'kwarg_1') + return map(str, count()) + + it = user_function('arg_1', 'arg_2', kwarg_1='kwarg_1') + actual = list(it) + expected = ['1', '3', '5', '7', '9'] + self.assertEqual(actual, expected) + + def test_result_index(self): + def stringify(*args, **kwargs): + self.assertEqual(args[0], 'arg_0') + iterable = args[1] + self.assertEqual(args[2], 'arg_2') + self.assertEqual(kwargs['kwarg_1'], 'kwarg_1') + return map(str, iterable) + + stringifier = mi.make_decorator(stringify, result_index=1) + + @stringifier('arg_0', 'arg_2', kwarg_1='kwarg_1') + def user_function(n): + return count(n) + + it = user_function(1) + actual = mi.take(5, it) + expected = ['1', '2', '3', '4', '5'] + self.assertEqual(actual, expected) + + def test_wrap_class(self): + seeker = mi.make_decorator(mi.seekable) + + @seeker() + def user_function(n): + return map(str, range(n)) + + it = user_function(5) + self.assertEqual(list(it), ['0', '1', '2', '3', '4']) + + it.seek(0) + self.assertEqual(list(it), ['0', '1', '2', '3', '4']) + + +class MapReduceTests(TestCase): + def test_default(self): + iterable = (str(x) for x in range(5)) + keyfunc = lambda x: int(x) // 2 + actual = sorted(mi.map_reduce(iterable, keyfunc).items()) + expected = [(0, ['0', '1']), (1, ['2', '3']), (2, ['4'])] + self.assertEqual(actual, expected) + + def test_valuefunc(self): + iterable = (str(x) for x in range(5)) + keyfunc = lambda x: int(x) // 2 + valuefunc = int + actual = sorted(mi.map_reduce(iterable, keyfunc, valuefunc).items()) + expected = [(0, [0, 1]), (1, [2, 3]), (2, [4])] + self.assertEqual(actual, expected) + + def test_reducefunc(self): + iterable = (str(x) for x in range(5)) + keyfunc = lambda x: int(x) // 2 + valuefunc = int + reducefunc = lambda value_list: reduce(mul, value_list, 1) + actual = sorted( + mi.map_reduce(iterable, keyfunc, valuefunc, reducefunc).items() + ) + expected = [(0, 0), (1, 6), (2, 4)] + self.assertEqual(actual, expected) + + def test_ret(self): + d = mi.map_reduce([1, 0, 2, 0, 1, 0], bool) + self.assertEqual(d, {False: [0, 0, 0], True: [1, 2, 1]}) + self.assertRaises(KeyError, lambda: d[None].append(1)) + + +class RlocateTests(TestCase): + def test_default_pred(self): + iterable = [0, 1, 1, 0, 1, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [4, 2, 1] + self.assertEqual(actual, expected) + + def test_no_matches(self): + iterable = [0, 0, 0] + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it)) + expected = [] + self.assertEqual(actual, expected) + + def test_custom_pred(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda x: x == '0' + for it in (iterable[:], iter(iterable)): + actual = list(mi.rlocate(it, pred)) + expected = [6, 5, 3, 0] + self.assertEqual(actual, expected) + + def test_efficient_reversal(self): + iterable = range(9 ** 9) # Is efficiently reversible + target = 9 ** 9 - 2 + pred = lambda x: x == target # Find-able from the right + actual = next(mi.rlocate(iterable, pred)) + self.assertEqual(actual, target) + + def test_window_size(self): + iterable = ['0', 1, 1, '0', 1, '0', '0'] + pred = lambda *args: args == ('0', 1) + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(it, pred, window_size=2)) + expected = [3, 0] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = [1, 2, 3, 4] + pred = lambda a, b, c, d, e: True + for it in (iterable, iter(iterable)): + actual = list(mi.rlocate(iterable, pred, window_size=5)) + expected = [0] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = [1, 2, 3, 4] + pred = lambda: True + for it in (iterable, iter(iterable)): + with self.assertRaises(ValueError): + list(mi.locate(iterable, pred, window_size=0)) + + +class ReplaceTests(TestCase): + def test_basic(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes)) + expected = [1, 3, 5, 7, 9] + self.assertEqual(actual, expected) + + def test_count(self): + iterable = range(10) + pred = lambda x: x % 2 == 0 + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, count=4)) + expected = [1, 3, 5, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size(self): + iterable = range(10) + pred = lambda *args: args == (0, 1, 2) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_end(self): + iterable = range(10) + pred = lambda *args: args == (7, 8, 9) + substitutes = [] + actual = list(mi.replace(iterable, pred, substitutes, window_size=3)) + expected = [0, 1, 2, 3, 4, 5, 6] + self.assertEqual(actual, expected) + + def test_window_size_count(self): + iterable = range(10) + pred = lambda *args: (args == (0, 1, 2)) or (args == (7, 8, 9)) + substitutes = [] + actual = list( + mi.replace(iterable, pred, substitutes, count=1, window_size=3) + ) + expected = [3, 4, 5, 6, 7, 8, 9] + self.assertEqual(actual, expected) + + def test_window_size_large(self): + iterable = range(4) + pred = lambda a, b, c, d, e: True + substitutes = [5, 6, 7] + actual = list(mi.replace(iterable, pred, substitutes, window_size=5)) + expected = [5, 6, 7] + self.assertEqual(actual, expected) + + def test_window_size_zero(self): + iterable = range(10) + pred = lambda *args: True + substitutes = [] + with self.assertRaises(ValueError): + list(mi.replace(iterable, pred, substitutes, window_size=0)) + + def test_iterable_substitutes(self): + iterable = range(5) + pred = lambda x: x % 2 == 0 + substitutes = iter('__') + actual = list(mi.replace(iterable, pred, substitutes)) + expected = ['_', '_', 1, '_', '_', 3, '_', '_'] + self.assertEqual(actual, expected) diff --git a/pipenv/vendor/more_itertools/tests/test_recipes.py b/pipenv/vendor/more_itertools/tests/test_recipes.py new file mode 100644 index 00000000..b3cfb62f --- /dev/null +++ b/pipenv/vendor/more_itertools/tests/test_recipes.py @@ -0,0 +1,616 @@ +from doctest import DocTestSuite +from unittest import TestCase + +from itertools import combinations +from six.moves import range + +import more_itertools as mi + + +def load_tests(loader, tests, ignore): + # Add the doctests + tests.addTests(DocTestSuite('more_itertools.recipes')) + return tests + + +class AccumulateTests(TestCase): + """Tests for ``accumulate()``""" + + def test_empty(self): + """Test that an empty input returns an empty output""" + self.assertEqual(list(mi.accumulate([])), []) + + def test_default(self): + """Test accumulate with the default function (addition)""" + self.assertEqual(list(mi.accumulate([1, 2, 3])), [1, 3, 6]) + + def test_bogus_function(self): + """Test accumulate with an invalid function""" + with self.assertRaises(TypeError): + list(mi.accumulate([1, 2, 3], func=lambda x: x)) + + def test_custom_function(self): + """Test accumulate with a custom function""" + self.assertEqual( + list(mi.accumulate((1, 2, 3, 2, 1), func=max)), [1, 2, 3, 3, 3] + ) + + +class TakeTests(TestCase): + """Tests for ``take()``""" + + def test_simple_take(self): + """Test basic usage""" + t = mi.take(5, range(10)) + self.assertEqual(t, [0, 1, 2, 3, 4]) + + def test_null_take(self): + """Check the null case""" + t = mi.take(0, range(10)) + self.assertEqual(t, []) + + def test_negative_take(self): + """Make sure taking negative items results in a ValueError""" + self.assertRaises(ValueError, lambda: mi.take(-3, range(10))) + + def test_take_too_much(self): + """Taking more than an iterator has remaining should return what the + iterator has remaining. + + """ + t = mi.take(10, range(5)) + self.assertEqual(t, [0, 1, 2, 3, 4]) + + +class TabulateTests(TestCase): + """Tests for ``tabulate()``""" + + def test_simple_tabulate(self): + """Test the happy path""" + t = mi.tabulate(lambda x: x) + f = tuple([next(t) for _ in range(3)]) + self.assertEqual(f, (0, 1, 2)) + + def test_count(self): + """Ensure tabulate accepts specific count""" + t = mi.tabulate(lambda x: 2 * x, -1) + f = (next(t), next(t), next(t)) + self.assertEqual(f, (-2, 0, 2)) + + +class TailTests(TestCase): + """Tests for ``tail()``""" + + def test_greater(self): + """Length of iterable is greater than requested tail""" + self.assertEqual(list(mi.tail(3, 'ABCDEFG')), ['E', 'F', 'G']) + + def test_equal(self): + """Length of iterable is equal to the requested tail""" + self.assertEqual( + list(mi.tail(7, 'ABCDEFG')), ['A', 'B', 'C', 'D', 'E', 'F', 'G'] + ) + + def test_less(self): + """Length of iterable is less than requested tail""" + self.assertEqual( + list(mi.tail(8, 'ABCDEFG')), ['A', 'B', 'C', 'D', 'E', 'F', 'G'] + ) + + +class ConsumeTests(TestCase): + """Tests for ``consume()``""" + + def test_sanity(self): + """Test basic functionality""" + r = (x for x in range(10)) + mi.consume(r, 3) + self.assertEqual(3, next(r)) + + def test_null_consume(self): + """Check the null case""" + r = (x for x in range(10)) + mi.consume(r, 0) + self.assertEqual(0, next(r)) + + def test_negative_consume(self): + """Check that negative consumsion throws an error""" + r = (x for x in range(10)) + self.assertRaises(ValueError, lambda: mi.consume(r, -1)) + + def test_total_consume(self): + """Check that iterator is totally consumed by default""" + r = (x for x in range(10)) + mi.consume(r) + self.assertRaises(StopIteration, lambda: next(r)) + + +class NthTests(TestCase): + """Tests for ``nth()``""" + + def test_basic(self): + """Make sure the nth item is returned""" + l = range(10) + for i, v in enumerate(l): + self.assertEqual(mi.nth(l, i), v) + + def test_default(self): + """Ensure a default value is returned when nth item not found""" + l = range(3) + self.assertEqual(mi.nth(l, 100, "zebra"), "zebra") + + def test_negative_item_raises(self): + """Ensure asking for a negative item raises an exception""" + self.assertRaises(ValueError, lambda: mi.nth(range(10), -3)) + + +class AllEqualTests(TestCase): + """Tests for ``all_equal()``""" + + def test_true(self): + """Everything is equal""" + self.assertTrue(mi.all_equal('aaaaaa')) + self.assertTrue(mi.all_equal([0, 0, 0, 0])) + + def test_false(self): + """Not everything is equal""" + self.assertFalse(mi.all_equal('aaaaab')) + self.assertFalse(mi.all_equal([0, 0, 0, 1])) + + def test_tricky(self): + """Not everything is identical, but everything is equal""" + items = [1, complex(1, 0), 1.0] + self.assertTrue(mi.all_equal(items)) + + def test_empty(self): + """Return True if the iterable is empty""" + self.assertTrue(mi.all_equal('')) + self.assertTrue(mi.all_equal([])) + + def test_one(self): + """Return True if the iterable is singular""" + self.assertTrue(mi.all_equal('0')) + self.assertTrue(mi.all_equal([0])) + + +class QuantifyTests(TestCase): + """Tests for ``quantify()``""" + + def test_happy_path(self): + """Make sure True count is returned""" + q = [True, False, True] + self.assertEqual(mi.quantify(q), 2) + + def test_custom_predicate(self): + """Ensure non-default predicates return as expected""" + q = range(10) + self.assertEqual(mi.quantify(q, lambda x: x % 2 == 0), 5) + + +class PadnoneTests(TestCase): + """Tests for ``padnone()``""" + + def test_happy_path(self): + """wrapper iterator should return None indefinitely""" + r = range(2) + p = mi.padnone(r) + self.assertEqual([0, 1, None, None], [next(p) for _ in range(4)]) + + +class NcyclesTests(TestCase): + """Tests for ``nyclces()``""" + + def test_happy_path(self): + """cycle a sequence three times""" + r = ["a", "b", "c"] + n = mi.ncycles(r, 3) + self.assertEqual( + ["a", "b", "c", "a", "b", "c", "a", "b", "c"], + list(n) + ) + + def test_null_case(self): + """asking for 0 cycles should return an empty iterator""" + n = mi.ncycles(range(100), 0) + self.assertRaises(StopIteration, lambda: next(n)) + + def test_pathalogical_case(self): + """asking for negative cycles should return an empty iterator""" + n = mi.ncycles(range(100), -10) + self.assertRaises(StopIteration, lambda: next(n)) + + +class DotproductTests(TestCase): + """Tests for ``dotproduct()``'""" + + def test_happy_path(self): + """simple dotproduct example""" + self.assertEqual(400, mi.dotproduct([10, 10], [20, 20])) + + +class FlattenTests(TestCase): + """Tests for ``flatten()``""" + + def test_basic_usage(self): + """ensure list of lists is flattened one level""" + f = [[0, 1, 2], [3, 4, 5]] + self.assertEqual(list(range(6)), list(mi.flatten(f))) + + def test_single_level(self): + """ensure list of lists is flattened only one level""" + f = [[0, [1, 2]], [[3, 4], 5]] + self.assertEqual([0, [1, 2], [3, 4], 5], list(mi.flatten(f))) + + +class RepeatfuncTests(TestCase): + """Tests for ``repeatfunc()``""" + + def test_simple_repeat(self): + """test simple repeated functions""" + r = mi.repeatfunc(lambda: 5) + self.assertEqual([5, 5, 5, 5, 5], [next(r) for _ in range(5)]) + + def test_finite_repeat(self): + """ensure limited repeat when times is provided""" + r = mi.repeatfunc(lambda: 5, times=5) + self.assertEqual([5, 5, 5, 5, 5], list(r)) + + def test_added_arguments(self): + """ensure arguments are applied to the function""" + r = mi.repeatfunc(lambda x: x, 2, 3) + self.assertEqual([3, 3], list(r)) + + def test_null_times(self): + """repeat 0 should return an empty iterator""" + r = mi.repeatfunc(range, 0, 3) + self.assertRaises(StopIteration, lambda: next(r)) + + +class PairwiseTests(TestCase): + """Tests for ``pairwise()``""" + + def test_base_case(self): + """ensure an iterable will return pairwise""" + p = mi.pairwise([1, 2, 3]) + self.assertEqual([(1, 2), (2, 3)], list(p)) + + def test_short_case(self): + """ensure an empty iterator if there's not enough values to pair""" + p = mi.pairwise("a") + self.assertRaises(StopIteration, lambda: next(p)) + + +class GrouperTests(TestCase): + """Tests for ``grouper()``""" + + def test_even(self): + """Test when group size divides evenly into the length of + the iterable. + + """ + self.assertEqual( + list(mi.grouper(3, 'ABCDEF')), [('A', 'B', 'C'), ('D', 'E', 'F')] + ) + + def test_odd(self): + """Test when group size does not divide evenly into the length of the + iterable. + + """ + self.assertEqual( + list(mi.grouper(3, 'ABCDE')), [('A', 'B', 'C'), ('D', 'E', None)] + ) + + def test_fill_value(self): + """Test that the fill value is used to pad the final group""" + self.assertEqual( + list(mi.grouper(3, 'ABCDE', 'x')), + [('A', 'B', 'C'), ('D', 'E', 'x')] + ) + + +class RoundrobinTests(TestCase): + """Tests for ``roundrobin()``""" + + def test_even_groups(self): + """Ensure ordered output from evenly populated iterables""" + self.assertEqual( + list(mi.roundrobin('ABC', [1, 2, 3], range(3))), + ['A', 1, 0, 'B', 2, 1, 'C', 3, 2] + ) + + def test_uneven_groups(self): + """Ensure ordered output from unevenly populated iterables""" + self.assertEqual( + list(mi.roundrobin('ABCD', [1, 2], range(0))), + ['A', 1, 'B', 2, 'C', 'D'] + ) + + +class PartitionTests(TestCase): + """Tests for ``partition()``""" + + def test_bool(self): + """Test when pred() returns a boolean""" + lesser, greater = mi.partition(lambda x: x > 5, range(10)) + self.assertEqual(list(lesser), [0, 1, 2, 3, 4, 5]) + self.assertEqual(list(greater), [6, 7, 8, 9]) + + def test_arbitrary(self): + """Test when pred() returns an integer""" + divisibles, remainders = mi.partition(lambda x: x % 3, range(10)) + self.assertEqual(list(divisibles), [0, 3, 6, 9]) + self.assertEqual(list(remainders), [1, 2, 4, 5, 7, 8]) + + +class PowersetTests(TestCase): + """Tests for ``powerset()``""" + + def test_combinatorics(self): + """Ensure a proper enumeration""" + p = mi.powerset([1, 2, 3]) + self.assertEqual( + list(p), + [(), (1,), (2,), (3,), (1, 2), (1, 3), (2, 3), (1, 2, 3)] + ) + + +class UniqueEverseenTests(TestCase): + """Tests for ``unique_everseen()``""" + + def test_everseen(self): + """ensure duplicate elements are ignored""" + u = mi.unique_everseen('AAAABBBBCCDAABBB') + self.assertEqual( + ['A', 'B', 'C', 'D'], + list(u) + ) + + def test_custom_key(self): + """ensure the custom key comparison works""" + u = mi.unique_everseen('aAbACCc', key=str.lower) + self.assertEqual(list('abC'), list(u)) + + def test_unhashable(self): + """ensure things work for unhashable items""" + iterable = ['a', [1, 2, 3], [1, 2, 3], 'a'] + u = mi.unique_everseen(iterable) + self.assertEqual(list(u), ['a', [1, 2, 3]]) + + def test_unhashable_key(self): + """ensure things work for unhashable items with a custom key""" + iterable = ['a', [1, 2, 3], [1, 2, 3], 'a'] + u = mi.unique_everseen(iterable, key=lambda x: x) + self.assertEqual(list(u), ['a', [1, 2, 3]]) + + +class UniqueJustseenTests(TestCase): + """Tests for ``unique_justseen()``""" + + def test_justseen(self): + """ensure only last item is remembered""" + u = mi.unique_justseen('AAAABBBCCDABB') + self.assertEqual(list('ABCDAB'), list(u)) + + def test_custom_key(self): + """ensure the custom key comparison works""" + u = mi.unique_justseen('AABCcAD', str.lower) + self.assertEqual(list('ABCAD'), list(u)) + + +class IterExceptTests(TestCase): + """Tests for ``iter_except()``""" + + def test_exact_exception(self): + """ensure the exact specified exception is caught""" + l = [1, 2, 3] + i = mi.iter_except(l.pop, IndexError) + self.assertEqual(list(i), [3, 2, 1]) + + def test_generic_exception(self): + """ensure the generic exception can be caught""" + l = [1, 2] + i = mi.iter_except(l.pop, Exception) + self.assertEqual(list(i), [2, 1]) + + def test_uncaught_exception_is_raised(self): + """ensure a non-specified exception is raised""" + l = [1, 2, 3] + i = mi.iter_except(l.pop, KeyError) + self.assertRaises(IndexError, lambda: list(i)) + + def test_first(self): + """ensure first is run before the function""" + l = [1, 2, 3] + f = lambda: 25 + i = mi.iter_except(l.pop, IndexError, f) + self.assertEqual(list(i), [25, 3, 2, 1]) + + +class FirstTrueTests(TestCase): + """Tests for ``first_true()``""" + + def test_something_true(self): + """Test with no keywords""" + self.assertEqual(mi.first_true(range(10)), 1) + + def test_nothing_true(self): + """Test default return value.""" + self.assertIsNone(mi.first_true([0, 0, 0])) + + def test_default(self): + """Test with a default keyword""" + self.assertEqual(mi.first_true([0, 0, 0], default='!'), '!') + + def test_pred(self): + """Test with a custom predicate""" + self.assertEqual( + mi.first_true([2, 4, 6], pred=lambda x: x % 3 == 0), 6 + ) + + +class RandomProductTests(TestCase): + """Tests for ``random_product()`` + + Since random.choice() has different results with the same seed across + python versions 2.x and 3.x, these tests use highly probably events to + create predictable outcomes across platforms. + """ + + def test_simple_lists(self): + """Ensure that one item is chosen from each list in each pair. + Also ensure that each item from each list eventually appears in + the chosen combinations. + + Odds are roughly 1 in 7.1 * 10e16 that one item from either list will + not be chosen after 100 samplings of one item from each list. Just to + be safe, better use a known random seed, too. + + """ + nums = [1, 2, 3] + lets = ['a', 'b', 'c'] + n, m = zip(*[mi.random_product(nums, lets) for _ in range(100)]) + n, m = set(n), set(m) + self.assertEqual(n, set(nums)) + self.assertEqual(m, set(lets)) + self.assertEqual(len(n), len(nums)) + self.assertEqual(len(m), len(lets)) + + def test_list_with_repeat(self): + """ensure multiple items are chosen, and that they appear to be chosen + from one list then the next, in proper order. + + """ + nums = [1, 2, 3] + lets = ['a', 'b', 'c'] + r = list(mi.random_product(nums, lets, repeat=100)) + self.assertEqual(2 * 100, len(r)) + n, m = set(r[::2]), set(r[1::2]) + self.assertEqual(n, set(nums)) + self.assertEqual(m, set(lets)) + self.assertEqual(len(n), len(nums)) + self.assertEqual(len(m), len(lets)) + + +class RandomPermutationTests(TestCase): + """Tests for ``random_permutation()``""" + + def test_full_permutation(self): + """ensure every item from the iterable is returned in a new ordering + + 15 elements have a 1 in 1.3 * 10e12 of appearing in sorted order, so + we fix a seed value just to be sure. + + """ + i = range(15) + r = mi.random_permutation(i) + self.assertEqual(set(i), set(r)) + if i == r: + raise AssertionError("Values were not permuted") + + def test_partial_permutation(self): + """ensure all returned items are from the iterable, that the returned + permutation is of the desired length, and that all items eventually + get returned. + + Sampling 100 permutations of length 5 from a set of 15 leaves a + (2/3)^100 chance that an item will not be chosen. Multiplied by 15 + items, there is a 1 in 2.6e16 chance that at least 1 item will not + show up in the resulting output. Using a random seed will fix that. + + """ + items = range(15) + item_set = set(items) + all_items = set() + for _ in range(100): + permutation = mi.random_permutation(items, 5) + self.assertEqual(len(permutation), 5) + permutation_set = set(permutation) + self.assertLessEqual(permutation_set, item_set) + all_items |= permutation_set + self.assertEqual(all_items, item_set) + + +class RandomCombinationTests(TestCase): + """Tests for ``random_combination()``""" + + def test_pseudorandomness(self): + """ensure different subsets of the iterable get returned over many + samplings of random combinations""" + items = range(15) + all_items = set() + for _ in range(50): + combination = mi.random_combination(items, 5) + all_items |= set(combination) + self.assertEqual(all_items, set(items)) + + def test_no_replacement(self): + """ensure that elements are sampled without replacement""" + items = range(15) + for _ in range(50): + combination = mi.random_combination(items, len(items)) + self.assertEqual(len(combination), len(set(combination))) + self.assertRaises( + ValueError, lambda: mi.random_combination(items, len(items) + 1) + ) + + +class RandomCombinationWithReplacementTests(TestCase): + """Tests for ``random_combination_with_replacement()``""" + + def test_replacement(self): + """ensure that elements are sampled with replacement""" + items = range(5) + combo = mi.random_combination_with_replacement(items, len(items) * 2) + self.assertEqual(2 * len(items), len(combo)) + if len(set(combo)) == len(combo): + raise AssertionError("Combination contained no duplicates") + + def test_pseudorandomness(self): + """ensure different subsets of the iterable get returned over many + samplings of random combinations""" + items = range(15) + all_items = set() + for _ in range(50): + combination = mi.random_combination_with_replacement(items, 5) + all_items |= set(combination) + self.assertEqual(all_items, set(items)) + + +class NthCombinationTests(TestCase): + def test_basic(self): + iterable = 'abcdefg' + r = 4 + for index, expected in enumerate(combinations(iterable, r)): + actual = mi.nth_combination(iterable, r, index) + self.assertEqual(actual, expected) + + def test_long(self): + actual = mi.nth_combination(range(180), 4, 2000000) + expected = (2, 12, 35, 126) + self.assertEqual(actual, expected) + + def test_invalid_r(self): + for r in (-1, 3): + with self.assertRaises(ValueError): + mi.nth_combination([], r, 0) + + def test_invalid_index(self): + with self.assertRaises(IndexError): + mi.nth_combination('abcdefg', 3, -36) + + +class PrependTests(TestCase): + def test_basic(self): + value = 'a' + iterator = iter('bcdefg') + actual = list(mi.prepend(value, iterator)) + expected = list('abcdefg') + self.assertEqual(actual, expected) + + def test_multiple(self): + value = 'ab' + iterator = iter('cdefg') + actual = tuple(mi.prepend(value, iterator)) + expected = ('ab',) + tuple('cdefg') + self.assertEqual(actual, expected) diff --git a/pipenv/vendor/orderedmultidict/LICENSE.md b/pipenv/vendor/orderedmultidict/LICENSE.md new file mode 100644 index 00000000..fd832f40 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/LICENSE.md @@ -0,0 +1,31 @@ +Build Amazing Things. + +*** + +### Unlicense + +This is free and unencumbered software released into the public\ +domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute\ +this software, either in source code form or as a compiled binary, for any\ +purpose, commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of\ +this software dedicate any and all copyright interest in the software to the\ +public domain. We make this dedication for the benefit of the public at\ +large and to the detriment of our heirs and successors. We intend this\ +dedication to be an overt act of relinquishment in perpetuity of all\ +present and future rights to this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF\ +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\ +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\ +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\ +SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\ +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT\ +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\ +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\ +SOFTWARE. + +For more information, please refer to <https://unlicense.org/> diff --git a/pipenv/vendor/orderedmultidict/__init__.py b/pipenv/vendor/orderedmultidict/__init__.py new file mode 100755 index 00000000..d122e35c --- /dev/null +++ b/pipenv/vendor/orderedmultidict/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) +# + +from os.path import dirname, join as pjoin + +from .orderedmultidict import * # noqa + +# Import all variables in __version__.py without explicit imports. +meta = {} +with open(pjoin(dirname(__file__), '__version__.py')) as f: + exec(f.read(), meta) +globals().update(dict((k, v) for k, v in meta.items() if k not in globals())) diff --git a/pipenv/vendor/orderedmultidict/__version__.py b/pipenv/vendor/orderedmultidict/__version__.py new file mode 100755 index 00000000..29e47e48 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/__version__.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) +# + +__title__ = 'orderedmultidict' +__version__ = '1.0.1' +__license__ = 'Unlicense' +__author__ = 'Ansgar Grunseid' +__contact__ = 'grunseid@gmail.com' +__description__ = 'Ordered Multivalue Dictionary' +__url__ = 'https://github.com/gruns/orderedmultidict' diff --git a/pipenv/vendor/orderedmultidict/itemlist.py b/pipenv/vendor/orderedmultidict/itemlist.py new file mode 100755 index 00000000..9524a560 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/itemlist.py @@ -0,0 +1,158 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) +# + +from __future__ import absolute_import + +from six.moves import zip_longest + +_absent = object() # Marker that means no parameter was provided. + + +class itemnode(object): + + """ + Dictionary key:value items wrapped in a node to be members of itemlist, the + doubly linked list defined below. + """ + + def __init__(self, prev=None, next=None, key=_absent, value=_absent): + self.prev = prev + self.next = next + self.key = key + self.value = value + + +class itemlist(object): + + """ + Doubly linked list of itemnodes. + + This class is used as the key:value item storage of orderedmultidict. + Methods below were only added as needed for use with orderedmultidict, so + some otherwise common list methods may be missing. + """ + + def __init__(self, items=[]): + self.root = itemnode() + self.root.next = self.root.prev = self.root + self.size = 0 + + for key, value in items: + self.append(key, value) + + def append(self, key, value): + tail = self.root.prev if self.root.prev is not self.root else self.root + node = itemnode(tail, self.root, key=key, value=value) + tail.next = node + self.root.prev = node + self.size += 1 + return node + + def removenode(self, node): + node.prev.next = node.next + node.next.prev = node.prev + self.size -= 1 + return self + + def clear(self): + for node, key, value in self: + self.removenode(node) + return self + + def items(self): + return list(self.iteritems()) + + def keys(self): + return list(self.iterkeys()) + + def values(self): + return list(self.itervalues()) + + def iteritems(self): + for node, key, value in self: + yield key, value + + def iterkeys(self): + for node, key, value in self: + yield key + + def itervalues(self): + for node, key, value in self: + yield value + + def reverse(self): + for node, key, value in self: + node.prev, node.next = node.next, node.prev + self.root.prev, self.root.next = self.root.next, self.root.prev + return self + + def __len__(self): + return self.size + + def __iter__(self): + current = self.root.next + while current and current is not self.root: + # Record current.next here in case current.next changes after the + # yield and before we return for the next iteration. For example, + # methods like reverse() will change current.next() before yield + # gets executed again. + nextnode = current.next + yield current, current.key, current.value + current = nextnode + + def __contains__(self, item): + """ + Params: + item: Can either be a (key,value) tuple or an itemnode reference. + """ + node = key = value = _absent + if hasattr(item, '__len__') and callable(item.__len__): + if len(item) == 2: + key, value = item + elif len(item) == 3: + node, key, value = item + else: + node = item + + if node is not _absent or _absent not in [key, value]: + for selfnode, selfkey, selfvalue in self: + if ((node is _absent and key == selfkey and value == selfvalue) + or (node is not _absent and node == selfnode)): + return True + return False + + def __getitem__(self, index): + # Only support direct access to the first or last element, as this is + # all orderedmultidict needs for now. + if index == 0 and self.root.next is not self.root: + return self.root.next + elif index == -1 and self.root.prev is not self.root: + return self.root.prev + raise IndexError(index) + + def __delitem__(self, index): + self.removenode(self[index]) + + def __eq__(self, other): + for (n1, key1, value1), (n2, key2, value2) in zip_longest(self, other): + if key1 != key2 or value1 != value2: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __nonzero__(self): + return self.size > 0 + + def __str__(self): + return '[%s]' % self.items() diff --git a/pipenv/vendor/orderedmultidict/orderedmultidict.py b/pipenv/vendor/orderedmultidict/orderedmultidict.py new file mode 100755 index 00000000..6cd55c0f --- /dev/null +++ b/pipenv/vendor/orderedmultidict/orderedmultidict.py @@ -0,0 +1,833 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) +# + +from __future__ import absolute_import + +import sys +from itertools import chain + +import six +from six.moves import map, zip_longest + +from .itemlist import itemlist + +if six.PY2: + from collections import MutableMapping +else: + from collections.abc import MutableMapping +try: + from collections import OrderedDict as odict # Python 2.7 and later. +except ImportError: + from ordereddict import OrderedDict as odict # Python 2.6 and earlier. + + +_absent = object() # Marker that means no parameter was provided. +_items_attr = 'items' if sys.version_info[0] >= 3 else 'iteritems' + + +def callable_attr(obj, attr): + return hasattr(obj, attr) and callable(getattr(obj, attr)) + + +# +# TODO(grun): Create a subclass of list that values(), getlist(), allitems(), +# etc return that the user can manipulate directly to control the omdict() +# object. +# +# For example, users should be able to do things like +# +# omd = omdict([(1,1), (1,11)]) +# omd.values(1).append('sup') +# omd.allitems() == [(1,1), (1,11), (1,'sup')] +# omd.values(1).remove(11) +# omd.allitems() == [(1,1), (1,'sup')] +# omd.values(1).extend(['two', 'more']) +# omd.allitems() == [(1,1), (1,'sup'), (1,'two'), (1,'more')] +# +# or +# +# omd = omdict([(1,1), (1,11)]) +# omd.allitems().extend([(2,2), (2,22)]) +# omd.allitems() == [(1,1), (1,11), (2,2), (2,22)]) +# +# or +# +# omd = omdict() +# omd.values(1) = [1, 11] +# omd.allitems() == [(1,1), (1,11)] +# omd.values(1) = list(map(lambda i: i * -10, omd.values(1))) +# omd.allitems() == [(1,-10), (1,-110)] +# omd.allitems() = filter(lambda (k,v): v > -100, omd.allitems()) +# omd.allitems() == [(1,-10)] +# +# etc. +# +# To accomplish this, subclass list in such a manner that each list element is +# really a two tuple, where the first tuple value is the actual value and the +# second tuple value is a reference to the itemlist node for that value. Users +# only interact with the first tuple values, the actual values, but behind the +# scenes when an element is modified, deleted, inserted, etc, the according +# itemlist nodes are modified, deleted, inserted, etc accordingly. In this +# manner, users can manipulate omdict objects directly through direct list +# manipulation. +# +# Once accomplished, some methods become redundant and should be removed in +# favor of the more intuitive direct value list manipulation. Such redundant +# methods include getlist() (removed in favor of values()?), addlist(), and +# setlist(). +# +# With the removal of many of the 'list' methods, think about renaming all +# remaining 'list' methods to 'values' methods, like poplist() -> popvalues(), +# poplistitem() -> popvaluesitem(), etc. This would be an easy switch for most +# methods, but wouldn't fit others so well. For example, iterlists() would +# become itervalues(), a name extremely similar to iterallvalues() but quite +# different in function. +# + + +class omdict(MutableMapping): + + """ + Ordered Multivalue Dictionary. + + A multivalue dictionary is a dictionary that can store multiple values per + key. An ordered multivalue dictionary is a multivalue dictionary that + retains the order of insertions and deletions. + + Internally, items are stored in a doubly linked list, self._items. A + dictionary, self._map, is also maintained and stores an ordered list of + linked list node references, one for each value associated with that key. + + Standard dict methods interact with the first value associated with a given + key. This means that omdict retains method parity with dict, and a dict + object can be replaced with an omdict object and all interaction will + behave identically. All dict methods that retain parity with omdict are: + + get(), setdefault(), pop(), popitem(), + clear(), copy(), update(), fromkeys(), len() + __getitem__(), __setitem__(), __delitem__(), __contains__(), + items(), keys(), values(), iteritems(), iterkeys(), itervalues(), + + Optional parameters have been added to some dict methods, but because the + added parameters are optional, existing use remains unaffected. An optional + <key> parameter has been added to these methods: + + items(), values(), iteritems(), itervalues() + + New methods have also been added to omdict. Methods with 'list' in their + name interact with lists of values, and methods with 'all' in their name + interact with all items in the dictionary, including multiple items with + the same key. + + The new omdict methods are: + + load(), size(), reverse(), + getlist(), add(), addlist(), set(), setlist(), setdefaultlist(), + poplist(), popvalue(), popvalues(), popitem(), poplistitem(), + allitems(), allkeys(), allvalues(), lists(), listitems(), + iterallitems(), iterallkeys(), iterallvalues(), iterlists(), + iterlistitems() + + Explanations and examples of the new methods above can be found in the + function comments below and online at + + https://github.com/gruns/orderedmultidict + + Additional omdict information and documentation can also be found at the + above url. + """ + + def __init__(self, *args, **kwargs): + # Doubly linked list of itemnodes. Each itemnode stores a key:value + # item. + self._items = itemlist() + + # Ordered dictionary of keys and itemnode references. Each itemnode + # reference points to one of that keys values. + self._map = odict() + + self.load(*args, **kwargs) + + def load(self, *args, **kwargs): + """ + Clear all existing key:value items and import all key:value items from + <mapping>. If multiple values exist for the same key in <mapping>, they + are all be imported. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.load([(4,4), (4,44), (5,5)]) + omd.allitems() == [(4,4), (4,44), (5,5)] + + Returns: <self>. + """ + self.clear() + self.updateall(*args, **kwargs) + return self + + def copy(self): + return self.__class__(self.allitems()) + + def clear(self): + self._map.clear() + self._items.clear() + + def size(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.size() == 5 + + Returns: Total number of items, including multiple items with the same + key. + """ + return len(self._items) + + @classmethod + def fromkeys(cls, iterable, value=None): + return cls([(key, value) for key in iterable]) + + def has_key(self, key): + return key in self + + def update(self, *args, **kwargs): + self._update_updateall(True, *args, **kwargs) + + def updateall(self, *args, **kwargs): + """ + Update this dictionary with the items from <mapping>, replacing + existing key:value items with shared keys before adding new key:value + items. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.updateall([(2,'two'), (1,'one'), (2,222), (1,111)]) + omd.allitems() == [(1, 'one'), (2, 'two'), (2, 222), (1, 111)] + + Returns: <self>. + """ + self._update_updateall(False, *args, **kwargs) + return self + + def _update_updateall(self, replace_at_most_one, *args, **kwargs): + # Bin the items in <args> and <kwargs> into <replacements> or + # <leftovers>. Items in <replacements> are new values to replace old + # values for a given key, and items in <leftovers> are new items to be + # added. + replacements, leftovers = dict(), [] + for mapping in chain(args, [kwargs]): + self._bin_update_items( + self._items_iterator(mapping), replace_at_most_one, + replacements, leftovers) + + # First, replace existing values for each key. + for key, values in six.iteritems(replacements): + self.setlist(key, values) + # Then, add the leftover items to the end of the list of all items. + for key, value in leftovers: + self.add(key, value) + + def _bin_update_items(self, items, replace_at_most_one, + replacements, leftovers): + """ + <replacements and <leftovers> are modified directly, ala pass by + reference. + """ + for key, value in items: + # If there are existing items with key <key> that have yet to be + # marked for replacement, mark that item's value to be replaced by + # <value> by appending it to <replacements>. + if key in self and key not in replacements: + replacements[key] = [value] + elif (key in self and not replace_at_most_one and + len(replacements[key]) < len(self.values(key))): + replacements[key].append(value) + else: + if replace_at_most_one: + replacements[key] = [value] + else: + leftovers.append((key, value)) + + def _items_iterator(self, container): + cont = container + iterator = iter(cont) + if callable_attr(cont, 'iterallitems'): + iterator = cont.iterallitems() + elif callable_attr(cont, 'allitems'): + iterator = iter(cont.allitems()) + elif callable_attr(cont, 'iteritems'): + iterator = cont.iteritems() + elif callable_attr(cont, 'items'): + iterator = iter(cont.items()) + return iterator + + def get(self, key, default=None): + if key in self: + return self._map[key][0].value + return default + + def getlist(self, key, default=[]): + """ + Returns: The list of values for <key> if <key> is in the dictionary, + else <default>. If <default> is not provided, an empty list is + returned. + """ + if key in self: + return [node.value for node in self._map[key]] + return default + + def setdefault(self, key, default=None): + if key in self: + return self[key] + self.add(key, default) + return default + + def setdefaultlist(self, key, defaultlist=[None]): + """ + Similar to setdefault() except <defaultlist> is a list of values to set + for <key>. If <key> already exists, its existing list of values is + returned. + + If <key> isn't a key and <defaultlist> is an empty list, [], no values + are added for <key> and <key> will not be added as a key. + + Returns: List of <key>'s values if <key> exists in the dictionary, + otherwise <default>. + """ + if key in self: + return self.getlist(key) + self.addlist(key, defaultlist) + return defaultlist + + def add(self, key, value=None): + """ + Add <value> to the list of values for <key>. If <key> is not in the + dictionary, then <value> is added as the sole value for <key>. + + Example: + omd = omdict() + omd.add(1, 1) # omd.allitems() == [(1,1)] + omd.add(1, 11) # omd.allitems() == [(1,1), (1,11)] + omd.add(2, 2) # omd.allitems() == [(1,1), (1,11), (2,2)] + + Returns: <self>. + """ + self._map.setdefault(key, []) + node = self._items.append(key, value) + self._map[key].append(node) + return self + + def addlist(self, key, valuelist=[]): + """ + Add the values in <valuelist> to the list of values for <key>. If <key> + is not in the dictionary, the values in <valuelist> become the values + for <key>. + + Example: + omd = omdict([(1,1)]) + omd.addlist(1, [11, 111]) + omd.allitems() == [(1, 1), (1, 11), (1, 111)] + omd.addlist(2, [2]) + omd.allitems() == [(1, 1), (1, 11), (1, 111), (2, 2)] + + Returns: <self>. + """ + for value in valuelist: + self.add(key, value) + return self + + def set(self, key, value=None): + """ + Sets <key>'s value to <value>. Identical in function to __setitem__(). + + Returns: <self>. + """ + self[key] = value + return self + + def setlist(self, key, values): + """ + Sets <key>'s list of values to <values>. Existing items with key <key> + are first replaced with new values from <values>. Any remaining old + items that haven't been replaced with new values are deleted, and any + new values from <values> that don't have corresponding items with <key> + to replace are appended to the end of the list of all items. + + If values is an empty list, [], <key> is deleted, equivalent in action + to del self[<key>]. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.setlist(1, [11, 111]) + omd.allitems() == [(1,11), (2,2), (1,111)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, [None]) + omd.allitems() == [(1,None), (2,2)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, []) + omd.allitems() == [(2,2)] + + Returns: <self>. + """ + if not values and key in self: + self.pop(key) + else: + it = zip_longest( + list(self._map.get(key, [])), values, fillvalue=_absent) + for node, value in it: + if node is not _absent and value is not _absent: + node.value = value + elif node is _absent: + self.add(key, value) + elif value is _absent: + self._map[key].remove(node) + self._items.removenode(node) + return self + + def removevalues(self, key, values): + """ + Removes all <values> from the values of <key>. If <key> has no + remaining values after removevalues(), the key is popped. + + Example: + omd = omdict([(1, 1), (1, 11), (1, 1), (1, 111)]) + omd.removevalues(1, [1, 111]) + omd.allitems() == [(1, 11)] + + Returns: <self>. + """ + self.setlist(key, [v for v in self.getlist(key) if v not in values]) + return self + + def pop(self, key, default=_absent): + if key in self: + return self.poplist(key)[0] + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def poplist(self, key, default=_absent): + """ + If <key> is in the dictionary, pop it and return its list of values. If + <key> is not in the dictionary, return <default>. KeyError is raised if + <default> is not provided and <key> is not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplist(1) == [1, 11, 111] + omd.allitems() == [(2,2), (3,3)] + omd.poplist(2) == [2] + omd.allitems() == [(3,3)] + + Raises: KeyError if <key> isn't in the dictionary and <default> isn't + provided. + Returns: List of <key>'s values. + """ + if key in self: + values = self.getlist(key) + del self._map[key] + for node, nodekey, nodevalue in self._items: + if nodekey == key: + self._items.removenode(node) + return values + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popvalue(self, key, value=_absent, default=_absent, last=True): + """ + If <value> is provided, pops the first or last (key,value) item in the + dictionary if <key> is in the dictionary. + + If <value> is not provided, pops the first or last value for <key> if + <key> is in the dictionary. + + If <key> no longer has any values after a popvalue() call, <key> is + removed from the dictionary. If <key> isn't in the dictionary and + <default> was provided, return default. KeyError is raised if <default> + is not provided and <key> is not in the dictionary. ValueError is + raised if <value> is provided but isn't a value for <key>. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3), (2,22)]) + omd.popvalue(1) == 111 + omd.allitems() == [(1,11), (1,111), (2,2), (3,3), (2,22)] + omd.popvalue(1, last=False) == 1 + omd.allitems() == [(1,11), (2,2), (3,3), (2,22)] + omd.popvalue(2, 2) == 2 + omd.allitems() == [(1,11), (3,3), (2,22)] + omd.popvalue(1, 11) == 11 + omd.allitems() == [(3,3), (2,22)] + omd.popvalue('not a key', default='sup') == 'sup' + + Params: + last: Boolean whether to return <key>'s first value (<last> is False) + or last value (<last> is True). + Raises: + KeyError if <key> isn't in the dictionary and <default> isn't + provided. + ValueError if <value> isn't a value for <key>. + Returns: The first or last of <key>'s values. + """ + def pop_node_with_index(key, index): + node = self._map[key].pop(index) + if not self._map[key]: + del self._map[key] + self._items.removenode(node) + return node + + if key in self: + if value is not _absent: + if last: + pos = self.values(key)[::-1].index(value) + else: + pos = self.values(key).index(value) + if pos == -1: + raise ValueError(value) + else: + index = (len(self.values(key)) - 1 - pos) if last else pos + return pop_node_with_index(key, index).value + else: + return pop_node_with_index(key, -1 if last else 0).value + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popitem(self, fromall=False, last=True): + """ + Pop and return a key:value item. + + If <fromall> is False, items()[0] is popped if <last> is False or + items()[-1] is popped if <last> is True. All remaining items with the + same key are removed. + + If <fromall> is True, allitems()[0] is popped if <last> is False or + allitems()[-1] is popped if <last> is True. Any remaining items with + the same key remain. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem() == (3,3) + omd.popitem(fromall=False, last=False) == (1,1) + omd.popitem(fromall=False, last=False) == (2,2) + + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem(fromall=True, last=False) == (1,1) + omd.popitem(fromall=True, last=False) == (1,11) + omd.popitem(fromall=True, last=True) == (3,3) + omd.popitem(fromall=True, last=False) == (1,111) + + Params: + fromall: Whether to pop an item from items() (<fromall> is True) or + allitems() (<fromall> is False). + last: Boolean whether to pop the first item or last item of items() + or allitems(). + Raises: KeyError if the dictionary is empty. + Returns: The first or last item from item() or allitem(). + """ + if not self._items: + raise KeyError('popitem(): %s is empty' % self.__class__.__name__) + + if fromall: + node = self._items[-1 if last else 0] + key = node.key + return key, self.popvalue(key, last=last) + else: + key = list(self._map.keys())[-1 if last else 0] + return key, self.pop(key) + + def poplistitem(self, last=True): + """ + Pop and return a key:valuelist item comprised of a key and that key's + list of values. If <last> is False, a key:valuelist item comprised of + keys()[0] and its list of values is popped and returned. If <last> is + True, a key:valuelist item comprised of keys()[-1] and its list of + values is popped and returned. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplistitem(last=True) == (3,[3]) + omd.poplistitem(last=False) == (1,[1,11,111]) + + Params: + last: Boolean whether to pop the first or last key and its associated + list of values. + Raises: KeyError if the dictionary is empty. + Returns: A two-tuple comprised of the first or last key and its + associated list of values. + """ + if not self._items: + s = 'poplistitem(): %s is empty' % self.__class__.__name__ + raise KeyError(s) + + key = self.keys()[-1 if last else 0] + return key, self.poplist(key) + + def items(self, key=_absent): + """ + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: List created from iteritems(<key>). Only items with key <key> + are returned if <key> is provided and is a dictionary key. + """ + return list(self.iteritems(key)) + + def keys(self): + return list(self.iterkeys()) + + def values(self, key=_absent): + """ + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: List created from itervalues(<key>).If <key> is provided and + is a dictionary key, only values of items with key <key> are + returned. + """ + if key is not _absent and key in self._map: + return self.getlist(key) + return list(self.itervalues()) + + def lists(self): + """ + Returns: List created from iterlists(). + """ + return list(self.iterlists()) + + def listitems(self): + """ + Returns: List created from iterlistitems(). + """ + return list(self.iterlistitems()) + + def iteritems(self, key=_absent): + """ + Parity with dict.iteritems() except the optional <key> parameter has + been added. If <key> is provided, only items with the provided key are + iterated over. KeyError is raised if <key> is provided and not in the + dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iteritems(1) -> (1,1) -> (1,11) -> (1,111) + omd.iteritems() -> (1,1) -> (2,2) -> (3,3) + + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: An iterator over the items() of the dictionary, or only items + with the key <key> if <key> is provided. + """ + if key is not _absent: + if key in self: + items = [(node.key, node.value) for node in self._map[key]] + return iter(items) + raise KeyError(key) + items = six.iteritems(self._map) + return iter((key, nodes[0].value) for (key, nodes) in items) + + def iterkeys(self): + return six.iterkeys(self._map) + + def itervalues(self, key=_absent): + """ + Parity with dict.itervalues() except the optional <key> parameter has + been added. If <key> is provided, only values from items with the + provided key are iterated over. KeyError is raised if <key> is provided + and not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.itervalues(1) -> 1 -> 11 -> 111 + omd.itervalues() -> 1 -> 11 -> 111 -> 2 -> 3 + + Raises: KeyError if <key> is provided and isn't in the dictionary. + Returns: An iterator over the values() of the dictionary, or only the + values of key <key> if <key> is provided. + """ + if key is not _absent: + if key in self: + return iter([node.value for node in self._map[key]]) + raise KeyError(key) + return iter([nodes[0].value for nodes in six.itervalues(self._map)]) + + def allitems(self, key=_absent): + ''' + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: List created from iterallitems(<key>). + ''' + return list(self.iterallitems(key)) + + def allkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allkeys() == [1,1,1,2,3] + + Returns: List created from iterallkeys(). + ''' + return list(self.iterallkeys()) + + def allvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allvalues() == [1,11,111,2,3] + omd.allvalues(1) == [1,11,111] + + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: List created from iterallvalues(<key>). + ''' + return list(self.iterallvalues(key)) + + def iterallitems(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallitems() == (1,1) -> (1,11) -> (1,111) -> (2,2) -> (3,3) + omd.iterallitems(1) == (1,1) -> (1,11) -> (1,111) + + Raises: KeyError if <key> is provided and not in the dictionary. + Returns: An iterator over every item in the diciontary. If <key> is + provided, only items with the key <key> are iterated over. + ''' + if key is not _absent: + # Raises KeyError if <key> is not in self._map. + return self.iteritems(key) + return self._items.iteritems() + + def iterallkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallkeys() == 1 -> 1 -> 1 -> 2 -> 3 + + Returns: An iterator over the keys of every item in the dictionary. + ''' + return self._items.iterkeys() + + def iterallvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallvalues() == 1 -> 11 -> 111 -> 2 -> 3 + + Returns: An iterator over the values of every item in the dictionary. + ''' + if key is not _absent: + if key in self: + return iter(self.getlist(key)) + raise KeyError(key) + return self._items.itervalues() + + def iterlists(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlists() -> [1,11,111] -> [2] -> [3] + + Returns: An iterator over the list comprised of the lists of values for + each key. + ''' + return map(lambda key: self.getlist(key), self) + + def iterlistitems(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlistitems() -> (1,[1,11,111]) -> (2,[2]) -> (3,[3]) + + Returns: An iterator over the list of key:valuelist items. + """ + return map(lambda key: (key, self.getlist(key)), self) + + def reverse(self): + """ + Reverse the order of all items in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.reverse() + omd.allitems() == [(3,3), (2,2), (1,111), (1,11), (1,1)] + + Returns: <self>. + """ + for key in six.iterkeys(self._map): + self._map[key].reverse() + self._items.reverse() + return self + + def __eq__(self, other): + if callable_attr(other, 'iterallitems'): + myiter, otheriter = self.iterallitems(), other.iterallitems() + for i1, i2 in zip_longest(myiter, otheriter, fillvalue=_absent): + if i1 != i2 or i1 is _absent or i2 is _absent: + return False + elif not hasattr(other, '__len__') or not hasattr(other, _items_attr): + return False + # Ignore order so we can compare ordered omdicts with unordered dicts. + else: + if len(self) != len(other): + return False + for key, value in six.iteritems(other): + if self.get(key, _absent) != value: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __len__(self): + return len(self._map) + + def __iter__(self): + for key in self.iterkeys(): + yield key + + def __contains__(self, key): + return key in self._map + + def __getitem__(self, key): + if key in self: + return self.get(key) + raise KeyError(key) + + def __setitem__(self, key, value): + self.setlist(key, [value]) + + def __delitem__(self, key): + return self.pop(key) + + def __nonzero__(self): + return bool(self._map) + + def __str__(self): + return '{%s}' % ', '.join( + map(lambda p: '%r: %r' % (p[0], p[1]), self.iterallitems())) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.allitems()) + + def __or__(self, other): + return self.__class__(chain(_get_items(self), _get_items(other))) + + def __ior__(self, other): + for k, v in _get_items(other): + self.add(k, value=v) + return self + + +def _get_items(mapping): + """Find item iterator for an object.""" + names = ('iterallitems', 'allitems', 'iteritems', 'items') + exist = (n for n in names if callable_attr(mapping, n)) + for a in exist: + return getattr(mapping, a)() + raise TypeError( + "Object {} has no compatible items interface.".format(mapping)) diff --git a/pipenv/vendor/packaging/LICENSE.APACHE b/pipenv/vendor/packaging/LICENSE.APACHE index 4947287f..f433b1a5 100644 --- a/pipenv/vendor/packaging/LICENSE.APACHE +++ b/pipenv/vendor/packaging/LICENSE.APACHE @@ -174,4 +174,4 @@ incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. - END OF TERMS AND CONDITIONS \ No newline at end of file + END OF TERMS AND CONDITIONS diff --git a/pipenv/vendor/packaging/__about__.py b/pipenv/vendor/packaging/__about__.py index 21fc6ce3..5161d141 100644 --- a/pipenv/vendor/packaging/__about__.py +++ b/pipenv/vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "20.3" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/vendor/packaging/__init__.py b/pipenv/vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/vendor/packaging/__init__.py +++ b/pipenv/vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/vendor/packaging/_compat.py b/pipenv/vendor/packaging/_compat.py index 210bb80b..a145f7ee 100644 --- a/pipenv/vendor/packaging/_compat.py +++ b/pipenv/vendor/packaging/_compat.py @@ -5,6 +5,11 @@ from __future__ import absolute_import, division, print_function import sys +from ._typing import MYPY_CHECK_RUNNING + +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import Any, Dict, Tuple, Type + PY2 = sys.version_info[0] == 2 PY3 = sys.version_info[0] == 3 @@ -12,19 +17,22 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): + # type: (Type[Any], Tuple[Type[Any], ...]) -> Any """ Create a base class with a metaclass. """ # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): + class metaclass(meta): # type: ignore def __new__(cls, name, this_bases, d): + # type: (Type[Any], str, Tuple[Any], Dict[Any, Any]) -> Any return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/vendor/packaging/_structures.py b/pipenv/vendor/packaging/_structures.py index e9fc4a04..800d5c55 100644 --- a/pipenv/vendor/packaging/_structures.py +++ b/pipenv/vendor/packaging/_structures.py @@ -4,67 +4,83 @@ from __future__ import absolute_import, division, print_function -class Infinity(object): - +class InfinityType(object): def __repr__(self): + # type: () -> str return "Infinity" def __hash__(self): + # type: () -> int return hash(repr(self)) def __lt__(self, other): + # type: (object) -> bool return False def __le__(self, other): + # type: (object) -> bool return False def __eq__(self, other): + # type: (object) -> bool return isinstance(other, self.__class__) def __ne__(self, other): + # type: (object) -> bool return not isinstance(other, self.__class__) def __gt__(self, other): + # type: (object) -> bool return True def __ge__(self, other): + # type: (object) -> bool return True def __neg__(self): + # type: (object) -> NegativeInfinityType return NegativeInfinity -Infinity = Infinity() +Infinity = InfinityType() -class NegativeInfinity(object): - +class NegativeInfinityType(object): def __repr__(self): + # type: () -> str return "-Infinity" def __hash__(self): + # type: () -> int return hash(repr(self)) def __lt__(self, other): + # type: (object) -> bool return True def __le__(self, other): + # type: (object) -> bool return True def __eq__(self, other): + # type: (object) -> bool return isinstance(other, self.__class__) def __ne__(self, other): + # type: (object) -> bool return not isinstance(other, self.__class__) def __gt__(self, other): + # type: (object) -> bool return False def __ge__(self, other): + # type: (object) -> bool return False def __neg__(self): + # type: (object) -> InfinityType return Infinity -NegativeInfinity = NegativeInfinity() +NegativeInfinity = NegativeInfinityType() diff --git a/pipenv/vendor/packaging/_typing.py b/pipenv/vendor/packaging/_typing.py new file mode 100644 index 00000000..dc6dfce7 --- /dev/null +++ b/pipenv/vendor/packaging/_typing.py @@ -0,0 +1,39 @@ +"""For neatly implementing static typing in packaging. + +`mypy` - the static type analysis tool we use - uses the `typing` module, which +provides core functionality fundamental to mypy's functioning. + +Generally, `typing` would be imported at runtime and used in that fashion - +it acts as a no-op at runtime and does not have any run-time overhead by +design. + +As it turns out, `typing` is not vendorable - it uses separate sources for +Python 2/Python 3. Thus, this codebase can not expect it to be present. +To work around this, mypy allows the typing import to be behind a False-y +optional to prevent it from running at runtime and type-comments can be used +to remove the need for the types to be accessible directly during runtime. + +This module provides the False-y guard in a nicely named fashion so that a +curious maintainer can reach here to read this. + +In packaging, all static-typing related imports should be guarded as follows: + + from packaging._typing import MYPY_CHECK_RUNNING + + if MYPY_CHECK_RUNNING: + from typing import ... + +Ref: https://github.com/python/mypy/issues/3216 +""" + +MYPY_CHECK_RUNNING = False + +if MYPY_CHECK_RUNNING: # pragma: no cover + import typing + + cast = typing.cast +else: + # typing's cast() is needed at runtime, but we don't want to import typing. + # Thus, we use a dummy no-op version, which we tell mypy to ignore. + def cast(type_, value): # type: ignore + return value diff --git a/pipenv/vendor/packaging/markers.py b/pipenv/vendor/packaging/markers.py index 5fdf510c..f0174711 100644 --- a/pipenv/vendor/packaging/markers.py +++ b/pipenv/vendor/packaging/markers.py @@ -13,12 +13,21 @@ from pyparsing import ZeroOrMore, Group, Forward, QuotedString from pyparsing import Literal as L # noqa from ._compat import string_types +from ._typing import MYPY_CHECK_RUNNING from .specifiers import Specifier, InvalidSpecifier +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import Any, Callable, Dict, List, Optional, Tuple, Union + + Operator = Callable[[str, str], bool] + __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,77 +51,73 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): + # type: (Any) -> None self.value = value def __str__(self): + # type: () -> str return str(self.value) def __repr__(self): + # type: () -> str return "<{0}({1!r})>".format(self.__class__.__name__, str(self)) def serialize(self): + # type: () -> str raise NotImplementedError class Variable(Node): - def serialize(self): + # type: () -> str return str(self) class Value(Node): - def serialize(self): + # type: () -> str return '"{0}"'.format(self) class Op(Node): - def serialize(self): + # type: () -> str return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") # PEP-345 + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # undocumented setuptools legacy + | L("extra") # PEP-508 ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -139,6 +144,7 @@ MARKER = stringStart + MARKER_EXPR + stringEnd def _coerce_parse_result(results): + # type: (Union[ParseResults, List[Any]]) -> List[Any] if isinstance(results, ParseResults): return [_coerce_parse_result(i) for i in results] else: @@ -146,14 +152,19 @@ def _coerce_parse_result(results): def _format_marker(marker, first=True): + # type: (Union[List[str], Tuple[Node, ...], str], Optional[bool]) -> str + assert isinstance(marker, (list, tuple, string_types)) # Sometimes we have a structure like [[...]] which is a single item list # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -177,10 +188,11 @@ _operators = { "!=": operator.ne, ">=": operator.ge, ">": operator.gt, -} +} # type: Dict[str, Operator] def _eval_op(lhs, op, rhs): + # type: (str, Op, str) -> bool try: spec = Specifier("".join([op.serialize(), rhs])) except InvalidSpecifier: @@ -188,7 +200,7 @@ def _eval_op(lhs, op, rhs): else: return spec.contains(lhs) - oper = _operators.get(op.serialize()) + oper = _operators.get(op.serialize()) # type: Optional[Operator] if oper is None: raise UndefinedComparison( "Undefined {0!r} on {1!r} and {2!r}.".format(op, lhs, rhs) @@ -197,13 +209,18 @@ def _eval_op(lhs, op, rhs): return oper(lhs, rhs) -_undefined = object() +class Undefined(object): + pass + + +_undefined = Undefined() def _get_env(environment, name): - value = environment.get(name, _undefined) + # type: (Dict[str, str], str) -> str + value = environment.get(name, _undefined) # type: Union[str, Undefined] - if value is _undefined: + if isinstance(value, Undefined): raise UndefinedEnvironmentName( "{0!r} does not exist in evaluation environment.".format(name) ) @@ -212,7 +229,8 @@ def _get_env(environment, name): def _evaluate_markers(markers, environment): - groups = [[]] + # type: (List[Any], Dict[str, str]) -> bool + groups = [[]] # type: List[List[bool]] for marker in markers: assert isinstance(marker, (list, tuple, string_types)) @@ -239,20 +257,25 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + # type: (sys._version_info) -> str + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): - iver = format_full_version(sys.implementation.version) - implementation_name = sys.implementation.name + # type: () -> Dict[str, str] + if hasattr(sys, "implementation"): + # Ignoring the `sys.implementation` reference for type checking due to + # mypy not liking that the attribute doesn't exist in Python 2.7 when + # run with the `--py27` flag. + iver = format_full_version(sys.implementation.version) # type: ignore + implementation_name = sys.implementation.name # type: ignore else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -264,28 +287,32 @@ def default_environment(): "platform_version": platform.version(), "python_full_version": platform.python_version(), "platform_python_implementation": platform.python_implementation(), - "python_version": platform.python_version()[:3], + "python_version": ".".join(platform.python_version_tuple()[:2]), "sys_platform": sys.platform, } class Marker(object): - def __init__(self, marker): + # type: (str) -> None try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): + # type: () -> str return _format_marker(self._markers) def __repr__(self): + # type: () -> str return "<Marker({0!r})>".format(str(self)) def evaluate(self, environment=None): + # type: (Optional[Dict[str, str]]) -> bool """Evaluate a marker. Return the boolean from evaluating the given marker against the diff --git a/pipenv/vendor/packaging/py.typed b/pipenv/vendor/packaging/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/pipenv/vendor/packaging/requirements.py b/pipenv/vendor/packaging/requirements.py index e8008a6d..1b547927 100644 --- a/pipenv/vendor/packaging/requirements.py +++ b/pipenv/vendor/packaging/requirements.py @@ -11,9 +11,13 @@ from pyparsing import ZeroOrMore, Word, Optional, Regex, Combine from pyparsing import Literal as L # noqa from six.moves.urllib import parse as urlparse +from ._typing import MYPY_CHECK_RUNNING from .markers import MARKER_EXPR, Marker from .specifiers import LegacySpecifier, Specifier, SpecifierSet +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import List + class InvalidRequirement(ValueError): """ @@ -38,8 +42,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +52,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +71,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -89,18 +93,25 @@ class Requirement(object): # TODO: Can we normalize the name and extra name? def __init__(self, requirement_string): + # type: (str) -> None try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -110,7 +121,8 @@ class Requirement(object): self.marker = req.marker if req.marker else None def __str__(self): - parts = [self.name] + # type: () -> str + parts = [self.name] # type: List[str] if self.extras: parts.append("[{0}]".format(",".join(sorted(self.extras)))) @@ -120,6 +132,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) @@ -127,4 +141,5 @@ class Requirement(object): return "".join(parts) def __repr__(self): + # type: () -> str return "<Requirement({0!r})>".format(str(self)) diff --git a/pipenv/vendor/packaging/specifiers.py b/pipenv/vendor/packaging/specifiers.py index 4c798999..94987486 100644 --- a/pipenv/vendor/packaging/specifiers.py +++ b/pipenv/vendor/packaging/specifiers.py @@ -9,8 +9,26 @@ import itertools import re from ._compat import string_types, with_metaclass +from ._typing import MYPY_CHECK_RUNNING from .version import Version, LegacyVersion, parse +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import ( + List, + Dict, + Union, + Iterable, + Iterator, + Optional, + Callable, + Tuple, + FrozenSet, + ) + + ParsedVersion = Union[Version, LegacyVersion] + UnparsedVersion = Union[Version, LegacyVersion, str] + CallableOperator = Callable[[ParsedVersion, str], bool] + class InvalidSpecifier(ValueError): """ @@ -18,10 +36,10 @@ class InvalidSpecifier(ValueError): """ -class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - +class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): # type: ignore @abc.abstractmethod def __str__(self): + # type: () -> str """ Returns the str representation of this Specifier like object. This should be representative of the Specifier itself. @@ -29,12 +47,14 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): @abc.abstractmethod def __hash__(self): + # type: () -> int """ Returns a hash value for this Specifier like object. """ @abc.abstractmethod def __eq__(self, other): + # type: (object) -> bool """ Returns a boolean representing whether or not the two Specifier like objects are equal. @@ -42,6 +62,7 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): @abc.abstractmethod def __ne__(self, other): + # type: (object) -> bool """ Returns a boolean representing whether or not the two Specifier like objects are not equal. @@ -49,6 +70,7 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): @abc.abstractproperty def prereleases(self): + # type: () -> Optional[bool] """ Returns whether or not pre-releases as a whole are allowed by this specifier. @@ -56,6 +78,7 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): @prereleases.setter def prereleases(self, value): + # type: (bool) -> None """ Sets whether or not pre-releases as a whole are allowed by this specifier. @@ -63,12 +86,14 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): @abc.abstractmethod def contains(self, item, prereleases=None): + # type: (str, Optional[bool]) -> bool """ Determines if the given item is contained within this specifier. """ @abc.abstractmethod def filter(self, iterable, prereleases=None): + # type: (Iterable[UnparsedVersion], Optional[bool]) -> Iterable[UnparsedVersion] """ Takes an iterable of items and filters them so that only items which are contained within this specifier are allowed in it. @@ -77,9 +102,10 @@ class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): class _IndividualSpecifier(BaseSpecifier): - _operators = {} + _operators = {} # type: Dict[str, str] def __init__(self, spec="", prereleases=None): + # type: (str, Optional[bool]) -> None match = self._regex.search(spec) if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) @@ -87,34 +113,34 @@ class _IndividualSpecifier(BaseSpecifier): self._spec = ( match.group("operator").strip(), match.group("version").strip(), - ) + ) # type: Tuple[str, str] # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases def __repr__(self): + # type: () -> str pre = ( ", prereleases={0!r}".format(self.prereleases) if self._prereleases is not None else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): + # type: () -> str return "{0}{1}".format(*self._spec) def __hash__(self): + # type: () -> int return hash(self._spec) def __eq__(self, other): + # type: (object) -> bool if isinstance(other, string_types): try: - other = self.__class__(other) + other = self.__class__(str(other)) except InvalidSpecifier: return NotImplemented elif not isinstance(other, self.__class__): @@ -123,9 +149,10 @@ class _IndividualSpecifier(BaseSpecifier): return self._spec == other._spec def __ne__(self, other): + # type: (object) -> bool if isinstance(other, string_types): try: - other = self.__class__(other) + other = self.__class__(str(other)) except InvalidSpecifier: return NotImplemented elif not isinstance(other, self.__class__): @@ -134,52 +161,67 @@ class _IndividualSpecifier(BaseSpecifier): return self._spec != other._spec def _get_operator(self, op): - return getattr(self, "_compare_{0}".format(self._operators[op])) + # type: (str) -> CallableOperator + operator_callable = getattr( + self, "_compare_{0}".format(self._operators[op]) + ) # type: CallableOperator + return operator_callable def _coerce_version(self, version): + # type: (UnparsedVersion) -> ParsedVersion if not isinstance(version, (LegacyVersion, Version)): version = parse(version) return version @property def operator(self): + # type: () -> str return self._spec[0] @property def version(self): + # type: () -> str return self._spec[1] @property def prereleases(self): + # type: () -> Optional[bool] return self._prereleases @prereleases.setter def prereleases(self, value): + # type: (bool) -> None self._prereleases = value def __contains__(self, item): + # type: (str) -> bool return self.contains(item) def contains(self, item, prereleases=None): + # type: (UnparsedVersion, Optional[bool]) -> bool + # Determine if prereleases are to be allowed or not. if prereleases is None: prereleases = self.prereleases # Normalize item to a Version or LegacyVersion, this allows us to have # a shortcut for ``"2.0" in Specifier(">=2") - item = self._coerce_version(item) + normalized_item = self._coerce_version(item) # Determine if we should be supporting prereleases in this specifier # or not, if we do not support prereleases than we can short circuit # logic if this version is a prereleases. - if item.is_prerelease and not prereleases: + if normalized_item.is_prerelease and not prereleases: return False # Actually do the comparison to determine if this item is contained # within this Specifier or not. - return self._get_operator(self.operator)(item, self.version) + operator_callable = self._get_operator(self.operator) # type: CallableOperator + return operator_callable(normalized_item, self.version) def filter(self, iterable, prereleases=None): + # type: (Iterable[UnparsedVersion], Optional[bool]) -> Iterable[UnparsedVersion] + yielded = False found_prereleases = [] @@ -194,8 +236,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +256,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P<operator>(==|!=|<=|>=|<|>)) \s* (?P<version> @@ -225,10 +267,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -240,42 +280,53 @@ class LegacySpecifier(_IndividualSpecifier): } def _coerce_version(self, version): + # type: (Union[ParsedVersion, str]) -> LegacyVersion if not isinstance(version, LegacyVersion): version = LegacyVersion(str(version)) return version def _compare_equal(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective == self._coerce_version(spec) def _compare_not_equal(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective != self._coerce_version(spec) def _compare_less_than_equal(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective <= self._coerce_version(spec) def _compare_greater_than_equal(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective >= self._coerce_version(spec) def _compare_less_than(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective < self._coerce_version(spec) def _compare_greater_than(self, prospective, spec): + # type: (LegacyVersion, str) -> bool return prospective > self._coerce_version(spec) -def _require_version_compare(fn): +def _require_version_compare( + fn # type: (Callable[[Specifier, ParsedVersion, str], bool]) +): + # type: (...) -> Callable[[Specifier, ParsedVersion, str], bool] @functools.wraps(fn) def wrapped(self, prospective, spec): + # type: (Specifier, ParsedVersion, str) -> bool if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P<operator>(~=|==|!=|<=|>=|<|>|===)) (?P<version> (?: @@ -367,10 +418,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -385,6 +434,8 @@ class Specifier(_IndividualSpecifier): @_require_version_compare def _compare_compatible(self, prospective, spec): + # type: (ParsedVersion, str) -> bool + # Compatible releases have an equivalent combination of >= and ==. That # is that ~=2.2 is equivalent to >=2.2,==2.*. This allows us to # implement this in terms of the other specifiers instead of @@ -397,8 +448,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,61 +457,73 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): + # type: (ParsedVersion, str) -> bool + # We need special logic to handle prefix matching if spec.endswith(".*"): # In the case of prefix matching we want to ignore local segment. prospective = Version(prospective.public) # Split the spec out by dots, and pretend that there is an implicit # dot in between a release segment and a pre-release segment. - spec = _version_split(spec[:-2]) # Remove the trailing .* + split_spec = _version_split(spec[:-2]) # Remove the trailing .* # Split the prospective version out by dots, and pretend that there # is an implicit dot in between a release segment and a pre-release # segment. - prospective = _version_split(str(prospective)) + split_prospective = _version_split(str(prospective)) # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + shortened_prospective = split_prospective[: len(split_spec)] # Pad out our two sides with zeros so that they both equal the same # length. - spec, prospective = _pad_version(spec, prospective) + padded_spec, padded_prospective = _pad_version( + split_spec, shortened_prospective + ) + + return padded_prospective == padded_spec else: # Convert our spec string into a Version - spec = Version(spec) + spec_version = Version(spec) # If the specifier does not have a local segment, then we want to # act as if the prospective version also does not have a local # segment. - if not spec.local: + if not spec_version.local: prospective = Version(prospective.public) - return prospective == spec + return prospective == spec_version @_require_version_compare def _compare_not_equal(self, prospective, spec): + # type: (ParsedVersion, str) -> bool return not self._compare_equal(prospective, spec) @_require_version_compare def _compare_less_than_equal(self, prospective, spec): + # type: (ParsedVersion, str) -> bool return prospective <= Version(spec) @_require_version_compare def _compare_greater_than_equal(self, prospective, spec): + # type: (ParsedVersion, str) -> bool return prospective >= Version(spec) @_require_version_compare - def _compare_less_than(self, prospective, spec): + def _compare_less_than(self, prospective, spec_str): + # type: (ParsedVersion, str) -> bool + # Convert our spec to a Version instance, since we'll want to work with # it as a version. - spec = Version(spec) + spec = Version(spec_str) # Check to see if the prospective version is less than the spec # version. If it's not we can short circuit and just return False now @@ -483,10 +545,12 @@ class Specifier(_IndividualSpecifier): return True @_require_version_compare - def _compare_greater_than(self, prospective, spec): + def _compare_greater_than(self, prospective, spec_str): + # type: (ParsedVersion, str) -> bool + # Convert our spec to a Version instance, since we'll want to work with # it as a version. - spec = Version(spec) + spec = Version(spec_str) # Check to see if the prospective version is greater than the spec # version. If it's not we can short circuit and just return False now @@ -514,10 +578,13 @@ class Specifier(_IndividualSpecifier): return True def _compare_arbitrary(self, prospective, spec): + # type: (Version, str) -> bool return str(prospective).lower() == str(spec).lower() @property def prereleases(self): + # type: () -> bool + # If there is an explicit prereleases set for this, then we'll just # blindly use that. if self._prereleases is not None: @@ -542,6 +609,7 @@ class Specifier(_IndividualSpecifier): @prereleases.setter def prereleases(self, value): + # type: (bool) -> None self._prereleases = value @@ -549,7 +617,8 @@ _prefix_regex = re.compile(r"^([0-9]+)((?:a|b|c|rc)[0-9]+)$") def _version_split(version): - result = [] + # type: (str) -> List[str] + result = [] # type: List[str] for item in version.split("."): match = _prefix_regex.search(item) if match: @@ -560,6 +629,7 @@ def _version_split(version): def _pad_version(left, right): + # type: (List[str], List[str]) -> Tuple[List[str], List[str]] left_split, right_split = [], [] # Get the release segment of our versions @@ -567,36 +637,28 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): - # Split on , to break each indidivual specifier into it's own item, and + # type: (str, Optional[bool]) -> None + + # Split on , to break each individual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. - specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] + split_specifiers = [s.strip() for s in specifiers.split(",") if s.strip()] # Parsed each individual specifier, attempting first to make it a # Specifier and falling back to a LegacySpecifier. parsed = set() - for specifier in specifiers: + for specifier in split_specifiers: try: parsed.add(Specifier(specifier)) except InvalidSpecifier: @@ -610,6 +672,7 @@ class SpecifierSet(BaseSpecifier): self._prereleases = prereleases def __repr__(self): + # type: () -> str pre = ( ", prereleases={0!r}".format(self.prereleases) if self._prereleases is not None @@ -619,12 +682,15 @@ class SpecifierSet(BaseSpecifier): return "<SpecifierSet({0!r}{1})>".format(str(self), pre) def __str__(self): + # type: () -> str return ",".join(sorted(str(s) for s in self._specs)) def __hash__(self): + # type: () -> int return hash(self._specs) def __and__(self, other): + # type: (Union[SpecifierSet, str]) -> SpecifierSet if isinstance(other, string_types): other = SpecifierSet(other) elif not isinstance(other, SpecifierSet): @@ -648,9 +714,8 @@ class SpecifierSet(BaseSpecifier): return specifier def __eq__(self, other): - if isinstance(other, string_types): - other = SpecifierSet(other) - elif isinstance(other, _IndividualSpecifier): + # type: (object) -> bool + if isinstance(other, (string_types, _IndividualSpecifier)): other = SpecifierSet(str(other)) elif not isinstance(other, SpecifierSet): return NotImplemented @@ -658,9 +723,8 @@ class SpecifierSet(BaseSpecifier): return self._specs == other._specs def __ne__(self, other): - if isinstance(other, string_types): - other = SpecifierSet(other) - elif isinstance(other, _IndividualSpecifier): + # type: (object) -> bool + if isinstance(other, (string_types, _IndividualSpecifier)): other = SpecifierSet(str(other)) elif not isinstance(other, SpecifierSet): return NotImplemented @@ -668,13 +732,17 @@ class SpecifierSet(BaseSpecifier): return self._specs != other._specs def __len__(self): + # type: () -> int return len(self._specs) def __iter__(self): + # type: () -> Iterator[FrozenSet[_IndividualSpecifier]] return iter(self._specs) @property def prereleases(self): + # type: () -> Optional[bool] + # If we have been given an explicit prerelease modifier, then we'll # pass that through here. if self._prereleases is not None: @@ -692,12 +760,16 @@ class SpecifierSet(BaseSpecifier): @prereleases.setter def prereleases(self, value): + # type: (bool) -> None self._prereleases = value def __contains__(self, item): + # type: (Union[ParsedVersion, str]) -> bool return self.contains(item) def contains(self, item, prereleases=None): + # type: (Union[ParsedVersion, str], Optional[bool]) -> bool + # Ensure that our item is a Version or LegacyVersion instance. if not isinstance(item, (LegacyVersion, Version)): item = parse(item) @@ -721,12 +793,15 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) + + def filter( + self, + iterable, # type: Iterable[Union[ParsedVersion, str]] + prereleases=None, # type: Optional[bool] + ): + # type: (...) -> Iterable[Union[ParsedVersion, str]] - def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing # one for this particular filter call, then we'll use whatever the # SpecifierSet thinks for whether or not we should support prereleases. @@ -744,8 +819,8 @@ class SpecifierSet(BaseSpecifier): # which will filter out any pre-releases, unless there are no final # releases, and which will filter out LegacyVersion in general. else: - filtered = [] - found_prereleases = [] + filtered = [] # type: List[Union[ParsedVersion, str]] + found_prereleases = [] # type: List[Union[ParsedVersion, str]] for item in iterable: # Ensure that we some kind of Version class for this item. diff --git a/pipenv/vendor/packaging/tags.py b/pipenv/vendor/packaging/tags.py new file mode 100644 index 00000000..300faab8 --- /dev/null +++ b/pipenv/vendor/packaging/tags.py @@ -0,0 +1,739 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import + +import distutils.util + +try: + from importlib.machinery import EXTENSION_SUFFIXES +except ImportError: # pragma: no cover + import imp + + EXTENSION_SUFFIXES = [x[0] for x in imp.get_suffixes()] + del imp +import logging +import os +import platform +import re +import struct +import sys +import sysconfig +import warnings + +from ._typing import MYPY_CHECK_RUNNING, cast + +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import ( + Dict, + FrozenSet, + IO, + Iterable, + Iterator, + List, + Optional, + Sequence, + Tuple, + Union, + ) + + PythonVersion = Sequence[int] + MacVersion = Tuple[int, int] + GlibcVersion = Tuple[int, int] + + +logger = logging.getLogger(__name__) + +INTERPRETER_SHORT_NAMES = { + "python": "py", # Generic. + "cpython": "cp", + "pypy": "pp", + "ironpython": "ip", + "jython": "jy", +} # type: Dict[str, str] + + +_32_BIT_INTERPRETER = sys.maxsize <= 2 ** 32 + + +class Tag(object): + + __slots__ = ["_interpreter", "_abi", "_platform"] + + def __init__(self, interpreter, abi, platform): + # type: (str, str, str) -> None + self._interpreter = interpreter.lower() + self._abi = abi.lower() + self._platform = platform.lower() + + @property + def interpreter(self): + # type: () -> str + return self._interpreter + + @property + def abi(self): + # type: () -> str + return self._abi + + @property + def platform(self): + # type: () -> str + return self._platform + + def __eq__(self, other): + # type: (object) -> bool + if not isinstance(other, Tag): + return NotImplemented + + return ( + (self.platform == other.platform) + and (self.abi == other.abi) + and (self.interpreter == other.interpreter) + ) + + def __hash__(self): + # type: () -> int + return hash((self._interpreter, self._abi, self._platform)) + + def __str__(self): + # type: () -> str + return "{}-{}-{}".format(self._interpreter, self._abi, self._platform) + + def __repr__(self): + # type: () -> str + return "<{self} @ {self_id}>".format(self=self, self_id=id(self)) + + +def parse_tag(tag): + # type: (str) -> FrozenSet[Tag] + tags = set() + interpreters, abis, platforms = tag.split("-") + for interpreter in interpreters.split("."): + for abi in abis.split("."): + for platform_ in platforms.split("."): + tags.add(Tag(interpreter, abi, platform_)) + return frozenset(tags) + + +def _warn_keyword_parameter(func_name, kwargs): + # type: (str, Dict[str, bool]) -> bool + """ + Backwards-compatibility with Python 2.7 to allow treating 'warn' as keyword-only. + """ + if not kwargs: + return False + elif len(kwargs) > 1 or "warn" not in kwargs: + kwargs.pop("warn", None) + arg = next(iter(kwargs.keys())) + raise TypeError( + "{}() got an unexpected keyword argument {!r}".format(func_name, arg) + ) + return kwargs["warn"] + + +def _get_config_var(name, warn=False): + # type: (str, bool) -> Union[int, str, None] + value = sysconfig.get_config_var(name) + if value is None and warn: + logger.debug( + "Config variable '%s' is unset, Python ABI tag may be incorrect", name + ) + return value + + +def _normalize_string(string): + # type: (str) -> str + return string.replace(".", "_").replace("-", "_") + + +def _abi3_applies(python_version): + # type: (PythonVersion) -> bool + """ + Determine if the Python version supports abi3. + + PEP 384 was first implemented in Python 3.2. + """ + return len(python_version) > 1 and tuple(python_version) >= (3, 2) + + +def _cpython_abis(py_version, warn=False): + # type: (PythonVersion, bool) -> List[str] + py_version = tuple(py_version) # To allow for version comparison. + abis = [] + version = _version_nodot(py_version[:2]) + debug = pymalloc = ucs4 = "" + with_debug = _get_config_var("Py_DEBUG", warn) + has_refcount = hasattr(sys, "gettotalrefcount") + # Windows doesn't set Py_DEBUG, so checking for support of debug-compiled + # extension modules is the best option. + # https://github.com/pypa/pip/issues/3383#issuecomment-173267692 + has_ext = "_d.pyd" in EXTENSION_SUFFIXES + if with_debug or (with_debug is None and (has_refcount or has_ext)): + debug = "d" + if py_version < (3, 8): + with_pymalloc = _get_config_var("WITH_PYMALLOC", warn) + if with_pymalloc or with_pymalloc is None: + pymalloc = "m" + if py_version < (3, 3): + unicode_size = _get_config_var("Py_UNICODE_SIZE", warn) + if unicode_size == 4 or ( + unicode_size is None and sys.maxunicode == 0x10FFFF + ): + ucs4 = "u" + elif debug: + # Debug builds can also load "normal" extension modules. + # We can also assume no UCS-4 or pymalloc requirement. + abis.append("cp{version}".format(version=version)) + abis.insert( + 0, + "cp{version}{debug}{pymalloc}{ucs4}".format( + version=version, debug=debug, pymalloc=pymalloc, ucs4=ucs4 + ), + ) + return abis + + +def cpython_tags( + python_version=None, # type: Optional[PythonVersion] + abis=None, # type: Optional[Iterable[str]] + platforms=None, # type: Optional[Iterable[str]] + **kwargs # type: bool +): + # type: (...) -> Iterator[Tag] + """ + Yields the tags for a CPython interpreter. + + The tags consist of: + - cp<python_version>-<abi>-<platform> + - cp<python_version>-abi3-<platform> + - cp<python_version>-none-<platform> + - cp<less than python_version>-abi3-<platform> # Older Python versions down to 3.2. + + If python_version only specifies a major version then user-provided ABIs and + the 'none' ABItag will be used. + + If 'abi3' or 'none' are specified in 'abis' then they will be yielded at + their normal position and not at the beginning. + """ + warn = _warn_keyword_parameter("cpython_tags", kwargs) + if not python_version: + python_version = sys.version_info[:2] + + interpreter = "cp{}".format(_version_nodot(python_version[:2])) + + if abis is None: + if len(python_version) > 1: + abis = _cpython_abis(python_version, warn) + else: + abis = [] + abis = list(abis) + # 'abi3' and 'none' are explicitly handled later. + for explicit_abi in ("abi3", "none"): + try: + abis.remove(explicit_abi) + except ValueError: + pass + + platforms = list(platforms or _platform_tags()) + for abi in abis: + for platform_ in platforms: + yield Tag(interpreter, abi, platform_) + if _abi3_applies(python_version): + for tag in (Tag(interpreter, "abi3", platform_) for platform_ in platforms): + yield tag + for tag in (Tag(interpreter, "none", platform_) for platform_ in platforms): + yield tag + + if _abi3_applies(python_version): + for minor_version in range(python_version[1] - 1, 1, -1): + for platform_ in platforms: + interpreter = "cp{version}".format( + version=_version_nodot((python_version[0], minor_version)) + ) + yield Tag(interpreter, "abi3", platform_) + + +def _generic_abi(): + # type: () -> Iterator[str] + abi = sysconfig.get_config_var("SOABI") + if abi: + yield _normalize_string(abi) + + +def generic_tags( + interpreter=None, # type: Optional[str] + abis=None, # type: Optional[Iterable[str]] + platforms=None, # type: Optional[Iterable[str]] + **kwargs # type: bool +): + # type: (...) -> Iterator[Tag] + """ + Yields the tags for a generic interpreter. + + The tags consist of: + - <interpreter>-<abi>-<platform> + + The "none" ABI will be added if it was not explicitly provided. + """ + warn = _warn_keyword_parameter("generic_tags", kwargs) + if not interpreter: + interp_name = interpreter_name() + interp_version = interpreter_version(warn=warn) + interpreter = "".join([interp_name, interp_version]) + if abis is None: + abis = _generic_abi() + platforms = list(platforms or _platform_tags()) + abis = list(abis) + if "none" not in abis: + abis.append("none") + for abi in abis: + for platform_ in platforms: + yield Tag(interpreter, abi, platform_) + + +def _py_interpreter_range(py_version): + # type: (PythonVersion) -> Iterator[str] + """ + Yields Python versions in descending order. + + After the latest version, the major-only version will be yielded, and then + all previous versions of that major version. + """ + if len(py_version) > 1: + yield "py{version}".format(version=_version_nodot(py_version[:2])) + yield "py{major}".format(major=py_version[0]) + if len(py_version) > 1: + for minor in range(py_version[1] - 1, -1, -1): + yield "py{version}".format(version=_version_nodot((py_version[0], minor))) + + +def compatible_tags( + python_version=None, # type: Optional[PythonVersion] + interpreter=None, # type: Optional[str] + platforms=None, # type: Optional[Iterable[str]] +): + # type: (...) -> Iterator[Tag] + """ + Yields the sequence of tags that are compatible with a specific version of Python. + + The tags consist of: + - py*-none-<platform> + - <interpreter>-none-any # ... if `interpreter` is provided. + - py*-none-any + """ + if not python_version: + python_version = sys.version_info[:2] + platforms = list(platforms or _platform_tags()) + for version in _py_interpreter_range(python_version): + for platform_ in platforms: + yield Tag(version, "none", platform_) + if interpreter: + yield Tag(interpreter, "none", "any") + for version in _py_interpreter_range(python_version): + yield Tag(version, "none", "any") + + +def _mac_arch(arch, is_32bit=_32_BIT_INTERPRETER): + # type: (str, bool) -> str + if not is_32bit: + return arch + + if arch.startswith("ppc"): + return "ppc" + + return "i386" + + +def _mac_binary_formats(version, cpu_arch): + # type: (MacVersion, str) -> List[str] + formats = [cpu_arch] + if cpu_arch == "x86_64": + if version < (10, 4): + return [] + formats.extend(["intel", "fat64", "fat32"]) + + elif cpu_arch == "i386": + if version < (10, 4): + return [] + formats.extend(["intel", "fat32", "fat"]) + + elif cpu_arch == "ppc64": + # TODO: Need to care about 32-bit PPC for ppc64 through 10.2? + if version > (10, 5) or version < (10, 4): + return [] + formats.append("fat64") + + elif cpu_arch == "ppc": + if version > (10, 6): + return [] + formats.extend(["fat32", "fat"]) + + formats.append("universal") + return formats + + +def mac_platforms(version=None, arch=None): + # type: (Optional[MacVersion], Optional[str]) -> Iterator[str] + """ + Yields the platform tags for a macOS system. + + The `version` parameter is a two-item tuple specifying the macOS version to + generate platform tags for. The `arch` parameter is the CPU architecture to + generate platform tags for. Both parameters default to the appropriate value + for the current system. + """ + version_str, _, cpu_arch = platform.mac_ver() # type: ignore + if version is None: + version = cast("MacVersion", tuple(map(int, version_str.split(".")[:2]))) + else: + version = version + if arch is None: + arch = _mac_arch(cpu_arch) + else: + arch = arch + for minor_version in range(version[1], -1, -1): + compat_version = version[0], minor_version + binary_formats = _mac_binary_formats(compat_version, arch) + for binary_format in binary_formats: + yield "macosx_{major}_{minor}_{binary_format}".format( + major=compat_version[0], + minor=compat_version[1], + binary_format=binary_format, + ) + + +# From PEP 513. +def _is_manylinux_compatible(name, glibc_version): + # type: (str, GlibcVersion) -> bool + # Check for presence of _manylinux module. + try: + import _manylinux # noqa + + return bool(getattr(_manylinux, name + "_compatible")) + except (ImportError, AttributeError): + # Fall through to heuristic check below. + pass + + return _have_compatible_glibc(*glibc_version) + + +def _glibc_version_string(): + # type: () -> Optional[str] + # Returns glibc version string, or None if not using glibc. + return _glibc_version_string_confstr() or _glibc_version_string_ctypes() + + +def _glibc_version_string_confstr(): + # type: () -> Optional[str] + """ + Primary implementation of glibc_version_string using os.confstr. + """ + # os.confstr is quite a bit faster than ctypes.DLL. It's also less likely + # to be broken or missing. This strategy is used in the standard library + # platform module. + # https://github.com/python/cpython/blob/fcf1d003bf4f0100c9d0921ff3d70e1127ca1b71/Lib/platform.py#L175-L183 + try: + # os.confstr("CS_GNU_LIBC_VERSION") returns a string like "glibc 2.17". + version_string = os.confstr( # type: ignore[attr-defined] # noqa: F821 + "CS_GNU_LIBC_VERSION" + ) + assert version_string is not None + _, version = version_string.split() # type: Tuple[str, str] + except (AssertionError, AttributeError, OSError, ValueError): + # os.confstr() or CS_GNU_LIBC_VERSION not available (or a bad value)... + return None + return version + + +def _glibc_version_string_ctypes(): + # type: () -> Optional[str] + """ + Fallback implementation of glibc_version_string using ctypes. + """ + try: + import ctypes + except ImportError: + return None + + # ctypes.CDLL(None) internally calls dlopen(NULL), and as the dlopen + # manpage says, "If filename is NULL, then the returned handle is for the + # main program". This way we can let the linker do the work to figure out + # which libc our process is actually using. + # + # Note: typeshed is wrong here so we are ignoring this line. + process_namespace = ctypes.CDLL(None) # type: ignore + try: + gnu_get_libc_version = process_namespace.gnu_get_libc_version + except AttributeError: + # Symbol doesn't exist -> therefore, we are not linked to + # glibc. + return None + + # Call gnu_get_libc_version, which returns a string like "2.5" + gnu_get_libc_version.restype = ctypes.c_char_p + version_str = gnu_get_libc_version() # type: str + # py2 / py3 compatibility: + if not isinstance(version_str, str): + version_str = version_str.decode("ascii") + + return version_str + + +# Separated out from have_compatible_glibc for easier unit testing. +def _check_glibc_version(version_str, required_major, minimum_minor): + # type: (str, int, int) -> bool + # Parse string and check against requested version. + # + # We use a regexp instead of str.split because we want to discard any + # random junk that might come after the minor version -- this might happen + # in patched/forked versions of glibc (e.g. Linaro's version of glibc + # uses version strings like "2.20-2014.11"). See gh-3588. + m = re.match(r"(?P<major>[0-9]+)\.(?P<minor>[0-9]+)", version_str) + if not m: + warnings.warn( + "Expected glibc version with 2 components major.minor," + " got: %s" % version_str, + RuntimeWarning, + ) + return False + return ( + int(m.group("major")) == required_major + and int(m.group("minor")) >= minimum_minor + ) + + +def _have_compatible_glibc(required_major, minimum_minor): + # type: (int, int) -> bool + version_str = _glibc_version_string() + if version_str is None: + return False + return _check_glibc_version(version_str, required_major, minimum_minor) + + +# Python does not provide platform information at sufficient granularity to +# identify the architecture of the running executable in some cases, so we +# determine it dynamically by reading the information from the running +# process. This only applies on Linux, which uses the ELF format. +class _ELFFileHeader(object): + # https://en.wikipedia.org/wiki/Executable_and_Linkable_Format#File_header + class _InvalidELFFileHeader(ValueError): + """ + An invalid ELF file header was found. + """ + + ELF_MAGIC_NUMBER = 0x7F454C46 + ELFCLASS32 = 1 + ELFCLASS64 = 2 + ELFDATA2LSB = 1 + ELFDATA2MSB = 2 + EM_386 = 3 + EM_S390 = 22 + EM_ARM = 40 + EM_X86_64 = 62 + EF_ARM_ABIMASK = 0xFF000000 + EF_ARM_ABI_VER5 = 0x05000000 + EF_ARM_ABI_FLOAT_HARD = 0x00000400 + + def __init__(self, file): + # type: (IO[bytes]) -> None + def unpack(fmt): + # type: (str) -> int + try: + result, = struct.unpack( + fmt, file.read(struct.calcsize(fmt)) + ) # type: (int, ) + except struct.error: + raise _ELFFileHeader._InvalidELFFileHeader() + return result + + self.e_ident_magic = unpack(">I") + if self.e_ident_magic != self.ELF_MAGIC_NUMBER: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_class = unpack("B") + if self.e_ident_class not in {self.ELFCLASS32, self.ELFCLASS64}: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_data = unpack("B") + if self.e_ident_data not in {self.ELFDATA2LSB, self.ELFDATA2MSB}: + raise _ELFFileHeader._InvalidELFFileHeader() + self.e_ident_version = unpack("B") + self.e_ident_osabi = unpack("B") + self.e_ident_abiversion = unpack("B") + self.e_ident_pad = file.read(7) + format_h = "<H" if self.e_ident_data == self.ELFDATA2LSB else ">H" + format_i = "<I" if self.e_ident_data == self.ELFDATA2LSB else ">I" + format_q = "<Q" if self.e_ident_data == self.ELFDATA2LSB else ">Q" + format_p = format_i if self.e_ident_class == self.ELFCLASS32 else format_q + self.e_type = unpack(format_h) + self.e_machine = unpack(format_h) + self.e_version = unpack(format_i) + self.e_entry = unpack(format_p) + self.e_phoff = unpack(format_p) + self.e_shoff = unpack(format_p) + self.e_flags = unpack(format_i) + self.e_ehsize = unpack(format_h) + self.e_phentsize = unpack(format_h) + self.e_phnum = unpack(format_h) + self.e_shentsize = unpack(format_h) + self.e_shnum = unpack(format_h) + self.e_shstrndx = unpack(format_h) + + +def _get_elf_header(): + # type: () -> Optional[_ELFFileHeader] + try: + with open(sys.executable, "rb") as f: + elf_header = _ELFFileHeader(f) + except (IOError, OSError, TypeError, _ELFFileHeader._InvalidELFFileHeader): + return None + return elf_header + + +def _is_linux_armhf(): + # type: () -> bool + # hard-float ABI can be detected from the ELF header of the running + # process + # https://static.docs.arm.com/ihi0044/g/aaelf32.pdf + elf_header = _get_elf_header() + if elf_header is None: + return False + result = elf_header.e_ident_class == elf_header.ELFCLASS32 + result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB + result &= elf_header.e_machine == elf_header.EM_ARM + result &= ( + elf_header.e_flags & elf_header.EF_ARM_ABIMASK + ) == elf_header.EF_ARM_ABI_VER5 + result &= ( + elf_header.e_flags & elf_header.EF_ARM_ABI_FLOAT_HARD + ) == elf_header.EF_ARM_ABI_FLOAT_HARD + return result + + +def _is_linux_i686(): + # type: () -> bool + elf_header = _get_elf_header() + if elf_header is None: + return False + result = elf_header.e_ident_class == elf_header.ELFCLASS32 + result &= elf_header.e_ident_data == elf_header.ELFDATA2LSB + result &= elf_header.e_machine == elf_header.EM_386 + return result + + +def _have_compatible_manylinux_abi(arch): + # type: (str) -> bool + if arch == "armv7l": + return _is_linux_armhf() + if arch == "i686": + return _is_linux_i686() + return True + + +def _linux_platforms(is_32bit=_32_BIT_INTERPRETER): + # type: (bool) -> Iterator[str] + linux = _normalize_string(distutils.util.get_platform()) + if is_32bit: + if linux == "linux_x86_64": + linux = "linux_i686" + elif linux == "linux_aarch64": + linux = "linux_armv7l" + manylinux_support = [] + _, arch = linux.split("_", 1) + if _have_compatible_manylinux_abi(arch): + if arch in {"x86_64", "i686", "aarch64", "armv7l", "ppc64", "ppc64le", "s390x"}: + manylinux_support.append( + ("manylinux2014", (2, 17)) + ) # CentOS 7 w/ glibc 2.17 (PEP 599) + if arch in {"x86_64", "i686"}: + manylinux_support.append( + ("manylinux2010", (2, 12)) + ) # CentOS 6 w/ glibc 2.12 (PEP 571) + manylinux_support.append( + ("manylinux1", (2, 5)) + ) # CentOS 5 w/ glibc 2.5 (PEP 513) + manylinux_support_iter = iter(manylinux_support) + for name, glibc_version in manylinux_support_iter: + if _is_manylinux_compatible(name, glibc_version): + yield linux.replace("linux", name) + break + # Support for a later manylinux implies support for an earlier version. + for name, _ in manylinux_support_iter: + yield linux.replace("linux", name) + yield linux + + +def _generic_platforms(): + # type: () -> Iterator[str] + yield _normalize_string(distutils.util.get_platform()) + + +def _platform_tags(): + # type: () -> Iterator[str] + """ + Provides the platform tags for this installation. + """ + if platform.system() == "Darwin": + return mac_platforms() + elif platform.system() == "Linux": + return _linux_platforms() + else: + return _generic_platforms() + + +def interpreter_name(): + # type: () -> str + """ + Returns the name of the running interpreter. + """ + try: + name = sys.implementation.name # type: ignore + except AttributeError: # pragma: no cover + # Python 2.7 compatibility. + name = platform.python_implementation().lower() + return INTERPRETER_SHORT_NAMES.get(name) or name + + +def interpreter_version(**kwargs): + # type: (bool) -> str + """ + Returns the version of the running interpreter. + """ + warn = _warn_keyword_parameter("interpreter_version", kwargs) + version = _get_config_var("py_version_nodot", warn=warn) + if version: + version = str(version) + else: + version = _version_nodot(sys.version_info[:2]) + return version + + +def _version_nodot(version): + # type: (PythonVersion) -> str + if any(v >= 10 for v in version): + sep = "_" + else: + sep = "" + return sep.join(map(str, version)) + + +def sys_tags(**kwargs): + # type: (bool) -> Iterator[Tag] + """ + Returns the sequence of tag triples for the running interpreter. + + The order of the sequence corresponds to priority order for the + interpreter, from most to least important. + """ + warn = _warn_keyword_parameter("sys_tags", kwargs) + + interp_name = interpreter_name() + if interp_name == "cp": + for tag in cpython_tags(warn=warn): + yield tag + else: + for tag in generic_tags(): + yield tag + + for tag in compatible_tags(): + yield tag diff --git a/pipenv/vendor/packaging/utils.py b/pipenv/vendor/packaging/utils.py index 4b94a82f..44f1bf98 100644 --- a/pipenv/vendor/packaging/utils.py +++ b/pipenv/vendor/packaging/utils.py @@ -5,28 +5,33 @@ from __future__ import absolute_import, division, print_function import re +from ._typing import MYPY_CHECK_RUNNING from .version import InvalidVersion, Version +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import Union _canonicalize_regex = re.compile(r"[-_.]+") def canonicalize_name(name): + # type: (str) -> str # This is taken from PEP 503. return _canonicalize_regex.sub("-", name).lower() -def canonicalize_version(version): +def canonicalize_version(_version): + # type: (str) -> Union[Version, str] """ - This is very similar to Version.__str__, but has one subtle differences + This is very similar to Version.__str__, but has one subtle difference with the way it handles the release segment. """ try: - version = Version(version) + version = Version(_version) except InvalidVersion: # Legacy versions cannot be normalized - return version + return _version parts = [] @@ -36,13 +41,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/vendor/packaging/version.py b/pipenv/vendor/packaging/version.py index 6ed5cbbd..f39a2a12 100644 --- a/pipenv/vendor/packaging/version.py +++ b/pipenv/vendor/packaging/version.py @@ -7,21 +7,46 @@ import collections import itertools import re -from ._structures import Infinity +from ._structures import Infinity, NegativeInfinity +from ._typing import MYPY_CHECK_RUNNING +if MYPY_CHECK_RUNNING: # pragma: no cover + from typing import Callable, Iterator, List, Optional, SupportsInt, Tuple, Union -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] + from ._structures import InfinityType, NegativeInfinityType + + InfiniteTypes = Union[InfinityType, NegativeInfinityType] + PrePostDevType = Union[InfiniteTypes, Tuple[str, int]] + SubLocalType = Union[InfiniteTypes, int, str] + LocalType = Union[ + NegativeInfinityType, + Tuple[ + Union[ + SubLocalType, + Tuple[SubLocalType, str], + Tuple[NegativeInfinityType, SubLocalType], + ], + ..., + ], + ] + CmpKey = Tuple[ + int, Tuple[int, ...], PrePostDevType, PrePostDevType, PrePostDevType, LocalType + ] + LegacyCmpKey = Tuple[int, Tuple[str, ...]] + VersionComparisonMethod = Callable[ + [Union[CmpKey, LegacyCmpKey], Union[CmpKey, LegacyCmpKey]], bool + ] + +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) def parse(version): + # type: (str) -> Union[LegacyVersion, Version] """ Parse the given version string and return either a :class:`Version` object or a :class:`LegacyVersion` object depending on if the given version is @@ -40,29 +65,38 @@ class InvalidVersion(ValueError): class _BaseVersion(object): + _key = None # type: Union[CmpKey, LegacyCmpKey] def __hash__(self): + # type: () -> int return hash(self._key) def __lt__(self, other): + # type: (_BaseVersion) -> bool return self._compare(other, lambda s, o: s < o) def __le__(self, other): + # type: (_BaseVersion) -> bool return self._compare(other, lambda s, o: s <= o) def __eq__(self, other): + # type: (object) -> bool return self._compare(other, lambda s, o: s == o) def __ge__(self, other): + # type: (_BaseVersion) -> bool return self._compare(other, lambda s, o: s >= o) def __gt__(self, other): + # type: (_BaseVersion) -> bool return self._compare(other, lambda s, o: s > o) def __ne__(self, other): + # type: (object) -> bool return self._compare(other, lambda s, o: s != o) def _compare(self, other, method): + # type: (object, VersionComparisonMethod) -> Union[bool, NotImplemented] if not isinstance(other, _BaseVersion): return NotImplemented @@ -70,72 +104,88 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): + # type: (str) -> None self._version = str(version) self._key = _legacy_cmpkey(self._version) def __str__(self): + # type: () -> str return self._version def __repr__(self): + # type: () -> str return "<LegacyVersion({0})>".format(repr(str(self))) @property def public(self): + # type: () -> str return self._version @property def base_version(self): + # type: () -> str return self._version @property def epoch(self): + # type: () -> int return -1 @property def release(self): + # type: () -> None return None @property def pre(self): + # type: () -> None return None @property def post(self): + # type: () -> None return None @property def dev(self): + # type: () -> None return None @property def local(self): + # type: () -> None return None @property def is_prerelease(self): + # type: () -> bool return False @property def is_postrelease(self): + # type: () -> bool return False @property def is_devrelease(self): + # type: () -> bool return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } def _parse_version_parts(s): + # type: (str) -> Iterator[str] for part in _legacy_version_component_re.split(s): part = _legacy_version_replacement_map.get(part, part) @@ -153,6 +203,8 @@ def _parse_version_parts(s): def _legacy_cmpkey(version): + # type: (str) -> LegacyCmpKey + # We hardcode an epoch of -1 here. A PEP 440 version can only have a epoch # greater than or equal to 0. This will effectively put the LegacyVersion, # which uses the defacto standard originally implemented by setuptools, @@ -161,7 +213,7 @@ def _legacy_cmpkey(version): # This scheme is taken from pkg_resources.parse_version setuptools prior to # it's adoption of the packaging library. - parts = [] + parts = [] # type: List[str] for part in _parse_version_parts(version.lower()): if part.startswith("*"): # remove "-" before a prerelease tag @@ -174,9 +226,8 @@ def _legacy_cmpkey(version): parts.pop() parts.append(part) - parts = tuple(parts) - return epoch, parts + return epoch, tuple(parts) # Deliberately not anchored to the start and end of the string, to make it @@ -215,12 +266,11 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): + # type: (str) -> None + # Validate the version and parse it into pieces match = self._regex.search(version) if not match: @@ -230,18 +280,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -256,9 +299,11 @@ class Version(_BaseVersion): ) def __repr__(self): + # type: () -> str return "<Version({0})>".format(repr(str(self))) def __str__(self): + # type: () -> str parts = [] # Epoch @@ -288,26 +333,35 @@ class Version(_BaseVersion): @property def epoch(self): - return self._version.epoch + # type: () -> int + _epoch = self._version.epoch # type: int + return _epoch @property def release(self): - return self._version.release + # type: () -> Tuple[int, ...] + _release = self._version.release # type: Tuple[int, ...] + return _release @property def pre(self): - return self._version.pre + # type: () -> Optional[Tuple[str, int]] + _pre = self._version.pre # type: Optional[Tuple[str, int]] + return _pre @property def post(self): + # type: () -> Optional[Tuple[str, int]] return self._version.post[1] if self._version.post else None @property def dev(self): + # type: () -> Optional[Tuple[str, int]] return self._version.dev[1] if self._version.dev else None @property def local(self): + # type: () -> Optional[str] if self._version.local: return ".".join(str(x) for x in self._version.local) else: @@ -315,10 +369,12 @@ class Version(_BaseVersion): @property def public(self): + # type: () -> str return str(self).split("+", 1)[0] @property def base_version(self): + # type: () -> str parts = [] # Epoch @@ -332,18 +388,41 @@ class Version(_BaseVersion): @property def is_prerelease(self): + # type: () -> bool return self.dev is not None or self.pre is not None @property def is_postrelease(self): + # type: () -> bool return self.post is not None @property def is_devrelease(self): + # type: () -> bool return self.dev is not None + @property + def major(self): + # type: () -> int + return self.release[0] if len(self.release) >= 1 else 0 + + @property + def minor(self): + # type: () -> int + return self.release[1] if len(self.release) >= 2 else 0 + + @property + def micro(self): + # type: () -> int + return self.release[2] if len(self.release) >= 3 else 0 + + +def _parse_letter_version( + letter, # type: str + number, # type: Union[str, bytes, SupportsInt] +): + # type: (...) -> Optional[Tuple[str, int]] -def _parse_letter_version(letter, number): if letter: # We consider there to be an implicit 0 in a pre-release if there is # not a numeral associated with it. @@ -373,11 +452,14 @@ def _parse_letter_version(letter, number): return letter, int(number) + return None + _local_version_separators = re.compile(r"[\._-]") def _parse_local_version(local): + # type: (str) -> Optional[LocalType] """ Takes a string like abc.1.twelve and turns it into ("abc", 1, "twelve"). """ @@ -386,21 +468,26 @@ def _parse_local_version(local): part.lower() if not part.isdigit() else int(part) for part in _local_version_separators.split(local) ) + return None -def _cmpkey(epoch, release, pre, post, dev, local): +def _cmpkey( + epoch, # type: int + release, # type: Tuple[int, ...] + pre, # type: Optional[Tuple[str, int]] + post, # type: Optional[Tuple[str, int]] + dev, # type: Optional[Tuple[str, int]] + local, # type: Optional[Tuple[SubLocalType]] +): + # type: (...) -> CmpKey + # When we compare a release version, we want to compare it with all of the # trailing zeros removed. So we'll use a reverse the list, drop all the now # leading zeros until we come to something non zero, then take the rest # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. - release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + _release = tuple( + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -408,23 +495,31 @@ def _cmpkey(epoch, release, pre, post, dev, local): # if there is not a pre or a post segment. If we have one of those then # the normal sorting rules will handle this case correctly. if pre is None and post is None and dev is not None: - pre = -Infinity + _pre = NegativeInfinity # type: PrePostDevType # Versions without a pre-release (except as noted above) should sort after # those with one. elif pre is None: - pre = Infinity + _pre = Infinity + else: + _pre = pre # Versions without a post segment should sort before those with one. if post is None: - post = -Infinity + _post = NegativeInfinity # type: PrePostDevType + + else: + _post = post # Versions without a development segment should sort after those with one. if dev is None: - dev = Infinity + _dev = Infinity # type: PrePostDevType + + else: + _dev = dev if local is None: # Versions without a local segment should sort before those with one. - local = -Infinity + _local = NegativeInfinity # type: LocalType else: # Versions with a local segment need that segment parsed to implement # the sorting rules in PEP440. @@ -433,9 +528,8 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local + _local = tuple( + (i, "") if isinstance(i, int) else (NegativeInfinity, i) for i in local ) - return epoch, release, pre, post, dev, local + return epoch, _release, _pre, _post, _dev, _local diff --git a/pipenv/vendor/parse.LICENSE b/pipenv/vendor/parse.LICENSE index 3163ad6d..6c73b16c 100644 --- a/pipenv/vendor/parse.LICENSE +++ b/pipenv/vendor/parse.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2012-2013 Richard Jones <richard@python.org> +Copyright (c) 2012-2019 Richard Jones <richard@python.org> Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal @@ -7,8 +7,8 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, @@ -17,5 +17,3 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. - - diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index 7f9f0786..9c8cae70 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -3,7 +3,7 @@ r'''Parse strings using a specification based on the Python format() syntax. ``parse()`` is the opposite of ``format()`` The module is set up to only export ``parse()``, ``search()``, ``findall()``, -and ``with_pattern()`` when ``import *`` is used: +and ``with_pattern()`` when ``import \*`` is used: >>> from parse import * @@ -21,7 +21,7 @@ Or to search a string for some pattern: Or find all the occurrences of some pattern in a string: ->>> ''.join(r.fixed[0] for r in findall(">{}<", "<p>the <b>bold</b> text</p>")) +>>> ''.join(r[0] for r in findall(">{}<", "<p>the <b>bold</b> text</p>")) 'the bold text' If you're going to use the same pattern to match lots of strings you can @@ -78,9 +78,11 @@ Some simple parse() format string examples: {'item': 'hand grenade'} >>> print(r['item']) hand grenade +>>> 'item' in r +True -Dotted names and indexes are possible though the application must make -additional sense of the result: +Note that `in` only works if you have named fields. Dotted names and indexes +are possible though the application must make additional sense of the result: >>> r = parse("Mmm, {food.type}, I love it!", "Mmm, spam, I love it!") >>> print(r) @@ -127,43 +129,45 @@ The differences between `parse()` and `format()` are: In addition some regular expression character group types "D", "w", "W", "s" and "S" are also available. - The "e" and "g" types are case-insensitive so there is not need for - the "E" or "G" types. + the "E" or "G" types. The "e" type handles Fortran formatted numbers (no + leading 0 before the decimal point). ===== =========================================== ======== Type Characters Matched Output ===== =========================================== ======== - w Letters and underscore str - W Non-letter and underscore str - s Whitespace str - S Non-whitespace str - d Digits (effectively integer numbers) int - D Non-digit str - n Numbers with thousands separators (, or .) int - % Percentage (converted to value/100.0) float - f Fixed-point numbers float - F Decimal numbers Decimal - e Floating-point numbers with exponent float +l Letters (ASCII) str +w Letters, numbers and underscore str +W Not letters, numbers and underscore str +s Whitespace str +S Non-whitespace str +d Digits (effectively integer numbers) int +D Non-digit str +n Numbers with thousands separators (, or .) int +% Percentage (converted to value/100.0) float +f Fixed-point numbers float +F Decimal numbers Decimal +e Floating-point numbers with exponent float e.g. 1.1e-10, NAN (all case insensitive) - g General number format (either d, f or e) float - b Binary numbers int - o Octal numbers int - x Hexadecimal numbers (lower and upper case) int - ti ISO 8601 format date/time datetime +g General number format (either d, f or e) float +b Binary numbers int +o Octal numbers int +x Hexadecimal numbers (lower and upper case) int +ti ISO 8601 format date/time datetime e.g. 1972-01-20T10:21:36Z ("T" and "Z" optional) - te RFC2822 e-mail format date/time datetime +te RFC2822 e-mail format date/time datetime e.g. Mon, 20 Jan 1972 10:21:36 +1000 - tg Global (day/month) format date/time datetime +tg Global (day/month) format date/time datetime e.g. 20/1/1972 10:21:36 AM +1:00 - ta US (month/day) format date/time datetime +ta US (month/day) format date/time datetime e.g. 1/20/1972 10:21:36 PM +10:30 - tc ctime() format date/time datetime +tc ctime() format date/time datetime e.g. Sun Sep 16 01:03:52 1973 - th HTTP log format date/time datetime +th HTTP log format date/time datetime e.g. 21/Nov/2011:00:07:11 +0000 - ts Linux system log format date/time datetime +ts Linux system log format date/time datetime e.g. Nov 9 03:37:44 - tt Time time +tt Time time e.g. 10:21:36 PM -5:30 ===== =========================================== ======== @@ -342,6 +346,21 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.15.0 Several fixes for parsing non-base 10 numbers (thanks @vladikcomper) +- 1.14.0 More broad acceptance of Fortran number format (thanks @purpleskyfall) +- 1.13.1 Project metadata correction. +- 1.13.0 Handle Fortran formatted numbers with no leading 0 before decimal + point (thanks @purpleskyfall). + Handle comparison of FixedTzOffset with other types of object. +- 1.12.1 Actually use the `case_sensitive` arg in compile (thanks @jacquev6) +- 1.12.0 Do not assume closing brace when an opening one is found (thanks @mattsep) +- 1.11.1 Revert having unicode char in docstring, it breaks Bamboo builds(?!) +- 1.11.0 Implement `__contains__` for Result instances. +- 1.10.0 Introduce a "letters" matcher, since "w" matches numbers + also. +- 1.9.1 Fix deprecation warnings around backslashes in regex strings + (thanks Mickael Schoentgen). Also fix some documentation formatting + issues. - 1.9.0 We now honor precision and width specifiers when parsing numbers and strings, allowing parsing of concatenated elements of fixed width (thanks Julia Signell) @@ -400,12 +419,12 @@ the pattern, the actual match represents the shortest successful match for and removed the restriction on mixing fixed-position and named fields - 1.0.0 initial release -This code is copyright 2012-2017 Richard Jones <richard@python.org> +This code is copyright 2012-2019 Richard Jones <richard@python.org> See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.9.0' +__version__ = '1.15.0' # yes, I now have two problems import re @@ -421,7 +440,7 @@ log = logging.getLogger(__name__) def with_pattern(pattern, regex_group_count=None): - """Attach a regular expression pattern matcher to a custom type converter + r"""Attach a regular expression pattern matcher to a custom type converter function. This annotates the type converter with the :attr:`pattern` attribute. @@ -449,15 +468,16 @@ def with_pattern(pattern, regex_group_count=None): return decorator -def int_convert(base): +def int_convert(base=None): '''Convert a string to an integer. The string may start with a sign. - It may be of a base other than 10. + It may be of a base other than 2, 8, 10 or 16. - If may start with a base indicator, 0#nnnn, which we assume should - override the specified base. + If base isn't specified, it will be detected automatically based + on a string format. When string starts with a base indicator, 0#nnnn, + it overrides the default base of 10. It may also have other non-numeric characters that we can ignore. ''' @@ -466,19 +486,28 @@ def int_convert(base): def f(string, match, base=base): if string[0] == '-': sign = -1 + number_start = 1 + elif string[0] == '+': + sign = 1 + number_start = 1 else: sign = 1 + number_start = 0 - if string[0] == '0' and len(string) > 2: - if string[1] in 'bB': - base = 2 - elif string[1] in 'oO': - base = 8 - elif string[1] in 'xX': - base = 16 - else: - # just go with the base specifed - pass + # If base wasn't specified, detect it automatically + if base is None: + + # Assume decimal number, unless different base is detected + base = 10 + + # For number formats starting with 0b, 0o, 0x, use corresponding base ... + if string[number_start] == '0' and len(string) - number_start > 2: + if string[number_start+1] in 'bB': + base = 2 + elif string[number_start+1] in 'oO': + base = 8 + elif string[number_start+1] in 'xX': + base = 16 chars = CHARS[:base] string = re.sub('[^%s]' % chars, '', string.lower()) @@ -513,6 +542,8 @@ class FixedTzOffset(tzinfo): return self.ZERO def __eq__(self, other): + if not isinstance(other, FixedTzOffset): + return False return self._name == other._name and self._offset == other._offset @@ -530,9 +561,9 @@ MONTHS_MAP = dict( Nov=11, November=11, Dec=12, December=12 ) -DAYS_PAT = '(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' -MONTHS_PAT = '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' -ALL_MONTHS_PAT = '(%s)' % '|'.join(MONTHS_MAP) +DAYS_PAT = r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' +MONTHS_PAT = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' +ALL_MONTHS_PAT = r'(%s)' % '|'.join(MONTHS_MAP) TIME_PAT = r'(\d{1,2}:\d{1,2}(:\d{1,2}(\.\d+)?)?)' AM_PAT = r'(\s+[AP]M)' TZ_PAT = r'(\s+[-+]\d\d?:?\d\d)' @@ -550,11 +581,11 @@ def date_convert(string, match, ymd=None, mdy=None, dmy=None, m=groups[mm] d=groups[dd] elif ymd is not None: - y, m, d = re.split('[-/\s]', groups[ymd]) + y, m, d = re.split(r'[-/\s]', groups[ymd]) elif mdy is not None: - m, d, y = re.split('[-/\s]', groups[mdy]) + m, d, y = re.split(r'[-/\s]', groups[mdy]) elif dmy is not None: - d, m, y = re.split('[-/\s]', groups[dmy]) + d, m, y = re.split(r'[-/\s]', groups[dmy]) elif d_m_y is not None: d, m, y = d_m_y d = groups[d] @@ -636,10 +667,10 @@ class RepeatedNameError(ValueError): # note: {} are handled separately # note: I don't use r'' here because Sublime Text 2 syntax highlight has a fit -REGEX_SAFETY = re.compile('([?\\\\.[\]()*+\^$!\|])') +REGEX_SAFETY = re.compile(r'([?\\\\.[\]()*+\^$!\|])') # allowed field types -ALLOWED_TYPES = set(list('nbox%fFegwWdDsS') + +ALLOWED_TYPES = set(list('nbox%fFegwWdDsSl') + ['t' + c for c in 'ieahgcts']) @@ -745,7 +776,7 @@ class Parser(object): @property def _match_re(self): if self.__match_re is None: - expression = '^%s$' % self._expression + expression = r'^%s$' % self._expression try: self.__match_re = re.compile(expression, self._re_flags) except AssertionError: @@ -875,7 +906,7 @@ class Parser(object): e.append(r'\{') elif part == '}}': e.append(r'\}') - elif part[0] == '{': + elif part[0] == '{' and part[-1] == '}': # this will be a braces-delimited field to handle e.append(self._handle_field(part)) else: @@ -923,16 +954,16 @@ class Parser(object): name, self._name_types[name])) group = self._name_to_group_map[name] # match previously-seen value - return '(?P=%s)' % group + return r'(?P=%s)' % group else: group = self._to_group_name(name) self._name_types[name] = format self._named_fields.append(group) # this will become a group, which must not contain dots - wrap = '(?P<%s>%%s)' % group + wrap = r'(?P<%s>%%s)' % group else: self._fixed_fields.append(self._group_index) - wrap = '(%s)' + wrap = r'(%s)' if ':' in field: format = field[1:] group = self._group_index @@ -940,14 +971,14 @@ class Parser(object): # simplest case: no type specifier ({} or {name}) if not format: self._group_index += 1 - return wrap % '.+?' + return wrap % r'.+?' # decode the format specification format = extract_format(format, self._extra_types) # figure type conversions, if any type = format['type'] - is_numeric = type and type in 'n%fegdobh' + is_numeric = type and type in 'n%fegdobx' if type in self._extra_types: type_converter = self._extra_types[type] s = getattr(type_converter, 'pattern', r'.+?') @@ -960,19 +991,19 @@ class Parser(object): return type_converter(string) self._type_conversions[group] = f elif type == 'n': - s = '\d{1,3}([,.]\d{3})*' + s = r'\d{1,3}([,.]\d{3})*' self._group_index += 1 self._type_conversions[group] = int_convert(10) elif type == 'b': - s = '(0[bB])?[01]+' + s = r'(0[bB])?[01]+' self._type_conversions[group] = int_convert(2) self._group_index += 1 elif type == 'o': - s = '(0[oO])?[0-7]+' + s = r'(0[oO])?[0-7]+' self._type_conversions[group] = int_convert(8) self._group_index += 1 elif type == 'x': - s = '(0[xX])?[0-9a-fA-F]+' + s = r'(0[xX])?[0-9a-fA-F]+' self._type_conversions[group] = int_convert(16) self._group_index += 1 elif type == '%': @@ -980,13 +1011,13 @@ class Parser(object): self._group_index += 1 self._type_conversions[group] = percentage elif type == 'f': - s = r'\d+\.\d+' + s = r'\d*\.\d+' self._type_conversions[group] = lambda s, m: float(s) elif type == 'F': - s = r'\d+\.\d+' + s = r'\d*\.\d+' self._type_conversions[group] = lambda s, m: Decimal(s) elif type == 'e': - s = r'\d+\.\d+[eE][-+]?\d+|nan|NAN|[-+]?inf|[-+]?INF' + s = r'\d*\.\d+[eE][-+]?\d+|nan|NAN|[-+]?inf|[-+]?INF' self._type_conversions[group] = lambda s, m: float(s) elif type == 'g': s = r'\d+(\.\d+)?([eE][-+]?\d+)?|nan|NAN|[-+]?inf|[-+]?INF' @@ -994,11 +1025,11 @@ class Parser(object): self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': if format.get('width'): - width = '{1,%s}' % int(format['width']) + width = r'{1,%s}' % int(format['width']) else: width = '+' - s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) - self._type_conversions[group] = int_convert(10) + s = r'\d{w}|[-+ ]?0[xX][0-9a-fA-F]{w}|[-+ ]?0[bB][01]{w}|[-+ ]?0[oO][0-7]{w}'.format(w=width) + self._type_conversions[group] = int_convert() # do not specify numeber base, determine it automatically elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ TIME_PAT @@ -1055,18 +1086,19 @@ class Parser(object): self._type_conversions[group] = partial(date_convert, mm=n+1, dd=n+3, hms=n + 5) self._group_index += 5 - + elif type == 'l': + s = r'[A-Za-z]+' elif type: s = r'\%s+' % type elif format.get('precision'): if format.get('width'): - s = '.{%s,%s}?' % (format['width'], format['precision']) + s = r'.{%s,%s}?' % (format['width'], format['precision']) else: - s = '.{1,%s}?' % format['precision'] + s = r'.{1,%s}?' % format['precision'] elif format.get('width'): - s = '.{%s,}?' % format['width'] + s = r'.{%s,}?' % format['width'] else: - s = '.+?' + s = r'.+?' align = format['align'] fill = format['fill'] @@ -1079,7 +1111,7 @@ class Parser(object): # configurable fill defaulting to "0" if not fill: fill = '0' - s = '%s*' % fill + s + s = r'%s*' % fill + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s @@ -1101,7 +1133,7 @@ class Parser(object): if not align: align = '>' - if fill in '.\+?*[](){}^$': + if fill in r'.\+?*[](){}^$': fill = '\\' + fill # align "=" has been handled @@ -1118,8 +1150,11 @@ class Parser(object): class Result(object): '''The result of a parse() or search(). - Fixed results may be looked up using result[index]. Named results may be - looked up using result['name']. + Fixed results may be looked up using `result[index]`. + + Named results may be looked up using `result['name']`. + + Named results may be tested for existence using `'name' in result`. ''' def __init__(self, fixed, named, spans): self.fixed = fixed @@ -1135,6 +1170,9 @@ class Result(object): return '<%s %r %r>' % (self.__class__.__name__, self.fixed, self.named) + def __contains__(self, name): + return name in self.named + class Match(object): '''The result of a parse() or search() if no results are generated. @@ -1292,10 +1330,10 @@ def compile(format, extra_types=None, case_sensitive=False): Returns a Parser instance. ''' - return Parser(format, extra_types=extra_types) + return Parser(format, extra_types=extra_types, case_sensitive=case_sensitive) -# Copyright (c) 2012-2013 Richard Jones <richard@python.org> +# Copyright (c) 2012-2020 Richard Jones <richard@python.org> # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/passa/actions/clean.py b/pipenv/vendor/passa/actions/clean.py index 3570e4db..5c19b31c 100644 --- a/pipenv/vendor/passa/actions/clean.py +++ b/pipenv/vendor/passa/actions/clean.py @@ -3,11 +3,11 @@ from __future__ import absolute_import, print_function, unicode_literals -def clean(project, dev=False): +def clean(project, default=True, dev=False): from passa.models.synchronizers import Cleaner from passa.operations.sync import clean - cleaner = Cleaner(project, default=True, develop=dev) + cleaner = Cleaner(project, default=default, develop=dev) success = clean(cleaner) if not success: diff --git a/pipenv/vendor/passa/cli/add.py b/pipenv/vendor/passa/cli/add.py index d5596cde..077149f0 100644 --- a/pipenv/vendor/passa/cli/add.py +++ b/pipenv/vendor/passa/cli/add.py @@ -20,7 +20,8 @@ class Command(BaseCommand): packages=options.packages, editables=options.editables, project=options.project, - dev=options.dev + dev=options.dev, + sync=options.sync ) diff --git a/pipenv/vendor/passa/cli/options.py b/pipenv/vendor/passa/cli/options.py index f8ba1fe7..f20b612a 100644 --- a/pipenv/vendor/passa/cli/options.py +++ b/pipenv/vendor/passa/cli/options.py @@ -20,13 +20,13 @@ class Project(passa.models.projects.Project): pipfile = root.joinpath("Pipfile") if not pipfile.is_file(): raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), + "project", "{0!r} is not a Pipfile project".format(root), ) try: super(Project, self).__init__(root.as_posix(), *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), + "project", "failed to parse Pipfile: {0!r}".format(str(e)), ) def __name__(self): diff --git a/pipenv/vendor/pathlib2/__init__.py b/pipenv/vendor/pathlib2/__init__.py index 2eb41e30..d5a47a66 100644 --- a/pipenv/vendor/pathlib2/__init__.py +++ b/pipenv/vendor/pathlib2/__init__.py @@ -12,12 +12,18 @@ import posixpath import re import six import sys -from collections import Sequence -from errno import EINVAL, ENOENT, ENOTDIR, EEXIST, EPERM, EACCES -from operator import attrgetter +from errno import EINVAL, ENOENT, ENOTDIR, EBADF +from errno import EEXIST, EPERM, EACCES +from operator import attrgetter from stat import ( S_ISDIR, S_ISLNK, S_ISREG, S_ISSOCK, S_ISBLK, S_ISCHR, S_ISFIFO) + +try: + from collections.abc import Sequence +except ImportError: + from collections import Sequence + try: from urllib import quote as urlquote_from_bytes except ImportError: @@ -54,6 +60,18 @@ __all__ = [ # Internals # +# EBADF - guard agains macOS `stat` throwing EBADF +_IGNORED_ERROS = (ENOENT, ENOTDIR, EBADF) + +_IGNORED_WINERRORS = ( + 21, # ERROR_NOT_READY - drive exists but is not accessible +) + + +def _ignore_error(exception): + return (getattr(exception, 'errno', None) in _IGNORED_ERROS or + getattr(exception, 'winerror', None) in _IGNORED_WINERRORS) + def _py2_fsencode(parts): # py2 => minimal unicode support @@ -90,9 +108,25 @@ def _try_except_filenotfounderror(try_func, except_func): try_func() except FileNotFoundError as exc: except_func(exc) + elif os.name != 'nt': + try: + try_func() + except EnvironmentError as exc: + if exc.errno != ENOENT: + raise + else: + except_func(exc) else: try: try_func() + except WindowsError as exc: + # errno contains winerror + # 2 = file not found + # 3 = path not found + if exc.errno not in (2, 3): + raise + else: + except_func(exc) except EnvironmentError as exc: if exc.errno != ENOENT: raise @@ -336,16 +370,26 @@ class _WindowsFlavour(_Flavour): else: # End of the path after the first one not found tail_parts = [] + + def _try_func(): + result[0] = self._ext_to_normal(_getfinalpathname(s)) + # if there was no exception, set flag to 0 + result[1] = 0 + + def _exc_func(exc): + pass + while True: - try: - s = self._ext_to_normal(_getfinalpathname(s)) - except FileNotFoundError: + result = [None, 1] + _try_except_filenotfounderror(_try_func, _exc_func) + if result[1] == 1: # file not found exception raised previous_s = s s, tail = os.path.split(s) tail_parts.append(tail) if previous_s == s: return path else: + s = result[0] return os.path.join(s, *reversed(tail_parts)) # Means fallback on absolute return None @@ -715,7 +759,13 @@ class _RecursiveWildcardSelector(_Selector): def try_iter(): entries = list(scandir(parent_path)) for entry in entries: - if entry.is_dir() and not entry.is_symlink(): + entry_is_dir = False + try: + entry_is_dir = entry.is_dir() + except OSError as e: + if not _ignore_error(e): + raise + if entry_is_dir and not entry.is_symlink(): path = parent_path._make_child_relpath(entry.name) for p in self._iterate_directories(path, is_dir, scandir): yield p @@ -832,7 +882,11 @@ class PurePath(object): # also handle unicode for PY2 (six.text_type = unicode) elif six.PY2 and isinstance(a, six.text_type): # cast to str using filesystem encoding - parts.append(a.encode(sys.getfilesystemencoding())) + # note: in rare circumstances, on Python < 3.2, + # getfilesystemencoding can return None, in that + # case fall back to ascii + parts.append(a.encode( + sys.getfilesystemencoding() or "ascii")) else: raise TypeError( "argument should be a str object or an os.PathLike " @@ -1026,8 +1080,9 @@ class PurePath(object): self._parts[:-1] + [name]) def with_suffix(self, suffix): - """Return a new path with the file suffix changed (or added, if - none). + """Return a new path with the file suffix changed. If the path + has no suffix, add given suffix. If the given suffix is an empty + string, remove the suffix from the path. """ # XXX if suffix is None, should the current suffix be removed? f = self._flavour @@ -1173,6 +1228,11 @@ class PurePosixPath(PurePath): class PureWindowsPath(PurePath): + """PurePath subclass for Windows systems. + + On a Windows system, instantiating a PurePath should return this object. + However, you can also instantiate it directly on any system. + """ _flavour = _windows_flavour __slots__ = () @@ -1181,6 +1241,14 @@ class PureWindowsPath(PurePath): class Path(PurePath): + """PurePath subclass that can make system calls. + + Path represents a filesystem path but unlike PurePath, also offers + methods to do system calls on path objects. Depending on your system, + instantiating a Path will return either a PosixPath or a WindowsPath + object. You can also instantiate a PosixPath or WindowsPath directly, + but cannot instantiate a WindowsPath on a POSIX system or vice versa. + """ __slots__ = ( '_accessor', '_closed', @@ -1286,7 +1354,7 @@ class Path(PurePath): def glob(self, pattern): """Iterate over this subtree and yield all existing files (of any - kind, including directories) matching the given pattern. + kind, including directories) matching the given relative pattern. """ if not pattern: raise ValueError("Unacceptable pattern: {0!r}".format(pattern)) @@ -1300,7 +1368,8 @@ class Path(PurePath): def rglob(self, pattern): """Recursively yield all existing files (of any kind, including - directories) matching the given pattern, anywhere in this subtree. + directories) matching the given relative pattern, anywhere in + this subtree. """ pattern = self._flavour.casefold(pattern) drv, root, pattern_parts = self._flavour.parse_parts((pattern,)) @@ -1339,9 +1408,20 @@ class Path(PurePath): s = self._flavour.resolve(self, strict=strict) if s is None: # No symlink resolution => for consistency, raise an error if - # the path doesn't exist or is forbidden - self.stat() + # the path is forbidden + # but not raise error if file does not exist (see issue #54). + + def _try_func(): + self.stat() + + def _exc_func(exc): + pass + + _try_except_filenotfounderror(_try_func, _exc_func) s = str(self.absolute()) + else: + # ensure s is a string (normpath requires this on older python) + s = str(s) # Now we have no symlinks in the path, it's safe to normalize it. normed = self._flavour.pathmod.normpath(s) obj = self._from_parts((normed,), init=False) @@ -1463,6 +1543,8 @@ class Path(PurePath): try: _try_except_filenotfounderror(_try_func, _exc_func) except OSError: + # Cannot rely on checking for EEXIST, since the operating system + # could give priority to other errors like EACCES or EROFS if not exist_ok or not self.is_dir(): raise @@ -1548,9 +1630,12 @@ class Path(PurePath): try: self.stat() except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise return False + except ValueError: + # Non-encodable path + return False return True def is_dir(self): @@ -1560,11 +1645,14 @@ class Path(PurePath): try: return S_ISDIR(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False def is_file(self): """ @@ -1574,11 +1662,35 @@ class Path(PurePath): try: return S_ISREG(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False + + def is_mount(self): + """ + Check if this path is a POSIX mount point + """ + # Need to exist and be a dir + if not self.exists() or not self.is_dir(): + return False + + parent = Path(self.parent) + try: + parent_dev = parent.stat().st_dev + except OSError: + return False + + dev = self.stat().st_dev + if dev != parent_dev: + return True + ino = self.stat().st_ino + parent_ino = parent.stat().st_ino + return ino == parent_ino def is_symlink(self): """ @@ -1587,10 +1699,13 @@ class Path(PurePath): try: return S_ISLNK(self.lstat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist return False + except ValueError: + # Non-encodable path + return False def is_block_device(self): """ @@ -1599,11 +1714,14 @@ class Path(PurePath): try: return S_ISBLK(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False def is_char_device(self): """ @@ -1612,11 +1730,14 @@ class Path(PurePath): try: return S_ISCHR(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False def is_fifo(self): """ @@ -1625,11 +1746,14 @@ class Path(PurePath): try: return S_ISFIFO(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False def is_socket(self): """ @@ -1638,11 +1762,14 @@ class Path(PurePath): try: return S_ISSOCK(self.stat().st_mode) except OSError as e: - if e.errno not in (ENOENT, ENOTDIR): + if not _ignore_error(e): raise # Path doesn't exist or is a broken symlink # (see https://bitbucket.org/pitrou/pathlib/issue/12/) return False + except ValueError: + # Non-encodable path + return False def expanduser(self): """ Return a new path with expanded ~ and ~user constructs @@ -1657,10 +1784,18 @@ class Path(PurePath): class PosixPath(Path, PurePosixPath): + """Path subclass for non-Windows systems. + + On a POSIX system, instantiating a Path should return this object. + """ __slots__ = () class WindowsPath(Path, PureWindowsPath): + """Path subclass for Windows systems. + + On a Windows system, instantiating a Path should return this object. + """ __slots__ = () def owner(self): @@ -1668,3 +1803,7 @@ class WindowsPath(Path, PureWindowsPath): def group(self): raise NotImplementedError("Path.group() is unsupported on this system") + + def is_mount(self): + raise NotImplementedError( + "Path.is_mount() is unsupported on this system") diff --git a/pipenv/patched/notpip/_vendor/lockfile/LICENSE b/pipenv/vendor/pep517/LICENSE similarity index 60% rename from pipenv/patched/notpip/_vendor/lockfile/LICENSE rename to pipenv/vendor/pep517/LICENSE index 610c0793..b0ae9dbc 100644 --- a/pipenv/patched/notpip/_vendor/lockfile/LICENSE +++ b/pipenv/vendor/pep517/LICENSE @@ -1,12 +1,12 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php +The MIT License (MIT) -Copyright (c) 2007 Skip Montanaro. +Copyright (c) 2017 Thomas Kluyver Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to -deal in the Software without restriction, including without limitation the -rights to use, copy, modify, merge, publish, distribute, sublicense, and/or -sell copies of the Software, and to permit persons to whom the Software is +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in @@ -16,6 +16,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING -FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS -IN THE SOFTWARE. +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pep517/__init__.py b/pipenv/vendor/pep517/__init__.py new file mode 100644 index 00000000..7355b68a --- /dev/null +++ b/pipenv/vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.8.2' diff --git a/pipenv/vendor/pep517/_in_process.py b/pipenv/vendor/pep517/_in_process.py new file mode 100644 index 00000000..a536b03e --- /dev/null +++ b/pipenv/vendor/pep517/_in_process.py @@ -0,0 +1,280 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variables: + PEP517_BUILD_BACKEND=entry.point:spec + PEP517_BACKEND_PATH=paths (separated with os.pathsep) +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import json +import os +import os.path +from os.path import join as pjoin +import re +import shutil +import sys +import traceback + +# This file is run as a script, and `import compat` is not zip-safe, so we +# include write_json() and read_json() from compat.py. +# +# Handle reading and writing JSON in UTF-8, on Python 3 and 2. + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) + + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + def __init__(self, traceback): + self.traceback = traceback + + +class BackendInvalid(Exception): + """Raised if the backend is invalid""" + def __init__(self, message): + self.message = message + + +class HookMissing(Exception): + """Raised if a hook is missing and we are not executing the fallback""" + + +def contained_in(filename, directory): + """Test if a file is located within the given directory.""" + filename = os.path.normcase(os.path.abspath(filename)) + directory = os.path.normcase(os.path.abspath(directory)) + return os.path.commonprefix([filename, directory]) == directory + + +def _build_backend(): + """Find and load the build backend""" + # Add in-tree backend directories to the front of sys.path. + backend_path = os.environ.get('PEP517_BACKEND_PATH') + if backend_path: + extra_pathitems = backend_path.split(os.pathsep) + sys.path[:0] = extra_pathitems + + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable(traceback.format_exc()) + + if backend_path: + if not any( + contained_in(obj.__file__, path) + for path in extra_pathitems + ): + raise BackendInvalid("Backend was not loaded from backend-path") + + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + + +def prepare_metadata_for_build_wheel( + metadata_directory, config_settings, _allow_fallback): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined, + unless _allow_fallback is False in which case HookMissing is raised. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + if not _allow_fallback: + raise HookMissing() + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + def __init__(self, traceback): + self.traceback = traceback + + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation(traceback.format_exc()) + + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except BackendUnavailable as e: + json_out['no_backend'] = True + json_out['traceback'] = e.traceback + except BackendInvalid as e: + json_out['backend_invalid'] = True + json_out['backend_error'] = e.message + except GotUnsupportedOperation as e: + json_out['unsupported'] = True + json_out['traceback'] = e.traceback + except HookMissing: + json_out['hook_missing'] = True + + write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/build.py b/pipenv/vendor/pep517/build.py new file mode 100644 index 00000000..7618c78c --- /dev/null +++ b/pipenv/vendor/pep517/build.py @@ -0,0 +1,124 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import toml +import shutil + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller +from .dirtools import tempdir, mkdir_p +from .compat import FileNotFoundError + +log = logging.getLogger(__name__) + + +def validate_system(system): + """ + Ensure build system has the requisite fields. + """ + required = {'requires', 'build-backend'} + if not (required <= set(system)): + message = "Missing required fields: {missing}".format( + missing=required-set(system), + ) + raise ValueError(message) + + +def load_system(source_dir): + """ + Load the build system from a source dir (pyproject.toml). + """ + pyproject = os.path.join(source_dir, 'pyproject.toml') + with open(pyproject) as f: + pyproject_data = toml.load(f) + return pyproject_data['build-system'] + + +def compat_system(source_dir): + """ + Given a source dir, attempt to get a build system backend + and requirements from pyproject.toml. Fallback to + setuptools but only if the file was not found or a build + system was not indicated. + """ + try: + system = load_system(source_dir) + except (FileNotFoundError, KeyError): + system = {} + system.setdefault( + 'build-backend', + 'setuptools.build_meta:__legacy__', + ) + system.setdefault('requires', ['setuptools', 'wheel']) + return system + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def build(source_dir, dist, dest=None, system=None): + system = system or load_system(source_dir) + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + validate_system(system) + hooks = Pep517HookCaller( + source_dir, system['build-backend'], system.get('backend-path') + ) + + with BuildEnvironment() as env: + env.pip_install(system['requires']) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/vendor/pep517/check.py b/pipenv/vendor/pep517/check.py new file mode 100644 index 00000000..9e0c0682 --- /dev/null +++ b/pipenv/vendor/pep517/check.py @@ -0,0 +1,203 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from toml import TomlDecodeError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +def check_build_sdist(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except Exception: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error( + "Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check_build_wheel(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except Exception: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + backend_path = buildsys.get('backend-path') + log.info('Loaded pyproject.toml') + except (TomlDecodeError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend, backend_path) + + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument( + 'source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} + + +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/colorlog.py b/pipenv/vendor/pep517/colorlog.py new file mode 100644 index 00000000..69c8a59d --- /dev/null +++ b/pipenv/vendor/pep517/colorlog.py @@ -0,0 +1,115 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is + # worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + mlen = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/vendor/pep517/compat.py b/pipenv/vendor/pep517/compat.py new file mode 100644 index 00000000..8432acb7 --- /dev/null +++ b/pipenv/vendor/pep517/compat.py @@ -0,0 +1,34 @@ +"""Python 2/3 compatibility""" +import json +import sys + + +# Handle reading and writing JSON in UTF-8, on Python 3 and 2. + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) + + +# FileNotFoundError + +try: + FileNotFoundError = FileNotFoundError +except NameError: + FileNotFoundError = IOError diff --git a/pipenv/vendor/pep517/dirtools.py b/pipenv/vendor/pep517/dirtools.py new file mode 100644 index 00000000..58c6ca0c --- /dev/null +++ b/pipenv/vendor/pep517/dirtools.py @@ -0,0 +1,44 @@ +import os +import io +import contextlib +import tempfile +import shutil +import errno +import zipfile + + +@contextlib.contextmanager +def tempdir(): + """Create a temporary directory in a context manager.""" + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def dir_to_zipfile(root): + """Construct an in-memory zip file for a directory.""" + buffer = io.BytesIO() + zip_file = zipfile.ZipFile(buffer, 'w') + for root, dirs, files in os.walk(root): + for path in dirs: + fs_path = os.path.join(root, path) + rel_path = os.path.relpath(fs_path, root) + zip_file.writestr(rel_path + '/', '') + for path in files: + fs_path = os.path.join(root, path) + rel_path = os.path.relpath(fs_path, root) + zip_file.write(fs_path, rel_path) + return zip_file diff --git a/pipenv/vendor/pep517/envbuild.py b/pipenv/vendor/pep517/envbuild.py new file mode 100644 index 00000000..cacd2b12 --- /dev/null +++ b/pipenv/vendor/pep517/envbuild.py @@ -0,0 +1,167 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +import toml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller, LoggerWrapper + +log = logging.getLogger(__name__) + + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = toml.load(f) + buildsys = pyproject_data['build-system'] + return ( + buildsys['requires'], + buildsys['build-backend'], + buildsys.get('backend-path'), + ) + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + cmd = [ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs) + check_call( + cmd, + stdout=LoggerWrapper(log, logging.INFO), + stderr=LoggerWrapper(log, logging.ERROR), + ) + + def __exit__(self, exc_type, exc_val, exc_tb): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend, backend_path = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend, backend_path) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend, backend_path = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend, backend_path) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/vendor/pep517/meta.py b/pipenv/vendor/pep517/meta.py new file mode 100644 index 00000000..d525de5c --- /dev/null +++ b/pipenv/vendor/pep517/meta.py @@ -0,0 +1,92 @@ +"""Build metadata for a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import shutil +import functools + +try: + import importlib.metadata as imp_meta +except ImportError: + import importlib_metadata as imp_meta + +try: + from zipfile import Path +except ImportError: + from zipp import Path + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller, quiet_subprocess_runner +from .dirtools import tempdir, mkdir_p, dir_to_zipfile +from .build import validate_system, load_system, compat_system + +log = logging.getLogger(__name__) + + +def _prep_meta(hooks, env, dest): + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build metadata in %s', td) + filename = hooks.prepare_metadata_for_build_wheel(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def build(source_dir='.', dest=None, system=None): + system = system or load_system(source_dir) + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + validate_system(system) + hooks = Pep517HookCaller( + source_dir, system['build-backend'], system.get('backend-path') + ) + + with hooks.subprocess_runner(quiet_subprocess_runner): + with BuildEnvironment() as env: + env.pip_install(system['requires']) + _prep_meta(hooks, env, dest) + + +def build_as_zip(builder=build): + with tempdir() as out_dir: + builder(dest=out_dir) + return dir_to_zipfile(out_dir) + + +def load(root): + """ + Given a source directory (root) of a package, + return an importlib.metadata.Distribution object + with metadata build from that package. + """ + root = os.path.expanduser(root) + system = compat_system(root) + builder = functools.partial(build, source_dir=root, system=system) + path = Path(build_as_zip(builder)) + return imp_meta.PathDistribution(path) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(): + args = parser.parse_args() + build(args.source_dir, args.out_dir) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/wrappers.py b/pipenv/vendor/pep517/wrappers.py new file mode 100644 index 00000000..00a3d1a7 --- /dev/null +++ b/pipenv/vendor/pep517/wrappers.py @@ -0,0 +1,308 @@ +import threading +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call, check_output, STDOUT +import sys +from tempfile import mkdtemp + +from . import compat + + +try: + import importlib.resources as resources + + def _in_proc_script_path(): + return resources.path(__package__, '_in_process.py') +except ImportError: + @contextmanager + def _in_proc_script_path(): + yield pjoin(dirname(abspath(__file__)), '_in_process.py') + + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + def __init__(self, traceback): + self.traceback = traceback + + +class BackendInvalid(Exception): + """Will be raised if the backend is invalid.""" + def __init__(self, backend_name, backend_path, message): + self.backend_name = backend_name + self.backend_path = backend_path + self.message = message + + +class HookMissing(Exception): + """Will be raised on missing hooks.""" + def __init__(self, hook_name): + super(HookMissing, self).__init__(hook_name) + self.hook_name = hook_name + + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + def __init__(self, traceback): + self.traceback = traceback + + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + +def quiet_subprocess_runner(cmd, cwd=None, extra_environ=None): + """A method of calling the wrapper subprocess while suppressing output.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_output(cmd, cwd=cwd, env=env, stderr=STDOUT) + + +def norm_and_check(source_tree, requested): + """Normalise and check a backend path. + + Ensure that the requested backend path is specified as a relative path, + and resolves to a location under the given source tree. + + Return an absolute version of the requested path. + """ + if os.path.isabs(requested): + raise ValueError("paths must be relative") + + abs_source = os.path.abspath(source_tree) + abs_requested = os.path.normpath(os.path.join(abs_source, requested)) + # We have to use commonprefix for Python 2.7 compatibility. So we + # normalise case to avoid problems because commonprefix is a character + # based comparison :-( + norm_source = os.path.normcase(abs_source) + norm_requested = os.path.normcase(abs_requested) + if os.path.commonprefix([norm_source, norm_requested]) != norm_source: + raise ValueError("paths must be inside source tree") + + return abs_requested + + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + build_backend : The build backend spec, as per PEP 517, from + pyproject.toml. + backend_path : The backend path, as per PEP 517, from pyproject.toml. + runner : A callable that invokes the wrapper subprocess. + + The 'runner', if provided, must expect the following: + cmd : a list of strings representing the command and arguments to + execute, as would be passed to e.g. 'subprocess.check_call'. + cwd : a string representing the working directory that must be + used for the subprocess. Corresponds to the provided source_dir. + extra_environ : a dict mapping environment variable names to values + which must be set for the subprocess execution. + """ + def __init__( + self, + source_dir, + build_backend, + backend_path=None, + runner=None, + ): + if runner is None: + runner = default_subprocess_runner + + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + if backend_path: + backend_path = [ + norm_and_check(self.source_dir, p) for p in backend_path + ] + self.backend_path = backend_path + self._subprocess_runner = runner + + @contextmanager + def subprocess_runner(self, runner): + """A context manager for temporarily overriding the default subprocess + runner. + """ + prev = self._subprocess_runner + self._subprocess_runner = runner + try: + yield + finally: + self._subprocess_runner = prev + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None, + _allow_fallback=True): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that (unless _allow_fallback is + False). + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + '_allow_fallback': _allow_fallback, + }) + + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + def _call_hook(self, hook_name, kwargs): + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + # For backend_path, we use sys.getfilesystemencoding. + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + extra_environ = {'PEP517_BUILD_BACKEND': build_backend} + + if self.backend_path: + backend_path = os.pathsep.join(self.backend_path) + if sys.version_info[0] == 2: + backend_path = backend_path.encode(sys.getfilesystemencoding()) + extra_environ['PEP517_BACKEND_PATH'] = backend_path + + with tempdir() as td: + hook_input = {'kwargs': kwargs} + compat.write_json(hook_input, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + with _in_proc_script_path() as script: + self._subprocess_runner( + [sys.executable, str(script), hook_name, td], + cwd=self.source_dir, + extra_environ=extra_environ + ) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation(data.get('traceback', '')) + if data.get('no_backend'): + raise BackendUnavailable(data.get('traceback', '')) + if data.get('backend_invalid'): + raise BackendInvalid( + backend_name=self.build_backend, + backend_path=self.backend_path, + message=data.get('backend_error', '') + ) + if data.get('hook_missing'): + raise HookMissing(hook_name) + return data['return_val'] + + +class LoggerWrapper(threading.Thread): + """ + Read messages from a pipe and redirect them + to a logger (see python's logging module). + """ + + def __init__(self, logger, level): + threading.Thread.__init__(self) + self.daemon = True + + self.logger = logger + self.level = level + + # create the pipe and reader + self.fd_read, self.fd_write = os.pipe() + self.reader = os.fdopen(self.fd_read) + + self.start() + + def fileno(self): + return self.fd_write + + @staticmethod + def remove_newline(msg): + return msg[:-1] if msg.endswith(os.linesep) else msg + + def run(self): + for line in self.reader: + self._write(self.remove_newline(line)) + + def _write(self, message): + self.logger.log(self.level, message) diff --git a/pipenv/vendor/pexpect/__init__.py b/pipenv/vendor/pexpect/__init__.py index 2a18d191..7e304537 100644 --- a/pipenv/vendor/pexpect/__init__.py +++ b/pipenv/vendor/pexpect/__init__.py @@ -75,7 +75,7 @@ if sys.platform != 'win32': from .pty_spawn import spawn, spawnu from .run import run, runu -__version__ = '4.6.0' +__version__ = '4.8.0' __revision__ = '' __all__ = ['ExceptionPexpect', 'EOF', 'TIMEOUT', 'spawn', 'spawnu', 'run', 'runu', 'which', 'split_command_line', '__version__', '__revision__'] diff --git a/pipenv/vendor/pexpect/_async.py b/pipenv/vendor/pexpect/_async.py index bdd515b1..dfbfeef5 100644 --- a/pipenv/vendor/pexpect/_async.py +++ b/pipenv/vendor/pexpect/_async.py @@ -1,5 +1,6 @@ import asyncio import errno +import signal from pexpect import EOF @@ -7,10 +8,7 @@ from pexpect import EOF def expect_async(expecter, timeout=None): # First process data that was previously read - if it maches, we don't need # async stuff. - previously_read = expecter.spawn.buffer - expecter.spawn._buffer = expecter.spawn.buffer_type() - expecter.spawn._before = expecter.spawn.buffer_type() - idx = expecter.new_data(previously_read) + idx = expecter.existing_data() if idx is not None: return idx if not expecter.spawn.async_pw_transport: @@ -29,6 +27,23 @@ def expect_async(expecter, timeout=None): transport.pause_reading() return expecter.timeout(e) +@asyncio.coroutine +def repl_run_command_async(repl, cmdlines, timeout=-1): + res = [] + repl.child.sendline(cmdlines[0]) + for line in cmdlines[1:]: + yield from repl._expect_prompt(timeout=timeout, async_=True) + res.append(repl.child.before) + repl.child.sendline(line) + + # Command was fully submitted, now wait for the next prompt + prompt_idx = yield from repl._expect_prompt(timeout=timeout, async_=True) + if prompt_idx == 1: + # We got the continuation prompt - command was incomplete + repl.child.kill(signal.SIGINT) + yield from repl._expect_prompt(timeout=1, async_=True) + raise ValueError("Continuation prompt found - input was incomplete:") + return u''.join(res + [repl.child.before]) class PatternWaiter(asyncio.Protocol): transport = None @@ -41,7 +56,7 @@ class PatternWaiter(asyncio.Protocol): if not self.fut.done(): self.fut.set_result(result) self.transport.pause_reading() - + def error(self, exc): if not self.fut.done(): self.fut.set_exception(exc) @@ -49,13 +64,14 @@ class PatternWaiter(asyncio.Protocol): def connection_made(self, transport): self.transport = transport - + def data_received(self, data): spawn = self.expecter.spawn s = spawn._decoder.decode(data) spawn._log(s, 'read') if self.fut.done(): + spawn._before.write(s) spawn._buffer.write(s) return @@ -67,7 +83,7 @@ class PatternWaiter(asyncio.Protocol): except Exception as e: self.expecter.errored() self.error(e) - + def eof_received(self): # N.B. If this gets called, async will close the pipe (the spawn object) # for us @@ -78,7 +94,7 @@ class PatternWaiter(asyncio.Protocol): self.error(e) else: self.found(index) - + def connection_lost(self, exc): if isinstance(exc, OSError) and exc.errno == errno.EIO: # We may get here without eof_received being called, e.g on Linux diff --git a/pipenv/vendor/pexpect/expect.py b/pipenv/vendor/pexpect/expect.py index 1c0275b4..d3409db9 100644 --- a/pipenv/vendor/pexpect/expect.py +++ b/pipenv/vendor/pexpect/expect.py @@ -6,45 +6,101 @@ class Expecter(object): def __init__(self, spawn, searcher, searchwindowsize=-1): self.spawn = spawn self.searcher = searcher + # A value of -1 means to use the figure from spawn, which should + # be None or a positive number. if searchwindowsize == -1: searchwindowsize = spawn.searchwindowsize self.searchwindowsize = searchwindowsize + self.lookback = None + if hasattr(searcher, 'longest_string'): + self.lookback = searcher.longest_string - def new_data(self, data): + def do_search(self, window, freshlen): spawn = self.spawn searcher = self.searcher - - pos = spawn._buffer.tell() - spawn._buffer.write(data) - spawn._before.write(data) - - # determine which chunk of data to search; if a windowsize is - # specified, this is the *new* data + the preceding <windowsize> bytes - if self.searchwindowsize: - spawn._buffer.seek(max(0, pos - self.searchwindowsize)) - window = spawn._buffer.read(self.searchwindowsize + len(data)) - else: - # otherwise, search the whole buffer (really slow for large datasets) - window = spawn.buffer - index = searcher.search(window, len(data)) + if freshlen > len(window): + freshlen = len(window) + index = searcher.search(window, freshlen, self.searchwindowsize) if index >= 0: spawn._buffer = spawn.buffer_type() spawn._buffer.write(window[searcher.end:]) - spawn.before = spawn._before.getvalue()[0:-(len(window) - searcher.start)] + spawn.before = spawn._before.getvalue()[ + 0:-(len(window) - searcher.start)] spawn._before = spawn.buffer_type() - spawn.after = window[searcher.start: searcher.end] + spawn._before.write(window[searcher.end:]) + spawn.after = window[searcher.start:searcher.end] spawn.match = searcher.match spawn.match_index = index # Found a match return index - elif self.searchwindowsize: - spawn._buffer = spawn.buffer_type() - spawn._buffer.write(window) + elif self.searchwindowsize or self.lookback: + maintain = self.searchwindowsize or self.lookback + if spawn._buffer.tell() > maintain: + spawn._buffer = spawn.buffer_type() + spawn._buffer.write(window[-maintain:]) + + def existing_data(self): + # First call from a new call to expect_loop or expect_async. + # self.searchwindowsize may have changed. + # Treat all data as fresh. + spawn = self.spawn + before_len = spawn._before.tell() + buf_len = spawn._buffer.tell() + freshlen = before_len + if before_len > buf_len: + if not self.searchwindowsize: + spawn._buffer = spawn.buffer_type() + window = spawn._before.getvalue() + spawn._buffer.write(window) + elif buf_len < self.searchwindowsize: + spawn._buffer = spawn.buffer_type() + spawn._before.seek( + max(0, before_len - self.searchwindowsize)) + window = spawn._before.read() + spawn._buffer.write(window) + else: + spawn._buffer.seek(max(0, buf_len - self.searchwindowsize)) + window = spawn._buffer.read() + else: + if self.searchwindowsize: + spawn._buffer.seek(max(0, buf_len - self.searchwindowsize)) + window = spawn._buffer.read() + else: + window = spawn._buffer.getvalue() + return self.do_search(window, freshlen) + + def new_data(self, data): + # A subsequent call, after a call to existing_data. + spawn = self.spawn + freshlen = len(data) + spawn._before.write(data) + if not self.searchwindowsize: + if self.lookback: + # search lookback + new data. + old_len = spawn._buffer.tell() + spawn._buffer.write(data) + spawn._buffer.seek(max(0, old_len - self.lookback)) + window = spawn._buffer.read() + else: + # copy the whole buffer (really slow for large datasets). + spawn._buffer.write(data) + window = spawn.buffer + else: + if len(data) >= self.searchwindowsize or not spawn._buffer.tell(): + window = data[-self.searchwindowsize:] + spawn._buffer = spawn.buffer_type() + spawn._buffer.write(window[-self.searchwindowsize:]) + else: + spawn._buffer.write(data) + new_len = spawn._buffer.tell() + spawn._buffer.seek(max(0, new_len - self.searchwindowsize)) + window = spawn._buffer.read() + return self.do_search(window, freshlen) def eof(self, err=None): spawn = self.spawn - spawn.before = spawn.buffer + spawn.before = spawn._before.getvalue() spawn._buffer = spawn.buffer_type() spawn._before = spawn.buffer_type() spawn.after = EOF @@ -60,12 +116,15 @@ class Expecter(object): msg += '\nsearcher: %s' % self.searcher if err is not None: msg = str(err) + '\n' + msg - raise EOF(msg) - + + exc = EOF(msg) + exc.__cause__ = None # in Python 3.x we can use "raise exc from None" + raise exc + def timeout(self, err=None): spawn = self.spawn - spawn.before = spawn.buffer + spawn.before = spawn._before.getvalue() spawn.after = TIMEOUT index = self.searcher.timeout_index if index >= 0: @@ -79,15 +138,18 @@ class Expecter(object): msg += '\nsearcher: %s' % self.searcher if err is not None: msg = str(err) + '\n' + msg - raise TIMEOUT(msg) + + exc = TIMEOUT(msg) + exc.__cause__ = None # in Python 3.x we can use "raise exc from None" + raise exc def errored(self): spawn = self.spawn - spawn.before = spawn.buffer + spawn.before = spawn._before.getvalue() spawn.after = None spawn.match = None spawn.match_index = None - + def expect_loop(self, timeout=-1): """Blocking expect""" spawn = self.spawn @@ -96,14 +158,10 @@ class Expecter(object): end_time = time.time() + timeout try: - incoming = spawn.buffer - spawn._buffer = spawn.buffer_type() - spawn._before = spawn.buffer_type() + idx = self.existing_data() + if idx is not None: + return idx while True: - idx = self.new_data(incoming) - # Keep reading until exception or return. - if idx is not None: - return idx # No match at this point if (timeout is not None) and (timeout < 0): return self.timeout() @@ -111,6 +169,10 @@ class Expecter(object): incoming = spawn.read_nonblocking(spawn.maxread, timeout) if self.spawn.delayafterread is not None: time.sleep(self.spawn.delayafterread) + idx = self.new_data(incoming) + # Keep reading until exception or return. + if idx is not None: + return idx if timeout is not None: timeout = end_time - time.time() except EOF as e: @@ -148,6 +210,7 @@ class searcher_string(object): self.eof_index = -1 self.timeout_index = -1 self._strings = [] + self.longest_string = 0 for n, s in enumerate(strings): if s is EOF: self.eof_index = n @@ -156,6 +219,8 @@ class searcher_string(object): self.timeout_index = n continue self._strings.append((n, s)) + if len(s) > self.longest_string: + self.longest_string = len(s) def __str__(self): '''This returns a human-readable string that represents the state of @@ -244,7 +309,7 @@ class searcher_re(object): self.eof_index = -1 self.timeout_index = -1 self._searches = [] - for n, s in zip(list(range(len(patterns))), patterns): + for n, s in enumerate(patterns): if s is EOF: self.eof_index = n continue diff --git a/pipenv/vendor/pexpect/pty_spawn.py b/pipenv/vendor/pexpect/pty_spawn.py index e0e2b54f..8e28ca7c 100644 --- a/pipenv/vendor/pexpect/pty_spawn.py +++ b/pipenv/vendor/pexpect/pty_spawn.py @@ -191,6 +191,7 @@ class spawn(SpawnBase): self.STDIN_FILENO = pty.STDIN_FILENO self.STDOUT_FILENO = pty.STDOUT_FILENO self.STDERR_FILENO = pty.STDERR_FILENO + self.str_last_chars = 100 self.cwd = cwd self.env = env self.echo = echo @@ -212,8 +213,8 @@ class spawn(SpawnBase): s.append(repr(self)) s.append('command: ' + str(self.command)) s.append('args: %r' % (self.args,)) - s.append('buffer (last 100 chars): %r' % self.buffer[-100:]) - s.append('before (last 100 chars): %r' % self.before[-100:] if self.before else '') + s.append('buffer (last %s chars): %r' % (self.str_last_chars,self.buffer[-self.str_last_chars:])) + s.append('before (last %s chars): %r' % (self.str_last_chars,self.before[-self.str_last_chars:] if self.before else '')) s.append('after: %r' % (self.after,)) s.append('match: %r' % (self.match,)) s.append('match_index: ' + str(self.match_index)) @@ -430,61 +431,83 @@ class spawn(SpawnBase): available right away then one character will be returned immediately. It will not wait for 30 seconds for another 99 characters to come in. - This is a wrapper around os.read(). It uses select.select() to - implement the timeout. ''' + On the other hand, if there are bytes available to read immediately, + all those bytes will be read (up to the buffer size). So, if the + buffer size is 1 megabyte and there is 1 megabyte of data available + to read, the buffer will be filled, regardless of timeout. + + This is a wrapper around os.read(). It uses select.select() or + select.poll() to implement the timeout. ''' if self.closed: raise ValueError('I/O operation on closed file.') + if self.use_poll: + def select(timeout): + return poll_ignore_interrupts([self.child_fd], timeout) + else: + def select(timeout): + return select_ignore_interrupts([self.child_fd], [], [], timeout)[0] + + # If there is data available to read right now, read as much as + # we can. We do this to increase performance if there are a lot + # of bytes to be read. This also avoids calling isalive() too + # often. See also: + # * https://github.com/pexpect/pexpect/pull/304 + # * http://trac.sagemath.org/ticket/10295 + if select(0): + try: + incoming = super(spawn, self).read_nonblocking(size) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + raise + while len(incoming) < size and select(0): + try: + incoming += super(spawn, self).read_nonblocking(size - len(incoming)) + except EOF: + # Maybe the child is dead: update some attributes in that case + self.isalive() + # Don't raise EOF, just return what we read so far. + return incoming + return incoming + if timeout == -1: timeout = self.timeout - # Note that some systems such as Solaris do not give an EOF when - # the child dies. In fact, you can still try to read - # from the child_fd -- it will block forever or until TIMEOUT. - # For this case, I test isalive() before doing any reading. - # If isalive() is false, then I pretend that this is the same as EOF. if not self.isalive(): - # timeout of 0 means "poll" - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts([self.child_fd], [], [], 0) - if not r: - self.flag_eof = True - raise EOF('End Of File (EOF). Braindead platform.') + # The process is dead, but there may or may not be data + # available to read. Note that some systems such as Solaris + # do not give an EOF when the child dies. In fact, you can + # still try to read from the child_fd -- it will block + # forever or until TIMEOUT. For that reason, it's important + # to do this check before calling select() with timeout. + if select(0): + return super(spawn, self).read_nonblocking(size) + self.flag_eof = True + raise EOF('End Of File (EOF). Braindead platform.') elif self.__irix_hack: # Irix takes a long time before it realizes a child was terminated. + # Make sure that the timeout is at least 2 seconds. # FIXME So does this mean Irix systems are forced to always have # FIXME a 2 second delay when calling read_nonblocking? That sucks. - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts([self.child_fd], [], [], 2) - if not r and not self.isalive(): - self.flag_eof = True - raise EOF('End Of File (EOF). Slow platform.') - if self.use_poll: - r = poll_ignore_interrupts([self.child_fd], timeout) - else: - r, w, e = select_ignore_interrupts( - [self.child_fd], [], [], timeout - ) + if timeout is not None and timeout < 2: + timeout = 2 - if not r: - if not self.isalive(): - # Some platforms, such as Irix, will claim that their - # processes are alive; timeout on the select; and - # then finally admit that they are not alive. - self.flag_eof = True - raise EOF('End of File (EOF). Very slow platform.') - else: - raise TIMEOUT('Timeout exceeded.') - - if self.child_fd in r: + # Because of the select(0) check above, we know that no data + # is available right now. But if a non-zero timeout is given + # (possibly timeout=None), we call select() with a timeout. + if (timeout != 0) and select(timeout): return super(spawn, self).read_nonblocking(size) - raise ExceptionPexpect('Reached an unexpected state.') # pragma: no cover + if not self.isalive(): + # Some platforms, such as Irix, will claim that their + # processes are alive; timeout on the select; and + # then finally admit that they are not alive. + self.flag_eof = True + raise EOF('End of File (EOF). Very slow platform.') + else: + raise TIMEOUT('Timeout exceeded.') def write(self, s): '''This is similar to send() except that there is no return value. @@ -730,10 +753,14 @@ class spawn(SpawnBase): child process in interact mode is duplicated to the given log. You may pass in optional input and output filter functions. These - functions should take a string and return a string. The output_filter - will be passed all the output from the child process. The input_filter - will be passed all the keyboard input from the user. The input_filter - is run BEFORE the check for the escape_character. + functions should take bytes array and return bytes array too. Even + with ``encoding='utf-8'`` support, meth:`interact` will always pass + input_filter and output_filter bytes. You may need to wrap your + function to decode and encode back to UTF-8. + + The output_filter will be passed all the output from the child process. + The input_filter will be passed all the keyboard input from the user. + The input_filter is run BEFORE the check for the escape_character. Note that if you change the window size of the parent the SIGWINCH signal will not be passed through to the child. If you want the child diff --git a/pipenv/vendor/pexpect/pxssh.py b/pipenv/vendor/pexpect/pxssh.py index ef2e9118..3d53bd97 100644 --- a/pipenv/vendor/pexpect/pxssh.py +++ b/pipenv/vendor/pexpect/pxssh.py @@ -109,7 +109,7 @@ class pxssh (spawn): username = raw_input('username: ') password = getpass.getpass('password: ') s.login (hostname, username, password) - + `debug_command_string` is only for the test suite to confirm that the string generated for SSH is correct, using this will not allow you to do anything other than get a string back from `pxssh.pxssh.login()`. @@ -118,12 +118,12 @@ class pxssh (spawn): def __init__ (self, timeout=30, maxread=2000, searchwindowsize=None, logfile=None, cwd=None, env=None, ignore_sighup=True, echo=True, options={}, encoding=None, codec_errors='strict', - debug_command_string=False): + debug_command_string=False, use_poll=False): spawn.__init__(self, None, timeout=timeout, maxread=maxread, searchwindowsize=searchwindowsize, logfile=logfile, cwd=cwd, env=env, ignore_sighup=ignore_sighup, echo=echo, - encoding=encoding, codec_errors=codec_errors) + encoding=encoding, codec_errors=codec_errors, use_poll=use_poll) self.name = '<pxssh>' @@ -154,7 +154,7 @@ class pxssh (spawn): # Unsetting SSH_ASKPASS on the remote side doesn't disable it! Annoying! #self.SSH_OPTS = "-x -o'RSAAuthentication=no' -o 'PubkeyAuthentication=no'" self.force_password = False - + self.debug_command_string = debug_command_string # User defined SSH options, eg, @@ -220,7 +220,7 @@ class pxssh (spawn): can take 12 seconds. Low latency connections are more likely to fail with a low sync_multiplier. Best case sync time gets worse with a high sync multiplier (500 ms with default). ''' - + # All of these timing pace values are magic. # I came up with these based on what seemed reliable for # connecting to a heavily loaded machine I have. @@ -253,20 +253,19 @@ class pxssh (spawn): ### TODO: This is getting messy and I'm pretty sure this isn't perfect. ### TODO: I need to draw a flow chart for this. ### TODO: Unit tests for SSH tunnels, remote SSH command exec, disabling original prompt sync - def login (self, server, username, password='', terminal_type='ansi', + def login (self, server, username=None, password='', terminal_type='ansi', original_prompt=r"[#$]", login_timeout=10, port=None, auto_prompt_reset=True, ssh_key=None, quiet=True, sync_multiplier=1, check_local_ip=True, password_regex=r'(?i)(?:password:)|(?:passphrase for key)', ssh_tunnels={}, spawn_local_ssh=True, - sync_original_prompt=True, ssh_config=None): + sync_original_prompt=True, ssh_config=None, cmd='ssh'): '''This logs the user into the given server. - It uses - 'original_prompt' to try to find the prompt right after login. When it - finds the prompt it immediately tries to reset the prompt to something - more easily matched. The default 'original_prompt' is very optimistic - and is easily fooled. It's more reliable to try to match the original + It uses 'original_prompt' to try to find the prompt right after login. + When it finds the prompt it immediately tries to reset the prompt to + something more easily matched. The default 'original_prompt' is very + optimistic and is easily fooled. It's more reliable to try to match the original prompt as exactly as possible to prevent false matches by server strings such as the "Message Of The Day". On many systems you can disable the MOTD on the remote server by creating a zero-length file @@ -284,27 +283,31 @@ class pxssh (spawn): uses a unique prompt in the :meth:`prompt` method. If the original prompt is not reset then this will disable the :meth:`prompt` method unless you manually set the :attr:`PROMPT` attribute. - + Set ``password_regex`` if there is a MOTD message with `password` in it. Changing this is like playing in traffic, don't (p)expect it to match straight away. - + If you require to connect to another SSH server from the your original SSH connection set ``spawn_local_ssh`` to `False` and this will use your current session to do so. Setting this option to `False` and not having an active session will trigger an error. - + Set ``ssh_key`` to a file path to an SSH private key to use that SSH key for the session authentication. Set ``ssh_key`` to `True` to force passing the current SSH authentication socket to the desired ``hostname``. - + Set ``ssh_config`` to a file path string of an SSH client config file to pass that file to the client to handle itself. You may set any options you wish in here, however doing so will require you to post extra information that you may not want to if you run into issues. + + Alter the ``cmd`` to change the ssh client used, or to prepend it with network + namespaces. For example ```cmd="ip netns exec vlan2 ssh"``` to execute the ssh in + network namespace named ```vlan```. ''' - + session_regex_array = ["(?i)are you sure you want to continue connecting", original_prompt, password_regex, "(?i)permission denied", "(?i)terminal type", TIMEOUT] session_init_regex_array = [] session_init_regex_array.extend(session_regex_array) @@ -320,7 +323,7 @@ class pxssh (spawn): if ssh_config is not None: if spawn_local_ssh and not os.path.isfile(ssh_config): raise ExceptionPxssh('SSH config does not exist or is not a file.') - ssh_options = ssh_options + '-F ' + ssh_config + ssh_options = ssh_options + ' -F ' + ssh_config if port is not None: ssh_options = ssh_options + ' -p %s'%(str(port)) if ssh_key is not None: @@ -331,7 +334,7 @@ class pxssh (spawn): if spawn_local_ssh and not os.path.isfile(ssh_key): raise ExceptionPxssh('private ssh key does not exist or is not a file.') ssh_options = ssh_options + ' -i %s' % (ssh_key) - + # SSH tunnels, make sure you know what you're putting into the lists # under each heading. Do not expect these to open 100% of the time, # The port you're requesting might be bound. @@ -354,7 +357,42 @@ class pxssh (spawn): if spawn_local_ssh==False: tunnel = quote(str(tunnel)) ssh_options = ssh_options + ' -' + cmd_type + ' ' + str(tunnel) - cmd = "ssh %s -l %s %s" % (ssh_options, username, server) + + if username is not None: + ssh_options = ssh_options + ' -l ' + username + elif ssh_config is None: + raise TypeError('login() needs either a username or an ssh_config') + else: # make sure ssh_config has an entry for the server with a username + with open(ssh_config, 'rt') as f: + lines = [l.strip() for l in f.readlines()] + + server_regex = r'^Host\s+%s\s*$' % server + user_regex = r'^User\s+\w+\s*$' + config_has_server = False + server_has_username = False + for line in lines: + if not config_has_server and re.match(server_regex, line, re.IGNORECASE): + config_has_server = True + elif config_has_server and 'hostname' in line.lower(): + pass + elif config_has_server and 'host' in line.lower(): + server_has_username = False # insurance + break # we have left the relevant section + elif config_has_server and re.match(user_regex, line, re.IGNORECASE): + server_has_username = True + break + + if lines: + del line + + del lines + + if not config_has_server: + raise TypeError('login() ssh_config has no Host entry for %s' % server) + elif not server_has_username: + raise TypeError('login() ssh_config has no user entry for %s' % server) + + cmd += " %s %s" % (ssh_options, server) if self.debug_command_string: return(cmd) diff --git a/pipenv/vendor/pexpect/replwrap.py b/pipenv/vendor/pexpect/replwrap.py index ed0e657d..c930f1e4 100644 --- a/pipenv/vendor/pexpect/replwrap.py +++ b/pipenv/vendor/pexpect/replwrap.py @@ -61,11 +61,11 @@ class REPLWrapper(object): self.child.expect(orig_prompt) self.child.sendline(prompt_change) - def _expect_prompt(self, timeout=-1): + def _expect_prompt(self, timeout=-1, async_=False): return self.child.expect_exact([self.prompt, self.continuation_prompt], - timeout=timeout) + timeout=timeout, async_=async_) - def run_command(self, command, timeout=-1): + def run_command(self, command, timeout=-1, async_=False): """Send a command to the REPL, wait for and return output. :param str command: The command to send. Trailing newlines are not needed. @@ -75,6 +75,10 @@ class REPLWrapper(object): :param int timeout: How long to wait for the next prompt. -1 means the default from the :class:`pexpect.spawn` object (default 30 seconds). None means to wait indefinitely. + :param bool async_: On Python 3.4, or Python 3.3 with asyncio + installed, passing ``async_=True`` will make this return an + :mod:`asyncio` Future, which you can yield from to get the same + result that this method would normally give directly. """ # Split up multiline commands and feed them in bit-by-bit cmdlines = command.splitlines() @@ -84,6 +88,10 @@ class REPLWrapper(object): if not cmdlines: raise ValueError("No command was given") + if async_: + from ._async import repl_run_command_async + return repl_run_command_async(self, cmdlines, timeout) + res = [] self.child.sendline(cmdlines[0]) for line in cmdlines[1:]: diff --git a/pipenv/vendor/pexpect/run.py b/pipenv/vendor/pexpect/run.py index d9dfe76b..ff288a12 100644 --- a/pipenv/vendor/pexpect/run.py +++ b/pipenv/vendor/pexpect/run.py @@ -67,7 +67,7 @@ def run(command, timeout=30, withexitstatus=False, events=None, contains patterns and responses. Whenever one of the patterns is seen in the command output, run() will send the associated response string. So, run() in the above example can be also written as: - + run("mencoder dvd://1 -o video.avi -oac copy -ovc copy", events=[(TIMEOUT,print_ticks)], timeout=5) diff --git a/pipenv/vendor/pexpect/screen.py b/pipenv/vendor/pexpect/screen.py index 5ab45b94..79f95c4e 100644 --- a/pipenv/vendor/pexpect/screen.py +++ b/pipenv/vendor/pexpect/screen.py @@ -90,7 +90,7 @@ class screen: self.encoding = encoding self.encoding_errors = encoding_errors if encoding is not None: - self.decoder = codecs.getincrementaldecoder(encoding)(encoding_errors) + self.decoder = codecs.getincrementaldecoder(encoding)(encoding_errors) else: self.decoder = None self.cur_r = 1 diff --git a/pipenv/vendor/pexpect/spawnbase.py b/pipenv/vendor/pexpect/spawnbase.py index 63c0b420..59e90576 100644 --- a/pipenv/vendor/pexpect/spawnbase.py +++ b/pipenv/vendor/pexpect/spawnbase.py @@ -120,6 +120,9 @@ class SpawnBase(object): self.async_pw_transport = None # This is the read buffer. See maxread. self._buffer = self.buffer_type() + # The buffer may be trimmed for efficiency reasons. This is the + # untrimmed buffer, used to create the before attribute. + self._before = self.buffer_type() def _log(self, s, direction): if self.logfile is not None: diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py index 22bf130f..3a0188c9 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py @@ -3,22 +3,30 @@ from __future__ import absolute_import import sys -__version__ = '0.3.2' - from . import shims - -old_module = sys.modules[__name__] +__version__ = "0.5.2" -module = sys.modules[__name__] = shims._new() +if "pip_shims" in sys.modules: + # mainly to keep a reference to the old module on hand so it doesn't get + # weakref'd away + if __name__ != "pip_shims": + del sys.modules["pip_shims"] +if __name__ in sys.modules: + old_module = sys.modules[__name__] + + +module = sys.modules[__name__] = sys.modules["pip_shims"] = shims._new() module.shims = shims -module.__dict__.update({ - '__file__': __file__, - '__package__': "pip_shims", - '__path__': __path__, - '__doc__': __doc__, - '__all__': module.__all__ + ['shims',], - '__version__': __version__, - '__name__': __name__ -}) +module.__dict__.update( + { + "__file__": __file__, + "__package__": "pip_shims", + "__path__": __path__, + "__doc__": __doc__, + "__all__": module.__all__ + ["shims"], + "__version__": __version__, + "__name__": __name__, + } +) diff --git a/pipenv/vendor/pip_shims/compat.py b/pipenv/vendor/pip_shims/compat.py new file mode 100644 index 00000000..0c125321 --- /dev/null +++ b/pipenv/vendor/pip_shims/compat.py @@ -0,0 +1,1579 @@ +# -*- coding=utf-8 -*- +""" +Backports and helper functionality to support using new functionality. +""" +from __future__ import absolute_import, print_function + +import atexit +import contextlib +import functools +import inspect +import os +import re +import sys +import types + +import six +from packaging import specifiers + +from .environment import MYPY_RUNNING +from .utils import ( + call_function_with_correct_args, + filter_allowed_args, + get_allowed_args, + get_method_args, + nullcontext, + suppress_setattr, +) + +if sys.version_info[:2] < (3, 5): + from pipenv.vendor.vistir.compat import TemporaryDirectory +else: + from tempfile import TemporaryDirectory + +if six.PY3: + from contextlib import ExitStack +else: + from pipenv.vendor.contextlib2 import ExitStack + + +if MYPY_RUNNING: + from optparse import Values + from requests import Session + from typing import ( + Any, + Callable, + Dict, + Generator, + Generic, + Iterable, + Iterator, + List, + Optional, + Tuple, + Type, + TypeVar, + Union, + ) + from .utils import TShimmedPath, TShim, TShimmedFunc + + TFinder = TypeVar("TFinder") + TResolver = TypeVar("TResolver") + TReqTracker = TypeVar("TReqTracker") + TReqSet = TypeVar("TReqSet") + TLink = TypeVar("TLink") + TSession = TypeVar("TSession", bound=Session) + TCommand = TypeVar("TCommand", covariant=True) + TCommandInstance = TypeVar("TCommandInstance") + TCmdDict = Dict[str, Union[Tuple[str, str, str], TCommandInstance]] + TInstallRequirement = TypeVar("TInstallRequirement") + TFormatControl = TypeVar("TFormatControl") + TShimmedCmdDict = Union[TShim, TCmdDict] + TWheelCache = TypeVar("TWheelCache") + TPreparer = TypeVar("TPreparer") + + +class SearchScope(object): + def __init__(self, find_links=None, index_urls=None): + self.index_urls = index_urls if index_urls else [] + self.find_links = find_links + + @classmethod + def create(cls, find_links=None, index_urls=None): + if not index_urls: + index_urls = ["https://pypi.org/simple"] + return cls(find_links=find_links, index_urls=index_urls) + + +class SelectionPreferences(object): + def __init__( + self, + allow_yanked=True, + allow_all_prereleases=False, + format_control=None, + prefer_binary=False, + ignore_requires_python=False, + ): + self.allow_yanked = allow_yanked + self.allow_all_prereleases = allow_all_prereleases + self.format_control = format_control + self.prefer_binary = prefer_binary + self.ignore_requires_python = ignore_requires_python + + +class TargetPython(object): + fallback_get_tags = None # type: Optional[TShimmedFunc] + + def __init__( + self, + platform=None, # type: Optional[str] + py_version_info=None, # type: Optional[Tuple[int, ...]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + ): + # type: (...) -> None + self._given_py_version_info = py_version_info + if py_version_info is None: + py_version_info = sys.version_info[:3] + elif len(py_version_info) < 3: + py_version_info += (3 - len(py_version_info)) * (0,) + else: + py_version_info = py_version_info[:3] + py_version = ".".join(map(str, py_version_info[:2])) + self.abi = abi + self.implementation = implementation + self.platform = platform + self.py_version = py_version + self.py_version_info = py_version_info + self._valid_tags = None + + def get_tags(self): + if self._valid_tags is None and self.fallback_get_tags: + fallback_func = resolve_possible_shim(self.fallback_get_tags) + versions = None + if self._given_py_version_info: + versions = ["".join(map(str, self._given_py_version_info[:2]))] + self._valid_tags = fallback_func( + versions=versions, + platform=self.platform, + abi=self.abi, + impl=self.implementation, + ) + return self._valid_tags + + +class CandidatePreferences(object): + def __init__(self, prefer_binary=False, allow_all_prereleases=False): + self.prefer_binary = prefer_binary + self.allow_all_prereleases = allow_all_prereleases + + +class LinkCollector(object): + def __init__(self, session=None, search_scope=None): + self.session = session + self.search_scope = search_scope + + +class CandidateEvaluator(object): + @classmethod + def create( + cls, + project_name, # type: str + target_python=None, # type: Optional[TargetPython] + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + specifier=None, # type: Optional[specifiers.BaseSpecifier] + hashes=None, # type: Optional[Any] + ): + if target_python is None: + target_python = TargetPython() + if specifier is None: + specifier = specifiers.SpecifierSet() + + supported_tags = target_python.get_tags() + + return cls( + project_name=project_name, + supported_tags=supported_tags, + specifier=specifier, + prefer_binary=prefer_binary, + allow_all_prereleases=allow_all_prereleases, + hashes=hashes, + ) + + def __init__( + self, + project_name, # type: str + supported_tags, # type: List[Any] + specifier, # type: specifiers.BaseSpecifier + prefer_binary=False, # type: bool + allow_all_prereleases=False, # type: bool + hashes=None, # type: Optional[Any] + ): + self._allow_all_prereleases = allow_all_prereleases + self._hashes = hashes + self._prefer_binary = prefer_binary + self._project_name = project_name + self._specifier = specifier + self._supported_tags = supported_tags + + +class LinkEvaluator(object): + def __init__( + self, + allow_yanked, + project_name, + canonical_name, + formats, + target_python, + ignore_requires_python=False, + ignore_compatibility=True, + ): + self._allow_yanked = allow_yanked + self._canonical_name = canonical_name + self._ignore_requires_python = ignore_requires_python + self._formats = formats + self._target_python = target_python + self._ignore_compatibility = ignore_compatibility + + self.project_name = project_name + + +class InvalidWheelFilename(Exception): + """Wheel Filename is Invalid""" + + +class Wheel(object): + wheel_file_re = re.compile( + r"""^(?P<namever>(?P<name>.+?)-(?P<ver>.*?)) + ((-(?P<build>\d[^-]*?))?-(?P<pyver>.+?)-(?P<abi>.+?)-(?P<plat>.+?) + \.whl|\.dist-info)$""", + re.VERBOSE, + ) + + def __init__(self, filename): + # type: (str) -> None + wheel_info = self.wheel_file_re.match(filename) + if not wheel_info: + raise InvalidWheelFilename("%s is not a valid wheel filename." % filename) + self.filename = filename + self.name = wheel_info.group("name").replace("_", "-") + # we'll assume "_" means "-" due to wheel naming scheme + # (https://github.com/pypa/pip/issues/1150) + self.version = wheel_info.group("ver").replace("_", "-") + self.build_tag = wheel_info.group("build") + self.pyversions = wheel_info.group("pyver").split(".") + self.abis = wheel_info.group("abi").split(".") + self.plats = wheel_info.group("plat").split(".") + + # All the tag combinations from this file + self.file_tags = { + (x, y, z) for x in self.pyversions for y in self.abis for z in self.plats + } + + def get_formatted_file_tags(self): + # type: () -> List[str] + """ + Return the wheel's tags as a sorted list of strings. + """ + return sorted("-".join(tag) for tag in self.file_tags) + + def support_index_min(self, tags): + # type: (List[Any]) -> int + """ + Return the lowest index that one of the wheel's file_tag combinations + achieves in the given list of supported tags. + + For example, if there are 8 supported tags and one of the file tags + is first in the list, then return 0. + + :param tags: the PEP 425 tags to check the wheel against, in order + with most preferred first. + :raises ValueError: If none of the wheel's file tags match one of + the supported tags. + """ + return min(tags.index(tag) for tag in self.file_tags if tag in tags) + + def supported(self, tags): + # type: (List[Any]) -> bool + """ + Return whether the wheel is compatible with one of the given tags. + + :param tags: the PEP 425 tags to check the wheel against. + """ + return not self.file_tags.isdisjoint(tags) + + +def resolve_possible_shim(target): + # type: (TShimmedFunc) -> Optional[Union[Type, Callable]] + if target is None: + return target + if getattr(target, "shim", None) and isinstance( + target.shim, (types.MethodType, types.FunctionType) + ): + return target.shim() + return target + + +@contextlib.contextmanager +def temp_environ(): + """Allow the ability to set os.environ temporarily""" + environ = dict(os.environ) + try: + yield + finally: + os.environ.clear() + os.environ.update(environ) + + +@contextlib.contextmanager +def get_requirement_tracker(req_tracker_creator=None): + # type: (Optional[Callable]) -> Generator[Optional[TReqTracker], None, None] + root = os.environ.get("PIP_REQ_TRACKER") + if not req_tracker_creator: + yield None + else: + req_tracker_args = [] + _, required_args = get_method_args(req_tracker_creator.__init__) # type: ignore + with ExitStack() as ctx: + if root is None: + root = ctx.enter_context(TemporaryDirectory(prefix="req-tracker")) + if root: + root = str(root) + ctx.enter_context(temp_environ()) + os.environ["PIP_REQ_TRACKER"] = root + if required_args is not None and "root" in required_args: + req_tracker_args.append(root) + with req_tracker_creator(*req_tracker_args) as tracker: + yield tracker + + +@contextlib.contextmanager +def ensure_resolution_dirs(**kwargs): + # type: (Any) -> Iterator[Dict[str, Any]] + """ + Ensures that the proper directories are scaffolded and present in the provided kwargs + for performing dependency resolution via pip. + + :return: A new kwargs dictionary with scaffolded directories for **build_dir**, **src_dir**, + **download_dir**, and **wheel_download_dir** added to the key value pairs. + :rtype: Dict[str, Any] + """ + keys = ("build_dir", "src_dir", "download_dir", "wheel_download_dir") + if not any(kwargs.get(key) is None for key in keys): + yield kwargs + else: + with TemporaryDirectory(prefix="pip-shims-") as base_dir: + for key in keys: + if kwargs.get(key) is not None: + continue + target = os.path.join(base_dir, key) + os.makedirs(target) + kwargs[key] = target + yield kwargs + + +@contextlib.contextmanager +def wheel_cache( + cache_dir=None, # type: str + format_control=None, # type: Any + wheel_cache_provider=None, # type: TShimmedFunc + format_control_provider=None, # type: Optional[TShimmedFunc] + tempdir_manager_provider=None, # type: TShimmedFunc +): + tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) + format_control_provider = resolve_possible_shim(format_control_provider) + if not format_control and not format_control_provider: + raise TypeError("Format control or provider needed for wheel cache!") + if not format_control: + format_control = format_control_provider(None, None) + with ExitStack() as ctx: + ctx.enter_context(tempdir_manager_provider()) + wheel_cache = wheel_cache_provider(cache_dir, format_control) + yield wheel_cache + + +def partial_command(shimmed_path, cmd_mapping=None): + # type: (Type, Optional[TShimmedCmdDict]) -> Union[Type[TCommandInstance], functools.partial] + """ + Maps a default set of arguments across all members of a + :class:`~pip_shims.models.ShimmedPath` instance, specifically for + :class:`~pip._internal.command.Command` instances which need + `summary` and `name` arguments. + + :param :class:`~pip_shims.models.ShimmedPath` shimmed_path: A + :class:`~pip_shims.models.ShimmedCollection` instance + :param Any cmd_mapping: A reference to use for mapping against, e.g. an + import that depends on pip also + :return: A dictionary mapping new arguments to their default values + :rtype: Dict[str, str] + """ + basecls = shimmed_path.shim() + resolved_cmd_mapping = None # type: Optional[Dict[str, Any]] + cmd_mapping = resolve_possible_shim(cmd_mapping) + if cmd_mapping is not None and isinstance(cmd_mapping, dict): + resolved_cmd_mapping = cmd_mapping.copy() + base_args = [] # type: List[str] + for root_cls in basecls.mro(): + if root_cls.__name__ == "Command": + _, root_init_args = get_method_args(root_cls.__init__) + if root_init_args is not None: + base_args = root_init_args.args + needs_name_and_summary = any(arg in base_args for arg in ("name", "summary")) + if not needs_name_and_summary: + basecls.name = shimmed_path.name + return basecls + elif ( + not resolved_cmd_mapping + and needs_name_and_summary + and getattr(functools, "partialmethod", None) + ): + new_init = functools.partial( + basecls.__init__, name=shimmed_path.name, summary="Summary" + ) + basecls.__init__ = new_init + result = basecls + assert resolved_cmd_mapping is not None + for command_name, command_info in resolved_cmd_mapping.items(): + if getattr(command_info, "class_name", None) == shimmed_path.name: + summary = getattr(command_info, "summary", "Command summary") + result = functools.partial(basecls, command_name, summary) + break + return result + + +def get_session( + install_cmd_provider=None, # type: Optional[TShimmedFunc] + install_cmd=None, # type: TCommandInstance + options=None, # type: Optional[Values] +): + # type: (...) -> TSession + session = None # type: Optional[TSession] + if install_cmd is None: + assert install_cmd_provider is not None + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_cmd = install_cmd_provider() + if options is None: + options, _ = install_cmd.parser.parse_args([]) # type: ignore + session = install_cmd._build_session(options) # type: ignore + assert session is not None + atexit.register(session.close) + return session + + +def populate_options( + install_command=None, # type: TCommandInstance + options=None, # type: Optional[Values] + **kwargs # type: Any +): + # (...) -> Tuple[Dict[str, Any], Values] + results = {} + if install_command is None and options is None: + raise TypeError("Must pass either options or InstallCommand to populate options") + if options is None and install_command is not None: + options, _ = install_command.parser.parse_args([]) # type: ignore + options_dict = options.__dict__ + for provided_key, provided_value in kwargs.items(): + if provided_key == "isolated": + options_key = "isolated_mode" + elif provided_key == "source_dir": + options_key = "src_dir" + else: + options_key = provided_key + if provided_key in options_dict and provided_value is not None: + setattr(options, options_key, provided_value) + results[provided_key] = provided_value + elif getattr(options, options_key, None) is not None: + results[provided_key] = getattr(options, options_key) + else: + results[provided_key] = provided_value + return results, options + + +def get_requirement_set( + install_command=None, # type: Optional[TCommandInstance] + req_set_provider=None, # type: Optional[TShimmedFunc] + build_dir=None, # type: Optional[str] + src_dir=None, # type: Optional[str] + download_dir=None, # type: Optional[str] + wheel_download_dir=None, # type: Optional[str] + session=None, # type: Optional[TSession] + wheel_cache=None, # type: Optional[TWheelCache] + upgrade=False, # type: bool + upgrade_strategy=None, # type: Optional[str] + ignore_installed=False, # type: bool + ignore_dependencies=False, # type: bool + force_reinstall=False, # type: bool + use_user_site=False, # type: bool + isolated=False, # type: bool + ignore_requires_python=False, # type: bool + require_hashes=None, # type: bool + cache_dir=None, # type: Optional[str] + options=None, # type: Optional[Values] + install_cmd_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] +): + # (...) -> TRequirementSet + """ + Creates a requirement set from the supplied parameters. + + Not all parameters are passed through for all pip versions, but any + invalid parameters will be ignored if they are not needed to generate a + requirement set on the current pip version. + + :param :class:`~pip_shims.models.ShimmedPathCollection` wheel_cache_provider: A + context manager provider which resolves to a `WheelCache` instance + :param install_command: A :class:`~pip._internal.commands.install.InstallCommand` + instance which is used to generate the finder. + :param :class:`~pip_shims.models.ShimmedPathCollection` req_set_provider: A provider + to build requirement set instances. + :param str build_dir: The directory to build requirements in. Removed in pip 10, + defeaults to None + :param str source_dir: The directory to use for source requirements. Removed in + pip 10, defaults to None + :param str download_dir: The directory to download requirement artifacts to. Removed + in pip 10, defaults to None + :param str wheel_download_dir: The directory to download wheels to. Removed in pip + 10, defaults ot None + :param :class:`~requests.Session` session: The pip session to use. Removed in pip 10, + defaults to None + :param WheelCache wheel_cache: The pip WheelCache instance to use for caching wheels. + Removed in pip 10, defaults to None + :param bool upgrade: Whether to try to upgrade existing requirements. Removed in pip + 10, defaults to False. + :param str upgrade_strategy: The upgrade strategy to use, e.g. "only-if-needed". + Removed in pip 10, defaults to None. + :param bool ignore_installed: Whether to ignore installed packages when resolving. + Removed in pip 10, defaults to False. + :param bool ignore_dependencies: Whether to ignore dependencies of requirements + when resolving. Removed in pip 10, defaults to False. + :param bool force_reinstall: Whether to force reinstall of packages when resolving. + Removed in pip 10, defaults to False. + :param bool use_user_site: Whether to use user site packages when resolving. Removed + in pip 10, defaults to False. + :param bool isolated: Whether to resolve in isolation. Removed in pip 10, defaults + to False. + :param bool ignore_requires_python: Removed in pip 10, defaults to False. + :param bool require_hashes: Whether to require hashes when resolving. Defaults to + False. + :param Values options: An :class:`~optparse.Values` instance from an install cmd + :param install_cmd_provider: A shim for providing new install command instances. + :type install_cmd_provider: :class:`~pip_shims.models.ShimmedPathCollection` + :return: A new requirement set instance + :rtype: :class:`~pip._internal.req.req_set.RequirementSet` + """ + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) + req_set_provider = resolve_possible_shim(req_set_provider) + if install_command is None: + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_command = install_cmd_provider() + required_args = inspect.getargs( + req_set_provider.__init__.__code__ + ).args # type: ignore + results, options = populate_options( + install_command, + options, + build_dir=build_dir, + src_dir=src_dir, + download_dir=download_dir, + upgrade=upgrade, + upgrade_strategy=upgrade_strategy, + ignore_installed=ignore_installed, + ignore_dependencies=ignore_dependencies, + force_reinstall=force_reinstall, + use_user_site=use_user_site, + isolated=isolated, + ignore_requires_python=ignore_requires_python, + require_hashes=require_hashes, + cache_dir=cache_dir, + ) + if session is None and "session" in required_args: + session = get_session(install_cmd=install_command, options=options) + with ExitStack() as stack: + if wheel_cache is None: + wheel_cache = stack.enter_context(wheel_cache_provider(cache_dir=cache_dir)) + results["wheel_cache"] = wheel_cache + results["session"] = session + results["wheel_download_dir"] = wheel_download_dir + return call_function_with_correct_args(req_set_provider, **results) + + +def get_package_finder( + install_cmd=None, # type: Optional[TCommand] + options=None, # type: Optional[Values] + session=None, # type: Optional[TSession] + platform=None, # type: Optional[str] + python_versions=None, # type: Optional[Tuple[str, ...]] + abi=None, # type: Optional[str] + implementation=None, # type: Optional[str] + target_python=None, # type: Optional[Any] + ignore_requires_python=None, # type: Optional[bool] + target_python_builder=None, # type: Optional[TShimmedFunc] + install_cmd_provider=None, # type: Optional[TShimmedFunc] +): + # type: (...) -> TFinder + """Shim for compatibility to generate package finders. + + Build and return a :class:`~pip._internal.index.package_finder.PackageFinder` + instance using the :class:`~pip._internal.commands.install.InstallCommand` helper + method to construct the finder, shimmed with backports as needed for compatibility. + + :param install_cmd_provider: A shim for providing new install command instances. + :type install_cmd_provider: :class:`~pip_shims.models.ShimmedPathCollection` + :param install_cmd: A :class:`~pip._internal.commands.install.InstallCommand` + instance which is used to generate the finder. + :param optparse.Values options: An optional :class:`optparse.Values` instance + generated by calling `install_cmd.parser.parse_args()` typically. + :param session: An optional session instance, can be created by the `install_cmd`. + :param Optional[str] platform: An optional platform string, e.g. linux_x86_64 + :param Optional[Tuple[str, ...]] python_versions: A tuple of 2-digit strings + representing python versions, e.g. ("27", "35", "36", "37"...) + :param Optional[str] abi: The target abi to support, e.g. "cp38" + :param Optional[str] implementation: An optional implementation string for limiting + searches to a specific implementation, e.g. "cp" or "py" + :param target_python: A :class:`~pip._internal.models.target_python.TargetPython` + instance (will be translated to alternate arguments if necessary on incompatible + pip versions). + :param Optional[bool] ignore_requires_python: Whether to ignore `requires_python` + on resulting candidates, only valid after pip version 19.3.1 + :param target_python_builder: A 'TargetPython' builder (e.g. the class itself, + uninstantiated) + :return: A :class:`pip._internal.index.package_finder.PackageFinder` instance + :rtype: :class:`pip._internal.index.package_finder.PackageFinder` + + :Example: + + >>> from pip_shims.shims import InstallCommand, get_package_finder + >>> install_cmd = InstallCommand() + >>> finder = get_package_finder( + ... install_cmd, python_versions=("27", "35", "36", "37", "38"), implementation=" + cp" + ... ) + >>> candidates = finder.find_all_candidates("requests") + >>> requests_222 = next(iter(c for c in candidates if c.version.public == "2.22.0")) + >>> requests_222 + <InstallationCandidate('requests', <Version('2.22.0')>, <Link https://files.pythonhos + ted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/r + equests-2.22.0-py2.py3-none-any.whl#sha256=9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9 + a590f48c010551dc6c4b31 (from https://pypi.org/simple/requests/) (requires-python:>=2. + 7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*)>)> + """ + if install_cmd is None: + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_cmd = install_cmd_provider() + if options is None: + options, _ = install_cmd.parser.parse_args([]) # type: ignore + if session is None: + session = get_session(install_cmd=install_cmd, options=options) # type: ignore + builder_args = inspect.getargs( + install_cmd._build_package_finder.__code__ + ) # type: ignore + build_kwargs = {"options": options, "session": session} + expects_targetpython = "target_python" in builder_args.args + received_python = any(arg for arg in [platform, python_versions, abi, implementation]) + if expects_targetpython and received_python and not target_python: + if target_python_builder is None: + target_python_builder = TargetPython + py_version_info = None + if python_versions: + py_version_info_python = max(python_versions) + py_version_info = tuple([int(part) for part in py_version_info_python]) + target_python = target_python_builder( + platform=platform, + abi=abi, + implementation=implementation, + py_version_info=py_version_info, + ) + build_kwargs["target_python"] = target_python + elif any( + arg in builder_args.args + for arg in ["platform", "python_versions", "abi", "implementation"] + ): + if target_python and not received_python: + tags = target_python.get_tags() + version_impl = {t[0] for t in tags} + # impls = set([v[:2] for v in version_impl]) + # impls.remove("py") + # impl = next(iter(impls), "py") if not target_python + versions = {v[2:] for v in version_impl} + build_kwargs.update( + { + "platform": target_python.platform, + "python_versions": versions, + "abi": target_python.abi, + "implementation": target_python.implementation, + } + ) + if ( + ignore_requires_python is not None + and "ignore_requires_python" in builder_args.args + ): + build_kwargs["ignore_requires_python"] = ignore_requires_python + return install_cmd._build_package_finder(**build_kwargs) # type: ignore + + +def shim_unpack( + unpack_fn, # type: TShimmedFunc + download_dir, # type str + tempdir_manager_provider, # type: TShimmedFunc + ireq=None, # type: Optional[Any] + link=None, # type: Optional[Any] + location=None, # type Optional[str], + hashes=None, # type: Optional[Any] + progress_bar="off", # type: str + only_download=None, # type: Optional[bool] + downloader_provider=None, # type: Optional[TShimmedFunc] + session=None, # type: Optional[Any] +): + # (...) -> None + """ + Accepts all parameters that have been valid to pass + to :func:`pip._internal.download.unpack_url` and selects or + drops parameters as needed before invoking the provided + callable. + + :param unpack_fn: A callable or shim referring to the pip implementation + :type unpack_fn: Callable + :param str download_dir: The directory to download the file to + :param TShimmedFunc tempdir_manager_provider: A callable or shim referring to + `global_tempdir_manager` function from pip or a shimmed no-op context manager + :param Optional[:class:`~pip._internal.req.req_install.InstallRequirement`] ireq: + an Install Requirement instance, defaults to None + :param Optional[:class:`~pip._internal.models.link.Link`] link: A Link instance, + defaults to None. + :param Optional[str] location: A location or source directory if the target is + a VCS url, defaults to None. + :param Optional[Any] hashes: A Hashes instance, defaults to None + :param str progress_bar: Indicates progress par usage during download, defatuls to + off. + :param Optional[bool] only_download: Whether to skip install, defaults to None. + :param Optional[ShimmedPathCollection] downloader_provider: A downloader class + to instantiate, if applicable. + :param Optional[`~requests.Session`] session: A PipSession instance, defaults to + None. + :return: The result of unpacking the url. + :rtype: None + """ + unpack_fn = resolve_possible_shim(unpack_fn) + downloader_provider = resolve_possible_shim(downloader_provider) + tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) + required_args = inspect.getargs(unpack_fn.__code__).args # type: ignore + unpack_kwargs = {"download_dir": download_dir} + with tempdir_manager_provider(): + if ireq: + if not link and ireq.link: + link = ireq.link + if only_download is None: + only_download = ireq.is_wheel + if hashes is None: + hashes = ireq.hashes(True) + if location is None and getattr(ireq, "source_dir", None): + location = ireq.source_dir + unpack_kwargs.update({"link": link, "location": location}) + if hashes is not None and "hashes" in required_args: + unpack_kwargs["hashes"] = hashes + if "progress_bar" in required_args: + unpack_kwargs["progress_bar"] = progress_bar + if only_download is not None and "only_download" in required_args: + unpack_kwargs["only_download"] = only_download + if session is not None and "session" in required_args: + unpack_kwargs["session"] = session + if "downloader" in required_args and downloader_provider is not None: + assert session is not None + assert progress_bar is not None + unpack_kwargs["downloader"] = downloader_provider(session, progress_bar) + return unpack_fn(**unpack_kwargs) # type: ignore + + +def _ensure_finder( + finder=None, # type: Optional[TFinder] + finder_provider=None, # type: Optional[Callable] + install_cmd=None, # type: Optional[TCommandInstance] + options=None, # type: Optional[Values] + session=None, # type: Optional[TSession] +): + if not any([finder, finder_provider, install_cmd]): + raise TypeError( + "RequirementPreparer requires a packagefinder but no InstallCommand" + " was provided to build one and none was passed in." + ) + if finder is not None: + return finder + else: + if session is None: + session = get_session(install_cmd=install_cmd, options=options) + if finder_provider is not None and options is not None: + finder_provider(options=options, session=session) + else: + finder = get_package_finder(install_cmd, options=options, session=session) + return finder + + +@contextlib.contextmanager +def make_preparer( + preparer_fn, # type: TShimmedFunc + req_tracker_fn=None, # type: Optional[TShimmedFunc] + build_dir=None, # type: Optional[str] + src_dir=None, # type: Optional[str] + download_dir=None, # type: Optional[str] + wheel_download_dir=None, # type: Optional[str] + progress_bar="off", # type: str + build_isolation=False, # type: bool + session=None, # type: Optional[TSession] + finder=None, # type: Optional[TFinder] + options=None, # type: Optional[Values] + require_hashes=None, # type: Optional[bool] + use_user_site=None, # type: Optional[bool] + req_tracker=None, # type: Optional[Union[TReqTracker, TShimmedFunc]] + install_cmd_provider=None, # type: Optional[TShimmedFunc] + downloader_provider=None, # type: Optional[TShimmedFunc] + install_cmd=None, # type: Optional[TCommandInstance] + finder_provider=None, # type: Optional[TShimmedFunc] +): + # (...) -> ContextManager + """ + Creates a requirement preparer for preparing pip requirements. + + Provides a compatibilty shim that accepts all previously valid arguments and + discards any that are no longer used. + + :raises TypeError: No requirement tracker provided and one cannot be generated + :raises TypeError: No valid sessions provided and one cannot be generated + :raises TypeError: No valid finders provided and one cannot be generated + :param TShimmedFunc preparer_fn: Callable or shim for generating preparers. + :param Optional[TShimmedFunc] req_tracker_fn: Callable or shim for generating + requirement trackers, defualts to None + :param Optional[str] build_dir: Directory for building packages and wheels, + defaults to None + :param Optional[str] src_dir: Directory to find or extract source files, defaults + to None + :param Optional[str] download_dir: Target directory to download files, defaults to + None + :param Optional[str] wheel_download_dir: Target directoryto download wheels, defaults + to None + :param str progress_bar: Whether to display a progress bar, defaults to off + :param bool build_isolation: Whether to build requirements in isolation, defaults + to False + :param Optional[TSession] session: Existing session to use for getting requirements, + defaults to None + :param Optional[TFinder] finder: The package finder to use during resolution, + defaults to None + :param Optional[Values] options: Pip options to use if needed, defaults to None + :param Optional[bool] require_hashes: Whether to require hashes for preparation + :param Optional[bool] use_user_site: Whether to use the user site directory for + preparing requirements + :param Optional[Union[TReqTracker, TShimmedFunc]] req_tracker: The requirement + tracker to use for building packages, defaults to None + :param Optional[TShimmedFunc] downloader_provider: A downloader provider + :param Optional[TCommandInstance] install_cmd: The install command used to create + the finder, session, and options if needed, defaults to None + :param Optional[TShimmedFunc] finder_provider: A package finder provider + :yield: A new requirement preparer instance + :rtype: ContextManager[:class:`~pip._internal.operations.prepare.RequirementPreparer`] + + :Example: + + >>> from pip_shims.shims import ( + ... InstallCommand, get_package_finder, make_preparer, get_requirement_tracker + ... ) + >>> install_cmd = InstallCommand() + >>> pip_options, _ = install_cmd.parser.parse_args([]) + >>> session = install_cmd._build_session(pip_options) + >>> finder = get_package_finder( + ... install_cmd, session=session, options=pip_options + ... ) + >>> with make_preparer( + ... options=pip_options, finder=finder, session=session, install_cmd=ic + ... ) as preparer: + ... print(preparer) + <pip._internal.operations.prepare.RequirementPreparer object at 0x7f8a2734be80> + """ + preparer_fn = resolve_possible_shim(preparer_fn) + downloader_provider = resolve_possible_shim(downloader_provider) + finder_provider = resolve_possible_shim(finder_provider) + required_args, required_kwargs = get_allowed_args(preparer_fn) # type: ignore + if not req_tracker and not req_tracker_fn and "req_tracker" in required_args: + raise TypeError("No requirement tracker and no req tracker generator found!") + if "downloader" in required_args and not downloader_provider: + raise TypeError("no downloader provided, but one is required to continue!") + req_tracker_fn = resolve_possible_shim(req_tracker_fn) + pip_options_created = options is None + session_is_required = "session" in required_args + finder_is_required = "finder" in required_args + downloader_is_required = "downloader" in required_args + options_map = { + "src_dir": src_dir, + "download_dir": download_dir, + "wheel_download_dir": wheel_download_dir, + "build_dir": build_dir, + "progress_bar": progress_bar, + "build_isolation": build_isolation, + "require_hashes": require_hashes, + "use_user_site": use_user_site, + } + if install_cmd is None: + assert install_cmd_provider is not None + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_cmd = install_cmd_provider() + preparer_args, options = populate_options(install_cmd, options, **options_map) + if options is not None and pip_options_created: + for k, v in options_map.items(): + suppress_setattr(options, k, v, filter_none=True) + if all([session is None, install_cmd is None, session_is_required]): + raise TypeError( + "Preparer requires a session instance which was not supplied and cannot be " + "created without an InstallCommand." + ) + elif all([session is None, session_is_required]): + session = get_session(install_cmd=install_cmd, options=options) + preparer_args["session"] = session + if finder_is_required: + finder = _ensure_finder( + finder=finder, + finder_provider=finder_provider, + install_cmd=install_cmd, + options=options, + session=session, + ) + preparer_args["finder"] = finder + if downloader_is_required: + preparer_args["downloader"] = downloader_provider(session, progress_bar) + req_tracker_fn = nullcontext if not req_tracker_fn else req_tracker_fn + with req_tracker_fn() as tracker_ctx: + if "req_tracker" in required_args: + req_tracker = tracker_ctx if req_tracker is None else req_tracker + preparer_args["req_tracker"] = req_tracker + + result = call_function_with_correct_args(preparer_fn, **preparer_args) + yield result + + +@contextlib.contextmanager +def _ensure_wheel_cache( + wheel_cache=None, # type: Optional[Type[TWheelCache]] + wheel_cache_provider=None, # type: Optional[Callable] + format_control=None, # type: Optional[TFormatControl] + format_control_provider=None, # type: Optional[Type[TShimmedFunc]] + options=None, # type: Optional[Values] + cache_dir=None, # type: Optional[str] +): + if wheel_cache is not None: + yield wheel_cache + elif wheel_cache_provider is not None: + with ExitStack() as stack: + cache_dir = getattr(options, "cache_dir", cache_dir) + format_control = getattr( + options, + "format_control", + format_control_provider(None, None), # TFormatControl + ) + wheel_cache = stack.enter_context( + wheel_cache_provider(cache_dir, format_control) + ) + yield wheel_cache + + +def get_ireq_output_path(wheel_cache, ireq): + if getattr(wheel_cache, "get_path_for_link", None): + return wheel_cache.get_path_for_link(ireq.link) + elif getattr(wheel_cache, "cached_wheel", None): + return wheel_cache.cached_wheel(ireq.link, ireq.name).url_without_fragment + + +def get_resolver( + resolver_fn, # type: TShimmedFunc + install_req_provider=None, # type: Optional[TShimmedFunc] + format_control_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] + finder=None, # type: Optional[TFinder] + upgrade_strategy="to-satisfy-only", # type: str + force_reinstall=None, # type: Optional[bool] + ignore_dependencies=None, # type: Optional[bool] + ignore_requires_python=None, # type: Optional[bool] + ignore_installed=True, # type: bool + use_user_site=False, # type: bool + isolated=None, # type: Optional[bool] + wheel_cache=None, # type: Optional[TWheelCache] + preparer=None, # type: Optional[TPreparer] + session=None, # type: Optional[TSession] + options=None, # type: Optional[Values] + make_install_req=None, # type: Optional[Callable] + install_cmd_provider=None, # type: Optional[TShimmedFunc] + install_cmd=None, # type: Optional[TCommandInstance] + use_pep517=True, # type: bool +): + # (...) -> TResolver + """ + A resolver creation compatibility shim for generating a resolver. + + Consumes any argument that was previously used to instantiate a + resolver, discards anything that is no longer valid. + + .. note:: This is only valid for **pip >= 10.0.0** + + :raises ValueError: A session is required but not provided and one cannot be created + :raises ValueError: A finder is required but not provided and one cannot be created + :raises ValueError: An install requirement provider is required and has not been + provided + :param TShimmedFunc resolver_fn: The resolver function used to create new resolver + instances. + :param TShimmedFunc install_req_provider: The provider function to use to generate + install requirements if needed. + :param TShimmedFunc format_control_provider: The provider function to use to generate + a format_control instance if needed. + :param TShimmedFunc wheel_cache_provider: The provider function to use to generate + a wheel cache if needed. + :param Optional[TFinder] finder: The package finder to use during resolution, + defaults to None. + :param str upgrade_strategy: Upgrade strategy to use, defaults to ``only-if-needed``. + :param Optional[bool] force_reinstall: Whether to simulate or assume package + reinstallation during resolution, defaults to None + :param Optional[bool] ignore_dependencies: Whether to ignore package dependencies, + defaults to None + :param Optional[bool] ignore_requires_python: Whether to ignore indicated + required_python versions on packages, defaults to None + :param bool ignore_installed: Whether to ignore installed packages during resolution, + defaults to True + :param bool use_user_site: Whether to use the user site location during resolution, + defaults to False + :param Optional[bool] isolated: Whether to isolate the resolution process, defaults + to None + :param Optional[TWheelCache] wheel_cache: The wheel cache to use, defaults to None + :param Optional[TPreparer] preparer: The requirement preparer to use, defaults to + None + :param Optional[TSession] session: Existing session to use for getting requirements, + defaults to None + :param Optional[Values] options: Pip options to use if needed, defaults to None + :param Optional[functools.partial] make_install_req: The partial function to pass in + to the resolver for actually generating install requirements, if necessary + :param Optional[TCommandInstance] install_cmd: The install command used to create + the finder, session, and options if needed, defaults to None. + :param bool use_pep517: Whether to use the pep517 build process. + :return: A new resolver instance. + :rtype: :class:`~pip._internal.legacy_resolve.Resolver` + + :Example: + + >>> import os + >>> from tempdir import TemporaryDirectory + >>> from pip_shims.shims import ( + ... InstallCommand, get_package_finder, make_preparer, get_requirement_tracker, + ... get_resolver, InstallRequirement, RequirementSet + ... ) + >>> install_cmd = InstallCommand() + >>> pip_options, _ = install_cmd.parser.parse_args([]) + >>> session = install_cmd._build_session(pip_options) + >>> finder = get_package_finder( + ... install_cmd, session=session, options=pip_options + ... ) + >>> wheel_cache = WheelCache(USER_CACHE_DIR, FormatControl(None, None)) + >>> with TemporaryDirectory() as temp_base: + ... reqset = RequirementSet() + ... ireq = InstallRequirement.from_line("requests") + ... ireq.is_direct = True + ... build_dir = os.path.join(temp_base, "build") + ... src_dir = os.path.join(temp_base, "src") + ... ireq.build_location(build_dir) + ... with make_preparer( + ... options=pip_options, finder=finder, session=session, + ... build_dir=build_dir, install_cmd=install_cmd, + ... ) as preparer: + ... resolver = get_resolver( + ... finder=finder, ignore_dependencies=False, ignore_requires_python=True, + ... preparer=preparer, session=session, options=pip_options, + ... install_cmd=install_cmd, wheel_cache=wheel_cache, + ... ) + ... resolver.require_hashes = False + ... reqset.add_requirement(ireq) + ... results = resolver.resolve(reqset) + ... #reqset.cleanup_files() + ... for result_req in reqset.requirements: + ... print(result_req) + requests + chardet + certifi + urllib3 + idna + """ + resolver_fn = resolve_possible_shim(resolver_fn) + install_req_provider = resolve_possible_shim(install_req_provider) + format_control_provider = resolve_possible_shim(format_control_provider) + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + required_args = inspect.getargs(resolver_fn.__init__.__code__).args # type: ignore + install_cmd_dependency_map = {"session": session, "finder": finder} + resolver_kwargs = {} # type: Dict[str, Any] + if install_cmd is None: + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_cmd = install_cmd_provider() + if options is None and install_cmd is not None: + options, _ = install_cmd.parser.parse_args([]) # type: ignore + for arg, val in install_cmd_dependency_map.items(): + if arg not in required_args: + continue + elif val is None and install_cmd is None: + raise TypeError( + "Preparer requires a {} but did not receive one " + "and cannot generate one".format(arg) + ) + elif arg == "session" and val is None: + val = get_session(install_cmd=install_cmd, options=options) + elif arg == "finder" and val is None: + val = get_package_finder(install_cmd, options=options, session=session) + resolver_kwargs[arg] = val + if "make_install_req" in required_args: + if make_install_req is None and install_req_provider is not None: + make_install_req_kwargs = { + "isolated": isolated, + "wheel_cache": wheel_cache, + "use_pep517": use_pep517, + } + factory_args, factory_kwargs = filter_allowed_args( + install_req_provider, **make_install_req_kwargs + ) + make_install_req = functools.partial( + install_req_provider, *factory_args, **factory_kwargs + ) + assert make_install_req is not None + resolver_kwargs["make_install_req"] = make_install_req + if "isolated" in required_args: + resolver_kwargs["isolated"] = isolated + resolver_kwargs.update( + { + "upgrade_strategy": upgrade_strategy, + "force_reinstall": force_reinstall, + "ignore_dependencies": ignore_dependencies, + "ignore_requires_python": ignore_requires_python, + "ignore_installed": ignore_installed, + "use_user_site": use_user_site, + "preparer": preparer, + } + ) + if "wheel_cache" in required_args: + with _ensure_wheel_cache( + wheel_cache=wheel_cache, + wheel_cache_provider=wheel_cache_provider, + format_control_provider=format_control_provider, + options=options, + ) as wheel_cache: + resolver_kwargs["wheel_cache"] = wheel_cache + return resolver_fn(**resolver_kwargs) # type: ignore + return resolver_fn(**resolver_kwargs) # type: ignore + + +def resolve( # noqa:C901 + ireq, # type: TInstallRequirement + reqset_provider=None, # type: Optional[TShimmedFunc] + req_tracker_provider=None, # type: Optional[TShimmedFunc] + install_cmd_provider=None, # type: Optional[TShimmedFunc] + install_command=None, # type: Optional[TCommand] + finder_provider=None, # type: Optional[TShimmedFunc] + resolver_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] + format_control_provider=None, # type: Optional[TShimmedFunc] + make_preparer_provider=None, # type: Optional[TShimmedFunc] + tempdir_manager_provider=None, # type: Optional[TShimmedFunc] + options=None, # type: Optional[Values] + session=None, # type: Optional[TSession] + resolver=None, # type: Optional[TResolver] + finder=None, # type: Optional[TFinder] + upgrade_strategy="to-satisfy-only", # type: str + force_reinstall=None, # type: Optional[bool] + ignore_dependencies=None, # type: Optional[bool] + ignore_requires_python=None, # type: Optional[bool] + ignore_installed=True, # type: bool + use_user_site=False, # type: bool + isolated=None, # type: Optional[bool] + build_dir=None, # type: Optional[str] + source_dir=None, # type: Optional[str] + download_dir=None, # type: Optional[str] + cache_dir=None, # type: Optional[str] + wheel_download_dir=None, # type: Optional[str] + wheel_cache=None, # type: Optional[TWheelCache] + require_hashes=None, # type: bool + check_supported_wheels=True, # type: bool +): + # (...) -> Set[TInstallRequirement] + """ + Resolves the provided **InstallRequirement**, returning a dictionary. + + Maps a dictionary of names to corresponding ``InstallRequirement`` values. + + :param :class:`~pip._internal.req.req_install.InstallRequirement` ireq: An + InstallRequirement to initiate the resolution process + :param :class:`~pip_shims.models.ShimmedPathCollection` reqset_provider: A provider + to build requirement set instances. + :param :class:`~pip_shims.models.ShimmedPathCollection` req_tracker_provider: A + provider to build requirement tracker instances + :param install_cmd_provider: A shim for providing new install command instances. + :type install_cmd_provider: :class:`~pip_shims.models.ShimmedPathCollection` + :param Optional[TCommandInstance] install_command: The install command used to + create the finder, session, and options if needed, defaults to None. + :param :class:`~pip_shims.models.ShimmedPathCollection` finder_provider: A provider + to package finder instances. + :param :class:`~pip_shims.models.ShimmedPathCollection` resolver_provider: A provider + to build resolver instances + :param TShimmedFunc wheel_cache_provider: The provider function to use to generate a + wheel cache if needed. + :param TShimmedFunc format_control_provider: The provider function to use to generate + a format_control instance if needed. + :param TShimmedFunc make_preparer_provider: Callable or shim for generating preparers. + :param Optional[TShimmedFunc] tempdir_manager_provider: Shim for generating tempdir + manager for pip temporary directories + :param Optional[Values] options: Pip options to use if needed, defaults to None + :param Optional[TSession] session: Existing session to use for getting requirements, + defaults to None + :param :class:`~pip._internal.legacy_resolve.Resolver` resolver: A pre-existing + resolver instance to use for resolution + :param Optional[TFinder] finder: The package finder to use during resolution, + defaults to None. + :param str upgrade_strategy: Upgrade strategy to use, defaults to ``only-if-needed``. + :param Optional[bool] force_reinstall: Whether to simulate or assume package + reinstallation during resolution, defaults to None + :param Optional[bool] ignore_dependencies: Whether to ignore package dependencies, + defaults to None + :param Optional[bool] ignore_requires_python: Whether to ignore indicated + required_python versions on packages, defaults to None + :param bool ignore_installed: Whether to ignore installed packages during + resolution, defaults to True + :param bool use_user_site: Whether to use the user site location during resolution, + defaults to False + :param Optional[bool] isolated: Whether to isolate the resolution process, defaults + to None + :param Optional[str] build_dir: Directory for building packages and wheels, defaults + to None + :param str source_dir: The directory to use for source requirements. Removed in pip + 10, defaults to None + :param Optional[str] download_dir: Target directory to download files, defaults to + None + :param str cache_dir: The cache directory to use for caching artifacts during + resolution + :param Optional[str] wheel_download_dir: Target directoryto download wheels, defaults + to None + :param Optional[TWheelCache] wheel_cache: The wheel cache to use, defaults to None + :param bool require_hashes: Whether to require hashes when resolving. Defaults to + False. + :param bool check_supported_wheels: Whether to check support of wheels before including + them in resolution. + :return: A dictionary mapping requirements to corresponding + :class:`~pip._internal.req.req_install.InstallRequirement`s + :rtype: :class:`~pip._internal.req.req_install.InstallRequirement` + + :Example: + + >>> from pip_shims.shims import resolve, InstallRequirement + >>> ireq = InstallRequirement.from_line("requests>=2.20") + >>> results = resolve(ireq) + >>> for k, v in results.items(): + ... print("{0}: {1!r}".format(k, v)) + requests: <InstallRequirement object: requests>=2.20 from https://files.pythonhosted. + org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/reque + sts-2.22.0-py2.py3-none-any.whl#sha256=9cf5292fcd0f598c671cfc1e0d7d1a7f13bb8085e9a590 + f48c010551dc6c4b31 editable=False> + idna: <InstallRequirement object: idna<2.9,>=2.5 from https://files.pythonhosted.org/ + packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8- + py2.py3-none-any.whl#sha256=ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432 + f7e4a3c (from requests>=2.20) editable=False> + urllib3: <InstallRequirement object: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 from htt + ps://files.pythonhosted.org/packages/b4/40/a9837291310ee1ccc242ceb6ebfd9eb21539649f19 + 3a7c8c86ba15b98539/urllib3-1.25.7-py2.py3-none-any.whl#sha256=a8a318824cc77d1fd4b2bec + 2ded92646630d7fe8619497b142c84a9e6f5a7293 (from requests>=2.20) editable=False> + chardet: <InstallRequirement object: chardet<3.1.0,>=3.0.2 from https://files.pythonh + osted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8 + /chardet-3.0.4-py2.py3-none-any.whl#sha256=fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed + 4531e3e15460124c106691 (from requests>=2.20) editable=False> + certifi: <InstallRequirement object: certifi>=2017.4.17 from https://files.pythonhost + ed.org/packages/18/b0/8146a4f8dd402f60744fa380bc73ca47303cccf8b9190fd16a827281eac2/ce + rtifi-2019.9.11-py2.py3-none-any.whl#sha256=fd7c7c74727ddcf00e9acd26bba8da604ffec95bf + 1c2144e67aff7a8b50e6cef (from requests>=2.20) editable=False> + """ + reqset_provider = resolve_possible_shim(reqset_provider) + finder_provider = resolve_possible_shim(finder_provider) + resolver_provider = resolve_possible_shim(resolver_provider) + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) + format_control_provider = resolve_possible_shim(format_control_provider) + make_preparer_provider = resolve_possible_shim(make_preparer_provider) + req_tracker_provider = resolve_possible_shim(req_tracker_provider) + install_cmd_provider = resolve_possible_shim(install_cmd_provider) + tempdir_manager_provider = resolve_possible_shim(tempdir_manager_provider) + if install_command is None: + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_command = install_cmd_provider() + kwarg_map = { + "upgrade_strategy": upgrade_strategy, + "force_reinstall": force_reinstall, + "ignore_dependencies": ignore_dependencies, + "ignore_requires_python": ignore_requires_python, + "ignore_installed": ignore_installed, + "use_user_site": use_user_site, + "isolated": isolated, + "build_dir": build_dir, + "src_dir": source_dir, + "download_dir": download_dir, + "require_hashes": require_hashes, + "cache_dir": cache_dir, + } + kwargs, options = populate_options(install_command, options, **kwarg_map) + with ExitStack() as ctx: + ctx.enter_context(tempdir_manager_provider()) + kwargs = ctx.enter_context( + ensure_resolution_dirs(wheel_download_dir=wheel_download_dir, **kwargs) + ) + wheel_download_dir = kwargs.pop("wheel_download_dir") + if session is None: + session = get_session(install_cmd=install_command, options=options) + if finder is None: + finder = finder_provider( + install_command, options=options, session=session + ) # type: ignore + format_control = getattr(options, "format_control", None) + if not format_control: + format_control = format_control_provider(None, None) # type: ignore + wheel_cache = ctx.enter_context( + wheel_cache_provider(kwargs["cache_dir"], format_control) + ) # type: ignore + ireq.is_direct = True # type: ignore + build_location_kwargs = {"build_dir": kwargs["build_dir"], "autodelete": True} + call_function_with_correct_args(ireq.build_location, **build_location_kwargs) + if reqset_provider is None: + raise TypeError( + "cannot resolve without a requirement set provider... failed!" + ) + reqset = reqset_provider( + install_command, + options=options, + session=session, + wheel_download_dir=wheel_download_dir, + **kwargs + ) # type: ignore + if getattr(reqset, "prepare_files", None): + reqset.add_requirement(ireq) + results = reqset.prepare_files(finder) + result = reqset.requirements + reqset.cleanup_files() + return result + if make_preparer_provider is None: + raise TypeError("Cannot create requirement preparer, cannot resolve!") + + preparer_args = { + "build_dir": kwargs["build_dir"], + "src_dir": kwargs["src_dir"], + "download_dir": kwargs["download_dir"], + "wheel_download_dir": wheel_download_dir, + "build_isolation": kwargs["isolated"], + "install_cmd": install_command, + "options": options, + "finder": finder, + "session": session, + "use_user_site": use_user_site, + "require_hashes": require_hashes, + } + if isinstance(req_tracker_provider, (types.FunctionType, functools.partial)): + preparer_args["req_tracker"] = ctx.enter_context(req_tracker_provider()) + resolver_keys = [ + "upgrade_strategy", + "force_reinstall", + "ignore_dependencies", + "ignore_installed", + "use_user_site", + "isolated", + "use_user_site", + ] + resolver_args = {key: kwargs[key] for key in resolver_keys if key in kwargs} + if resolver_provider is None: + raise TypeError("Cannot resolve without a resolver provider... failed!") + preparer = ctx.enter_context(make_preparer_provider(**preparer_args)) + resolver = resolver_provider( + finder=finder, + preparer=preparer, + session=session, + options=options, + install_cmd=install_command, + wheel_cache=wheel_cache, + **resolver_args + ) # type: ignore + resolver.require_hashes = kwargs.get("require_hashes", False) # type: ignore + _, required_resolver_args = get_method_args(resolver.resolve) + resolver_args = [] + if "requirement_set" in required_resolver_args.args: + reqset.add_requirement(ireq) + resolver_args.append(reqset) + elif "root_reqs" in required_resolver_args.args: + resolver_args.append([ireq]) + if "check_supported_wheels" in required_resolver_args.args: + resolver_args.append(check_supported_wheels) + result_reqset = resolver.resolve(*resolver_args) # type: ignore + if result_reqset is None: + result_reqset = reqset + results = result_reqset.requirements + cleanup_fn = getattr(reqset, "cleanup_files", None) + if cleanup_fn is not None: + cleanup_fn() + return results + + +def build_wheel( # noqa:C901 + req=None, # type: Optional[TInstallRequirement] + reqset=None, # type: Optional[Union[TReqSet, Iterable[TInstallRequirement]]] + output_dir=None, # type: Optional[str] + preparer=None, # type: Optional[TPreparer] + wheel_cache=None, # type: Optional[TWheelCache] + build_options=None, # type: Optional[List[str]] + global_options=None, # type: Optional[List[str]] + check_binary_allowed=None, # type: Optional[Callable[TInstallRequirement, bool]] + no_clean=False, # type: bool + session=None, # type: Optional[TSession] + finder=None, # type: Optional[TFinder] + install_command=None, # type: Optional[TCommand] + req_tracker=None, # type: Optional[TReqTracker] + build_dir=None, # type: Optional[str] + src_dir=None, # type: Optional[str] + download_dir=None, # type: Optional[str] + wheel_download_dir=None, # type: Optional[str] + cache_dir=None, # type: Optional[str] + use_user_site=False, # type: bool + use_pep517=None, # type: Optional[bool] + format_control_provider=None, # type: Optional[TShimmedFunc] + wheel_cache_provider=None, # type: Optional[TShimmedFunc] + preparer_provider=None, # type: Optional[TShimmedFunc] + wheel_builder_provider=None, # type: Optional[TShimmedFunc] + build_one_provider=None, # type: Optional[TShimmedFunc] + build_one_inside_env_provider=None, # type: Optional[TShimmedFunc] + build_many_provider=None, # type: Optional[TShimmedFunc] + install_command_provider=None, # type: Optional[TShimmedFunc] + finder_provider=None, # type: Optional[TShimmedFunc] + reqset_provider=None, # type: Optional[TShimmedFunc] +): + # type: (...) -> Generator[Union[str, Tuple[List[TInstallRequirement], ...]], None, None] + """ + Build a wheel or a set of wheels + + :raises TypeError: Raised when no requirements are provided + :param Optional[TInstallRequirement] req: An `InstallRequirement` to build + :param Optional[TReqSet] reqset: A `RequirementSet` instance (`pip<10`) or an + iterable of `InstallRequirement` instances (`pip>=10`) to build + :param Optional[str] output_dir: Target output directory, only useful when building + one wheel using pip>=20.0 + :param Optional[TPreparer] preparer: A preparer instance, defaults to None + :param Optional[TWheelCache] wheel_cache: A wheel cache instance, defaults to None + :param Optional[List[str]] build_options: A list of build options to pass in + :param Optional[List[str]] global_options: A list of global options to pass in + :param Optional[Callable[TInstallRequirement, bool]] check_binary_allowed: A callable + to check whether we are allowed to build and cache wheels for an ireq + :param bool no_clean: Whether to avoid cleaning up wheels + :param Optional[TSession] session: A `PipSession` instance to pass to create a + `finder` if necessary + :param Optional[TFinder] finder: A `PackageFinder` instance to use for generating a + `WheelBuilder` instance on `pip<20` + :param Optional[TCommandInstance] install_command: The install command used to + create the finder, session, and options if needed, defaults to None. + :param Optional[TReqTracker] req_tracker: An optional requirement tracker instance, + if one already exists + :param Optional[str] build_dir: Passthrough parameter for building preparer + :param Optional[str] src_dir: Passthrough parameter for building preparer + :param Optional[str] download_dir: Passthrough parameter for building preparer + :param Optional[str] wheel_download_dir: Passthrough parameter for building preparer + :param Optional[str] cache_dir: Passthrough cache directory for wheel cache options + :param bool use_user_site: Whether to use the user site directory when preparing + install requirements on `pip<20` + :param Optional[bool] use_pep517: When set to *True* or *False*, prefers building + with or without pep517 as specified, otherwise uses requirement preference. + Only works for single requirements. + :param Optional[TShimmedFunc] format_control_provider: A provider for the + `FormatControl` class + :param Optional[TShimmedFunc] wheel_cache_provider: A provider for the `WheelCache` + class + :param Optional[TShimmedFunc] preparer_provider: A provider for the + `RequirementPreparer` class + :param Optional[TShimmedFunc] wheel_builder_provider: A provider for the + `WheelBuilder` class, if it exists + :param Optional[TShimmedFunc] build_one_provider: A provider for the `_build_one` + function, if it exists + :param Optional[TShimmedFunc] build_one_inside_env_provider: A provider for the + `_build_one_inside_env` function, if it exists + :param Optional[TShimmedFunc] build_many_provider: A provider for the `build` + function, if it exists + :param Optional[TShimmedFunc] install_command_provider: A shim for providing new + install command instances + :param TShimmedFunc finder_provider: A provider to package finder instances + :param TShimmedFunc reqset_provider: A provider for requirement set generation + :return: A tuple of successful and failed install requirements or else a path to + a wheel + :rtype: Optional[Union[str, Tuple[List[TInstallRequirement], List[TInstallRequirement]]]] + """ + wheel_cache_provider = resolve_possible_shim(wheel_cache_provider) + preparer_provider = resolve_possible_shim(preparer_provider) + wheel_builder_provider = resolve_possible_shim(wheel_builder_provider) + build_one_provider = resolve_possible_shim(build_one_provider) + build_one_inside_env_provider = resolve_possible_shim(build_one_inside_env_provider) + build_many_provider = resolve_possible_shim(build_many_provider) + install_cmd_provider = resolve_possible_shim(install_command_provider) + format_control_provider = resolve_possible_shim(format_control_provider) + finder_provider = resolve_possible_shim(finder_provider) or get_package_finder + reqset_provider = resolve_possible_shim(reqset_provider) + global_options = [] if global_options is None else global_options + build_options = [] if build_options is None else build_options + options = None + kwarg_map = { + "cache_dir": cache_dir, + "src_dir": src_dir, + "download_dir": download_dir, + "wheel_download_dir": wheel_download_dir, + "build_dir": build_dir, + "use_user_site": use_user_site, + } + if not req and not reqset: + raise TypeError("Must provide either a requirement or requirement set to build") + with ExitStack() as ctx: + kwargs = kwarg_map.copy() + if wheel_cache is None and (reqset is not None or output_dir is None): + if install_command is None: + assert isinstance(install_cmd_provider, (type, functools.partial)) + install_command = install_cmd_provider() + kwargs, options = populate_options(install_command, options, **kwarg_map) + format_control = getattr(options, "format_control", None) + if not format_control: + format_control = format_control_provider(None, None) # type: ignore + wheel_cache = ctx.enter_context( + wheel_cache_provider(options.cache_dir, format_control) + ) + if req and not reqset and not output_dir: + output_dir = get_ireq_output_path(wheel_cache, req) + if not reqset and build_one_provider: + yield build_one_provider(req, output_dir, build_options, global_options) + elif build_many_provider: + yield build_many_provider( + reqset, wheel_cache, build_options, global_options, check_binary_allowed + ) + else: + builder_args, builder_kwargs = get_allowed_args(wheel_builder_provider) + if "requirement_set" in builder_args and not reqset: + reqset = reqset_provider() + if session is None and finder is None: + session = get_session(install_cmd=install_command, options=options) + finder = finder_provider( + install_command, options=options, session=session + ) + if preparer is None: + preparer_kwargs = { + "build_dir": kwargs["build_dir"], + "src_dir": kwargs["src_dir"], + "download_dir": kwargs["download_dir"], + "wheel_download_dir": kwargs["wheel_download_dir"], + "finder": finder, + "session": session + if session + else get_session(install_cmd=install_command, options=options), + "install_cmd": install_command, + "options": options, + "use_user_site": use_user_site, + "req_tracker": req_tracker, + } + preparer = ctx.enter_context(preparer_provider(**preparer_kwargs)) + check_bin = check_binary_allowed if check_binary_allowed else lambda x: True + builder_kwargs = { + "requirement_set": reqset, + "finder": finder, + "preparer": preparer, + "wheel_cache": wheel_cache, + "no_clean": no_clean, + "build_options": build_options, + "global_options": global_options, + "check_binary_allowed": check_bin, + } + builder = call_function_with_correct_args( + wheel_builder_provider, **builder_kwargs + ) + if req and not reqset: + if not output_dir: + output_dir = get_ireq_output_path(wheel_cache, req) + if use_pep517 is not None: + req.use_pep517 = use_pep517 + yield builder._build_one(req, output_dir) + else: + yield builder.build(reqset) diff --git a/pipenv/vendor/pip_shims/environment.py b/pipenv/vendor/pip_shims/environment.py new file mode 100644 index 00000000..2268ea26 --- /dev/null +++ b/pipenv/vendor/pip_shims/environment.py @@ -0,0 +1,45 @@ +# -*- coding=utf-8 -*- +""" +Module with functionality to learn about the environment. +""" +from __future__ import absolute_import + +import importlib +import os + + +def get_base_import_path(): + base_import_path = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") + return base_import_path + + +BASE_IMPORT_PATH = get_base_import_path() + + +def get_pip_version(import_path=BASE_IMPORT_PATH): + try: + pip = importlib.import_module(import_path) + except ImportError: + if import_path != "pip": + return get_pip_version(import_path="pip") + else: + import subprocess + + version = subprocess.check_output(["pip", "--version"]) + if version: + version = version.decode("utf-8").split()[1] + return version + return "0.0.0" + version = getattr(pip, "__version__", None) + return version + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) diff --git a/pipenv/vendor/pip_shims/models.py b/pipenv/vendor/pip_shims/models.py new file mode 100644 index 00000000..79871f5b --- /dev/null +++ b/pipenv/vendor/pip_shims/models.py @@ -0,0 +1,1223 @@ +# -*- coding: utf-8 -*- +""" +Helper module for shimming functionality across pip versions. +""" +from __future__ import absolute_import, print_function + +import collections +import functools +import importlib +import inspect +import operator +import sys +import types +import weakref + +import six + +from . import compat +from .environment import BASE_IMPORT_PATH, MYPY_RUNNING, get_pip_version +from .utils import ( + add_mixin_to_class, + apply_alias, + ensure_function, + fallback_is_artifact, + fallback_is_file_url, + fallback_is_vcs, + get_method_args, + has_property, + make_classmethod, + make_method, + nullcontext, + parse_version, + resolve_possible_shim, + set_default_kwargs, + split_package, + suppress_setattr, +) + +# format: off +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore # noqa +from six.moves import Sequence, Mapping # type: ignore # noqa # isort:skip + +# format: on + + +if MYPY_RUNNING: + import packaging.version + + Module = types.ModuleType + from typing import ( # noqa:F811 + Any, + Callable, + ContextManager, + Dict, + Iterable, + List, + Mapping, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + ) + + +PIP_VERSION_SET = { + "7.0.0", + "7.0.1", + "7.0.2", + "7.0.3", + "7.1.0", + "7.1.1", + "7.1.2", + "8.0.0", + "8.0.1", + "8.0.2", + "8.0.3", + "8.1.0", + "8.1.1", + "8.1.2", + "9.0.0", + "9.0.1", + "9.0.2", + "9.0.3", + "10.0.0", + "10.0.1", + "18.0", + "18.1", + "19.0", + "19.0.1", + "19.0.2", + "19.0.3", + "19.1", + "19.1.1", + "19.2", + "19.2.1", + "19.2.2", + "19.2.3", + "19.3", + "19.3.1", + "20.0", + "20.0.1", + "20.0.2", +} + + +ImportTypesBase = collections.namedtuple( + "ImportTypes", ["FUNCTION", "CLASS", "MODULE", "CONTEXTMANAGER"] +) + + +class ImportTypes(ImportTypesBase): + FUNCTION = 0 + CLASS = 1 + MODULE = 2 + CONTEXTMANAGER = 3 + METHOD = 4 + ATTRIBUTE = 5 + + +class PipVersion(Sequence): + def __init__( + self, + version, + round_prereleases_up=True, + base_import_path=None, + vendor_import_path="pip._vendor", + ): + # type: (str, bool, Optional[str], str) -> None + self.version = version + self.vendor_import_path = vendor_import_path + self.round_prereleases_up = round_prereleases_up + parsed_version = self._parse() + if round_prereleases_up and parsed_version.is_prerelease: + parsed_version._version = parsed_version._version._replace(dev=None, pre=None) + self.version = str(parsed_version) + parsed_version = self._parse() + if base_import_path is None: + if parsed_version >= parse_version("10.0.0"): + base_import_path = "{}._internal".format(BASE_IMPORT_PATH) + else: + base_import_path = "{}".format(BASE_IMPORT_PATH) + self.base_import_path = base_import_path + self.parsed_version = parsed_version + + @property + def version_tuple(self): + return tuple(self.parsed_version._version) + + @property + def version_key(self): + return self.parsed_version._key + + def is_valid(self, compared_to): + # type: (PipVersion) -> bool + return self == compared_to + + def __len__(self): + # type: () -> int + return len(self.version_tuple) + + def __getitem__(self, item): + return self.version_tuple[item] + + def _parse(self): + # type: () -> packaging.version._BaseVersion + return parse_version(self.version) + + def __hash__(self): + # type: () -> int + return hash(self.parsed_version) + + def __str__(self): + # type: () -> str + return "{!s}".format(self.parsed_version) + + def __repr__(self): + # type: () -> str + return ( + "<PipVersion {!r}, Path: {!r}, Vendor Path: {!r}, " "Parsed Version: {!r}>" + ).format( + self.version, + self.base_import_path, + self.vendor_import_path, + self.parsed_version, + ) + + def __gt__(self, other): + # type: (PipVersion) -> bool + return self.parsed_version > other.parsed_version + + def __lt__(self, other): + # type: (PipVersion) -> bool + return self.parsed_version < other.parsed_version + + def __le__(self, other): + # type: (PipVersion) -> bool + return self.parsed_version <= other.parsed_version + + def __ge__(self, other): + # type: (PipVersion) -> bool + return self.parsed_version >= other.parsed_version + + def __ne__(self, other): + # type: (object) -> bool + if not isinstance(other, PipVersion): + return NotImplemented + return self.parsed_version != other.parsed_version + + def __eq__(self, other): + # type: (object) -> bool + if not isinstance(other, PipVersion): + return NotImplemented + return self.parsed_version == other.parsed_version + + +version_cache = weakref.WeakValueDictionary() # type: Mapping[str, PipVersion] +CURRENT_PIP_VERSION = None # type: Optional[PipVersion] + + +def pip_version_lookup(version, *args, **kwargs): + # type: (str, Any, Any) -> PipVersion + try: + cached = version_cache.get(version) + except KeyError: + cached = None + if cached is not None: + return cached + pip_version = PipVersion(version, *args, **kwargs) + version_cache[version] = pip_version + return pip_version + + +def lookup_current_pip_version(): + # type: () -> PipVersion + global CURRENT_PIP_VERSION + if CURRENT_PIP_VERSION is not None: + return CURRENT_PIP_VERSION + CURRENT_PIP_VERSION = pip_version_lookup(get_pip_version()) + return CURRENT_PIP_VERSION + + +class PipVersionRange(Sequence): + def __init__(self, start, end): + # type: (PipVersion, PipVersion) -> None + if start > end: + raise ValueError("Start version must come before end version") + self._versions = (start, end) + + def __str__(self): + # type: () -> str + return "{!s} -> {!s}".format(self._versions[0], self._versions[-1]) + + @property + def base_import_paths(self): + # type: () -> Set[str] + return {version.base_import_path for version in self._versions} + + @property + def vendor_import_paths(self): + # type: () -> Set[str] + return {version.vendor_import_path for version in self._versions} + + def is_valid(self): + # type: () -> bool + return pip_version_lookup(get_pip_version()) in self + + def __contains__(self, item): + # type: (PipVersion) -> bool + if not isinstance(item, PipVersion): + raise TypeError("Need a PipVersion instance to compare") + return item >= self[0] and item <= self[-1] + + def __getitem__(self, item): + # type: (int) -> PipVersion + return self._versions[item] + + def __len__(self): + # type: () -> int + return len(self._versions) + + def __lt__(self, other): + # type: ("PipVersionRange") -> bool + return (other.is_valid() and not self.is_valid()) or ( + not (self.is_valid() or other.is_valid()) + or (self.is_valid() and other.is_valid()) + and self._versions[-1] < other._versions[-1] + ) + + def __hash__(self): + # type: () -> int + return hash(self._versions) + + +class ShimmedPath(object): + __modules = {} # type: Dict[str, Module] + + def __init__( + self, + name, # type: str + import_target, # type: str + import_type, # type: int + version_range, # type: PipVersionRange + provided_methods=None, # type: Optional[Dict[str, Callable]] + provided_functions=None, # type: Optional[Dict[str, Callable]] + provided_classmethods=None, # type: Optional[Dict[str, Callable]] + provided_contextmanagers=None, # type: Optional[Dict[str, Callable]] + provided_mixins=None, # type: Optional[List[Type]] + default_args=None, # type: Dict[str, Sequence[List[Any], Dict[str, Any]]] + ): + # type: (...) -> None + if provided_methods is None: + provided_methods = {} + if provided_classmethods is None: + provided_classmethods = {} + if provided_functions is None: + provided_functions = {} + if provided_contextmanagers is None: + provided_contextmanagers = {} + if provided_mixins is None: + provided_mixins = [] + if default_args is None: + default_args = {} + self.version_range = version_range + self.name = name + self.full_import_path = import_target + module_path, name_to_import = split_package(import_target) + self.module_path = module_path + self.name_to_import = name_to_import + self.import_type = import_type + self._imported = None # type: Optional[Module] + self._provided = None # type: Optional[Union[Module, Type, Callable, Any]] + self.provided_methods = provided_methods + self.provided_functions = provided_functions + self.provided_classmethods = provided_classmethods + self.provided_contextmanagers = provided_contextmanagers + self.provided_mixins = [m for m in provided_mixins if m is not None] + self.default_args = default_args + self.aliases = [] # type: List[List[str]] + self._shimmed = None # type: Optional[Any] + + def _as_tuple(self): + # type: () -> Tuple[str, PipVersionRange, str, int] + return (self.name, self.version_range, self.full_import_path, self.import_type) + + def alias(self, aliases): + # type: (List[str]) -> "ShimmedPath" + self.aliases.append(aliases) + return self + + @classmethod + def _import_module(cls, module): + # type: (str) -> Optional[Module] + if module in ShimmedPath.__modules: + result = ShimmedPath.__modules[module] + if result is not None: + return result + try: + imported = importlib.import_module(module) + except ImportError: + return None + else: + ShimmedPath.__modules[module] = imported + return imported + + @classmethod + def _parse_provides_dict( + cls, + provides, # type: Dict[str, Callable] + prepend_arg_to_callables=None, # type: Optional[str] + ): + # type: (...) -> Dict[str, Callable] + creating_methods = False + creating_classmethods = False + if prepend_arg_to_callables is not None: + if prepend_arg_to_callables == "self": + creating_methods = True + elif prepend_arg_to_callables == "cls": + creating_classmethods = True + provides_map = {} + for item_name, item_value in provides.items(): + if isinstance(item_value, ShimmedPath): + item_value = item_value.shim() + if inspect.isfunction(item_value): + callable_args = inspect.getargs(item_value.__code__).args + if "self" not in callable_args and creating_methods: + item_value = make_method(item_value)(item_name) + elif "cls" not in callable_args and creating_classmethods: + item_value = make_classmethod(item_value)(item_name) + elif isinstance(item_value, six.string_types): + module_path, name = split_package(item_value) + module = cls._import_module(module_path) + item_value = getattr(module, name, None) + if item_value is not None: + provides_map[item_name] = item_value + return provides_map + + def _update_default_kwargs(self, parent, provided): + # type: (Union[Module, None], Union[Type, Module]) -> Tuple[Optional[Module], Union[Type, Module]] # noqa + for func_name, defaults in self.default_args.items(): + # * Note that we set default args here because we have the + # * option to use it, even though currently we dont + # * so we are forcibly ignoring the linter warning about it + default_args, default_kwargs = defaults # noqa:W0612 + provided = set_default_kwargs( + provided, func_name, *default_args, **default_kwargs + ) + return parent, provided + + def _ensure_functions(self, provided): + # type: (Union[Module, Type, None]) -> Any + functions = self._parse_provides_dict(self.provided_functions) + if provided is None: + provided = __module__ # type: ignore # noqa:F821 + for funcname, func in functions.items(): + func = ensure_function(provided, funcname, func) + setattr(provided, funcname, func) + return provided + + def _ensure_methods(self, provided): + # type: (Type) -> Type + """Given a base class, a new name, and any number of functions to + attach, turns those functions into classmethods, attaches them, + and returns an updated class object. + """ + if not self.is_class: + return provided + if not inspect.isclass(provided): + raise TypeError("Provided argument is not a class: {!r}".format(provided)) + methods = self._parse_provides_dict( + self.provided_methods, prepend_arg_to_callables="self" + ) + classmethods = self._parse_provides_dict( + self.provided_classmethods, prepend_arg_to_callables="cls" + ) + if not methods and not classmethods: + return provided + new_functions = provided.__dict__.copy() + if classmethods: + new_functions.update( + { + method_name: clsmethod + for method_name, clsmethod in classmethods.items() + if method_name not in provided.__dict__ + } + ) + if methods: + new_functions.update( + { + method_name: method + for method_name, method in methods.items() + if method_name not in provided.__dict__ + } + ) + classname = provided.__name__ + if six.PY2: + classname = classname.encode(sys.getdefaultencoding()) + type_ = type(classname, (provided,), new_functions) + return type_ + + @property + def is_class(self): + # type: () -> bool + return self.import_type == ImportTypes.CLASS + + @property + def is_module(self): + # type: () -> bool + return self.import_type == ImportTypes.MODULE + + @property + def is_method(self): + # type: () -> bool + return self.import_type == ImportTypes.METHOD + + @property + def is_function(self): + # type: () -> bool + return self.import_type == ImportTypes.FUNCTION + + @property + def is_contextmanager(self): + # type: () -> bool + return self.import_type == ImportTypes.CONTEXTMANAGER + + @property + def is_attribute(self): + # type: () -> bool + return self.import_type == ImportTypes.ATTRIBUTE + + def __contains__(self, pip_version): + # type: (str) -> bool + return pip_version_lookup(pip_version) in self.version_range + + @property + def is_valid(self): + # type: () -> bool + return self.version_range.is_valid() + + @property + def sort_order(self): + # type: () -> int + return 1 if self.is_valid else 0 + + def _shim_base(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Any + result = getattr(imported, attribute_name, None) + return self._apply_aliases(imported, result) + + def _apply_aliases(self, imported, target): + # type: (Union[Module, None], Any) -> Any + for alias_list in self.aliases: + target = apply_alias(imported, target, *alias_list) + suppress_setattr(imported, self.name, target) + return target + + def _shim_parent(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Tuple[Optional[Module], Any] + result = self._shim_base(imported, attribute_name) + if result is not None: + imported, result = self._update_default_kwargs(imported, result) + suppress_setattr(imported, attribute_name, result) + return imported, result + + def update_sys_modules(self, imported): + # type: (Optional[Module]) -> None + if imported is None: + return None + if self.calculated_module_path in sys.modules: + del sys.modules[self.calculated_module_path] + sys.modules[self.calculated_module_path] = imported + + def shim_class(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Type + imported, result = self._shim_parent(imported, attribute_name) + if result is not None: + assert inspect.isclass(result) # noqa + result = self._ensure_methods(result) + if self.provided_mixins: + result = add_mixin_to_class(result, self.provided_mixins) + self._imported = imported + self._provided = result + self.update_sys_modules(imported) + if imported is not None: + ShimmedPath.__modules[imported.__name__] = imported + return result + + def shim_module(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Module + imported, result = self._shim_parent(imported, attribute_name) + if result is not None: + result = self._ensure_functions(result) + full_import_path = "{}.{}".format(self.calculated_module_path, attribute_name) + self._imported = imported + assert isinstance(result, types.ModuleType) + self._provided = result + if full_import_path in sys.modules: + del sys.modules[full_import_path] + sys.modules[full_import_path] = result + self.update_sys_modules(imported) + if imported is not None: + ShimmedPath.__modules[imported.__name__] = imported + return result # type: ignore + + def shim_function(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Callable + return self._shim_base(imported, attribute_name) + + def shim_attribute(self, imported, attribute_name): + # type: (Union[Module, None], Any) -> Any + return self._shim_base(imported, attribute_name) + + def shim_contextmanager(self, imported, attribute_name): + # type: (Union[Module, None], str) -> Callable + result = self._shim_base(imported, attribute_name) + if result is None: + result = nullcontext + suppress_setattr(imported, attribute_name, result) + self.update_sys_modules(imported) + return result + + @property + def shimmed(self): + # type: () -> Any + if self._shimmed is None: + self._shimmed = self.shim() + return self._shimmed + + def shim(self): + # type: () -> (Union[Module, Callable, ContextManager, Type]) + imported = self._import() + if self.is_class: + return self.shim_class(imported, self.name_to_import) + elif self.is_module: + return self.shim_module(imported, self.name_to_import) + elif self.is_contextmanager: + return self.shim_contextmanager(imported, self.name_to_import) + elif self.is_function: + return self.shim_function(imported, self.name_to_import) + elif self.is_attribute: + return self.shim_attribute(imported, self.name_to_import) + return self._shim_base(imported, self.name_to_import) + + @property + def calculated_module_path(self): + current_pip = lookup_current_pip_version() + prefix = current_pip.base_import_path + return ".".join([prefix, self.module_path]).rstrip(".") + + def _import(self, prefix=None): + # type: (Optional[str]) -> Optional[Module] + # TODO: Decide whether to use _imported and _shimmed or to set the shimmed + # always to _imported and never save the unshimmed module + if self._imported is not None: + return self._imported + result = self._import_module(self.calculated_module_path) + return result + + def __hash__(self): + # type: () -> int + return hash(self._as_tuple()) + + +class ShimmedPathCollection(object): + + __registry = {} # type: Dict[str, Any] + + def __init__(self, name, import_type, paths=None): + # type: (str, int, Optional[Sequence[ShimmedPath]]) -> None + self.name = name + self.import_type = import_type + self.paths = set() # type: Set[ShimmedPath] + self.top_path = None + self._default = None + self._default_args = {} # type: Dict[str, Sequence[List[Any], Dict[str, Any]]] + self.provided_methods = {} # type: Dict[str, Callable] + self.provided_functions = {} # type: Dict[str, Callable] + self.provided_contextmanagers = {} # type: Dict[str, Callable] + self.provided_classmethods = {} # type: Dict[str, Callable] + self.provided_mixins = [] # type: List[Type] + self.pre_shim_functions = [] # type: List[Callable] + self.aliases = [] # type: List[List[str]] + if paths is not None: + if isinstance(paths, six.string_types): + self.create_path(paths, version_start=lookup_current_pip_version()) + else: + self.paths.update(set(paths)) + self.register() + + def register(self): + # type: () -> None + self.__registry[self.name] = self + + @classmethod + def get_registry(cls): + # type: () -> Dict[str, "ShimmedPathCollection"] + return cls.__registry.copy() + + def add_path(self, path): + # type: (ShimmedPath) -> None + self.paths.add(path) + + def set_default(self, default): + # type: (Any) -> None + if isinstance(default, (ShimmedPath, ShimmedPathCollection)): + default = default.shim() + try: + default.__qualname__ = default.__name__ = self.name + except AttributeError: + pass + self._default = default + + def set_default_args(self, callable_name, *args, **kwargs): + # type: (str, Any, Any) -> None + self._default_args.update({callable_name: [args, kwargs]}) + + def provide_function(self, name, fn): + # type: (str, Union[Callable, ShimmedPath, ShimmedPathCollection]) -> None + if isinstance(fn, (ShimmedPath, ShimmedPathCollection)): + fn = resolve_possible_shim(fn) # type: ignore + self.provided_functions[name] = fn # type: ignore + + def provide_method(self, name, fn): + # type: (str, Union[Callable, ShimmedPath, ShimmedPathCollection, property]) -> None + if isinstance(fn, (ShimmedPath, ShimmedPathCollection)): + fn = resolve_possible_shim(fn) # type: ignore + self.provided_methods[name] = fn # type: ignore + + def alias(self, aliases): + # type: (List[str]) -> None + """ + Takes a list of methods, functions, attributes, etc and ensures they + all exist on the object pointing at the same referent. + + :param List[str] aliases: Names to map to the same functionality if they do not + exist. + :return: None + :rtype: None + """ + self.aliases.append(aliases) + + def add_mixin(self, mixin): + # type: (Optional[Union[Type, ShimmedPathCollection]]) -> None + if isinstance(mixin, ShimmedPathCollection): + mixin = mixin.shim() + if mixin is not None and inspect.isclass(mixin): + self.provided_mixins.append(mixin) + + def create_path(self, import_path, version_start, version_end=None): + # type: (str, str, Optional[str]) -> None + pip_version_start = pip_version_lookup(version_start) + if version_end is None: + version_end = "9999" + pip_version_end = pip_version_lookup(version_end) + version_range = PipVersionRange(pip_version_start, pip_version_end) + new_path = ShimmedPath( + self.name, + import_path, + self.import_type, + version_range, + self.provided_methods, + self.provided_functions, + self.provided_classmethods, + self.provided_contextmanagers, + self.provided_mixins, + self._default_args, + ) + if self.aliases: + for alias_list in self.aliases: + new_path.alias(alias_list) + self.add_path(new_path) + + def _sort_paths(self): + # type: () -> List[ShimmedPath] + return sorted(self.paths, key=operator.attrgetter("version_range"), reverse=True) + + def _get_top_path(self): + # type: () -> Optional[ShimmedPath] + return next(iter(self._sort_paths()), None) + + @classmethod + def traverse(cls, shim): + # type: (Union[ShimmedPath, ShimmedPathCollection, Any]) -> Any + if isinstance(shim, (ShimmedPath, ShimmedPathCollection)): + result = shim.shim() + return result + return shim + + def shim(self): + # type: () -> Any + top_path = self._get_top_path() # type: Union[ShimmedPath, None] + if not self.pre_shim_functions: + result = self.traverse(top_path) + else: + for fn in self.pre_shim_functions: + result = fn(top_path) + result = self.traverse(result) + if result == nullcontext and self._default is not None: + default_result = self.traverse(self._default) + if default_result: + return default_result + if result is None and self._default is not None: + result = self.traverse(self._default) + return result + + def pre_shim(self, fn): + # type: (Callable) -> None + self.pre_shim_functions.append(fn) + + +def import_pip(): + return importlib.import_module("pip") + + +_strip_extras = ShimmedPathCollection("_strip_extras", ImportTypes.FUNCTION) +_strip_extras.create_path("req.req_install._strip_extras", "7.0.0", "18.0.0") +_strip_extras.create_path("req.constructors._strip_extras", "18.1.0") + +cmdoptions = ShimmedPathCollection("cmdoptions", ImportTypes.MODULE) +cmdoptions.create_path("cli.cmdoptions", "18.1", "9999") +cmdoptions.create_path("cmdoptions", "7.0.0", "18.0") + +commands_dict = ShimmedPathCollection("commands_dict", ImportTypes.ATTRIBUTE) +commands_dict.create_path("commands.commands_dict", "7.0.0", "9999") + +SessionCommandMixin = ShimmedPathCollection("SessionCommandMixin", ImportTypes.CLASS) +SessionCommandMixin.create_path("cli.req_command.SessionCommandMixin", "19.3.0", "9999") + +Command = ShimmedPathCollection("Command", ImportTypes.CLASS) +Command.set_default_args("__init__", name="PipCommand", summary="Default pip command.") +Command.add_mixin(SessionCommandMixin) +Command.create_path("cli.base_command.Command", "18.1", "9999") +Command.create_path("basecommand.Command", "7.0.0", "18.0") + +ConfigOptionParser = ShimmedPathCollection("ConfigOptionParser", ImportTypes.CLASS) +ConfigOptionParser.create_path("cli.parser.ConfigOptionParser", "18.1", "9999") +ConfigOptionParser.create_path("baseparser.ConfigOptionParser", "7.0.0", "18.0") + +InstallCommand = ShimmedPathCollection("InstallCommand", ImportTypes.CLASS) +InstallCommand.pre_shim( + functools.partial(compat.partial_command, cmd_mapping=commands_dict) +) +InstallCommand.create_path("commands.install.InstallCommand", "7.0.0", "9999") + +DistributionNotFound = ShimmedPathCollection("DistributionNotFound", ImportTypes.CLASS) +DistributionNotFound.create_path("exceptions.DistributionNotFound", "7.0.0", "9999") + +FAVORITE_HASH = ShimmedPathCollection("FAVORITE_HASH", ImportTypes.ATTRIBUTE) +FAVORITE_HASH.create_path("utils.hashes.FAVORITE_HASH", "7.0.0", "9999") + +FormatControl = ShimmedPathCollection("FormatControl", ImportTypes.CLASS) +FormatControl.create_path("models.format_control.FormatControl", "18.1", "9999") +FormatControl.create_path("index.FormatControl", "7.0.0", "18.0") + +FrozenRequirement = ShimmedPathCollection("FrozenRequirement", ImportTypes.CLASS) +FrozenRequirement.create_path("FrozenRequirement", "7.0.0", "9.0.3") +FrozenRequirement.create_path("operations.freeze.FrozenRequirement", "10.0.0", "9999") + +get_installed_distributions = ShimmedPathCollection( + "get_installed_distributions", ImportTypes.FUNCTION +) +get_installed_distributions.create_path( + "utils.misc.get_installed_distributions", "10", "9999" +) +get_installed_distributions.create_path("utils.get_installed_distributions", "7", "9.0.3") + +get_supported = ShimmedPathCollection("get_supported", ImportTypes.FUNCTION) +get_supported.create_path("pep425tags.get_supported", "7.0.0", "9999") + +get_tags = ShimmedPathCollection("get_tags", ImportTypes.FUNCTION) +get_tags.create_path("pep425tags.get_tags", "7.0.0", "9999") + +index_group = ShimmedPathCollection("index_group", ImportTypes.FUNCTION) +index_group.create_path("cli.cmdoptions.index_group", "18.1", "9999") +index_group.create_path("cmdoptions.index_group", "7.0.0", "18.0") + +InstallationError = ShimmedPathCollection("InstallationError", ImportTypes.CLASS) +InstallationError.create_path("exceptions.InstallationError", "7.0.0", "9999") + +UninstallationError = ShimmedPathCollection("UninstallationError", ImportTypes.CLASS) +UninstallationError.create_path("exceptions.UninstallationError", "7.0.0", "9999") + +DistributionNotFound = ShimmedPathCollection("DistributionNotFound", ImportTypes.CLASS) +DistributionNotFound.create_path("exceptions.DistributionNotFound", "7.0.0", "9999") + +RequirementsFileParseError = ShimmedPathCollection( + "RequirementsFileParseError", ImportTypes.CLASS +) +RequirementsFileParseError.create_path( + "exceptions.RequirementsFileParseError", "7.0.0", "9999" +) + +BestVersionAlreadyInstalled = ShimmedPathCollection( + "BestVersionAlreadyInstalled", ImportTypes.CLASS +) +BestVersionAlreadyInstalled.create_path( + "exceptions.BestVersionAlreadyInstalled", "7.0.0", "9999" +) + +BadCommand = ShimmedPathCollection("BadCommand", ImportTypes.CLASS) +BadCommand.create_path("exceptions.BadCommand", "7.0.0", "9999") + +CommandError = ShimmedPathCollection("CommandError", ImportTypes.CLASS) +CommandError.create_path("exceptions.CommandError", "7.0.0", "9999") + +PreviousBuildDirError = ShimmedPathCollection("PreviousBuildDirError", ImportTypes.CLASS) +PreviousBuildDirError.create_path("exceptions.PreviousBuildDirError", "7.0.0", "9999") + +install_req_from_editable = ShimmedPathCollection( + "install_req_from_editable", ImportTypes.FUNCTION +) +install_req_from_editable.create_path( + "req.constructors.install_req_from_editable", "18.1", "9999" +) +install_req_from_editable.create_path( + "req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0" +) + +install_req_from_line = ShimmedPathCollection( + "install_req_from_line", ImportTypes.FUNCTION +) +install_req_from_line.create_path( + "req.constructors.install_req_from_line", "18.1", "9999" +) +install_req_from_line.create_path( + "req.req_install.InstallRequirement.from_line", "7.0.0", "18.0" +) + +install_req_from_req_string = ShimmedPathCollection( + "install_req_from_req_string", ImportTypes.FUNCTION +) +install_req_from_req_string.create_path( + "req.constructors.install_req_from_req_string", "19.0", "9999" +) + +InstallRequirement = ShimmedPathCollection("InstallRequirement", ImportTypes.CLASS) +InstallRequirement.provide_method("from_line", install_req_from_line) +InstallRequirement.provide_method("from_editable", install_req_from_editable) +InstallRequirement.alias(["build_location", "ensure_build_location"]) + +InstallRequirement.create_path("req.req_install.InstallRequirement", "7.0.0", "9999") + +is_archive_file = ShimmedPathCollection("is_archive_file", ImportTypes.FUNCTION) +is_archive_file.create_path("req.constructors.is_archive_file", "19.3", "9999") +is_archive_file.create_path("download.is_archive_file", "7.0.0", "19.2.3") + +is_file_url = ShimmedPathCollection("is_file_url", ImportTypes.FUNCTION) +is_file_url.set_default(fallback_is_file_url) +is_file_url.create_path("download.is_file_url", "7.0.0", "19.2.3") + +Downloader = ShimmedPathCollection("Downloader", ImportTypes.CLASS) +Downloader.create_path("network.download.Downloader", "19.3.9", "9999") + +unpack_url = ShimmedPathCollection("unpack_url", ImportTypes.FUNCTION) +unpack_url.create_path("download.unpack_url", "7.0.0", "19.3.9") +unpack_url.create_path("operations.prepare.unpack_url", "20.0", "9999") + +is_installable_dir = ShimmedPathCollection("is_installable_dir", ImportTypes.FUNCTION) +is_installable_dir.create_path("utils.misc.is_installable_dir", "10.0.0", "9999") +is_installable_dir.create_path("utils.is_installable_dir", "7.0.0", "9.0.3") + +Link = ShimmedPathCollection("Link", ImportTypes.CLASS) +Link.provide_method("is_vcs", property(fallback_is_vcs)) +Link.provide_method("is_artifact", property(fallback_is_artifact)) +Link.create_path("models.link.Link", "19.0.0", "9999") +Link.create_path("index.Link", "7.0.0", "18.1") + +make_abstract_dist = ShimmedPathCollection("make_abstract_dist", ImportTypes.FUNCTION) +make_abstract_dist.create_path( + "distributions.make_distribution_for_install_requirement", "20.0.0", "9999" +) +make_abstract_dist.create_path( + "distributions.make_distribution_for_install_requirement", "19.1.2", "19.3.9" +) +make_abstract_dist.create_path( + "operations.prepare.make_abstract_dist", "10.0.0", "19.1.1" +) +make_abstract_dist.create_path("req.req_set.make_abstract_dist", "7.0.0", "9.0.3") + +make_distribution_for_install_requirement = ShimmedPathCollection( + "make_distribution_for_install_requirement", ImportTypes.FUNCTION +) +make_distribution_for_install_requirement.create_path( + "distributions.make_distribution_for_install_requirement", "20.0.0", "9999" +) +make_distribution_for_install_requirement.create_path( + "distributions.make_distribution_for_install_requirement", "19.1.2", "19.9.9" +) + +make_option_group = ShimmedPathCollection("make_option_group", ImportTypes.FUNCTION) +make_option_group.create_path("cli.cmdoptions.make_option_group", "18.1", "9999") +make_option_group.create_path("cmdoptions.make_option_group", "7.0.0", "18.0") + +PackageFinder = ShimmedPathCollection("PackageFinder", ImportTypes.CLASS) +PackageFinder.create_path("index.PackageFinder", "7.0.0", "19.9") +PackageFinder.create_path("index.package_finder.PackageFinder", "20.0", "9999") + +CandidateEvaluator = ShimmedPathCollection("CandidateEvaluator", ImportTypes.CLASS) +CandidateEvaluator.set_default(compat.CandidateEvaluator) +CandidateEvaluator.create_path("index.CandidateEvaluator", "19.1.0", "19.3.9") +CandidateEvaluator.create_path("index.package_finder.CandidateEvaluator", "20.0", "9999") + +CandidatePreferences = ShimmedPathCollection("CandidatePreferences", ImportTypes.CLASS) +CandidatePreferences.set_default(compat.CandidatePreferences) +CandidatePreferences.create_path("index.CandidatePreferences", "19.2.0", "19.9") +CandidatePreferences.create_path( + "index.package_finder.CandidatePreferences", "20.0", "9999" +) + +LinkCollector = ShimmedPathCollection("LinkCollector", ImportTypes.CLASS) +LinkCollector.set_default(compat.LinkCollector) +LinkCollector.create_path("collector.LinkCollector", "19.3.0", "19.9") +LinkCollector.create_path("index.collector.LinkCollector", "20.0", "9999") + +LinkEvaluator = ShimmedPathCollection("LinkEvaluator", ImportTypes.CLASS) +LinkEvaluator.set_default(compat.LinkEvaluator) +LinkEvaluator.create_path("index.LinkEvaluator", "19.2.0", "19.9") +LinkEvaluator.create_path("index.package_finder.LinkEvaluator", "20.0", "9999") + +TargetPython = ShimmedPathCollection("TargetPython", ImportTypes.CLASS) +compat.TargetPython.fallback_get_tags = get_tags +TargetPython.set_default(compat.TargetPython) +TargetPython.create_path("models.target_python.TargetPython", "19.2.0", "9999") + +SearchScope = ShimmedPathCollection("SearchScope", ImportTypes.CLASS) +SearchScope.set_default(compat.SearchScope) +SearchScope.create_path("models.search_scope.SearchScope", "19.2.0", "9999") + +SelectionPreferences = ShimmedPathCollection("SelectionPreferences", ImportTypes.CLASS) +SelectionPreferences.set_default(compat.SelectionPreferences) +SelectionPreferences.create_path( + "models.selection_prefs.SelectionPreferences", "19.2.0", "9999" +) + +parse_requirements = ShimmedPathCollection("parse_requirements", ImportTypes.FUNCTION) +parse_requirements.create_path("req.req_file.parse_requirements", "7.0.0", "9999") + +path_to_url = ShimmedPathCollection("path_to_url", ImportTypes.FUNCTION) +path_to_url.create_path("download.path_to_url", "7.0.0", "19.2.3") +path_to_url.create_path("utils.urls.path_to_url", "19.3.0", "9999") + +PipError = ShimmedPathCollection("PipError", ImportTypes.CLASS) +PipError.create_path("exceptions.PipError", "7.0.0", "9999") + +RequirementPreparer = ShimmedPathCollection("RequirementPreparer", ImportTypes.CLASS) +RequirementPreparer.create_path("operations.prepare.RequirementPreparer", "7", "9999") + +RequirementSet = ShimmedPathCollection("RequirementSet", ImportTypes.CLASS) +RequirementSet.create_path("req.req_set.RequirementSet", "7.0.0", "9999") + +RequirementTracker = ShimmedPathCollection( + "RequirementTracker", ImportTypes.CONTEXTMANAGER +) +RequirementTracker.create_path("req.req_tracker.RequirementTracker", "7.0.0", "9999") + +TempDirectory = ShimmedPathCollection("TempDirectory", ImportTypes.CLASS) +TempDirectory.create_path("utils.temp_dir.TempDirectory", "7.0.0", "9999") + +global_tempdir_manager = ShimmedPathCollection( + "global_tempdir_manager", ImportTypes.CONTEXTMANAGER +) +global_tempdir_manager.create_path( + "utils.temp_dir.global_tempdir_manager", "7.0.0", "9999" +) + +shim_unpack = ShimmedPathCollection("shim_unpack", ImportTypes.FUNCTION) +shim_unpack.set_default( + functools.partial( + compat.shim_unpack, + unpack_fn=unpack_url, + downloader_provider=Downloader, + tempdir_manager_provider=global_tempdir_manager, + ) +) + +get_requirement_tracker = ShimmedPathCollection( + "get_requirement_tracker", ImportTypes.CONTEXTMANAGER +) +get_requirement_tracker.set_default( + functools.partial(compat.get_requirement_tracker, RequirementTracker.shim()) +) +get_requirement_tracker.create_path( + "req.req_tracker.get_requirement_tracker", "7.0.0", "9999" +) + +Resolver = ShimmedPathCollection("Resolver", ImportTypes.CLASS) +Resolver.create_path("resolve.Resolver", "7.0.0", "19.1.1") +Resolver.create_path("legacy_resolve.Resolver", "19.1.2", "20.0.89999") +Resolver.create_path("resolution.legacy.resolver.Resolver", "20.0.99999", "99999") + +SafeFileCache = ShimmedPathCollection("SafeFileCache", ImportTypes.CLASS) +SafeFileCache.create_path("network.cache.SafeFileCache", "19.3.0", "9999") +SafeFileCache.create_path("download.SafeFileCache", "7.0.0", "19.2.3") + +UninstallPathSet = ShimmedPathCollection("UninstallPathSet", ImportTypes.CLASS) +UninstallPathSet.create_path("req.req_uninstall.UninstallPathSet", "7.0.0", "9999") + +url_to_path = ShimmedPathCollection("url_to_path", ImportTypes.FUNCTION) +url_to_path.create_path("download.url_to_path", "7.0.0", "19.2.3") +url_to_path.create_path("utils.urls.url_to_path", "19.3.0", "9999") + +USER_CACHE_DIR = ShimmedPathCollection("USER_CACHE_DIR", ImportTypes.ATTRIBUTE) +USER_CACHE_DIR.create_path("locations.USER_CACHE_DIR", "7.0.0", "9999") + +VcsSupport = ShimmedPathCollection("VcsSupport", ImportTypes.CLASS) +VcsSupport.create_path("vcs.VcsSupport", "7.0.0", "19.1.1") +VcsSupport.create_path("vcs.versioncontrol.VcsSupport", "19.2", "9999") + +Wheel = ShimmedPathCollection("Wheel", ImportTypes.CLASS) +Wheel.create_path("wheel.Wheel", "7.0.0", "19.3.9") +Wheel.set_default(compat.Wheel) + +WheelCache = ShimmedPathCollection("WheelCache", ImportTypes.CLASS) +WheelCache.create_path("cache.WheelCache", "10.0.0", "9999") +WheelCache.create_path("wheel.WheelCache", "7", "9.0.3") + +WheelBuilder = ShimmedPathCollection("WheelBuilder", ImportTypes.CLASS) +WheelBuilder.create_path("wheel.WheelBuilder", "7.0.0", "19.9") + +build = ShimmedPathCollection("build", ImportTypes.FUNCTION) +build.create_path("wheel_builder.build", "19.9", "9999") + +build_one = ShimmedPathCollection("build_one", ImportTypes.FUNCTION) +build_one.create_path("wheel_builder._build_one", "19.9", "9999") + +build_one_inside_env = ShimmedPathCollection("build_one_inside_env", ImportTypes.FUNCTION) +build_one_inside_env.create_path("wheel_builder._build_one_inside_env", "19.9", "9999") + +AbstractDistribution = ShimmedPathCollection("AbstractDistribution", ImportTypes.CLASS) +AbstractDistribution.create_path( + "distributions.base.AbstractDistribution", "19.1.2", "9999" +) + +InstalledDistribution = ShimmedPathCollection("InstalledDistribution", ImportTypes.CLASS) +InstalledDistribution.create_path( + "distributions.installed.InstalledDistribution", "19.1.2", "9999" +) + +SourceDistribution = ShimmedPathCollection("SourceDistribution", ImportTypes.CLASS) +SourceDistribution.create_path("req.req_set.IsSDist", "7.0.0", "9.0.3") +SourceDistribution.create_path("operations.prepare.IsSDist", "10.0.0", "19.1.1") +SourceDistribution.create_path( + "distributions.source.SourceDistribution", "19.1.2", "19.2.3" +) +SourceDistribution.create_path( + "distributions.source.legacy.SourceDistribution", "19.3.0", "19.9" +) +SourceDistribution.create_path("distributions.sdist.SourceDistribution", "20.0", "9999") + +WheelDistribution = ShimmedPathCollection("WheelDistribution", ImportTypes.CLASS) +WheelDistribution.create_path("distributions.wheel.WheelDistribution", "19.1.2", "9999") + +Downloader = ShimmedPathCollection("Downloader", ImportTypes.CLASS) +Downloader.create_path("network.download.Downloader", "20.0.0", "9999") + +PyPI = ShimmedPathCollection("PyPI", ImportTypes.ATTRIBUTE) +PyPI.create_path("models.index.PyPI", "7.0.0", "9999") + +stdlib_pkgs = ShimmedPathCollection("stdlib_pkgs", ImportTypes.ATTRIBUTE) +stdlib_pkgs.create_path("utils.compat.stdlib_pkgs", "18.1", "9999") +stdlib_pkgs.create_path("compat.stdlib_pkgs", "7", "18.0") + +DEV_PKGS = ShimmedPathCollection("DEV_PKGS", ImportTypes.ATTRIBUTE) +DEV_PKGS.create_path("commands.freeze.DEV_PKGS", "9.0.0", "9999") +DEV_PKGS.set_default({"setuptools", "pip", "distribute", "wheel"}) + + +wheel_cache = ShimmedPathCollection("wheel_cache", ImportTypes.FUNCTION) +wheel_cache.set_default( + functools.partial( + compat.wheel_cache, + wheel_cache_provider=WheelCache, + tempdir_manager_provider=global_tempdir_manager, + format_control_provider=FormatControl, + ) +) + + +get_package_finder = ShimmedPathCollection("get_package_finder", ImportTypes.FUNCTION) +get_package_finder.set_default( + functools.partial( + compat.get_package_finder, + install_cmd_provider=InstallCommand, + target_python_builder=TargetPython.shim(), + ) +) + + +make_preparer = ShimmedPathCollection("make_preparer", ImportTypes.FUNCTION) +make_preparer.set_default( + functools.partial( + compat.make_preparer, + install_cmd_provider=InstallCommand, + preparer_fn=RequirementPreparer, + downloader_provider=Downloader, + req_tracker_fn=get_requirement_tracker, + finder_provider=get_package_finder, + ) +) + + +get_resolver = ShimmedPathCollection("get_resolver", ImportTypes.FUNCTION) +get_resolver.set_default( + functools.partial( + compat.get_resolver, + install_cmd_provider=InstallCommand, + resolver_fn=Resolver, + install_req_provider=install_req_from_req_string, + wheel_cache_provider=wheel_cache, + format_control_provider=FormatControl, + ) +) + + +get_requirement_set = ShimmedPathCollection("get_requirement_set", ImportTypes.FUNCTION) +get_requirement_set.set_default( + functools.partial( + compat.get_requirement_set, + install_cmd_provider=InstallCommand, + req_set_provider=RequirementSet, + wheel_cache_provider=wheel_cache, + ) +) + + +resolve = ShimmedPathCollection("resolve", ImportTypes.FUNCTION) +resolve.set_default( + functools.partial( + compat.resolve, + install_cmd_provider=InstallCommand, + reqset_provider=get_requirement_set, + finder_provider=get_package_finder, + resolver_provider=get_resolver, + wheel_cache_provider=wheel_cache, + format_control_provider=FormatControl, + make_preparer_provider=make_preparer, + req_tracker_provider=get_requirement_tracker, + tempdir_manager_provider=global_tempdir_manager, + ) +) + + +build_wheel = ShimmedPathCollection("build_wheel", ImportTypes.FUNCTION) +build_wheel.set_default( + functools.partial( + compat.build_wheel, + install_command_provider=InstallCommand, + wheel_cache_provider=wheel_cache, + wheel_builder_provider=WheelBuilder, + build_one_provider=build_one, + build_one_inside_env_provider=build_one_inside_env, + build_many_provider=build, + preparer_provider=make_preparer, + format_control_provider=FormatControl, + reqset_provider=get_requirement_set, + ) +) diff --git a/pipenv/vendor/pip_shims/shims.py b/pipenv/vendor/pip_shims/shims.py index 34de6906..fb029378 100644 --- a/pipenv/vendor/pip_shims/shims.py +++ b/pipenv/vendor/pip_shims/shims.py @@ -1,26 +1,43 @@ # -*- coding=utf-8 -*- -from collections import namedtuple -from contextlib import contextmanager -import importlib -import os +""" +Main module with magic self-replacement mechanisms to handle import speedups. +""" +from __future__ import absolute_import + import sys +import types + +from packaging.version import parse as parse_version + +from .models import ( + ShimmedPathCollection, + get_package_finder, + import_pip, + lookup_current_pip_version, +) -import six -six.add_move(six.MovedAttribute("Callable", "collections", "collections.abc")) -from six.moves import Callable +class _shims(types.ModuleType): + CURRENT_PIP_VERSION = str(lookup_current_pip_version()) - -class _shims(object): - CURRENT_PIP_VERSION = "18.1" - BASE_IMPORT_PATH = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") - path_info = namedtuple("PathInfo", "path start_version end_version") + @classmethod + def parse_version(cls, version): + return parse_version(version) def __dir__(self): result = list(self._locations.keys()) + list(self.__dict__.keys()) - result.extend(('__file__', '__doc__', '__all__', - '__docformat__', '__name__', '__path__', - '__package__', '__version__')) + result.extend( + ( + "__file__", + "__doc__", + "__all__", + "__docformat__", + "__name__", + "__path__", + "__package__", + "__version__", + ) + ) return result @classmethod @@ -32,310 +49,27 @@ class _shims(object): return list(self._locations.keys()) def __init__(self): - # from .utils import _parse, get_package, STRING_TYPES - from . import utils - self.utils = utils - self._parse = utils._parse - self.get_package = utils.get_package - self.STRING_TYPES = utils.STRING_TYPES - self._modules = { - "pip": importlib.import_module(self.BASE_IMPORT_PATH), - "pip_shims.utils": utils - } - self.pip_version = getattr(self._modules["pip"], "__version__") - version_types = ["post", "pre", "dev", "rc"] - if any(post in self.pip_version.rsplit(".")[-1] for post in version_types): - self.pip_version, _, _ = self.pip_version.rpartition(".") - self.parsed_pip_version = self._parse(self.pip_version) - self._contextmanagers = ("RequirementTracker",) - self._moves = { - "InstallRequirement": { - "from_editable": "install_req_from_editable", - "from_line": "install_req_from_line", - } - } - self._locations = { - "parse_version": ("index.parse_version", "7", "9999"), - "_strip_extras": ( - ("req.req_install._strip_extras", "7", "18.0"), - ("req.constructors._strip_extras", "18.1", "9999"), - ), - "cmdoptions": ( - ("cli.cmdoptions", "18.1", "9999"), - ("cmdoptions", "7.0.0", "18.0") - ), - "Command": ( - ("cli.base_command.Command", "18.1", "9999"), - ("basecommand.Command", "7.0.0", "18.0") - ), - "ConfigOptionParser": ( - ("cli.parser.ConfigOptionParser", "18.1", "9999"), - ("baseparser.ConfigOptionParser", "7.0.0", "18.0") - ), - "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), - "FAVORITE_HASH": ("utils.hashes.FAVORITE_HASH", "7.0.0", "9999"), - "FormatControl": ( - ("models.format_control.FormatControl", "18.1", "9999"), - ("index.FormatControl", "7.0.0", "18.0"), - ), - "FrozenRequirement": ( - ("FrozenRequirement", "7.0.0", "9.0.3"), - ("operations.freeze.FrozenRequirement", "10.0.0", "9999") - ), - "get_installed_distributions": ( - ("utils.misc.get_installed_distributions", "10", "9999"), - ("utils.get_installed_distributions", "7", "9.0.3") - ), - "index_group": ( - ("cli.cmdoptions.index_group", "18.1", "9999"), - ("cmdoptions.index_group", "7.0.0", "18.0") - ), - "InstallRequirement": ("req.req_install.InstallRequirement", "7.0.0", "9999"), - "InstallationError": ("exceptions.InstallationError", "7.0.0", "9999"), - "UninstallationError": ("exceptions.UninstallationError", "7.0.0", "9999"), - "DistributionNotFound": ("exceptions.DistributionNotFound", "7.0.0", "9999"), - "RequirementsFileParseError": ("exceptions.RequirementsFileParseError", "7.0.0", "9999"), - "BestVersionAlreadyInstalled": ("exceptions.BestVersionAlreadyInstalled", "7.0.0", "9999"), - "BadCommand": ("exceptions.BadCommand", "7.0.0", "9999"), - "CommandError": ("exceptions.CommandError", "7.0.0", "9999"), - "PreviousBuildDirError": ("exceptions.PreviousBuildDirError", "7.0.0", "9999"), - "install_req_from_editable": ( - ("req.constructors.install_req_from_editable", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_editable", "7.0.0", "18.0") - ), - "install_req_from_line": ( - ("req.constructors.install_req_from_line", "18.1", "9999"), - ("req.req_install.InstallRequirement.from_line", "7.0.0", "18.0") - ), - "is_archive_file": ("download.is_archive_file", "7.0.0", "9999"), - "is_file_url": ("download.is_file_url", "7.0.0", "9999"), - "unpack_url": ("download.unpack_url", "7.0.0", "9999"), - "is_installable_dir": ( - ("utils.misc.is_installable_dir", "10.0.0", "9999"), - ("utils.is_installable_dir", "7.0.0", "9.0.3") - ), - "Link": ("index.Link", "7.0.0", "9999"), - "make_abstract_dist": ( - ("operations.prepare.make_abstract_dist", "10.0.0", "9999"), - ("req.req_set.make_abstract_dist", "7.0.0", "9.0.3") - ), - "make_option_group": ( - ("cli.cmdoptions.make_option_group", "18.1", "9999"), - ("cmdoptions.make_option_group", "7.0.0", "18.0") - ), - "PackageFinder": ("index.PackageFinder", "7.0.0", "9999"), - "parse_requirements": ("req.req_file.parse_requirements", "7.0.0", "9999"), - "parse_version": ("index.parse_version", "7.0.0", "9999"), - "path_to_url": ("download.path_to_url", "7.0.0", "9999"), - "PipError": ("exceptions.PipError", "7.0.0", "9999"), - "RequirementPreparer": ("operations.prepare.RequirementPreparer", "7", "9999"), - "RequirementSet": ("req.req_set.RequirementSet", "7.0.0", "9999"), - "RequirementTracker": ("req.req_tracker.RequirementTracker", "7.0.0", "9999"), - "Resolver": ("resolve.Resolver", "7.0.0", "9999"), - "SafeFileCache": ("download.SafeFileCache", "7.0.0", "9999"), - "UninstallPathSet": ("req.req_uninstall.UninstallPathSet", "7.0.0", "9999"), - "url_to_path": ("download.url_to_path", "7.0.0", "9999"), - "USER_CACHE_DIR": ("locations.USER_CACHE_DIR", "7.0.0", "9999"), - "VcsSupport": ("vcs.VcsSupport", "7.0.0", "9999"), - "Wheel": ("wheel.Wheel", "7.0.0", "9999"), - "WheelCache": ( - ("cache.WheelCache", "10.0.0", "9999"), - ("wheel.WheelCache", "7", "9.0.3") - ), - "WheelBuilder": ("wheel.WheelBuilder", "7.0.0", "9999"), - "PyPI": ("models.index.PyPI", "7.0.0", "9999"), - } - - def _ensure_methods(self, cls, classname, *methods): - method_names = [m[0] for m in methods] - if all(getattr(cls, m, None) for m in method_names): - return cls - new_functions = {} - class BaseFunc(Callable): - def __init__(self, func_base, name, *args, **kwargs): - self.func = func_base - self.__name__ = self.__qualname__ = name - - def __call__(self, cls, *args, **kwargs): - return self.func(*args, **kwargs) - - for method_name, fn in methods: - new_functions[method_name] = classmethod(BaseFunc(fn, method_name)) - if six.PY2: - classname = classname.encode(sys.getdefaultencoding()) - type_ = type( - classname, - (cls,), - new_functions - ) - return type_ - - def _get_module_paths(self, module, base_path=None): - if not base_path: - base_path = self.BASE_IMPORT_PATH - module = self._locations[module] - if not isinstance(next(iter(module)), (tuple, list)): - module_paths = self.get_pathinfo(module) - else: - module_paths = [self.get_pathinfo(pth) for pth in module] - return self.sort_paths(module_paths, base_path) - - def _get_remapped_methods(self, moved_package): - original_base, original_target = moved_package - original_import = self._import(self._locations[original_target]) - old_to_new = {} - new_to_old = {} - for method_name, new_method_name in self._moves.get(original_target, {}).items(): - module_paths = self._get_module_paths(new_method_name) - target = next(iter( - sorted(set([ - tgt for mod, tgt in map(self.get_package, module_paths) - ]))), None - ) - old_to_new[method_name] = { - "target": target, - "name": new_method_name, - "location": self._locations[new_method_name], - "module": self._import(self._locations[new_method_name]) - } - new_to_old[new_method_name] = { - "target": original_target, - "name": method_name, - "location": self._locations[original_target], - "module": original_import - } - return (old_to_new, new_to_old) - - def _import_moved_module(self, moved_package): - old_to_new, new_to_old = self._get_remapped_methods(moved_package) - imported = None - method_map = [] - new_target = None - for old_method, remapped in old_to_new.items(): - new_name = remapped["name"] - new_target = new_to_old[new_name]["target"] - if not imported: - imported = self._modules[new_target] = new_to_old[new_name]["module"] - method_map.append((old_method, remapped["module"])) - if getattr(imported, "__class__", "") == type: - imported = self._ensure_methods( - imported, new_target, *method_map - ) - self._modules[new_target] = imported - if imported: - return imported - return - - def _check_moved_methods(self, search_pth, moves): - module_paths = [ - self.get_package(pth) for pth in self._get_module_paths(search_pth) - ] - moved_methods = [ - (base, target_cls) for base, target_cls - in module_paths if target_cls in moves - ] - return next(iter(moved_methods), None) + self.pip = import_pip() + self._locations = ShimmedPathCollection.get_registry() + self._locations["get_package_finder"] = get_package_finder + self.pip_version = str(lookup_current_pip_version()) + self.parsed_pip_version = lookup_current_pip_version() def __getattr__(self, *args, **kwargs): locations = super(_shims, self).__getattribute__("_locations") - contextmanagers = super(_shims, self).__getattribute__("_contextmanagers") - moves = super(_shims, self).__getattribute__("_moves") if args[0] in locations: - moved_package = self._check_moved_methods(args[0], moves) - if moved_package: - imported = self._import_moved_module(moved_package) - if imported: - return imported - else: - imported = self._import(locations[args[0]]) - if not imported and args[0] in contextmanagers: - return self.nullcontext - return imported + return locations[args[0]].shim() return super(_shims, self).__getattribute__(*args, **kwargs) - def is_valid(self, path_info_tuple): - if ( - path_info_tuple.start_version <= self.parsed_pip_version - and path_info_tuple.end_version >= self.parsed_pip_version - ): - return 1 - return 0 - - def sort_paths(self, module_paths, base_path): - if not isinstance(module_paths, list): - module_paths = [module_paths] - prefix_order = [pth.format(base_path) for pth in ["{0}._internal", "{0}"]] - # Pip 10 introduced the internal api division - if self._parse(self.pip_version) < self._parse("10.0.0"): - prefix_order = reversed(prefix_order) - paths = sorted(module_paths, key=self.is_valid, reverse=True) - search_order = [ - "{0}.{1}".format(p, pth.path) - for p in prefix_order - for pth in paths - if pth is not None - ] - return search_order - - def import_module(self, module): - if module in self._modules: - return self._modules[module] - try: - imported = importlib.import_module(module) - except ImportError: - imported = None - else: - self._modules[module] = imported - return imported - - def none_or_ctxmanager(self, pkg_name): - if pkg_name in self._contextmanagers: - return self.nullcontext - return None - - def get_package_from_modules(self, modules): - modules = [ - (package_name, self.import_module(m)) - for m, package_name in map(self.get_package, modules) - ] - imports = [ - getattr(m, pkg, self.none_or_ctxmanager(pkg)) for pkg, m in modules - if m is not None - ] - return next(iter(imports), None) - - def _import(self, module_paths, base_path=None): - if not base_path: - base_path = self.BASE_IMPORT_PATH - if not isinstance(next(iter(module_paths)), (tuple, list)): - module_paths = self.get_pathinfo(module_paths) - else: - module_paths = [self.get_pathinfo(pth) for pth in module_paths] - search_order = self.sort_paths(module_paths, base_path) - return self.get_package_from_modules(search_order) - - def do_import(self, *args, **kwargs): - return self._import(*args, **kwargs) - - @contextmanager - def nullcontext(self, *args, **kwargs): - try: - yield - finally: - pass - - def get_pathinfo(self, module_path): - assert isinstance(module_path, (list, tuple)) - module_path, start_version, end_version = module_path - return self.path_info(module_path, self._parse(start_version), self._parse(end_version)) - old_module = sys.modules[__name__] if __name__ in sys.modules else None module = sys.modules[__name__] = _shims() -module.__dict__.update({ - '__file__': __file__, - '__package__': __package__, - '__doc__': __doc__, - '__all__': module.__all__, - '__name__': __name__, -}) +module.__dict__.update( + { + "__file__": __file__, + "__package__": __package__, + "__doc__": __doc__, + "__all__": module.__all__, + "__name__": __name__, + } +) diff --git a/pipenv/vendor/pip_shims/utils.py b/pipenv/vendor/pip_shims/utils.py index d6101e21..162b4a20 100644 --- a/pipenv/vendor/pip_shims/utils.py +++ b/pipenv/vendor/pip_shims/utils.py @@ -1,13 +1,98 @@ # -*- coding=utf-8 -*- -from functools import wraps +""" +Shared utility functions which are not specific to any particular module. +""" +from __future__ import absolute_import + +import contextlib +import copy +import inspect import sys +from functools import wraps + +import packaging.version +import six + +from .environment import MYPY_RUNNING + +# format: off +six.add_move( + six.MovedAttribute("Callable", "collections", "collections.abc") +) # type: ignore # noqa +from six.moves import Callable # type: ignore # isort:skip # noqa + +# format: on + +if MYPY_RUNNING: + from types import ModuleType + from typing import ( + Any, + Dict, + Iterator, + List, + Optional, + Sequence, + Tuple, + Type, + TypeVar, + Union, + ) + + TShimmedPath = TypeVar("TShimmedPath") + TShimmedPathCollection = TypeVar("TShimmedPathCollection") + TShim = Union[TShimmedPath, TShimmedPathCollection] + TShimmedFunc = Union[TShimmedPath, TShimmedPathCollection, Callable, Type] + STRING_TYPES = (str,) if sys.version_info < (3, 0): - STRING_TYPES = STRING_TYPES + (unicode,) + STRING_TYPES = STRING_TYPES + (unicode,) # noqa:F821 + + +class BaseMethod(Callable): + def __init__(self, func_base, name, *args, **kwargs): + # type: (Callable, str, Any, Any) -> None + self.func = func_base + self.__name__ = self.__qualname__ = name + + def __call__(self, *args, **kwargs): + # type: (Any, Any) -> Any + return self.func(*args, **kwargs) + + +class BaseClassMethod(Callable): + def __init__(self, func_base, name, *args, **kwargs): + # type: (Callable, str, Any, Any) -> None + self.func = func_base + self.__name__ = self.__qualname__ = name + + def __call__(self, cls, *args, **kwargs): + # type: (Type, Any, Any) -> Any + return self.func(*args, **kwargs) + + +def make_method(fn): + # type: (Callable) -> Callable + @wraps(fn) + def method_creator(*args, **kwargs): + # type: (Any, Any) -> Callable + return BaseMethod(fn, *args, **kwargs) + + return method_creator + + +def make_classmethod(fn): + # type: (Callable) -> Callable + @wraps(fn) + def classmethod_creator(*args, **kwargs): + # type: (Any, Any) -> Callable + return classmethod(BaseClassMethod(fn, *args, **kwargs)) + + return classmethod_creator def memoize(obj): + # type: (Any) -> Callable cache = obj.cache = {} @wraps(obj) @@ -16,20 +101,365 @@ def memoize(obj): if key not in cache: cache[key] = obj(*args, **kwargs) return cache[key] + return memoizer @memoize def _parse(version): + # type: (str) -> Tuple[int, ...] if isinstance(version, STRING_TYPES): return tuple((int(i) for i in version.split("."))) return version -def get_package(module, subimport=None): +@memoize +def parse_version(version): + # type: (str) -> packaging.version._BaseVersion + if not isinstance(version, STRING_TYPES): + raise TypeError("Can only derive versions from string, got {0!r}".format(version)) + return packaging.version.parse(version) + + +@memoize +def split_package(module, subimport=None): + # type: (str, Optional[str]) -> Tuple[str, str] + """ + Used to determine what target to import. + + Either splits off the final segment or uses the provided sub-import to return a + 2-tuple of the import path and the target module or sub-path. + + :param str module: A package to import from + :param Optional[str] subimport: A class, function, or subpackage to import + :return: A 2-tuple of the corresponding import package and sub-import path + :rtype: Tuple[str, str] + + :Example: + + >>> from pip_shims.utils import split_package + >>> split_package("pip._internal.req.req_install", subimport="InstallRequirement") + ("pip._internal.req.req_install", "InstallRequirement") + >>> split_package("pip._internal.cli.base_command") + ("pip._internal.cli", "base_command") + """ package = None if subimport: package = subimport else: module, _, package = module.rpartition(".") return module, package + + +def get_method_args(target_method): + # type: (Callable) -> Tuple[Callable, Optional[inspect.Arguments]] + """ + Returns the arguments for a callable. + + :param Callable target_method: A callable to retrieve arguments for + :return: A 2-tuple of the original callable and its resulting arguments + :rtype: Tuple[Callable, Optional[inspect.Arguments]] + """ + inspected_args = None + try: + inspected_args = inspect.getargs(target_method.__code__) + except AttributeError: + target_func = getattr(target_method, "__func__", None) + if target_func is not None: + inspected_args = inspect.getargs(target_func.__code__) + else: + target_func = target_method + return target_func, inspected_args + + +def set_default_kwargs(basecls, method, *args, **default_kwargs): + # type: (Union[Type, ModuleType], Callable, Any, Any) -> Union[Type, ModuleType] # noqa + target_method = getattr(basecls, method, None) + if target_method is None: + return basecls + target_func, inspected_args = get_method_args(target_method) + if inspected_args is not None: + pos_args = inspected_args.args + else: + pos_args = [] + # Spit back the base class if we can't find matching arguments + # to put defaults in place of + if not any(arg in pos_args for arg in list(default_kwargs.keys())): + return basecls + prepended_defaults = tuple() # type: Tuple[Any, ...] + # iterate from the function's argument order to make sure we fill this + # out in the correct order + for arg in args: + prepended_defaults += (arg,) + for arg in pos_args: + if arg in default_kwargs: + prepended_defaults = prepended_defaults + (default_kwargs[arg],) + if not prepended_defaults: + return basecls + if six.PY2 and inspect.ismethod(target_method): + new_defaults = prepended_defaults + target_func.__defaults__ + target_method.__func__.__defaults__ = new_defaults + else: + new_defaults = prepended_defaults + target_method.__defaults__ + target_method.__defaults__ = new_defaults + setattr(basecls, method, target_method) + return basecls + + +def ensure_function(parent, funcname, func): + # type: (Union[ModuleType, Type, Callable, Any], str, Callable) -> Callable + """Given a module, a function name, and a function object, attaches the given + function to the module and ensures it is named properly according to the provided + argument + + :param Any parent: The parent to attack the function to + :param str funcname: The name to give the function + :param Callable func: The function to rename and attach to **parent** + :returns: The function with its name, qualname, etc set to mirror **parent** + :rtype: Callable + """ + qualname = funcname + if parent is None: + parent = __module__ # type: ignore # noqa:F821 + parent_is_module = inspect.ismodule(parent) + parent_is_class = inspect.isclass(parent) + module = None + if parent_is_module: + module = parent.__name__ + elif parent_is_class: + qualname = "{0}.{1}".format(parent.__name__, qualname) + module = getattr(parent, "__module__", None) + else: + module = getattr(parent, "__module__", None) + try: + func.__name__ = funcname + except AttributeError: + if getattr(func, "__func__", None) is not None: + func = func.__func__ + func.__name__ = funcname + func.__qualname__ = qualname + + func.__module__ = module + return func + + +def add_mixin_to_class(basecls, mixins): + # type: (Type, List[Type]) -> Type + """ + Given a class, adds the provided mixin classes as base classes and gives a new class + + :param Type basecls: An initial class to generate a new class from + :param List[Type] mixins: A list of mixins to add as base classes + :return: A new class with the provided mixins as base classes + :rtype: Type[basecls, *mixins] + """ + if not any(mixins): + return basecls + base_dict = basecls.__dict__.copy() + class_tuple = (basecls,) # type: Tuple[Type, ...] + for mixin in mixins: + if not mixin: + continue + mixin_dict = mixin.__dict__.copy() + base_dict.update(mixin_dict) + class_tuple = class_tuple + (mixin,) + base_dict.update(basecls.__dict__) + return type(basecls.__name__, class_tuple, base_dict) + + +def fallback_is_file_url(link): + # type: (Any) -> bool + return link.url.lower().startswith("file:") + + +def fallback_is_artifact(self): + # type: (Any) -> bool + return not getattr(self, "is_vcs", False) + + +def fallback_is_vcs(self): + # type: (Any) -> bool + return not getattr(self, "is_artifact", True) + + +def resolve_possible_shim(target): + # type: (TShimmedFunc) -> Optional[Union[Type, Callable]] + if target is None: + return target + if getattr(target, "shim", None): + return target.shim() + return target + + +@contextlib.contextmanager +def nullcontext(*args, **kwargs): + # type: (Any, Any) -> Iterator + try: + yield + finally: + pass + + +def has_property(target, name): + # type: (Any, str) -> bool + if getattr(target, name, None) is not None: + return True + return False + + +def apply_alias(imported, target, *aliases): + # type: (Union[ModuleType, Type, None], Any, Any) -> Any + """ + Given a target with attributes, point non-existant aliases at the first existing one + + :param Union[ModuleType, Type] imported: A Module or Class base + :param Any target: The target which is a member of **imported** and will have aliases + :param str aliases: A list of aliases, the first found attribute will be the basis + for all non-existant names which will be created as pointers + :return: The original target + :rtype: Any + """ + base_value = None # type: Optional[Any] + applied_aliases = set() + unapplied_aliases = set() + for alias in aliases: + if has_property(target, alias): + base_value = getattr(target, alias) + applied_aliases.add(alias) + else: + unapplied_aliases.add(alias) + is_callable = inspect.ismethod(base_value) or inspect.isfunction(base_value) + for alias in unapplied_aliases: + if is_callable: + func_copy = copy.deepcopy(base_value) + alias_value = ensure_function(imported, alias, func_copy) + else: + alias_value = base_value + setattr(target, alias, alias_value) + return target + + +def suppress_setattr(obj, attr, value, filter_none=False): + """ + Set an attribute, suppressing any exceptions and skipping the attempt on failure. + + :param Any obj: Object to set the attribute on + :param str attr: The attribute name to set + :param Any value: The value to set the attribute to + :param bool filter_none: [description], defaults to False + :return: Nothing + :rtype: None + + :Example: + + >>> class MyClass(object): + ... def __init__(self, name): + ... self.name = name + ... self.parent = None + ... def __repr__(self): + ... return "<{0!r} instance (name={1!r}, parent={2!r})>".format( + ... self.__class__.__name__, self.name, self.parent + ... ) + ... def __str__(self): + ... return self.name + >>> me = MyClass("Dan") + >>> dad = MyClass("John") + >>> grandfather = MyClass("Joe") + >>> suppress_setattr(dad, "parent", grandfather) + >>> dad + <'MyClass' instance (name='John', parent=<'MyClass' instance (name='Joe', parent=None + )>)> + >>> suppress_setattr(me, "parent", dad) + >>> me + <'MyClass' instance (name='Dan', parent=<'MyClass' instance (name='John', parent=<'My + Class' instance (name='Joe', parent=None)>)>)> + >>> suppress_setattr(me, "grandparent", grandfather) + >>> me + <'MyClass' instance (name='Dan', parent=<'MyClass' instance (name='John', parent=<'My + Class' instance (name='Joe', parent=None)>)>)> + """ + if filter_none and value is None: + pass + try: + setattr(obj, attr, value) + except Exception: # noqa + pass + + +def get_allowed_args(fn_or_class): + # type: (Union[Callable, Type]) -> Tuple[List[str], Dict[str, Any]] + """ + Given a callable or a class, returns the arguments and default kwargs passed in. + + :param Union[Callable, Type] fn_or_class: A function, method or class to inspect. + :return: A 2-tuple with a list of arguments and a dictionary of keywords mapped to + default values. + :rtype: Tuple[List[str], Dict[str, Any]] + """ + try: + signature = inspect.signature(fn_or_class) + except AttributeError: + import funcsigs + + signature = funcsigs.signature(fn_or_class) + args = [] + kwargs = {} + for arg, param in signature.parameters.items(): + if ( + param.kind in (param.POSITIONAL_OR_KEYWORD, param.POSITIONAL_ONLY) + ) and param.default is param.empty: + args.append(arg) + else: + kwargs[arg] = param.default if param.default is not param.empty else None + return args, kwargs + + +def call_function_with_correct_args(fn, **provided_kwargs): + # type: (Callable, Dict[str, Any]) -> Any + """ + Determines which arguments from **provided_kwargs** to call **fn** and calls it. + + Consumes a list of allowed arguments (e.g. from :func:`~inspect.getargs()`) and + uses it to determine which of the arguments in the provided kwargs should be passed + through to the given callable. + + :param Callable fn: A callable which has some dynamic arguments + :param List[str] allowed_args: A list of allowed arguments which can be passed to + the supplied function + :return: The result of calling the function + :rtype: Any + """ + # signature = inspect.signature(fn) + args = [] + kwargs = {} + func_args, func_kwargs = get_allowed_args(fn) + for arg in func_args: + args.append(provided_kwargs[arg]) + for arg in func_kwargs: + if not provided_kwargs.get(arg): + continue + kwargs[arg] = provided_kwargs[arg] + return fn(*args, **kwargs) + + +def filter_allowed_args(fn, **provided_kwargs): + # type: (Callable, Dict[str, Any]) -> Tuple[List[Any], Dict[str, Any]] + """ + Given a function and a kwarg mapping, return only those kwargs used in the function. + + :param Callable fn: A function to inspect + :param Dict[str, Any] kwargs: A mapping of kwargs to filter + :return: A new, filtered kwarg mapping + :rtype: Tuple[List[Any], Dict[str, Any]] + """ + args = [] + kwargs = {} + func_args, func_kwargs = get_allowed_args(fn) + for arg in func_args: + if arg in provided_kwargs: + args.append(provided_kwargs[arg]) + for arg in func_kwargs: + if arg not in provided_kwargs: + continue + kwargs[arg] = provided_kwargs[arg] + return args, kwargs diff --git a/pipenv/vendor/pipdeptree.py b/pipenv/vendor/pipdeptree.py index 2082fc8a..899118cc 100644 --- a/pipenv/vendor/pipdeptree.py +++ b/pipenv/vendor/pipdeptree.py @@ -22,7 +22,7 @@ import pkg_resources # from graphviz import backend, Digraph -__version__ = '0.13.0' +__version__ = '0.13.2' flatten = chain.from_iterable @@ -127,6 +127,13 @@ def guess_version(pkg_key, default='?'): return getattr(m, '__version__', default) +def frozen_req_from_dist(dist): + try: + return FrozenRequirement.from_dist(dist) + except TypeError: + return FrozenRequirement.from_dist(dist, []) + + class Package(object): """Abstract class for wrappers around objects that pip returns. @@ -154,7 +161,7 @@ class Package(object): @staticmethod def frozen_repr(obj): - fr = FrozenRequirement.from_dist(obj, []) + fr = frozen_req_from_dist(obj) return str(fr).strip() def __getattr__(self, key): @@ -563,7 +570,7 @@ def _get_args(): def main(): args = _get_args() pkgs = get_installed_distributions(local_only=args.local_only, - user_only=args.user_only) + user_only=args.user_only) dist_index = build_dist_index(pkgs) tree = construct_tree(dist_index) diff --git a/pipenv/vendor/pipreqs/__init__.py b/pipenv/vendor/pipreqs/__init__.py index e0aa5d8b..a6ce8933 100644 --- a/pipenv/vendor/pipreqs/__init__.py +++ b/pipenv/vendor/pipreqs/__init__.py @@ -2,4 +2,4 @@ __author__ = 'Vadim Kravcenko' __email__ = 'vadim.kravcenko@gmail.com' -__version__ = '0.4.9' +__version__ = '0.4.10' diff --git a/pipenv/vendor/pipreqs/mapping b/pipenv/vendor/pipreqs/mapping index 46cca297..6f5a469d 100644 --- a/pipenv/vendor/pipreqs/mapping +++ b/pipenv/vendor/pipreqs/mapping @@ -9,7 +9,8 @@ BeautifulSoupTests:BeautifulSoup BioSQL:biopython BuildbotStatusShields:BuildbotEightStatusShields ComputedAttribute:ExtensionClass -Crypto:pycrypto +Crypto:pycryptodome +Cryptodome:pycryptodomex FSM:pexpect FiftyOneDegrees:51degrees_mobile_detector_v3_wrapper GeoBaseMain:GeoBasesDev @@ -263,7 +264,6 @@ armstrong:armstrong.hatband armstrong:armstrong.templates.standard armstrong:armstrong.utils.backends armstrong:armstrong.utils.celery -arrow:arrow_fatisar arstecnica:arstecnica.raccoon.autobahn arstecnica:arstecnica.sqlalchemy.async article-downloader:article_downloader @@ -536,6 +536,7 @@ cassandra:cassandra_driver cassandralauncher:CassandraLauncher cc42:42qucc cerberus:Cerberus +cfnlint:cfn-lint chameleon:Chameleon charmtools:charm_tools chef:PyChef @@ -582,7 +583,9 @@ dateutil:python_dateutil dawg:DAWG deb822:python_debian debian:python_debian +decouple:python-decouple demo:webunit +demosongs:PySynth deployer:juju_deployer depot:filedepot devtools:tg.devtools @@ -698,6 +701,7 @@ html:pies2overrides htmloutput:nosehtmloutput http:pies2overrides hvad:django_hvad +krbV:krbv i99fix:199Fix igraph:python_igraph imdb:IMDbPY @@ -771,6 +775,8 @@ mimeparse:python_mimeparse minitage:minitage.paste minitage:minitage.recipe.common missingdrawables:android_missingdrawables +mixfiles:PySynth +mkfreq:PySynth mkrst_themes:2lazy2rest mockredis:mockredispy modargs:python_modargs @@ -786,11 +792,13 @@ monthdelta:MonthDelta mopidy:Mopidy mopytools:MoPyTools mptt:django_mptt +mpv:python-mpv mrbob:mr.bob msgpack:msgpack_python mutations:aino_mutations mws:amazon_mws mysql:mysql_connector_repackaged +MySQL-python:MySQLdb native_tags:django_native_tags ndg:ndg_httpsclient nereid:trytond_nereid @@ -800,7 +808,7 @@ nester:abofly nester:bssm_pythonSig novaclient:python_novaclient oauth2_provider:alauda_django_oauth -oauth2client:google_api_python_client +oauth2client:oauth2client odf:odfpy ometa:Parsley openid:python_openid @@ -821,12 +829,14 @@ past:future paste:PasteScript path:forked_path path:path.py +patricia:patricia-trie paver:Paver peak:ProxyTypes picasso:anderson.picasso picklefield:django-picklefield pilot:BigJob pivotal:pivotal_py +play_wav:PySynth playhouse:peewee plivoxml:plivo plone:plone.alterego @@ -910,9 +920,9 @@ plone:plone.z3cform plonetheme:plonetheme.barceloneta png:pypng polymorphic:django_polymorphic -portalocker:ConcurrentLogHandler postmark:python_postmark powerprompt:bash_powerprompt +prefetch:django-prefetch printList:AndrewList progressbar:progressbar2 progressbar:progressbar33 @@ -947,6 +957,14 @@ pyrimaa:AEI pysideuic:PySide pysqlite2:adhocracy_pysqlite pysqlite2:pysqlite +pysynth_b:PySynth +pysynth_beeper:PySynth +pysynth_c:PySynth +pysynth_d:PySynth +pysynth_e:PySynth +pysynth_p:PySynth +pysynth_s:PySynth +pysynth_samp:PySynth pythongettext:python_gettext pythonjsonlogger:python_json_logger pyutilib:PyUtilib @@ -1004,6 +1022,7 @@ singleton:pysingleton sittercommon:cerebrod skbio:scikit_bio sklearn:scikit_learn +slack:slackclient slugify:unicode_slugify smarkets:smk_python_sdk snappy:ctypes_snappy diff --git a/pipenv/vendor/pipreqs/pipreqs.py b/pipenv/vendor/pipreqs/pipreqs.py index 791168a9..4b817c3c 100644 --- a/pipenv/vendor/pipreqs/pipreqs.py +++ b/pipenv/vendor/pipreqs/pipreqs.py @@ -3,25 +3,38 @@ """pipreqs - Generate pip requirements.txt file based on imports Usage: - pipreqs [options] <path> + pipreqs [options] [<path>] + +Arguments: + <path> The path to the directory containing the application + files for which a requirements file should be + generated (defaults to the current working + directory). Options: - --use-local Use ONLY local package info instead of querying PyPI - --pypi-server <url> Use custom PyPi server - --proxy <url> Use Proxy, parameter will be passed to requests library. You can also just set the - environments parameter in your terminal: + --use-local Use ONLY local package info instead of querying PyPI. + --pypi-server <url> Use custom PyPi server. + --proxy <url> Use Proxy, parameter will be passed to requests + library. You can also just set the environments + parameter in your terminal: $ export HTTP_PROXY="http://10.10.1.10:3128" $ export HTTPS_PROXY="https://10.10.1.10:1080" - --debug Print debug information - --ignore <dirs>... Ignore extra directories, each separated by a comma + --debug Print debug information. + --ignore <dirs>... Ignore extra directories, each separated by a comma. + --no-follow-links Do not follow symbolic links in the project --encoding <charset> Use encoding parameter for file open --savepath <file> Save the list of requirements in the given file - --print Output the list of requirements in the standard output + --print Output the list of requirements in the standard + output. --force Overwrite existing requirements.txt - --diff <file> Compare modules in requirements.txt to project imports. - --clean <file> Clean up requirements.txt by removing modules that are not imported in project. + --diff <file> Compare modules in requirements.txt to project + imports. + --clean <file> Clean up requirements.txt by removing modules + that are not imported in project. + --no-pin Omit version of output packages. """ from __future__ import print_function, absolute_import +from contextlib import contextmanager import os import sys import re @@ -50,7 +63,39 @@ else: py2_exclude = ["concurrent", "concurrent.futures"] -def get_all_imports(path, encoding=None, extra_ignore_dirs=None): +@contextmanager +def _open(filename=None, mode='r'): + """Open a file or ``sys.stdout`` depending on the provided filename. + + Args: + filename (str): The path to the file that should be opened. If + ``None`` or ``'-'``, ``sys.stdout`` or ``sys.stdin`` is + returned depending on the desired mode. Defaults to ``None``. + mode (str): The mode that should be used to open the file. + + Yields: + A file handle. + + """ + if not filename or filename == '-': + if not mode or 'r' in mode: + file = sys.stdin + elif 'w' in mode: + file = sys.stdout + else: + raise ValueError('Invalid mode for file: {}'.format(mode)) + else: + file = open(filename, mode) + + try: + yield file + finally: + if file not in (sys.stdin, sys.stdout): + file.close() + + +def get_all_imports( + path, encoding=None, extra_ignore_dirs=None, follow_links=True): imports = set() raw_imports = set() candidates = [] @@ -63,7 +108,8 @@ def get_all_imports(path, encoding=None, extra_ignore_dirs=None): ignore_dirs_parsed.append(os.path.basename(os.path.realpath(e))) ignore_dirs.extend(ignore_dirs_parsed) - for root, dirs, files in os.walk(path): + walk = os.walk(path, followlinks=follow_links) + for root, dirs, files in walk: dirs[:] = [d for d in dirs if d not in ignore_dirs] candidates.append(os.path.basename(root)) @@ -71,42 +117,44 @@ def get_all_imports(path, encoding=None, extra_ignore_dirs=None): candidates += [os.path.splitext(fn)[0] for fn in files] for file_name in files: - with open_func(os.path.join(root, file_name), "r", encoding=encoding) as f: + file_name = os.path.join(root, file_name) + with open_func(file_name, "r", encoding=encoding) as f: contents = f.read() - try: - tree = ast.parse(contents) - for node in ast.walk(tree): - if isinstance(node, ast.Import): - for subnode in node.names: - raw_imports.add(subnode.name) - elif isinstance(node, ast.ImportFrom): - raw_imports.add(node.module) - except Exception as exc: - if ignore_errors: - traceback.print_exc(exc) - logging.warn("Failed on file: %s" % os.path.join(root, file_name)) - continue - else: - logging.error("Failed on file: %s" % os.path.join(root, file_name)) - raise exc - - + try: + tree = ast.parse(contents) + for node in ast.walk(tree): + if isinstance(node, ast.Import): + for subnode in node.names: + raw_imports.add(subnode.name) + elif isinstance(node, ast.ImportFrom): + raw_imports.add(node.module) + except Exception as exc: + if ignore_errors: + traceback.print_exc(exc) + logging.warn("Failed on file: %s" % file_name) + continue + else: + logging.error("Failed on file: %s" % file_name) + raise exc # Clean up imports for name in [n for n in raw_imports if n]: - # Sanity check: Name could have been None if the import statement was as from . import X + # Sanity check: Name could have been None if the import + # statement was as ``from . import X`` # Cleanup: We only want to first part of the import. - # Ex: from django.conf --> django.conf. But we only want django as an import + # Ex: from django.conf --> django.conf. But we only want django + # as an import. cleaned_name, _, _ = name.partition('.') imports.add(cleaned_name) - packages = set(imports) - set(set(candidates) & set(imports)) + packages = imports - (set(candidates) & imports) logging.debug('Found packages: {0}'.format(packages)) with open(join("stdlib"), "r") as f: - data = [x.strip() for x in f.readlines()] - data = [x for x in data if x not in py2_exclude] if py2 else data - return sorted(list(set(packages) - set(data))) + data = {x.strip() for x in f} + + data = {x for x in data if x not in py2_exclude} if py2 else data + return list(packages - data) def filter_line(l): @@ -114,32 +162,30 @@ def filter_line(l): def generate_requirements_file(path, imports): - with open(path, "w") as out_file: + with _open(path, "w") as out_file: logging.debug('Writing {num} requirements: {imports} to {file}'.format( num=len(imports), file=path, imports=", ".join([x['name'] for x in imports]) )) fmt = '{name}=={version}' - out_file.write('\n'.join(fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports) + '\n') + out_file.write('\n'.join( + fmt.format(**item) if item['version'] else '{name}'.format(**item) + for item in imports) + '\n') + def output_requirements(imports): - logging.debug('Writing {num} requirements: {imports} to stdout'.format( - num=len(imports), - imports=", ".join([x['name'] for x in imports]) - )) - fmt = '{name}=={version}' - print('\n'.join(fmt.format(**item) if item['version'] else '{name}'.format(**item) - for item in imports)) + generate_requirements_file('-', imports) -def get_imports_info(imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): +def get_imports_info( + imports, pypi_server="https://pypi.python.org/pypi/", proxy=None): result = [] for item in imports: try: - response = requests.get("{0}{1}/json".format(pypi_server, item), proxies=proxy) + response = requests.get( + "{0}{1}/json".format(pypi_server, item), proxies=proxy) if response.status_code == 200: if hasattr(response.content, 'decode'): data = json2package(response.content.decode()) @@ -163,11 +209,13 @@ def get_locally_installed_packages(encoding=None): for root, dirs, files in os.walk(path): for item in files: if "top_level" in item: - with open_func(os.path.join(root, item), "r", encoding=encoding) as f: + item = os.path.join(root, item) + with open_func(item, "r", encoding=encoding) as f: package = root.split(os.sep)[-1].split("-") try: package_import = f.read().strip().split("\n") - except: + except: # NOQA + # TODO: What errors do we intend to suppress here? continue for i_item in package_import: if ((i_item not in ignore) and @@ -203,18 +251,24 @@ def get_import_local(imports, encoding=None): def get_pkg_names(pkgs): - result = [] + """Get PyPI package names from a list of imports. + + Args: + pkgs (List[str]): List of import names. + + Returns: + List[str]: The corresponding PyPI package names. + + """ + result = set() with open(join("mapping"), "r") as f: - data = [x.strip().split(":") for x in f.readlines()] - for pkg in pkgs: - toappend = pkg - for item in data: - if item[0] == pkg: - toappend = item[1] - break - if toappend not in result: - result.append(toappend) - return result + data = dict(x.strip().split(":") for x in f) + for pkg in pkgs: + # Look up the mapped requirement. If a mapping isn't found, + # simply use the package name. + result.add(data.get(pkg, pkg)) + # Return a sorted list for backward compatibility. + return sorted(result, key=lambda s: s.lower()) def get_name_without_alias(name): @@ -228,6 +282,7 @@ def get_name_without_alias(name): def join(f): return os.path.join(os.path.dirname(__file__), f) + def parse_requirements(file_): """Parse a requirements formatted file. @@ -245,7 +300,9 @@ def parse_requirements(file_): tuple: The contents of the file, excluding comments. """ modules = [] - delim = ["<", ">", "=", "!", "~"] # https://www.python.org/dev/peps/pep-0508/#complete-grammar + # For the dependency identifier specification, see + # https://www.python.org/dev/peps/pep-0508/#complete-grammar + delim = ["<", ">", "=", "!", "~"] try: f = open_func(file_, "r") @@ -253,14 +310,16 @@ def parse_requirements(file_): logging.error("Failed on file: {}".format(file_)) raise else: - data = [x.strip() for x in f.readlines() if x != "\n"] - finally: - f.close() + try: + data = [x.strip() for x in f.readlines() if x != "\n"] + finally: + f.close() data = [x for x in data if x[0].isalpha()] for x in data: - if not any([y in x for y in delim]): # Check for modules w/o a specifier. + # Check for modules w/o a specifier. + if not any([y in x for y in delim]): modules.append({"name": x, "version": None}) for y in x: if y in delim: @@ -298,11 +357,13 @@ def compare_modules(file_, imports): def diff(file_, imports): - """Display the difference between modules in a file and imported modules.""" + """Display the difference between modules in a file and imported modules.""" # NOQA modules_not_imported = compare_modules(file_, imports) - logging.info("The following modules are in {} but do not seem to be imported: " - "{}".format(file_, ", ".join(x for x in modules_not_imported))) + logging.info( + "The following modules are in {} but do not seem to be imported: " + "{}".format(file_, ", ".join(x for x in modules_not_imported))) + def clean(file_, imports): """Remove modules that aren't imported in project from file.""" @@ -316,29 +377,36 @@ def clean(file_, imports): logging.error("Failed on file: {}".format(file_)) raise else: - for i in f.readlines(): - if re_remove.match(i) is None: - to_write.append(i) - f.seek(0) - f.truncate() + try: + for i in f.readlines(): + if re_remove.match(i) is None: + to_write.append(i) + f.seek(0) + f.truncate() - for i in to_write: - f.write(i) - finally: - f.close() + for i in to_write: + f.write(i) + finally: + f.close() logging.info("Successfully cleaned up requirements in " + file_) + def init(args): encoding = args.get('--encoding') extra_ignore_dirs = args.get('--ignore') + follow_links = not args.get('--no-follow-links') + input_path = args['<path>'] + if input_path is None: + input_path = os.path.abspath(os.curdir) if extra_ignore_dirs: extra_ignore_dirs = extra_ignore_dirs.split(',') - candidates = get_all_imports(args['<path>'], + candidates = get_all_imports(input_path, encoding=encoding, - extra_ignore_dirs=extra_ignore_dirs) + extra_ignore_dirs=extra_ignore_dirs, + follow_links=follow_links) candidates = get_pkg_names(candidates) logging.debug("Found imports: " + ", ".join(candidates)) pypi_server = "https://pypi.python.org/pypi/" @@ -364,7 +432,7 @@ def init(args): pypi_server=pypi_server) path = (args["--savepath"] if args["--savepath"] else - os.path.join(args['<path>'], "requirements.txt")) + os.path.join(input_path, "requirements.txt")) if args["--diff"]: diff(args["--diff"], imports) @@ -374,11 +442,17 @@ def init(args): clean(args["--clean"], imports) return - if not args["--print"] and not args["--savepath"] and not args["--force"] and os.path.exists(path): + if (not args["--print"] + and not args["--savepath"] + and not args["--force"] + and os.path.exists(path)): logging.warning("Requirements.txt already exists, " "use --force to overwrite it") return + if args.get('--no-pin'): + imports = [{'name': item["name"], 'version': ''} for item in imports] + if args["--print"]: output_requirements(imports) logging.info("Successfully output requirements") diff --git a/pipenv/vendor/pipreqs/stdlib b/pipenv/vendor/pipreqs/stdlib index 71edcc87..470fd5cc 100644 --- a/pipenv/vendor/pipreqs/stdlib +++ b/pipenv/vendor/pipreqs/stdlib @@ -1,6 +1,8 @@ __builtin__ __future__ __main__ +_dummy_thread +_thread _winreg abc aepack @@ -15,6 +17,7 @@ argparse array ast asynchat +asyncio asyncore atexit audioop @@ -28,6 +31,7 @@ binhex bisect bsddb buildtools +builtins bz2 calendar Carbon @@ -101,6 +105,7 @@ code codecs codeop collections +collections.abc ColorPicker colorsys commands @@ -108,12 +113,16 @@ compileall compiler compiler.ast compiler.visitor +concurrent +concurrent.futures ConfigParser +configparser contextlib Cookie cookielib copy copy_reg +copyreg cPickle cProfile crypt @@ -127,6 +136,9 @@ curses.textpad datetime dbhash dbm +dbm.dumb +dbm.gnu +dbm.ndbm decimal DEVICE difflib @@ -188,21 +200,27 @@ dummy_threading EasyDialogs email email.charset +email.contentmanager email.encoders email.errors email.generator email.header +email.headerregistry email.iterators email.message email.mime email.parser +email.policy email.utils encodings encodings.idna +encodings.mbcs encodings.utf_8_sig ensurepip +enum errno exceptions +faulthandler fcntl filecmp fileinput @@ -226,8 +244,8 @@ gensuitemodule getopt getpass gettext -gl GL +gl glob grp gzip @@ -236,9 +254,17 @@ heapq hmac hotshot hotshot.stats +html +html.entities +html.parser htmlentitydefs htmllib HTMLParser +http +http.client +http.cookiejar +http.cookies +http.server httplib ic icopen @@ -248,12 +274,17 @@ imgfile imghdr imp importlib +importlib.abc +importlib.machinery +importlib.util imputil inspect io +ipaddress itertools jpeg json +json.tool keyword lib2to3 linecache @@ -261,6 +292,7 @@ locale logging logging.config logging.handlers +lzma macerrors MacOS macostools @@ -301,11 +333,13 @@ os os.path ossaudiodev parser +pathlib pdb pickle pickletools pipes PixMapWrapper +pkg_resources pkgutil platform plistlib @@ -322,10 +356,12 @@ py_compile pyclbr pydoc Queue +queue quopri random re readline +reprlib resource rexec rfc822 @@ -335,7 +371,9 @@ runpy sched ScrolledText select +selectors sets +setuptools sgmllib sha shelve @@ -350,10 +388,12 @@ smtplib sndhdr socket SocketServer +socketserver spwd sqlite3 ssl stat +statistics statvfs string StringIO @@ -361,8 +401,8 @@ stringprep struct subprocess sunau -sunaudiodev SUNAUDIODEV +sunaudiodev symbol symtable sys @@ -374,6 +414,7 @@ telnetlib tempfile termios test +test.support test.test_support textwrap thread @@ -382,17 +423,30 @@ time timeit Tix Tkinter +tkinter +tkinter.scrolledtext +tkinter.tix +tkinter.ttk token tokenize trace traceback +tracemalloc ttk tty turtle +turtledemo types +typing unicodedata unittest +unittest.mock urllib +urllib.error +urllib.parse +urllib.request +urllib.response +urllib.robotparser urllib2 urlparse user @@ -401,6 +455,7 @@ UserList UserString uu uuid +venv videoreader W warnings @@ -408,6 +463,7 @@ wave weakref webbrowser whichdb +winreg winsound wsgiref wsgiref.handlers @@ -422,654 +478,17 @@ xml.dom.minidom xml.dom.pulldom xml.etree.ElementTree xml.parsers.expat +xml.parsers.expat.errors +xml.parsers.expat.model xml.sax xml.sax.handler xml.sax.saxutils xml.sax.xmlreader +xmlrpc +xmlrpc.client +xmlrpc.server xmlrpclib -zipfile -zipimport -zlib -__future__ -__main__ -_dummy_thread -_thread -abc -aifc -argparse -array -ast -asynchat -asyncio -asyncore -atexit -audioop -base64 -bdb -binascii -binhex -bisect -builtins -bz2 -calendar -cgi -cgitb -chunk -cmath -cmd -code -codecs -codeop -collections -collections.abc -colorsys -compileall -concurrent -concurrent.futures -configparser -contextlib -copy -copyreg -cProfile -crypt -csv -ctypes -curses -curses.ascii -curses.panel -curses.textpad -datetime -dbm -dbm.dumb -dbm.gnu -dbm.ndbm -decimal -difflib -dis -distutils -distutils.archive_util -distutils.bcppcompiler -distutils.ccompiler -distutils.cmd -distutils.command -distutils.command.bdist -distutils.command.bdist_dumb -distutils.command.bdist_msi -distutils.command.bdist_packager -distutils.command.bdist_rpm -distutils.command.bdist_wininst -distutils.command.build -distutils.command.build_clib -distutils.command.build_ext -distutils.command.build_py -distutils.command.build_scripts -distutils.command.check -distutils.command.clean -distutils.command.config -distutils.command.install -distutils.command.install_data -distutils.command.install_headers -distutils.command.install_lib -distutils.command.install_scripts -distutils.command.register -distutils.command.sdist -distutils.core -distutils.cygwinccompiler -distutils.debug -distutils.dep_util -distutils.dir_util -distutils.dist -distutils.errors -distutils.extension -distutils.fancy_getopt -distutils.file_util -distutils.filelist -distutils.log -distutils.msvccompiler -distutils.spawn -distutils.sysconfig -distutils.text_file -distutils.unixccompiler -distutils.util -distutils.version -doctest -dummy_threading -email -email.charset -email.contentmanager -email.encoders -email.errors -email.generator -email.header -email.headerregistry -email.iterators -email.message -email.mime -email.parser -email.policy -email.utils -encodings -encodings.idna -encodings.mbcs -encodings.utf_8_sig -ensurepip -enum -errno -faulthandler -fcntl -filecmp -fileinput -fnmatch -formatter -fpectl -fractions -ftplib -functools -gc -getopt -getpass -gettext -glob -grp -gzip -hashlib -heapq -hmac -html -html.entities -html.parser -http -http.client -http.cookiejar -http.cookies -http.server -imaplib -imghdr -imp -importlib -importlib.abc -importlib.machinery -importlib.util -inspect -io -ipaddress -itertools -json -keyword -lib2to3 -linecache -locale -logging -logging.config -logging.handlers -lzma -macpath -mailbox -mailcap -marshal -math -mimetypes -mmap -modulefinder -msilib -msvcrt -multiprocessing -multiprocessing.connection -multiprocessing.dummy -multiprocessing.managers -multiprocessing.pool -multiprocessing.sharedctypes -netrc -nis -nntplib -numbers -operator -optparse -os -os.path -ossaudiodev -parser -pathlib -pdb -pickle -pickletools -pipes -pkgutil -platform -plistlib -poplib -posix -pprint -profile -pstats -pty -pwd -py_compile -pyclbr -pydoc -queue -quopri -random -re -readline -reprlib -resource -rlcompleter -runpy -sched -select -selectors -shelve -shlex -shutil -signal -site -smtpd -smtplib -sndhdr -socket -socketserver -spwd -sqlite3 -ssl -stat -statistics -string -stringprep -struct -subprocess -sunau -symbol -symtable -sys -sysconfig -syslog -tabnanny -tarfile -telnetlib -tempfile -termios -test -test.support -textwrap -threading -time -timeit -tkinter -tkinter.scrolledtext -tkinter.tix -tkinter.ttk -token -tokenize -trace -traceback -tracemalloc -tty -turtle -turtledemo -types -unicodedata -unittest -unittest.mock -urllib -urllib.error -urllib.parse -urllib.request -urllib.response -urllib.robotparser -uu -uuid -venv -warnings -wave -weakref -webbrowser -winreg -winsound -wsgiref -wsgiref.handlers -wsgiref.headers -wsgiref.simple_server -wsgiref.util -wsgiref.validate -xdrlib -xml -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree.ElementTree -xml.parsers.expat -xml.parsers.expat.errors -xml.parsers.expat.model -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc -xmlrpc.client -xmlrpc.server -zipfile -zipimport -zlib -__future__ -__main__ -_dummy_thread -_thread -abc -aifc -argparse -array -ast -asynchat -asyncio -asyncore -atexit -audioop -base64 -bdb -binascii -binhex -bisect -builtins -bz2 -calendar -cgi -cgitb -chunk -cmath -cmd -code -codecs -codeop -collections -collections.abc -colorsys -compileall -concurrent -concurrent.futures -configparser -contextlib -copy -copyreg -cProfile -crypt -csv -ctypes -curses -curses.ascii -curses.panel -curses.textpad -datetime -dbm -dbm.dumb -dbm.gnu -dbm.ndbm -decimal -difflib -dis -distutils -distutils.archive_util -distutils.bcppcompiler -distutils.ccompiler -distutils.cmd -distutils.command -distutils.command.bdist -distutils.command.bdist_dumb -distutils.command.bdist_msi -distutils.command.bdist_packager -distutils.command.bdist_rpm -distutils.command.bdist_wininst -distutils.command.build -distutils.command.build_clib -distutils.command.build_ext -distutils.command.build_py -distutils.command.build_scripts -distutils.command.check -distutils.command.clean -distutils.command.config -distutils.command.install -distutils.command.install_data -distutils.command.install_headers -distutils.command.install_lib -distutils.command.install_scripts -distutils.command.register -distutils.command.sdist -distutils.core -distutils.cygwinccompiler -distutils.debug -distutils.dep_util -distutils.dir_util -distutils.dist -distutils.errors -distutils.extension -distutils.fancy_getopt -distutils.file_util -distutils.filelist -distutils.log -distutils.msvccompiler -distutils.spawn -distutils.sysconfig -distutils.text_file -distutils.unixccompiler -distutils.util -distutils.version -doctest -dummy_threading -email -email.charset -email.contentmanager -email.encoders -email.errors -email.generator -email.header -email.headerregistry -email.iterators -email.message -email.mime -email.parser -email.policy -email.utils -encodings -encodings.idna -encodings.mbcs -encodings.utf_8_sig -ensurepip -enum -errno -faulthandler -fcntl -filecmp -fileinput -fnmatch -formatter -fpectl -fractions -ftplib -functools -gc -getopt -getpass -gettext -glob -grp -gzip -hashlib -heapq -hmac -html -html.entities -html.parser -http -http.client -http.cookiejar -http.cookies -http.server -imaplib -imghdr -imp -importlib -importlib.abc -importlib.machinery -importlib.util -inspect -io -ipaddress -itertools -json -json.tool -keyword -lib2to3 -linecache -locale -logging -logging.config -logging.handlers -lzma -macpath -mailbox -mailcap -marshal -math -mimetypes -mmap -modulefinder -msilib -msvcrt -multiprocessing -multiprocessing.connection -multiprocessing.dummy -multiprocessing.managers -multiprocessing.pool -multiprocessing.sharedctypes -netrc -nis -nntplib -numbers -operator -optparse -os -os.path -ossaudiodev -parser -pathlib -pdb -pickle -pickletools -pipes -pkgutil -platform -plistlib -poplib -posix -pprint -profile -pstats -pty -pwd -py_compile -pyclbr -pydoc -queue -quopri -random -re -readline -reprlib -resource -rlcompleter -runpy -sched -select -selectors -shelve -shlex -shutil -signal -site -smtpd -smtplib -sndhdr -socket -socketserver -spwd -sqlite3 -ssl -stat -statistics -string -stringprep -struct -subprocess -sunau -symbol -symtable -sys -sysconfig -syslog -tabnanny -tarfile -telnetlib -tempfile -termios -test -test.support -textwrap -threading -time -timeit -tkinter -tkinter.scrolledtext -tkinter.tix -tkinter.ttk -token -tokenize -trace -traceback -tracemalloc -tty -turtle -turtledemo -types -unicodedata -unittest -unittest.mock -urllib -urllib.error -urllib.parse -urllib.request -urllib.response -urllib.robotparser -uu -uuid -venv -warnings -wave -weakref -webbrowser -winreg -winsound -wsgiref -wsgiref.handlers -wsgiref.headers -wsgiref.simple_server -wsgiref.util -wsgiref.validate -xdrlib -xml -xml.dom -xml.dom.minidom -xml.dom.pulldom -xml.etree.ElementTree -xml.parsers.expat -xml.parsers.expat.errors -xml.parsers.expat.model -xml.sax -xml.sax.handler -xml.sax.saxutils -xml.sax.xmlreader -xmlrpc -xmlrpc.client -xmlrpc.server +yp zipapp zipfile zipimport diff --git a/pipenv/vendor/plette/__init__.py b/pipenv/vendor/plette/__init__.py index 5daf460c..ba03cef5 100644 --- a/pipenv/vendor/plette/__init__.py +++ b/pipenv/vendor/plette/__init__.py @@ -3,7 +3,7 @@ __all__ = [ "Lockfile", "Pipfile", ] -__version__ = '0.2.3.dev0' +__version__ = '0.2.4.dev0' from .lockfiles import Lockfile from .pipfiles import Pipfile diff --git a/pipenv/vendor/plette/models/base.py b/pipenv/vendor/plette/models/base.py index fad0d09e..72cf372e 100644 --- a/pipenv/vendor/plette/models/base.py +++ b/pipenv/vendor/plette/models/base.py @@ -8,13 +8,24 @@ class ValidationError(ValueError): def __init__(self, value, validator): super(ValidationError, self).__init__(value) self.validator = validator + self.value = value + + def __str__(self): + return '{}\n{}'.format( + self.value, + '\n'.join( + '{}: {}'.format(k, e) + for k, errors in self.validator.errors.items() + for e in errors + ) + ) VALIDATORS = {} def validate(cls, data): - if not cerberus: # Skip validation if Cerberus is not available. + if not cerberus: # Skip validation if Cerberus is not available. return schema = cls.__SCHEMA__ key = id(schema) @@ -22,7 +33,7 @@ def validate(cls, data): v = VALIDATORS[key] except KeyError: v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) - if v.validate(dict(data), normalize=False): + if v.validate(data, normalize=False): return raise ValidationError(data, v) @@ -33,6 +44,7 @@ class DataView(object): Validates the input mapping on creation. A subclass is expected to provide a `__SCHEMA__` class attribute specifying a validator schema. """ + def __init__(self, data): self.validate(data) self._data = data @@ -42,9 +54,11 @@ class DataView(object): def __eq__(self, other): if not isinstance(other, type(self)): - raise TypeError("cannot compare {0!r} with {1!r}".format( - type(self).__name__, type(other).__name__, - )) + raise TypeError( + "cannot compare {0!r} with {1!r}".format( + type(self).__name__, type(other).__name__ + ) + ) return self._data == other._data def __getitem__(self, key): @@ -78,6 +92,7 @@ class DataViewCollection(DataView): You should not instantiate an instance from this class, but from one of its subclasses instead. """ + item_class = None def __repr__(self): @@ -103,6 +118,7 @@ class DataViewMapping(DataViewCollection): The keys are primitive values, while values are instances of `item_class`. """ + @classmethod def validate(cls, data): for d in data.values(): @@ -126,6 +142,7 @@ class DataViewSequence(DataViewCollection): Each entry is an instance of `item_class`. """ + @classmethod def validate(cls, data): for d in data: diff --git a/pipenv/vendor/pyparsing.py b/pipenv/vendor/pyparsing.py index cf38419b..581d5bbb 100644 --- a/pipenv/vendor/pyparsing.py +++ b/pipenv/vendor/pyparsing.py @@ -1,6 +1,7 @@ +# -*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{"<salutation>, <addressee>!"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'<ParserElement.__add__>} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``"<salutation>, <addressee>!"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'<ParserElement.__add__>` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pyparsing import Word, alphas @@ -49,33 +53,50 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString<ParserElement.parseString>} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'<And>}, L{'|'<MatchFirst>}, L{'^'<Or>}, and L{'&'<Each>} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'<And>`, :class:`'|'<MatchFirst>`, :class:`'^'<Or>`, + and :class:`'&'<Each>` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - access the parsed data, which is returned as a :class:`ParseResults` + object + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.2" -__versionTime__ = "29 Sep 2018 15:58 UTC" +__version__ = "2.4.7" +__versionTime__ = "30 Mar 2020 00:43 UTC" __author__ = "Paul McGuire <ptmcg@users.sourceforge.net>" import string @@ -90,6 +111,16 @@ import pprint import traceback import types from datetime import datetime +from operator import itemgetter +import itertools +from functools import wraps +from contextlib import contextmanager + +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse try: from _thread import RLock @@ -99,11 +130,11 @@ except ImportError: try: # Python 3 from collections.abc import Iterable - from collections.abc import MutableMapping + from collections.abc import MutableMapping, Mapping except ImportError: # Python 2.7 from collections import Iterable - from collections import MutableMapping + from collections import MutableMapping, Mapping try: from collections import OrderedDict as _OrderedDict @@ -113,28 +144,78 @@ except ImportError: except ImportError: _OrderedDict = None -#~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass -__all__ = [ -'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', -'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', -'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', -'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', -'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', -'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', -'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', -'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', -'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', -'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', -'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', -'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', -'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', -] +# version compatibility configuration +__compat__ = SimpleNamespace() +__compat__.__doc__ = """ + A cross-version compatibility configuration for pyparsing features that will be + released in a future version. By setting values in this configuration to True, + those features can be enabled in prior versions for compatibility development + and testing. + + - collect_all_And_tokens - flag to enable fix for Issue #63 that fixes erroneous grouping + of results names when an And expression is nested within an Or or MatchFirst; set to + True to enable bugfix released in pyparsing 2.3.0, or False to preserve + pre-2.3.0 handling of named results +""" +__compat__.collect_all_And_tokens = True + +__diag__ = SimpleNamespace() +__diag__.__doc__ = """ +Diagnostic configuration (all default to False) + - warn_multiple_tokens_in_named_alternation - flag to enable warnings when a results + name is defined on a MatchFirst or Or expression with one or more And subexpressions + (only warns if __compat__.collect_all_And_tokens is False) + - warn_ungrouped_named_tokens_in_collection - flag to enable warnings when a results + name is defined on a containing expression with ungrouped subexpressions that also + have results names + - warn_name_set_on_empty_Forward - flag to enable warnings whan a Forward is defined + with a results name, but has no contents defined + - warn_on_multiple_string_args_to_oneof - flag to enable warnings whan oneOf is + incorrectly called with multiple str arguments + - enable_debug_on_named_expressions - flag to auto-enable debug on all subsequent + calls to ParserElement.setName() +""" +__diag__.warn_multiple_tokens_in_named_alternation = False +__diag__.warn_ungrouped_named_tokens_in_collection = False +__diag__.warn_name_set_on_empty_Forward = False +__diag__.warn_on_multiple_string_args_to_oneof = False +__diag__.enable_debug_on_named_expressions = False +__diag__._all_names = [nm for nm in vars(__diag__) if nm.startswith("enable_") or nm.startswith("warn_")] + +def _enable_all_warnings(): + __diag__.warn_multiple_tokens_in_named_alternation = True + __diag__.warn_ungrouped_named_tokens_in_collection = True + __diag__.warn_name_set_on_empty_Forward = True + __diag__.warn_on_multiple_string_args_to_oneof = True +__diag__.enable_all_warnings = _enable_all_warnings + + +__all__ = ['__version__', '__versionTime__', '__author__', '__compat__', '__diag__', + 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', + 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', + 'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', + 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', + 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', + 'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', + 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', + 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', + 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', + 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', + 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', + 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', + 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', + 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', + 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', + 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', + 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation', 'locatedExpr', 'withClass', + 'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', + 'conditionAsParseAction', 're', + ] system_version = tuple(sys.version_info)[:3] PY_3 = system_version[0] == 3 @@ -142,6 +223,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,11 +234,13 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ - if isinstance(obj,unicode): + if isinstance(obj, unicode): return obj try: @@ -174,39 +258,50 @@ else: # build list of single arg builtins, tolerant of Python version, that can be used as parse actions singleArgBuiltins = [] import __builtin__ + for fname in "sum len sorted reversed list tuple set any all min max".split(): try: - singleArgBuiltins.append(getattr(__builtin__,fname)) + singleArgBuiltins.append(getattr(__builtin__, fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" # ampersand must be replaced first from_symbols = '&><"\'' - to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split()) - for from_,to_ in zip(from_symbols, to_symbols): + to_symbols = ('&' + s + ';' for s in "amp gt lt quot apos".split()) + for from_, to_ in zip(from_symbols, to_symbols): data = data.replace(from_, to_) return data -class _Constants(object): - pass - -alphas = string.ascii_uppercase + string.ascii_lowercase -nums = "0123456789" -hexnums = nums + "ABCDEFabcdef" -alphanums = alphas + nums -_bslash = chr(92) +alphas = string.ascii_uppercase + string.ascii_lowercase +nums = "0123456789" +hexnums = nums + "ABCDEFabcdef" +alphanums = alphas + nums +_bslash = chr(92) printables = "".join(c for c in string.printable if c not in string.whitespace) + +def conditionAsParseAction(fn, message=None, fatal=False): + msg = message if message is not None else "failed user-defined condition" + exc_type = ParseFatalException if fatal else ParseException + fn = _trim_arity(fn) + + @wraps(fn) + def pa(s, l, t): + if not bool(fn(s, l, t)): + raise exc_type(s, l, msg) + + return pa + class ParseBaseException(Exception): """base exception class for all parsing runtime exceptions""" # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible - def __init__( self, pstr, loc=0, msg=None, elem=None ): + def __init__(self, pstr, loc=0, msg=None, elem=None): self.loc = loc if msg is None: self.msg = pstr @@ -220,32 +315,39 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) - def __getattr__( self, aname ): + def __getattr__(self, aname): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ - if( aname == "lineno" ): - return lineno( self.loc, self.pstr ) - elif( aname in ("col", "column") ): - return col( self.loc, self.pstr ) - elif( aname == "line" ): - return line( self.loc, self.pstr ) + if aname == "lineno": + return lineno(self.loc, self.pstr) + elif aname in ("col", "column"): + return col(self.loc, self.pstr) + elif aname == "line": + return line(self.loc, self.pstr) else: raise AttributeError(aname) - def __str__( self ): - return "%s (at char %d), (line:%d, col:%d)" % \ - ( self.msg, self.loc, self.lineno, self.column ) - def __repr__( self ): + def __str__(self): + if self.pstr: + if self.loc >= len(self.pstr): + foundstr = ', found end of text' + else: + foundstr = (', found %r' % self.pstr[self.loc:self.loc + 1]).replace(r'\\', '\\') + else: + foundstr = '' + return ("%s%s (at char %d), (line:%d, col:%d)" % + (self.msg, foundstr, self.loc, self.lineno, self.column)) + def __repr__(self): return _ustr(self) - def markInputline( self, markerString = ">!<" ): + def markInputline(self, markerString=">!<"): """Extracts the exception line from the input string, and marks the location of the exception with a special symbol. """ @@ -262,22 +364,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff[0] + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', '<module>'): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +459,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop<And._ErrorStop>} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop<And._ErrorStop>` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,34 +480,38 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" - def __init__( self, parseElementList ): + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ + def __init__(self, parseElementList): self.parseElementTrace = parseElementList - def __str__( self ): + def __str__(self): return "RecursiveGrammarException: %s" % self.parseElementTrace class _ParseResultsWithOffset(object): - def __init__(self,p1,p2): - self.tup = (p1,p2) - def __getitem__(self,i): + def __init__(self, p1, p2): + self.tup = (p1, p2) + def __getitem__(self, i): return self.tup[i] def __repr__(self): return repr(self.tup[0]) - def setOffset(self,i): - self.tup = (self.tup[0],i) + def setOffset(self, i): + self.tup = (self.tup[0], i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.<resultsName>} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.<resultsName>`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +528,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -360,7 +542,7 @@ class ParseResults(object): - month: 12 - year: 1999 """ - def __new__(cls, toklist=None, name=None, asList=True, modal=True ): + def __new__(cls, toklist=None, name=None, asList=True, modal=True): if isinstance(toklist, cls): return toklist retobj = object.__new__(cls) @@ -369,7 +551,7 @@ class ParseResults(object): # Performance tuning: we construct a *lot* of these, so keep this # constructor as small and fast as possible - def __init__( self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance ): + def __init__(self, toklist=None, name=None, asList=True, modal=True, isinstance=isinstance): if self.__doinit: self.__doinit = False self.__name = None @@ -390,96 +572,104 @@ class ParseResults(object): if name is not None and name: if not modal: self.__accumNames[name] = 0 - if isinstance(name,int): - name = _ustr(name) # will always return a str, but use _ustr for consistency + if isinstance(name, int): + name = _ustr(name) # will always return a str, but use _ustr for consistency self.__name = name - if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None,'',[])): - if isinstance(toklist,basestring): - toklist = [ toklist ] + if not (isinstance(toklist, (type(None), basestring, list)) and toklist in (None, '', [])): + if isinstance(toklist, basestring): + toklist = [toklist] if asList: - if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + if isinstance(toklist, ParseResults): + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: - self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]), 0) self[name].__name = name else: try: self[name] = toklist[0] - except (KeyError,TypeError,IndexError): + except (KeyError, TypeError, IndexError): self[name] = toklist - def __getitem__( self, i ): - if isinstance( i, (int,slice) ): + def __getitem__(self, i): + if isinstance(i, (int, slice)): return self.__toklist[i] else: if i not in self.__accumNames: return self.__tokdict[i][-1][0] else: - return ParseResults([ v[0] for v in self.__tokdict[i] ]) + return ParseResults([v[0] for v in self.__tokdict[i]]) - def __setitem__( self, k, v, isinstance=isinstance ): - if isinstance(v,_ParseResultsWithOffset): - self.__tokdict[k] = self.__tokdict.get(k,list()) + [v] + def __setitem__(self, k, v, isinstance=isinstance): + if isinstance(v, _ParseResultsWithOffset): + self.__tokdict[k] = self.__tokdict.get(k, list()) + [v] sub = v[0] - elif isinstance(k,(int,slice)): + elif isinstance(k, (int, slice)): self.__toklist[k] = v sub = v else: - self.__tokdict[k] = self.__tokdict.get(k,list()) + [_ParseResultsWithOffset(v,0)] + self.__tokdict[k] = self.__tokdict.get(k, list()) + [_ParseResultsWithOffset(v, 0)] sub = v - if isinstance(sub,ParseResults): + if isinstance(sub, ParseResults): sub.__parent = wkref(self) - def __delitem__( self, i ): - if isinstance(i,(int,slice)): - mylen = len( self.__toklist ) + def __delitem__(self, i): + if isinstance(i, (int, slice)): + mylen = len(self.__toklist) del self.__toklist[i] # convert int to slice if isinstance(i, int): if i < 0: i += mylen - i = slice(i, i+1) + i = slice(i, i + 1) # get removed indices removed = list(range(*i.indices(mylen))) removed.reverse() # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): + for name, occurrences in self.__tokdict.items(): for j in removed: for k, (value, position) in enumerate(occurrences): occurrences[k] = _ParseResultsWithOffset(value, position - (position > j)) else: del self.__tokdict[i] - def __contains__( self, k ): + def __contains__(self, k): return k in self.__tokdict - def __len__( self ): return len( self.__toklist ) - def __bool__(self): return ( not not self.__toklist ) + def __len__(self): + return len(self.__toklist) + + def __bool__(self): + return (not not self.__toklist) __nonzero__ = __bool__ - def __iter__( self ): return iter( self.__toklist ) - def __reversed__( self ): return iter( self.__toklist[::-1] ) - def _iterkeys( self ): + + def __iter__(self): + return iter(self.__toklist) + + def __reversed__(self): + return iter(self.__toklist[::-1]) + + def _iterkeys(self): if hasattr(self.__tokdict, "iterkeys"): return self.__tokdict.iterkeys() else: return iter(self.__tokdict) - def _itervalues( self ): + def _itervalues(self): return (self[k] for k in self._iterkeys()) - - def _iteritems( self ): + + def _iteritems(self): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -491,35 +681,36 @@ class ParseResults(object): iteritems = _iteritems """Returns an iterator of all named result key-value tuples (Python 2.x only).""" - def keys( self ): + def keys(self): """Returns all named result keys (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.iterkeys()) - def values( self ): + def values(self): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - - def items( self ): + + def items(self): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) - def haskeys( self ): + def haskeys(self): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - - def pop( self, *args, **kwargs): + + def pop(self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +727,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -544,14 +737,14 @@ class ParseResults(object): """ if not args: args = [-1] - for k,v in kwargs.items(): + for k, v in kwargs.items(): if k == 'default': args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or - args[0] in self): + if (isinstance(args[0], int) + or len(args) == 1 + or args[0] in self): index = args[0] ret = self[index] del self[index] @@ -563,14 +756,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -582,13 +776,14 @@ class ParseResults(object): else: return defaultValue - def insert( self, index, insStr ): + def insert(self, index, insStr): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -598,17 +793,18 @@ class ParseResults(object): """ self.__toklist.insert(index, insStr) # fixup indices in token dictionary - for name,occurrences in self.__tokdict.items(): + for name, occurrences in self.__tokdict.items(): for k, (value, position) in enumerate(occurrences): occurrences[k] = _ParseResultsWithOffset(value, position + (position > index)) - def append( self, item ): + def append(self, item): """ Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -616,13 +812,14 @@ class ParseResults(object): """ self.__toklist.append(item) - def extend( self, itemseq ): + def extend(self, itemseq): """ Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -630,104 +827,98 @@ class ParseResults(object): print(patt.addParseAction(make_palindrome).parseString("lskdj sdlkjf lksd")) # -> 'lskdjsdlkjflksddsklfjkldsjdksl' """ if isinstance(itemseq, ParseResults): - self += itemseq + self.__iadd__(itemseq) else: self.__toklist.extend(itemseq) - def clear( self ): + def clear(self): """ Clear all elements and results names. """ del self.__toklist[:] self.__tokdict.clear() - def __getattr__( self, name ): + def __getattr__(self, name): try: return self[name] except KeyError: return "" - - if name in self.__tokdict: - if name not in self.__accumNames: - return self.__tokdict[name][-1][0] - else: - return ParseResults([ v[0] for v in self.__tokdict[name] ]) - else: - return "" - def __add__( self, other ): + def __add__(self, other): ret = self.copy() ret += other return ret - def __iadd__( self, other ): + def __iadd__(self, other): if other.__tokdict: offset = len(self.__toklist) - addoffset = lambda a: offset if a<0 else a+offset + addoffset = lambda a: offset if a < 0 else a + offset otheritems = other.__tokdict.items() - otherdictitems = [(k, _ParseResultsWithOffset(v[0],addoffset(v[1])) ) - for (k,vlist) in otheritems for v in vlist] - for k,v in otherdictitems: + otherdictitems = [(k, _ParseResultsWithOffset(v[0], addoffset(v[1]))) + for k, vlist in otheritems for v in vlist] + for k, v in otherdictitems: self[k] = v - if isinstance(v[0],ParseResults): + if isinstance(v[0], ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist - self.__accumNames.update( other.__accumNames ) + self.__accumNames.update(other.__accumNames) return self def __radd__(self, other): - if isinstance(other,int) and other == 0: + if isinstance(other, int) and other == 0: # useful for merging many ParseResults using sum() builtin return self.copy() else: # this may raise a TypeError - so be it return other + self - - def __repr__( self ): - return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) - def __str__( self ): + def __repr__(self): + return "(%s, %s)" % (repr(self.__toklist), repr(self.__tokdict)) + + def __str__(self): return '[' + ', '.join(_ustr(i) if isinstance(i, ParseResults) else repr(i) for i in self.__toklist) + ']' - def _asStringList( self, sep='' ): + def _asStringList(self, sep=''): out = [] for item in self.__toklist: if out and sep: out.append(sep) - if isinstance( item, ParseResults ): + if isinstance(item, ParseResults): out += item._asStringList() else: - out.append( _ustr(item) ) + out.append(_ustr(item)) return out - def asList( self ): + def asList(self): """ Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> <class 'pyparsing.ParseResults'> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> <class 'list'> ['sldkj', 'lsdkj', 'sldkj'] """ - return [res.asList() if isinstance(res,ParseResults) else res for res in self.__toklist] + return [res.asList() if isinstance(res, ParseResults) else res for res in self.__toklist] - def asDict( self ): + def asDict(self): """ Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> <class 'pyparsing.ParseResults'> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> <class 'dict'> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +931,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,28 +940,28 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - - return dict((k,toItem(v)) for k,v in item_fn()) - def copy( self ): + return dict((k, toItem(v)) for k, v in item_fn()) + + def copy(self): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ - ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret = ParseResults(self.__toklist) + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent - ret.__accumNames.update( self.__accumNames ) + ret.__accumNames.update(self.__accumNames) ret.__name = self.__name return ret - def asXML( self, doctag=None, namedItemsOnly=False, indent="", formatted=True ): + def asXML(self, doctag=None, namedItemsOnly=False, indent="", formatted=True): """ (Deprecated) Returns the parse results as XML. Tags are created for tokens and lists that have defined results names. """ nl = "\n" out = [] - namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items() - for v in vlist) + namedItems = dict((v[1], k) for (k, vlist) in self.__tokdict.items() + for v in vlist) nextLevelIndent = indent + " " # collapse out indents if formatting is not desired @@ -792,20 +983,20 @@ class ParseResults(object): else: selfTag = "ITEM" - out += [ nl, indent, "<", selfTag, ">" ] + out += [nl, indent, "<", selfTag, ">"] - for i,res in enumerate(self.__toklist): - if isinstance(res,ParseResults): + for i, res in enumerate(self.__toklist): + if isinstance(res, ParseResults): if i in namedItems: - out += [ res.asXML(namedItems[i], - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] + out += [res.asXML(namedItems[i], + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] else: - out += [ res.asXML(None, - namedItemsOnly and doctag is None, - nextLevelIndent, - formatted)] + out += [res.asXML(None, + namedItemsOnly and doctag is None, + nextLevelIndent, + formatted)] else: # individual token, see if there is a name for it resTag = None @@ -817,38 +1008,41 @@ class ParseResults(object): else: resTag = "ITEM" xmlBodyText = _xml_escape(_ustr(res)) - out += [ nl, nextLevelIndent, "<", resTag, ">", - xmlBodyText, - "</", resTag, ">" ] + out += [nl, nextLevelIndent, "<", resTag, ">", + xmlBodyText, + "</", resTag, ">"] - out += [ nl, indent, "</", selfTag, ">" ] + out += [nl, indent, "</", selfTag, ">"] return "".join(out) - def __lookup(self,sub): - for k,vlist in self.__tokdict.items(): - for v,loc in vlist: + def __lookup(self, sub): + for k, vlist in self.__tokdict.items(): + for v, loc in vlist: if sub is v: return k return None def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -861,26 +1055,29 @@ class ParseResults(object): return par.__lookup(self) else: return None - elif (len(self) == 1 and - len(self.__tokdict) == 1 and - next(iter(self.__tokdict.values()))[0][1] in (0,-1)): + elif (len(self) == 1 + and len(self.__tokdict) == 1 + and next(iter(self.__tokdict.values()))[0][1] in (0, -1)): return next(iter(self.__tokdict.keys())) else: return None - def dump(self, indent='', depth=0, full=True): + def dump(self, indent='', full=True, include_list=True, _depth=0): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -888,38 +1085,57 @@ class ParseResults(object): """ out = [] NL = '\n' - out.append( indent+_ustr(self.asList()) ) + if include_list: + out.append(indent + _ustr(self.asList())) + else: + out.append('') + if full: if self.haskeys(): - items = sorted((str(k), v) for k,v in self.items()) - for k,v in items: + items = sorted((str(k), v) for k, v in self.items()) + for k, v in items: if out: out.append(NL) - out.append( "%s%s- %s: " % (indent,(' '*depth), k) ) - if isinstance(v,ParseResults): + out.append("%s%s- %s: " % (indent, (' ' * _depth), k)) + if isinstance(v, ParseResults): if v: - out.append( v.dump(indent,depth+1) ) + out.append(v.dump(indent=indent, full=full, include_list=include_list, _depth=_depth + 1)) else: out.append(_ustr(v)) else: out.append(repr(v)) - elif any(isinstance(vv,ParseResults) for vv in self): + elif any(isinstance(vv, ParseResults) for vv in self): v = self - for i,vv in enumerate(v): - if isinstance(vv,ParseResults): - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) + for i, vv in enumerate(v): + if isinstance(vv, ParseResults): + out.append("\n%s%s[%d]:\n%s%s%s" % (indent, + (' ' * (_depth)), + i, + indent, + (' ' * (_depth + 1)), + vv.dump(indent=indent, + full=full, + include_list=include_list, + _depth=_depth + 1))) else: - out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + out.append("\n%s%s[%d]:\n%s%s%s" % (indent, + (' ' * (_depth)), + i, + indent, + (' ' * (_depth + 1)), + _ustr(vv))) + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint <https://docs.python.org/3/library/pprint.html>`_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint <https://docs.python.org/3/library/pprint.html#pprint.pprint>`_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1143,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -938,18 +1156,15 @@ class ParseResults(object): # add support for pickle protocol def __getstate__(self): - return ( self.__toklist, - ( self.__tokdict.copy(), - self.__parent is not None and self.__parent() or None, - self.__accumNames, - self.__name ) ) + return (self.__toklist, + (self.__tokdict.copy(), + self.__parent is not None and self.__parent() or None, + self.__accumNames, + self.__name)) - def __setstate__(self,state): + def __setstate__(self, state): self.__toklist = state[0] - (self.__tokdict, - par, - inAccumNames, - self.__name) = state[1] + self.__tokdict, par, inAccumNames, self.__name = state[1] self.__accumNames = {} self.__accumNames.update(inAccumNames) if par is not None: @@ -961,53 +1176,82 @@ class ParseResults(object): return self.__toklist, self.__name, self.__asList, self.__modal def __dir__(self): - return (dir(type(self)) + list(self.keys())) + return dir(type(self)) + list(self.keys()) + + @classmethod + def from_dict(cls, other, name=None): + """ + Helper classmethod to construct a ParseResults from a dict, preserving the + name-value relations as results names. If an optional 'name' argument is + given, a nested ParseResults will be returned + """ + def is_iterable(obj): + try: + iter(obj) + except Exception: + return False + else: + if PY_3: + return not isinstance(obj, (str, bytes)) + else: + return not isinstance(obj, basestring) + + ret = cls([]) + for k, v in other.items(): + if isinstance(v, Mapping): + ret += cls.from_dict(v, name=k) + else: + ret += cls([v], name=k, asList=is_iterable(v)) + if name is not None: + ret = cls([ret], name=name) + return ret MutableMapping.register(ParseResults) -def col (loc,strg): +def col (loc, strg): """Returns current column within a string, counting newlines as line separators. The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ``<TAB>`` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg - return 1 if 0<loc<len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) + return 1 if 0 < loc < len(s) and s[loc-1] == '\n' else loc - s.rfind("\n", 0, loc) -def lineno(loc,strg): +def lineno(loc, strg): """Returns current line number within a string, counting newlines as line separators. - The first line is number 1. + The first line is number 1. - Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ - return strg.count("\n",0,loc) + 1 + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ``<TAB>`` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ + return strg.count("\n", 0, loc) + 1 -def line( loc, strg ): +def line(loc, strg): """Returns the line of text containing loc within a string, counting newlines as line separators. """ lastCR = strg.rfind("\n", 0, loc) nextCR = strg.find("\n", loc) if nextCR >= 0: - return strg[lastCR+1:nextCR] + return strg[lastCR + 1:nextCR] else: - return strg[lastCR+1:] + return strg[lastCR + 1:] -def _defaultStartDebugAction( instring, loc, expr ): - print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))) +def _defaultStartDebugAction(instring, loc, expr): + print(("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % (lineno(loc, instring), col(loc, instring)))) -def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ): - print ("Matched " + _ustr(expr) + " -> " + str(toks.asList())) +def _defaultSuccessDebugAction(instring, startloc, endloc, expr, toks): + print("Matched " + _ustr(expr) + " -> " + str(toks.asList())) -def _defaultExceptionDebugAction( instring, loc, expr, exc ): - print ("Exception raised:" + _ustr(exc)) +def _defaultExceptionDebugAction(instring, loc, expr, exc): + print("Exception raised:" + _ustr(exc)) def nullDebugAction(*args): """'Do-nothing' debug action, to suppress debugging output during parsing.""" @@ -1038,16 +1282,16 @@ def nullDebugAction(*args): 'decorator to trim function calls to match the arity of the target' def _trim_arity(func, maxargs=2): if func in singleArgBuiltins: - return lambda s,l,t: func(t) + return lambda s, l, t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples - if system_version[:2] >= (3,5): + if system_version[:2] >= (3, 5): def extract_stack(limit=0): # special handling for Python 3.5.0 - extra deep call stack by 1 - offset = -3 if system_version == (3,5,0) else -2 - frame_summary = traceback.extract_stack(limit=-offset+limit-1)[offset] + offset = -3 if system_version == (3, 5, 0) else -2 + frame_summary = traceback.extract_stack(limit=-offset + limit - 1)[offset] return [frame_summary[:2]] def extract_tb(tb, limit=0): frames = traceback.extract_tb(tb, limit=limit) @@ -1056,15 +1300,15 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] - pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) + pa_call_line_synth = (this_line[0], this_line[1] + LINE_DIFF) def wrapper(*args): while 1: @@ -1082,7 +1326,10 @@ def _trim_arity(func, maxargs=2): if not extract_tb(tb, limit=2)[-1][:2] == pa_call_line_synth: raise finally: - del tb + try: + del tb + except NameError: + pass if limit[0] <= maxargs: limit[0] += 1 @@ -1092,7 +1339,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "<parse action>" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1100,20 +1347,22 @@ def _trim_arity(func, maxargs=2): return wrapper + class ParserElement(object): """Abstract base level parser element class.""" DEFAULT_WHITE_CHARS = " \n\t\r" verbose_stacktrace = False @staticmethod - def setDefaultWhitespaceChars( chars ): + def setDefaultWhitespaceChars(chars): r""" Overrides the default whitespace chars Example:: + # default whitespace chars are space, <TAB> and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,32 +1373,39 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ ParserElement._literalStringClass = cls - def __init__( self, savelist=False ): + @classmethod + def _trim_traceback(cls, tb): + while tb.tb_next: + tb = tb.tb_next + return tb + + def __init__(self, savelist=False): self.parseAction = list() self.failAction = None - #~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall + # ~ self.name = "<unknown>" # don't define self.name, let subclasses try/except upcall self.strRepr = None self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1159,116 +1415,134 @@ class ParserElement(object): self.mayIndexError = True # used to optimize exception handling for subclasses that don't advance parse index self.errmsg = "" self.modalResults = True # used to mark results names as modal (report only last) or cumulative (list all) - self.debugActions = ( None, None, None ) #custom debug actions + self.debugActions = (None, None, None) # custom debug actions self.re = None self.callPreparse = True # used to avoid redundant calls to preParse self.callDuringTry = False - def copy( self ): + def copy(self): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) - integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") - integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + integerK = integer.copy().addParseAction(lambda toks: toks[0] * 1024) + Suppress("K") + integerM = integer.copy().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: - integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + + integerM = integer().addParseAction(lambda toks: toks[0] * 1024 * 1024) + Suppress("M") """ - cpy = copy.copy( self ) + cpy = copy.copy(self) cpy.parseAction = self.parseAction[:] cpy.ignoreExprs = self.ignoreExprs[:] if self.copyDefaultWhiteChars: cpy.whiteChars = ParserElement.DEFAULT_WHITE_CHARS return cpy - def setName( self, name ): + def setName(self, name): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ self.name = name self.errmsg = "Expected " + self.name - if hasattr(self,"exception"): - self.exception.msg = self.errmsg + if __diag__.enable_debug_on_named_expressions: + self.setDebug() return self - def setResultsName( self, name, listAllMatches=False ): + def setResultsName(self, name, listAllMatches=False): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: date_str = integer("year") + '/' + integer("month") + '/' + integer("day") """ + return self._setResultsName(name, listAllMatches) + + def _setResultsName(self, name, listAllMatches=False): newself = self.copy() if name.endswith("*"): name = name[:-1] - listAllMatches=True + listAllMatches = True newself.resultsName = name newself.modalResults = not listAllMatches return newself - def setBreak(self,breakFlag = True): + def setBreak(self, breakFlag=True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: _parseMethod = self._parse def breaker(instring, loc, doActions=True, callPreParse=True): import pdb + # this call to pdb.set_trace() is intentional, not a checkin error pdb.set_trace() - return _parseMethod( instring, loc, doActions, callPreParse ) + return _parseMethod(instring, loc, doActions, callPreParse) breaker._originalParseMethod = _parseMethod self._parse = breaker else: - if hasattr(self._parse,"_originalParseMethod"): + if hasattr(self._parse, "_originalParseMethod"): self._parse = self._parse._originalParseMethod return self - def setParseAction( self, *fns, **kwargs ): + def setParseAction(self, *fns, **kwargs): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s, loc, toks)`` , + ``fn(loc, toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. + If None is passed as the parse action, all previously added parse actions for this + expression are cleared. + Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False``) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}<parseString>} for more information - on parsing strings containing C{<TAB>}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ``<TAB>`` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1281,30 +1555,36 @@ class ParserElement(object): # note that integer fields are now ints, not strings date_str.parseString("1999/12/31") # -> [1999, '/', 12, '/', 31] """ - self.parseAction = list(map(_trim_arity, list(fns))) - self.callDuringTry = kwargs.get("callDuringTry", False) + if list(fns) == [None,]: + self.parseAction = [] + else: + if not all(callable(fn) for fn in fns): + raise TypeError("parse actions must be callable") + self.parseAction = list(map(_trim_arity, list(fns))) + self.callDuringTry = kwargs.get("callDuringTry", False) return self - def addParseAction( self, *fns, **kwargs ): + def addParseAction(self, *fns, **kwargs): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}<setParseAction>}. - - See examples in L{I{copy}<copy>}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}<setParseAction>} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1312,45 +1592,42 @@ class ParserElement(object): result = date_str.parseString("1999/12/31") # -> Exception: Only support years 2000 and later (at char 0), (line:1, col:1) """ - msg = kwargs.get("message", "failed user-defined condition") - exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: - def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): - raise exc_type(s,l,msg) - self.parseAction.append(pa) + self.parseAction.append(conditionAsParseAction(fn, message=kwargs.get('message'), + fatal=kwargs.get('fatal', False))) + self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self - def setFailAction( self, fn ): + def setFailAction(self, fn): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s, loc, expr, err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self - def _skipIgnorables( self, instring, loc ): + def _skipIgnorables(self, instring, loc): exprsFound = True while exprsFound: exprsFound = False for e in self.ignoreExprs: try: while 1: - loc,dummy = e._parse( instring, loc ) + loc, dummy = e._parse(instring, loc) exprsFound = True except ParseException: pass return loc - def preParse( self, instring, loc ): + def preParse(self, instring, loc): if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) + loc = self._skipIgnorables(instring, loc) if self.skipWhitespace: wt = self.whiteChars @@ -1360,90 +1637,106 @@ class ParserElement(object): return loc - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): return loc, [] - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): return tokenlist - #~ @profile - def _parseNoCache( self, instring, loc, doActions=True, callPreParse=True ): - debugging = ( self.debug ) #and doActions ) + # ~ @profile + def _parseNoCache(self, instring, loc, doActions=True, callPreParse=True): + TRY, MATCH, FAIL = 0, 1, 2 + debugging = (self.debug) # and doActions) if debugging or self.failAction: - #~ print ("Match",self,"at loc",loc,"(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )) - if (self.debugActions[0] ): - self.debugActions[0]( instring, loc, self ) - if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) - else: - preloc = loc - tokensStart = preloc + # ~ print ("Match", self, "at loc", loc, "(%d, %d)" % (lineno(loc, instring), col(loc, instring))) + if self.debugActions[TRY]: + self.debugActions[TRY](instring, loc, self) try: - try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) - except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) - except ParseBaseException as err: - #~ print ("Exception raised:", err) - if self.debugActions[2]: - self.debugActions[2]( instring, tokensStart, self, err ) + if callPreParse and self.callPreparse: + preloc = self.preParse(instring, loc) + else: + preloc = loc + tokensStart = preloc + if self.mayIndexError or preloc >= len(instring): + try: + loc, tokens = self.parseImpl(instring, preloc, doActions) + except IndexError: + raise ParseException(instring, len(instring), self.errmsg, self) + else: + loc, tokens = self.parseImpl(instring, preloc, doActions) + except Exception as err: + # ~ print ("Exception raised:", err) + if self.debugActions[FAIL]: + self.debugActions[FAIL](instring, tokensStart, self, err) if self.failAction: - self.failAction( instring, tokensStart, self, err ) + self.failAction(instring, tokensStart, self, err) raise else: if callPreParse and self.callPreparse: - preloc = self.preParse( instring, loc ) + preloc = self.preParse(instring, loc) else: preloc = loc tokensStart = preloc if self.mayIndexError or preloc >= len(instring): try: - loc,tokens = self.parseImpl( instring, preloc, doActions ) + loc, tokens = self.parseImpl(instring, preloc, doActions) except IndexError: - raise ParseException( instring, len(instring), self.errmsg, self ) + raise ParseException(instring, len(instring), self.errmsg, self) else: - loc,tokens = self.parseImpl( instring, preloc, doActions ) + loc, tokens = self.parseImpl(instring, preloc, doActions) - tokens = self.postParse( instring, loc, tokens ) + tokens = self.postParse(instring, loc, tokens) - retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults ) + retTokens = ParseResults(tokens, self.resultsName, asList=self.saveAsList, modal=self.modalResults) if self.parseAction and (doActions or self.callDuringTry): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, + try: + tokens = fn(instring, tokensStart, retTokens) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: + retTokens = ParseResults(tokens, self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) - except ParseBaseException as err: - #~ print "Exception raised in user parse action:", err - if (self.debugActions[2] ): - self.debugActions[2]( instring, tokensStart, self, err ) + asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults) + except Exception as err: + # ~ print "Exception raised in user parse action:", err + if self.debugActions[FAIL]: + self.debugActions[FAIL](instring, tokensStart, self, err) raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: - retTokens = ParseResults( tokens, + try: + tokens = fn(instring, tokensStart, retTokens) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: + retTokens = ParseResults(tokens, self.resultsName, - asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), - modal=self.modalResults ) + asList=self.saveAsList and isinstance(tokens, (ParseResults, list)), + modal=self.modalResults) if debugging: - #~ print ("Matched",self,"->",retTokens.asList()) - if (self.debugActions[1] ): - self.debugActions[1]( instring, tokensStart, loc, self, retTokens ) + # ~ print ("Matched", self, "->", retTokens.asList()) + if self.debugActions[MATCH]: + self.debugActions[MATCH](instring, tokensStart, loc, self, retTokens) return loc, retTokens - def tryParse( self, instring, loc ): + def tryParse(self, instring, loc): try: - return self._parse( instring, loc, doActions=False )[0] + return self._parse(instring, loc, doActions=False)[0] except ParseFatalException: - raise ParseException( instring, loc, self.errmsg, self) - + raise ParseException(instring, loc, self.errmsg, self) + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1758,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1539,7 +1832,7 @@ class ParserElement(object): # this method gets repeatedly called during backtracking with the same arguments - # we can cache these arguments and save ourselves the trouble of re-parsing the contained expression - def _parseCache( self, instring, loc, doActions=True, callPreParse=True ): + def _parseCache(self, instring, loc, doActions=True, callPreParse=True): HIT, MISS = 0, 1 lookup = (self, instring, loc, callPreParse, doActions) with ParserElement.packrat_cache_lock: @@ -1560,7 +1853,7 @@ class ParserElement(object): ParserElement.packrat_cache_stats[HIT] += 1 if isinstance(value, Exception): raise value - return (value[0], value[1].copy()) + return value[0], value[1].copy() _parse = _parseNoCache @@ -1577,23 +1870,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1605,76 +1898,85 @@ class ParserElement(object): ParserElement.packrat_cache = ParserElement._FifoCache(cache_size_limit) ParserElement._parse = ParserElement._parseCache - def parseString( self, instring, parseAll=False ): + def parseString(self, instring, parseAll=False): """ Execute the parse expression with the given string. This is the main interface to the client code, once the complete expression has been built. - If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + Returns the parsed data as a :class:`ParseResults` object, which may be + accessed as a list, or as a dict or object with attributes if the given parser + includes results names. - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + If you want the grammar to require that the entire input string be + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). + + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}<parseWithTabs>}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s, loc, toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ ParserElement.resetCache() if not self.streamlined: self.streamline() - #~ self.saveAsList = True + # ~ self.saveAsList = True for e in self.ignoreExprs: e.streamline() if not self.keepTabs: instring = instring.expandtabs() try: - loc, tokens = self._parse( instring, 0 ) + loc, tokens = self._parse(instring, 0) if parseAll: - loc = self.preParse( instring, loc ) + loc = self.preParse(instring, loc) se = Empty() + StringEnd() - se._parse( instring, loc ) + se._parse(instring, loc) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc else: return tokens - def scanString( self, instring, maxMatches=_MAX_INT, overlap=False ): + def scanString(self, instring, maxMatches=_MAX_INT, overlap=False): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}<parseString>} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) - for tokens,start,end in Word(alphas).scanString(source): + for tokens, start, end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1701,16 +2003,16 @@ class ParserElement(object): try: while loc <= instrlen and matches < maxMatches: try: - preloc = preparseFn( instring, loc ) - nextLoc,tokens = parseFn( instring, preloc, callPreParse=False ) + preloc = preparseFn(instring, loc) + nextLoc, tokens = parseFn(instring, preloc, callPreParse=False) except ParseException: - loc = preloc+1 + loc = preloc + 1 else: if nextLoc > loc: matches += 1 yield tokens, preloc, nextLoc if overlap: - nextloc = preparseFn( instring, loc ) + nextloc = preparseFn(instring, loc) if nextloc > loc: loc = nextLoc else: @@ -1718,29 +2020,34 @@ class ParserElement(object): else: loc = nextLoc else: - loc = preloc+1 + loc = preloc + 1 except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc - def transformString( self, instring ): + def transformString(self, instring): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1749,169 +2056,206 @@ class ParserElement(object): # keep string locs straight between transformString and scanString self.keepTabs = True try: - for t,s,e in self.scanString( instring ): - out.append( instring[lastE:s] ) + for t, s, e in self.scanString(instring): + out.append(instring[lastE:s]) if t: - if isinstance(t,ParseResults): + if isinstance(t, ParseResults): out += t.asList() - elif isinstance(t,list): + elif isinstance(t, list): out += t else: out.append(t) lastE = e out.append(instring[lastE:]) out = [o for o in out if o] - return "".join(map(_ustr,_flatten(out))) + return "".join(map(_ustr, _flatten(out))) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc - def searchString( self, instring, maxMatches=_MAX_INT ): + def searchString(self, instring, maxMatches=_MAX_INT): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ try: - return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ]) + return ParseResults([t for t, s, e in self.scanString(instring, maxMatches)]) except ParseBaseException as exc: if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 last = 0 - for t,s,e in self.scanString(instring, maxMatches=maxsplit): + for t, s, e in self.scanString(instring, maxMatches=maxsplit): yield instring[last:s] if includeSeparators: yield t[0] last = e yield instring[last:] - def __add__(self, other ): + def __add__(self, other): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: - Hello, World! -> ['Hello', ',', 'World', '!'] - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return And( [ self, other ] ) - def __radd__(self, other ): + prints:: + + Hello, World! -> ['Hello', ',', 'World', '!'] + + ``...`` may be used as a parse expression as a short form of :class:`SkipTo`. + + Literal('start') + ... + Literal('end') + + is equivalent to: + + Literal('start') + SkipTo('end')("_skipped*") + Literal('end') + + Note that the skipped text is returned with '_skipped' as a results name, + and to support having multiple skips in the same parser, the value returned is + a list of all skipped text. """ - Implementation of + operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if other is Ellipsis: + return _PendingSkip(self) + + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) + return None + return And([self, other]) + + def __radd__(self, other): + """ + Implementation of + operator when left operand is not a :class:`ParserElement` + """ + if other is Ellipsis: + return SkipTo(self)("_skipped*") + self + + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) return None return other + self def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return self + And._ErrorStop() + other - def __rsub__(self, other ): + def __rsub__(self, other): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other - self - def __mul__(self,other): + def __mul__(self, other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min, max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n, None)`` or ``expr*(n, )`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None, n)`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None, None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1, None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None, n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None, n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None, n) + ~expr`` """ - if isinstance(other,int): - minElements, optElements = other,0 - elif isinstance(other,tuple): + if other is Ellipsis: + other = (0, None) + elif isinstance(other, tuple) and other[:1] == (Ellipsis,): + other = ((0, ) + other[1:] + (None,))[:2] + + if isinstance(other, int): + minElements, optElements = other, 0 + elif isinstance(other, tuple): + other = tuple(o if o is not Ellipsis else None for o in other) other = (other + (None, None))[:2] if other[0] is None: other = (0, other[1]) - if isinstance(other[0],int) and other[1] is None: + if isinstance(other[0], int) and other[1] is None: if other[0] == 0: return ZeroOrMore(self) if other[0] == 1: return OneOrMore(self) else: - return self*other[0] + ZeroOrMore(self) - elif isinstance(other[0],int) and isinstance(other[1],int): + return self * other[0] + ZeroOrMore(self) + elif isinstance(other[0], int) and isinstance(other[1], int): minElements, optElements = other optElements -= minElements else: - raise TypeError("cannot multiply 'ParserElement' and ('%s','%s') objects", type(other[0]),type(other[1])) + raise TypeError("cannot multiply 'ParserElement' and ('%s', '%s') objects", type(other[0]), type(other[1])) else: raise TypeError("cannot multiply 'ParserElement' and '%s' objects", type(other)) @@ -1920,145 +2264,190 @@ class ParserElement(object): if optElements < 0: raise ValueError("second tuple value must be greater or equal to first tuple value") if minElements == optElements == 0: - raise ValueError("cannot multiply ParserElement by 0 or (0,0)") + raise ValueError("cannot multiply ParserElement by 0 or (0, 0)") - if (optElements): + if optElements: def makeOptionalList(n): - if n>1: - return Optional(self + makeOptionalList(n-1)) + if n > 1: + return Optional(self + makeOptionalList(n - 1)) else: return Optional(self) if minElements: if minElements == 1: ret = self + makeOptionalList(optElements) else: - ret = And([self]*minElements) + makeOptionalList(optElements) + ret = And([self] * minElements) + makeOptionalList(optElements) else: ret = makeOptionalList(optElements) else: if minElements == 1: ret = self else: - ret = And([self]*minElements) + ret = And([self] * minElements) return ret def __rmul__(self, other): return self.__mul__(other) - def __or__(self, other ): + def __or__(self, other): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): - warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) - return None - return MatchFirst( [ self, other ] ) + if other is Ellipsis: + return _PendingSkip(self, must_skip=True) - def __ror__(self, other ): - """ - Implementation of | operator when left operand is not a C{L{ParserElement}} - """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) + return None + return MatchFirst([self, other]) + + def __ror__(self, other): + """ + Implementation of | operator when left operand is not a :class:`ParserElement` + """ + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): + warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), + SyntaxWarning, stacklevel=2) return None return other | self - def __xor__(self, other ): + def __xor__(self, other): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None - return Or( [ self, other ] ) + return Or([self, other]) - def __rxor__(self, other ): + def __rxor__(self, other): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other ^ self - def __and__(self, other ): + def __and__(self, other): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None - return Each( [ self, other ] ) + return Each([self, other]) - def __rand__(self, other ): + def __rand__(self, other): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - if not isinstance( other, ParserElement ): + if isinstance(other, basestring): + other = self._literalStringClass(other) + if not isinstance(other, ParserElement): warnings.warn("Cannot combine element of type %s with ParserElement" % type(other), - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) return None return other & self - def __invert__( self ): + def __invert__(self): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ - return NotAny( self ) + return NotAny(self) + + def __iter__(self): + # must implement __iter__ to override legacy use of sequential access to __getitem__ to + # iterate over a sequence + raise TypeError('%r object is not iterable' % self.__class__.__name__) + + def __getitem__(self, key): + """ + use ``[]`` indexing notation as a short form for expression repetition: + - ``expr[n]`` is equivalent to ``expr*n`` + - ``expr[m, n]`` is equivalent to ``expr*(m, n)`` + - ``expr[n, ...]`` or ``expr[n,]`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr[..., n]`` is equivalent to ``expr*(0, n)`` + (read as "0 to n instances of ``expr``") + - ``expr[...]`` and ``expr[0, ...]`` are equivalent to ``ZeroOrMore(expr)`` + - ``expr[1, ...]`` is equivalent to ``OneOrMore(expr)`` + ``None`` may be used in place of ``...``. + + Note that ``expr[..., n]`` and ``expr[m, n]``do not raise an exception + if more than ``n`` ``expr``s exist in the input stream. If this behavior is + desired, then write ``expr[..., n] + ~expr``. + """ + + # convert single arg keys to tuples + try: + if isinstance(key, str): + key = (key,) + iter(key) + except TypeError: + key = (key, key) + + if len(key) > 2: + warnings.warn("only 1 or 2 index arguments supported ({0}{1})".format(key[:5], + '... [{0}]'.format(len(key)) + if len(key) > 5 else '')) + + # clip to 2 elements + ret = self * tuple(key[:2]) + return ret def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent - userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas).setResultsName("name") + Word(nums + "-").setResultsName("socsecno") + userdata = Word(alphas)("name") + Word(nums + "-")("socsecno") """ if name is not None: - return self.setResultsName(name) + return self._setResultsName(name) else: return self.copy() - def suppress( self ): + def suppress(self): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ - return Suppress( self ) + return Suppress(self) - def leaveWhitespace( self ): + def leaveWhitespace(self): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False return self - def setWhitespaceChars( self, chars ): + def setWhitespaceChars(self, chars): """ Overrides the default whitespace chars """ @@ -2067,39 +2456,40 @@ class ParserElement(object): self.copyDefaultWhiteChars = False return self - def parseWithTabs( self ): + def parseWithTabs(self): """ - Overrides default behavior to expand C{<TAB>}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{<TAB>} characters. + Overrides default behavior to expand ``<TAB>``s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ``<TAB>`` characters. """ self.keepTabs = True return self - def ignore( self, other ): + def ignore(self, other): """ Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ if isinstance(other, basestring): other = Suppress(other) - if isinstance( other, Suppress ): + if isinstance(other, Suppress): if other not in self.ignoreExprs: self.ignoreExprs.append(other) else: - self.ignoreExprs.append( Suppress( other.copy() ) ) + self.ignoreExprs.append(Suppress(other.copy())) return self - def setDebugActions( self, startAction, successAction, exceptionAction ): + def setDebugActions(self, startAction, successAction, exceptionAction): """ Enable display of debugging messages while doing pattern matching. """ @@ -2109,22 +2499,24 @@ class ParserElement(object): self.debug = True return self - def setDebug( self, flag=True ): + def setDebug(self, flag=True): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,40 +2529,40 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match <exprname> at loc <n>(<line>,<col>)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match <exprname> at loc <n>(<line>,<col>)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: - self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) + self.setDebugActions(_defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction) else: self.debug = False return self - def __str__( self ): + def __str__(self): return self.name - def __repr__( self ): + def __repr__(self): return _ustr(self) - def streamline( self ): + def streamline(self): self.streamlined = True self.strRepr = None return self - def checkRecursion( self, parseElementList ): + def checkRecursion(self, parseElementList): pass - def validate( self, validateTrace=[] ): + def validate(self, validateTrace=None): """ Check defined expressions for valid structure, check for infinite recursive definitions. """ - self.checkRecursion( [] ) + self.checkRecursion([]) - def parseFile( self, file_or_filename, parseAll=False ): + def parseFile(self, file_or_filename, parseAll=False): """ Execute the parse expression on the given file or filename. If a filename is specified (instead of a file object), @@ -2187,39 +2579,43 @@ class ParserElement(object): if ParserElement.verbose_stacktrace: raise else: - # catch and re-raise exception from here, clears out pyparsing internal stack trace + # catch and re-raise exception from here, clearing out pyparsing internal stack trace + if getattr(exc, '__traceback__', None) is not None: + exc.__traceback__ = self._trim_traceback(exc.__traceback__) raise exc - def __eq__(self,other): - if isinstance(other, ParserElement): - return self is other or vars(self) == vars(other) + def __eq__(self, other): + if self is other: + return True elif isinstance(other, basestring): return self.matches(other) - else: - return super(ParserElement,self)==other + elif isinstance(other, ParserElement): + return vars(self) == vars(other) + return False - def __ne__(self,other): + def __ne__(self, other): return not (self == other) def __hash__(self): - return hash(id(self)) + return id(self) - def __req__(self,other): + def __req__(self, other): return self == other - def __rne__(self,other): + def __rne__(self, other): return not (self == other) def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2624,35 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None, + file=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output + - file - (default=``None``) optional file-like object to which test output will be written; + if None, will default to ``sys.stdout`` Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2676,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2696,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,36 +2718,41 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): tests = list(map(str.strip, tests.rstrip().splitlines())) if isinstance(comment, basestring): comment = Literal(comment) + if file is None: + file = sys.stdout + print_ = file.write + allResults = [] comments = [] success = True + NL = Literal(r'\n').addParseAction(replaceWith('\n')).ignore(quotedString) + BOM = u'\ufeff' for t in tests: if comment is not None and comment.matches(t, False) or comments and not t: comments.append(t) continue if not t: continue - out = ['\n'.join(comments), t] + out = ['\n' + '\n'.join(comments) if comments else '', t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = NL.transformString(t.lstrip(BOM)) result = self.parseString(t, parseAll=parseAll) - out.append(result.dump(full=fullDump)) - success = success and not failureTests except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: out.append(line(pe.loc, t)) - out.append(' '*(col(pe.loc,t)-1) + '^' + fatal) + out.append(' ' * (col(pe.loc, t) - 1) + '^' + fatal) else: - out.append(' '*pe.loc + '^' + fatal) + out.append(' ' * pe.loc + '^' + fatal) out.append("FAIL: " + str(pe)) success = success and failureTests result = pe @@ -2350,67 +2760,115 @@ class ParserElement(object): out.append("FAIL-EXCEPTION: " + str(exc)) success = success and failureTests result = exc + else: + success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + if isinstance(pp_value, ParseResults): + out.append(pp_value.dump()) + else: + out.append(str(pp_value)) + else: + out.append(result.dump()) + except Exception as e: + out.append(result.dump(full=fullDump)) + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) + else: + out.append(result.dump(full=fullDump)) if printResults: if fullDump: out.append('') - print('\n'.join(out)) + print_('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + +class _PendingSkip(ParserElement): + # internal placeholder class to hold a place were '...' is added to a parser element, + # once another ParserElement is added, this placeholder will be replaced with a SkipTo + def __init__(self, expr, must_skip=False): + super(_PendingSkip, self).__init__() + self.strRepr = str(expr + Empty()).replace('Empty', '...') + self.name = self.strRepr + self.anchor = expr + self.must_skip = must_skip + + def __add__(self, other): + skipper = SkipTo(other).setName("...")("_skipped*") + if self.must_skip: + def must_skip(t): + if not t._skipped or t._skipped.asList() == ['']: + del t[0] + t.pop("_skipped", None) + def show_skip(t): + if t._skipped.asList()[-1:] == ['']: + skipped = t.pop('_skipped') + t['_skipped'] = 'missing <' + repr(self.anchor) + '>' + return (self.anchor + skipper().addParseAction(must_skip) + | skipper().addParseAction(show_skip)) + other + + return self.anchor + skipper + other + + def __repr__(self): + return self.strRepr + + def parseImpl(self, *args): + raise Exception("use of `...` expression without following SkipTo target expression") + + class Token(ParserElement): + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. - """ - def __init__( self ): - super(Token,self).__init__( savelist=False ) + def __init__(self): + super(Token, self).__init__(savelist=False) class Empty(Token): + """An empty token, will always match. """ - An empty token, will always match. - """ - def __init__( self ): - super(Empty,self).__init__() + def __init__(self): + super(Empty, self).__init__() self.name = "Empty" self.mayReturnEmpty = True self.mayIndexError = False class NoMatch(Token): + """A token that will never match. """ - A token that will never match. - """ - def __init__( self ): - super(NoMatch,self).__init__() + def __init__(self): + super(NoMatch, self).__init__() self.name = "NoMatch" self.mayReturnEmpty = True self.mayIndexError = False self.errmsg = "Unmatchable token" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): raise ParseException(instring, loc, self.errmsg, self) class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ - def __init__( self, matchString ): - super(Literal,self).__init__() + def __init__(self, matchString): + super(Literal, self).__init__() self.match = matchString self.matchLen = len(matchString) try: @@ -2424,39 +2882,54 @@ class Literal(Token): self.mayReturnEmpty = False self.mayIndexError = False - # Performance tuning: this routine gets called a *lot* - # if this is a single character match string and the first character matches, - # short-circuit as quickly as possible, and avoid calling startswith - #~ @profile - def parseImpl( self, instring, loc, doActions=True ): - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) ): - return loc+self.matchLen, self.match + # Performance tuning: modify __class__ to select + # a parseImpl optimized for single-character check + if self.matchLen == 1 and type(self) is Literal: + self.__class__ = _SingleCharLiteral + + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] == self.firstMatchChar and instring.startswith(self.match, loc): + return loc + self.matchLen, self.match raise ParseException(instring, loc, self.errmsg, self) + +class _SingleCharLiteral(Literal): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] == self.firstMatchChar: + return loc + 1, self.match + raise ParseException(instring, loc, self.errmsg, self) + _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ - DEFAULT_KEYWORD_CHARS = alphanums+"_$" + DEFAULT_KEYWORD_CHARS = alphanums + "_$" - def __init__( self, matchString, identChars=None, caseless=False ): - super(Keyword,self).__init__() + def __init__(self, matchString, identChars=None, caseless=False): + super(Keyword, self).__init__() if identChars is None: identChars = Keyword.DEFAULT_KEYWORD_CHARS self.match = matchString @@ -2465,7 +2938,7 @@ class Keyword(Token): self.firstMatchChar = matchString[0] except IndexError: warnings.warn("null string passed to Keyword; use Empty() instead", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) self.name = '"%s"' % self.match self.errmsg = "Expected " + self.name self.mayReturnEmpty = False @@ -2476,86 +2949,94 @@ class Keyword(Token): identChars = identChars.upper() self.identChars = set(identChars) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.caseless: - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) and - (loc == 0 or instring[loc-1].upper() not in self.identChars) ): - return loc+self.matchLen, self.match + if ((instring[loc:loc + self.matchLen].upper() == self.caselessmatch) + and (loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen].upper() not in self.identChars) + and (loc == 0 + or instring[loc - 1].upper() not in self.identChars)): + return loc + self.matchLen, self.match + else: - if (instring[loc] == self.firstMatchChar and - (self.matchLen==1 or instring.startswith(self.match,loc)) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and - (loc == 0 or instring[loc-1] not in self.identChars) ): - return loc+self.matchLen, self.match + if instring[loc] == self.firstMatchChar: + if ((self.matchLen == 1 or instring.startswith(self.match, loc)) + and (loc >= len(instring) - self.matchLen + or instring[loc + self.matchLen] not in self.identChars) + and (loc == 0 or instring[loc - 1] not in self.identChars)): + return loc + self.matchLen, self.match + raise ParseException(instring, loc, self.errmsg, self) def copy(self): - c = super(Keyword,self).copy() + c = super(Keyword, self).copy() c.identChars = Keyword.DEFAULT_KEYWORD_CHARS return c @staticmethod - def setDefaultKeywordChars( chars ): + def setDefaultKeywordChars(chars): """Overrides the default Keyword chars """ Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ - def __init__( self, matchString ): - super(CaselessLiteral,self).__init__( matchString.upper() ) + def __init__(self, matchString): + super(CaselessLiteral, self).__init__(matchString.upper()) # Preserve the defining literal. self.returnString = matchString self.name = "'%s'" % self.returnString self.errmsg = "Expected " + self.name - def parseImpl( self, instring, loc, doActions=True ): - if instring[ loc:loc+self.matchLen ].upper() == self.match: - return loc+self.matchLen, self.returnString + def parseImpl(self, instring, loc, doActions=True): + if instring[loc:loc + self.matchLen].upper() == self.match: + return loc + self.matchLen, self.returnString raise ParseException(instring, loc, self.errmsg, self) class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: - OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) - """ - def __init__( self, matchString, identChars=None ): - super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] + + (Contrast with example for :class:`CaselessLiteral`.) + """ + def __init__(self, matchString, identChars=None): + super(CaselessKeyword, self).__init__(matchString, identChars, caseless=True) class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2568,7 +3049,7 @@ class CloseMatch(Token): patt.parseString("ATCAXCGAAXGGA") # -> (['ATCAXCGAAXGGA'], {'mismatches': [[4, 9]], 'original': ['ATCATCGAATGGA']}) """ def __init__(self, match_string, maxMismatches=1): - super(CloseMatch,self).__init__() + super(CloseMatch, self).__init__() self.name = match_string self.match_string = match_string self.maxMismatches = maxMismatches @@ -2576,7 +3057,7 @@ class CloseMatch(Token): self.mayIndexError = False self.mayReturnEmpty = False - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): start = loc instrlen = len(instring) maxloc = start + len(self.match_string) @@ -2587,8 +3068,8 @@ class CloseMatch(Token): mismatches = [] maxMismatches = self.maxMismatches - for match_stringloc,s_m in enumerate(zip(instring[loc:maxloc], self.match_string)): - src,mat = s_m + for match_stringloc, s_m in enumerate(zip(instring[loc:maxloc], match_string)): + src, mat = s_m if src != mat: mismatches.append(match_stringloc) if len(mismatches) > maxMismatches: @@ -2596,7 +3077,7 @@ class CloseMatch(Token): else: loc = match_stringloc + 1 results = ParseResults([instring[start:loc]]) - results['original'] = self.match_string + results['original'] = match_string results['mismatches'] = mismatches return loc, results @@ -2604,61 +3085,68 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' - hostname = Word(alphas, alphanums+'-') - + hostname = Word(alphas, alphanums + '-') + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ - def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ): - super(Word,self).__init__() + def __init__(self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None): + super(Word, self).__init__() if excludeChars: + excludeChars = set(excludeChars) initChars = ''.join(c for c in initChars if c not in excludeChars) if bodyChars: bodyChars = ''.join(c for c in bodyChars if c not in excludeChars) self.initCharsOrig = initChars self.initChars = set(initChars) - if bodyChars : + if bodyChars: self.bodyCharsOrig = bodyChars self.bodyChars = set(bodyChars) else: @@ -2686,34 +3174,28 @@ class Word(Token): self.mayIndexError = False self.asKeyword = asKeyword - if ' ' not in self.initCharsOrig+self.bodyCharsOrig and (min==1 and max==0 and exact==0): + if ' ' not in self.initCharsOrig + self.bodyCharsOrig and (min == 1 and max == 0 and exact == 0): if self.bodyCharsOrig == self.initCharsOrig: self.reString = "[%s]+" % _escapeRegexRangeChars(self.initCharsOrig) elif len(self.initCharsOrig) == 1: - self.reString = "%s[%s]*" % \ - (re.escape(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "%s[%s]*" % (re.escape(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) else: - self.reString = "[%s][%s]*" % \ - (_escapeRegexRangeChars(self.initCharsOrig), - _escapeRegexRangeChars(self.bodyCharsOrig),) + self.reString = "[%s][%s]*" % (_escapeRegexRangeChars(self.initCharsOrig), + _escapeRegexRangeChars(self.bodyCharsOrig),) if self.asKeyword: - self.reString = r"\b"+self.reString+r"\b" + self.reString = r"\b" + self.reString + r"\b" + try: - self.re = re.compile( self.reString ) + self.re = re.compile(self.reString) except Exception: self.re = None + else: + self.re_match = self.re.match + self.__class__ = _WordRegex - def parseImpl( self, instring, loc, doActions=True ): - if self.re: - result = self.re.match(instring,loc) - if not result: - raise ParseException(instring, loc, self.errmsg, self) - - loc = result.end() - return loc, result.group() - - if not(instring[ loc ] in self.initChars): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] not in self.initChars: raise ParseException(instring, loc, self.errmsg, self) start = loc @@ -2721,17 +3203,18 @@ class Word(Token): instrlen = len(instring) bodychars = self.bodyChars maxloc = start + self.maxLen - maxloc = min( maxloc, instrlen ) + maxloc = min(maxloc, instrlen) while loc < maxloc and instring[loc] in bodychars: loc += 1 throwException = False if loc - start < self.minLen: throwException = True - if self.maxSpecified and loc < instrlen and instring[loc] in bodychars: + elif self.maxSpecified and loc < instrlen and instring[loc] in bodychars: throwException = True - if self.asKeyword: - if (start>0 and instring[start-1] in bodychars) or (loc<instrlen and instring[loc] in bodychars): + elif self.asKeyword: + if (start > 0 and instring[start - 1] in bodychars + or loc < instrlen and instring[loc] in bodychars): throwException = True if throwException: @@ -2739,51 +3222,87 @@ class Word(Token): return loc, instring[start:loc] - def __str__( self ): + def __str__(self): try: - return super(Word,self).__str__() + return super(Word, self).__str__() except Exception: pass - if self.strRepr is None: def charsAsStr(s): - if len(s)>4: - return s[:4]+"..." + if len(s) > 4: + return s[:4] + "..." else: return s - if ( self.initCharsOrig != self.bodyCharsOrig ): - self.strRepr = "W:(%s,%s)" % ( charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig) ) + if self.initCharsOrig != self.bodyCharsOrig: + self.strRepr = "W:(%s, %s)" % (charsAsStr(self.initCharsOrig), charsAsStr(self.bodyCharsOrig)) else: self.strRepr = "W:(%s)" % charsAsStr(self.initCharsOrig) return self.strRepr +class _WordRegex(Word): + def parseImpl(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + return loc, result.group() + + +class Char(_WordRegex): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset, asKeyword=False, excludeChars=None): + super(Char, self).__init__(charset, exact=1, asKeyword=asKeyword, excludeChars=excludeChars) + self.reString = "[%s]" % _escapeRegexRangeChars(''.join(self.initChars)) + if asKeyword: + self.reString = r"\b%s\b" % self.reString + self.re = re.compile(self.reString) + self.re_match = self.re.match + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P<name>...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module <https://docs.python.org/3/library/re.html>`_. + If the given regex contains named groups (defined using ``(?P<name>...)``), + these will be preserved as named parse results. + + If instead of the Python stdlib re module you wish to use a different RE module + (such as the `regex` module), you can replace it by either building your + Regex object with a compiled RE that was compiled using regex: Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P<year>\d{4})-(?P<month>\d\d?)-(?P<day>\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + + # use regex module instead of stdlib re module to construct a Regex using + # a compiled regular expression + import regex + parser = pp.Regex(regex.compile(r'[0-9]')) + """ - compiledREtype = type(re.compile("[A-Z]")) - def __init__( self, pattern, flags=0, asGroupList=False, asMatch=False): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" - super(Regex,self).__init__() + def __init__(self, pattern, flags=0, asGroupList=False, asMatch=False): + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module <https://docs.python.org/3/library/re.html>`_ module for an + explanation of the acceptable patterns and flags. + """ + super(Regex, self).__init__() if isinstance(pattern, basestring): if not pattern: warnings.warn("null string passed to Regex; use Empty() instead", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) self.pattern = pattern self.flags = flags @@ -2793,46 +3312,64 @@ class Regex(Token): self.reString = self.pattern except sre_constants.error: warnings.warn("invalid pattern (%s) passed to Regex" % pattern, - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) raise - elif isinstance(pattern, Regex.compiledREtype): + elif hasattr(pattern, 'pattern') and hasattr(pattern, 'match'): self.re = pattern - self.pattern = \ - self.reString = str(pattern) + self.pattern = self.reString = pattern.pattern self.flags = flags - + else: - raise ValueError("Regex may only be constructed with a string or a compiled RE object") + raise TypeError("Regex may only be constructed with a string or a compiled RE object") + + self.re_match = self.re.match self.name = _ustr(self) self.errmsg = "Expected " + self.name self.mayIndexError = False - self.mayReturnEmpty = True + self.mayReturnEmpty = self.re_match("") is not None self.asGroupList = asGroupList self.asMatch = asMatch + if self.asGroupList: + self.parseImpl = self.parseImplAsGroupList + if self.asMatch: + self.parseImpl = self.parseImplAsMatch - def parseImpl( self, instring, loc, doActions=True ): - result = self.re.match(instring,loc) + def parseImpl(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) if not result: raise ParseException(instring, loc, self.errmsg, self) loc = result.end() + ret = ParseResults(result.group()) d = result.groupdict() - if self.asMatch: - ret = result - elif self.asGroupList: - ret = result.groups() - else: - ret = ParseResults(result.group()) - if d: - for k in d: - ret[k] = d[k] - return loc,ret + if d: + for k, v in d.items(): + ret[k] = v + return loc, ret - def __str__( self ): + def parseImplAsGroupList(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result.groups() + return loc, ret + + def parseImplAsMatch(self, instring, loc, doActions=True): + result = self.re_match(instring, loc) + if not result: + raise ParseException(instring, loc, self.errmsg, self) + + loc = result.end() + ret = result + return loc, ret + + def __str__(self): try: - return super(Regex,self).__str__() + return super(Regex, self).__str__() except Exception: pass @@ -2842,19 +3379,25 @@ class Regex(Token): return self.strRepr def sub(self, repl): - """ + r""" Return Regex with an attached parse action to transform the parsed - result as if called using C{re.sub(expr, repl, string)}. + result as if called using `re.sub(expr, repl, string) <https://docs.python.org/3/library/re.html#re.sub>`_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2</\1>") + print(make_html.transformString("h1:main title:")) + # prints "<h1>main title</h1>" """ if self.asGroupList: - warnings.warn("cannot use sub() with Regex(asGroupList=True)", - SyntaxWarning, stacklevel=2) + warnings.warn("cannot use sub() with Regex(asGroupList=True)", + SyntaxWarning, stacklevel=2) raise SyntaxError() if self.asMatch and callable(repl): - warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", - SyntaxWarning, stacklevel=2) - raise SyntaxError() + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + SyntaxWarning, stacklevel=2) + raise SyntaxError() if self.asMatch: def pa(tokens): @@ -2867,35 +3410,50 @@ class Regex(Token): class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None``) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None``) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False``) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True``) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True``) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] """ - def __init__( self, quoteChar, escChar=None, escQuote=None, multiline=False, unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): - super(QuotedString,self).__init__() + def __init__(self, quoteChar, escChar=None, escQuote=None, multiline=False, + unquoteResults=True, endQuoteChar=None, convertWhitespaceEscapes=True): + super(QuotedString, self).__init__() # remove white space from quote chars - wont work anyway quoteChar = quoteChar.strip() if not quoteChar: - warnings.warn("quoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + warnings.warn("quoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) raise SyntaxError() if endQuoteChar is None: @@ -2903,7 +3461,7 @@ class QuotedString(Token): else: endQuoteChar = endQuoteChar.strip() if not endQuoteChar: - warnings.warn("endQuoteChar cannot be the empty string",SyntaxWarning,stacklevel=2) + warnings.warn("endQuoteChar cannot be the empty string", SyntaxWarning, stacklevel=2) raise SyntaxError() self.quoteChar = quoteChar @@ -2918,35 +3476,34 @@ class QuotedString(Token): if multiline: self.flags = re.MULTILINE | re.DOTALL - self.pattern = r'%s(?:[^%s%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + self.pattern = r'%s(?:[^%s%s]' % (re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '')) else: self.flags = 0 - self.pattern = r'%s(?:[^%s\n\r%s]' % \ - ( re.escape(self.quoteChar), - _escapeRegexRangeChars(self.endQuoteChar[0]), - (escChar is not None and _escapeRegexRangeChars(escChar) or '') ) + self.pattern = r'%s(?:[^%s\n\r%s]' % (re.escape(self.quoteChar), + _escapeRegexRangeChars(self.endQuoteChar[0]), + (escChar is not None and _escapeRegexRangeChars(escChar) or '')) if len(self.endQuoteChar) > 1: self.pattern += ( '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]), - _escapeRegexRangeChars(self.endQuoteChar[i])) - for i in range(len(self.endQuoteChar)-1,0,-1)) + ')' - ) + _escapeRegexRangeChars(self.endQuoteChar[i])) + for i in range(len(self.endQuoteChar) - 1, 0, -1)) + ')') + if escQuote: self.pattern += (r'|(?:%s)' % re.escape(escQuote)) if escChar: self.pattern += (r'|(?:%s.)' % re.escape(escChar)) - self.escCharReplacePattern = re.escape(self.escChar)+"(.)" + self.escCharReplacePattern = re.escape(self.escChar) + "(.)" self.pattern += (r')*%s' % re.escape(self.endQuoteChar)) try: self.re = re.compile(self.pattern, self.flags) self.reString = self.pattern + self.re_match = self.re.match except sre_constants.error: warnings.warn("invalid pattern (%s) passed to Regex" % self.pattern, - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) raise self.name = _ustr(self) @@ -2954,8 +3511,8 @@ class QuotedString(Token): self.mayIndexError = False self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): - result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None + def parseImpl(self, instring, loc, doActions=True): + result = instring[loc] == self.firstQuoteChar and self.re_match(instring, loc) or None if not result: raise ParseException(instring, loc, self.errmsg, self) @@ -2965,18 +3522,18 @@ class QuotedString(Token): if self.unquoteResults: # strip off quotes - ret = ret[self.quoteCharLen:-self.endQuoteCharLen] + ret = ret[self.quoteCharLen: -self.endQuoteCharLen] - if isinstance(ret,basestring): + if isinstance(ret, basestring): # replace escaped whitespace if '\\' in ret and self.convertWhitespaceEscapes: ws_map = { - r'\t' : '\t', - r'\n' : '\n', - r'\f' : '\f', - r'\r' : '\r', + r'\t': '\t', + r'\n': '\n', + r'\f': '\f', + r'\r': '\r', } - for wslit,wschar in ws_map.items(): + for wslit, wschar in ws_map.items(): ret = ret.replace(wslit, wschar) # replace escaped characters @@ -2989,9 +3546,9 @@ class QuotedString(Token): return loc, ret - def __str__( self ): + def __str__(self): try: - return super(QuotedString,self).__str__() + return super(QuotedString, self).__str__() except Exception: pass @@ -3002,28 +3559,33 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ - def __init__( self, notChars, min=1, max=0, exact=0 ): - super(CharsNotIn,self).__init__() + def __init__(self, notChars, min=1, max=0, exact=0): + super(CharsNotIn, self).__init__() self.skipWhitespace = False self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError("cannot specify a minimum length < 1; use " + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3038,19 +3600,18 @@ class CharsNotIn(Token): self.name = _ustr(self) self.errmsg = "Expected " + self.name - self.mayReturnEmpty = ( self.minLen == 0 ) + self.mayReturnEmpty = (self.minLen == 0) self.mayIndexError = False - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if instring[loc] in self.notChars: raise ParseException(instring, loc, self.errmsg, self) start = loc loc += 1 notchars = self.notChars - maxlen = min( start+self.maxLen, len(instring) ) - while loc < maxlen and \ - (instring[loc] not in notchars): + maxlen = min(start + self.maxLen, len(instring)) + while loc < maxlen and instring[loc] not in notchars: loc += 1 if loc - start < self.minLen: @@ -3058,7 +3619,7 @@ class CharsNotIn(Token): return loc, instring[start:loc] - def __str__( self ): + def __str__(self): try: return super(CharsNotIn, self).__str__() except Exception: @@ -3073,25 +3634,44 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "<SPC>", - "\t": "<TAB>", - "\n": "<LF>", - "\r": "<CR>", - "\f": "<FF>", + ' ' : '<SP>', + '\t': '<TAB>', + '\n': '<LF>', + '\r': '<CR>', + '\f': '<FF>', + u'\u00A0': '<NBSP>', + u'\u1680': '<OGHAM_SPACE_MARK>', + u'\u180E': '<MONGOLIAN_VOWEL_SEPARATOR>', + u'\u2000': '<EN_QUAD>', + u'\u2001': '<EM_QUAD>', + u'\u2002': '<EN_SPACE>', + u'\u2003': '<EM_SPACE>', + u'\u2004': '<THREE-PER-EM_SPACE>', + u'\u2005': '<FOUR-PER-EM_SPACE>', + u'\u2006': '<SIX-PER-EM_SPACE>', + u'\u2007': '<FIGURE_SPACE>', + u'\u2008': '<PUNCTUATION_SPACE>', + u'\u2009': '<THIN_SPACE>', + u'\u200A': '<HAIR_SPACE>', + u'\u200B': '<ZERO_WIDTH_SPACE>', + u'\u202F': '<NNBSP>', + u'\u205F': '<MMSP>', + u'\u3000': '<IDEOGRAPHIC_SPACE>', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): - super(White,self).__init__() + super(White, self).__init__() self.matchWhite = ws - self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) ) - #~ self.leaveWhitespace() + self.setWhitespaceChars("".join(c for c in self.whiteChars if c not in self.matchWhite)) + # ~ self.leaveWhitespace() self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite)) self.mayReturnEmpty = True self.errmsg = "Expected " + self.name @@ -3107,13 +3687,13 @@ class White(Token): self.maxLen = exact self.minLen = exact - def parseImpl( self, instring, loc, doActions=True ): - if not(instring[ loc ] in self.matchWhite): + def parseImpl(self, instring, loc, doActions=True): + if instring[loc] not in self.matchWhite: raise ParseException(instring, loc, self.errmsg, self) start = loc loc += 1 maxloc = start + self.maxLen - maxloc = min( maxloc, len(instring) ) + maxloc = min(maxloc, len(instring)) while loc < maxloc and instring[loc] in self.matchWhite: loc += 1 @@ -3124,44 +3704,44 @@ class White(Token): class _PositionToken(Token): - def __init__( self ): - super(_PositionToken,self).__init__() - self.name=self.__class__.__name__ + def __init__(self): + super(_PositionToken, self).__init__() + self.name = self.__class__.__name__ self.mayReturnEmpty = True self.mayIndexError = False class GoToColumn(_PositionToken): + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ - Token to advance to a specific column of input text; useful for tabular report scraping. - """ - def __init__( self, colno ): - super(GoToColumn,self).__init__() + def __init__(self, colno): + super(GoToColumn, self).__init__() self.col = colno - def preParse( self, instring, loc ): - if col(loc,instring) != self.col: + def preParse(self, instring, loc): + if col(loc, instring) != self.col: instrlen = len(instring) if self.ignoreExprs: - loc = self._skipIgnorables( instring, loc ) - while loc < instrlen and instring[loc].isspace() and col( loc, instring ) != self.col : + loc = self._skipIgnorables(instring, loc) + while loc < instrlen and instring[loc].isspace() and col(loc, instring) != self.col: loc += 1 return loc - def parseImpl( self, instring, loc, doActions=True ): - thiscol = col( loc, instring ) + def parseImpl(self, instring, loc, doActions=True): + thiscol = col(loc, instring) if thiscol > self.col: - raise ParseException( instring, loc, "Text not in expected column", self ) + raise ParseException(instring, loc, "Text not in expected column", self) newloc = loc + self.col - thiscol - ret = instring[ loc: newloc ] + ret = instring[loc: newloc] return newloc, ret class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + r"""Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3171,262 +3751,305 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ - def __init__( self ): - super(LineStart,self).__init__() + def __init__(self): + super(LineStart, self).__init__() self.errmsg = "Expected start of line" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if col(loc, instring) == 1: return loc, [] raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): + """Matches if current position is at the end of a line within the + parse string """ - Matches if current position is at the end of a line within the parse string - """ - def __init__( self ): - super(LineEnd,self).__init__() - self.setWhitespaceChars( ParserElement.DEFAULT_WHITE_CHARS.replace("\n","") ) + def __init__(self): + super(LineEnd, self).__init__() + self.setWhitespaceChars(ParserElement.DEFAULT_WHITE_CHARS.replace("\n", "")) self.errmsg = "Expected end of line" - def parseImpl( self, instring, loc, doActions=True ): - if loc<len(instring): + def parseImpl(self, instring, loc, doActions=True): + if loc < len(instring): if instring[loc] == "\n": - return loc+1, "\n" + return loc + 1, "\n" else: raise ParseException(instring, loc, self.errmsg, self) elif loc == len(instring): - return loc+1, [] + return loc + 1, [] else: raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): + """Matches if current position is at the beginning of the parse + string """ - Matches if current position is at the beginning of the parse string - """ - def __init__( self ): - super(StringStart,self).__init__() + def __init__(self): + super(StringStart, self).__init__() self.errmsg = "Expected start of text" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc != 0: # see if entire string up to here is just whitespace and ignoreables - if loc != self.preParse( instring, 0 ): + if loc != self.preParse(instring, 0): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class StringEnd(_PositionToken): + """Matches if current position is at the end of the parse string """ - Matches if current position is at the end of the parse string - """ - def __init__( self ): - super(StringEnd,self).__init__() + def __init__(self): + super(StringEnd, self).__init__() self.errmsg = "Expected end of text" - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc < len(instring): raise ParseException(instring, loc, self.errmsg, self) elif loc == len(instring): - return loc+1, [] + return loc + 1, [] elif loc > len(instring): return loc, [] else: raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. - """ - def __init__(self, wordChars = printables): - super(WordStart,self).__init__() + def __init__(self, wordChars=printables): + super(WordStart, self).__init__() self.wordChars = set(wordChars) self.errmsg = "Not at the start of a word" - def parseImpl(self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if loc != 0: - if (instring[loc-1] in self.wordChars or - instring[loc] not in self.wordChars): + if (instring[loc - 1] in self.wordChars + or instring[loc] not in self.wordChars): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class WordEnd(_PositionToken): + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. - """ - def __init__(self, wordChars = printables): - super(WordEnd,self).__init__() + def __init__(self, wordChars=printables): + super(WordEnd, self).__init__() self.wordChars = set(wordChars) self.skipWhitespace = False self.errmsg = "Not at the end of a word" - def parseImpl(self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): instrlen = len(instring) - if instrlen>0 and loc<instrlen: + if instrlen > 0 and loc < instrlen: if (instring[loc] in self.wordChars or - instring[loc-1] not in self.wordChars): + instring[loc - 1] not in self.wordChars): raise ParseException(instring, loc, self.errmsg, self) return loc, [] class ParseExpression(ParserElement): + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. - """ - def __init__( self, exprs, savelist = False ): - super(ParseExpression,self).__init__(savelist) - if isinstance( exprs, _generatorType ): + def __init__(self, exprs, savelist=False): + super(ParseExpression, self).__init__(savelist) + if isinstance(exprs, _generatorType): exprs = list(exprs) - if isinstance( exprs, basestring ): - self.exprs = [ ParserElement._literalStringClass( exprs ) ] - elif isinstance( exprs, Iterable ): + if isinstance(exprs, basestring): + self.exprs = [self._literalStringClass(exprs)] + elif isinstance(exprs, ParserElement): + self.exprs = [exprs] + elif isinstance(exprs, Iterable): exprs = list(exprs) # if sequence of strings provided, wrap with Literal - if all(isinstance(expr, basestring) for expr in exprs): - exprs = map(ParserElement._literalStringClass, exprs) + if any(isinstance(expr, basestring) for expr in exprs): + exprs = (self._literalStringClass(e) if isinstance(e, basestring) else e for e in exprs) self.exprs = list(exprs) else: try: - self.exprs = list( exprs ) + self.exprs = list(exprs) except TypeError: - self.exprs = [ exprs ] + self.exprs = [exprs] self.callPreparse = False - def __getitem__( self, i ): - return self.exprs[i] - - def append( self, other ): - self.exprs.append( other ) + def append(self, other): + self.exprs.append(other) self.strRepr = None return self - def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + def leaveWhitespace(self): + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False - self.exprs = [ e.copy() for e in self.exprs ] + self.exprs = [e.copy() for e in self.exprs] for e in self.exprs: e.leaveWhitespace() return self - def ignore( self, other ): - if isinstance( other, Suppress ): + def ignore(self, other): + if isinstance(other, Suppress): if other not in self.ignoreExprs: - super( ParseExpression, self).ignore( other ) + super(ParseExpression, self).ignore(other) for e in self.exprs: - e.ignore( self.ignoreExprs[-1] ) + e.ignore(self.ignoreExprs[-1]) else: - super( ParseExpression, self).ignore( other ) + super(ParseExpression, self).ignore(other) for e in self.exprs: - e.ignore( self.ignoreExprs[-1] ) + e.ignore(self.ignoreExprs[-1]) return self - def __str__( self ): + def __str__(self): try: - return super(ParseExpression,self).__str__() + return super(ParseExpression, self).__str__() except Exception: pass if self.strRepr is None: - self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.exprs) ) + self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.exprs)) return self.strRepr - def streamline( self ): - super(ParseExpression,self).streamline() + def streamline(self): + super(ParseExpression, self).streamline() for e in self.exprs: e.streamline() - # collapse nested And's of the form And( And( And( a,b), c), d) to And( a,b,c,d ) + # collapse nested And's of the form And(And(And(a, b), c), d) to And(a, b, c, d) # but only if there are no parse actions or resultsNames on the nested And's # (likewise for Or's and MatchFirst's) - if ( len(self.exprs) == 2 ): + if len(self.exprs) == 2: other = self.exprs[0] - if ( isinstance( other, self.__class__ ) and - not(other.parseAction) and - other.resultsName is None and - not other.debug ): - self.exprs = other.exprs[:] + [ self.exprs[1] ] + if (isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug): + self.exprs = other.exprs[:] + [self.exprs[1]] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError other = self.exprs[-1] - if ( isinstance( other, self.__class__ ) and - not(other.parseAction) and - other.resultsName is None and - not other.debug ): + if (isinstance(other, self.__class__) + and not other.parseAction + and other.resultsName is None + and not other.debug): self.exprs = self.exprs[:-1] + other.exprs[:] self.strRepr = None self.mayReturnEmpty |= other.mayReturnEmpty self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self - def setResultsName( self, name, listAllMatches=False ): - ret = super(ParseExpression,self).setResultsName(name,listAllMatches) - return ret - - def validate( self, validateTrace=[] ): - tmp = validateTrace[:]+[self] + def validate(self, validateTrace=None): + tmp = (validateTrace if validateTrace is not None else [])[:] + [self] for e in self.exprs: e.validate(tmp) - self.checkRecursion( [] ) - + self.checkRecursion([]) + def copy(self): - ret = super(ParseExpression,self).copy() + ret = super(ParseExpression, self).copy() ret.exprs = [e.copy() for e in self.exprs] return ret + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_ungrouped_named_tokens_in_collection: + for e in self.exprs: + if isinstance(e, ParserElement) and e.resultsName: + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName), + stacklevel=3) + + return super(ParseExpression, self)._setResultsName(name, listAllMatches) + + class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) - expr = And([integer("id"),name_expr("name"),integer("age")]) + expr = And([integer("id"), name_expr("name"), integer("age")]) # more easily written as: expr = integer("id") + name_expr("name") + integer("age") """ class _ErrorStop(Empty): def __init__(self, *args, **kwargs): - super(And._ErrorStop,self).__init__(*args, **kwargs) + super(And._ErrorStop, self).__init__(*args, **kwargs) self.name = '-' self.leaveWhitespace() - def __init__( self, exprs, savelist = True ): - super(And,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=True): + exprs = list(exprs) + if exprs and Ellipsis in exprs: + tmp = [] + for i, expr in enumerate(exprs): + if expr is Ellipsis: + if i < len(exprs) - 1: + skipto_arg = (Empty() + exprs[i + 1]).exprs[-1] + tmp.append(SkipTo(skipto_arg)("_skipped*")) + else: + raise Exception("cannot construct And with sequence ending in ...") + else: + tmp.append(expr) + exprs[:] = tmp + super(And, self).__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) - self.setWhitespaceChars( self.exprs[0].whiteChars ) + self.setWhitespaceChars(self.exprs[0].whiteChars) self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + # collapse any _PendingSkip's + if self.exprs: + if any(isinstance(e, ParseExpression) and e.exprs and isinstance(e.exprs[-1], _PendingSkip) + for e in self.exprs[:-1]): + for i, e in enumerate(self.exprs[:-1]): + if e is None: + continue + if (isinstance(e, ParseExpression) + and e.exprs and isinstance(e.exprs[-1], _PendingSkip)): + e.exprs[-1] = e.exprs[-1] + self.exprs[i + 1] + self.exprs[i + 1] = None + self.exprs = [e for e in self.exprs if e is not None] + + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing - loc, resultlist = self.exprs[0]._parse( instring, loc, doActions, callPreParse=False ) + loc, resultlist = self.exprs[0]._parse(instring, loc, doActions, callPreParse=False) errorStop = False for e in self.exprs[1:]: if isinstance(e, And._ErrorStop): @@ -3434,7 +4057,7 @@ class And(ParseExpression): continue if errorStop: try: - loc, exprtokens = e._parse( instring, loc, doActions ) + loc, exprtokens = e._parse(instring, loc, doActions) except ParseSyntaxException: raise except ParseBaseException as pe: @@ -3443,25 +4066,25 @@ class And(ParseExpression): except IndexError: raise ParseSyntaxException(instring, len(instring), self.errmsg, self) else: - loc, exprtokens = e._parse( instring, loc, doActions ) + loc, exprtokens = e._parse(instring, loc, doActions) if exprtokens or exprtokens.haskeys(): resultlist += exprtokens return loc, resultlist - def __iadd__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #And( [ self, other ] ) + def __iadd__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # And([self, other]) - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) if not e.mayReturnEmpty: break - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3471,33 +4094,42 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ - def __init__( self, exprs, savelist = False ): - super(Or,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=False): + super(Or, self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) else: self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(Or, self).streamline() + if __compat__.collect_all_And_tokens: + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): maxExcLoc = -1 maxException = None matches = [] for e in self.exprs: try: - loc2 = e.tryParse( instring, loc ) + loc2 = e.tryParse(instring, loc) except ParseException as err: err.__traceback__ = None if err.loc > maxExcLoc: @@ -3505,22 +4137,45 @@ class Or(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) + maxException = ParseException(instring, len(instring), e.errmsg, self) maxExcLoc = len(instring) else: # save match among all matches, to retry longest to shortest matches.append((loc2, e)) if matches: - matches.sort(key=lambda x: -x[0]) - for _,e in matches: + # re-evaluate all matches in descending order of length of match, in case attached actions + # might change whether or how much they match of the input. + matches.sort(key=itemgetter(0), reverse=True) + + if not doActions: + # no further conditions or parse actions to change the selection of + # alternative, so the first match will be the best match + best_expr = matches[0][1] + return best_expr._parse(instring, loc, doActions) + + longest = -1, None + for loc1, expr1 in matches: + if loc1 <= longest[0]: + # already have a longer match than this one will deliver, we are done + return longest + try: - return e._parse( instring, loc, doActions ) + loc2, toks = expr1._parse(instring, loc, doActions) except ParseException as err: err.__traceback__ = None if err.loc > maxExcLoc: maxException = err maxExcLoc = err.loc + else: + if loc2 >= loc1: + return loc2, toks + # didn't match as much as before + elif loc2 > longest[0]: + longest = loc2, toks + + if longest != (-1, None): + return longest if maxException is not None: maxException.msg = self.errmsg @@ -3529,13 +4184,13 @@ class Or(ParseExpression): raise ParseException(instring, loc, "no defined alternatives to match", self) - def __ixor__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #Or( [ self, other ] ) + def __ixor__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # Or([self, other]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3543,21 +4198,33 @@ class Or(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) + + def _setResultsName(self, name, listAllMatches=False): + if (not __compat__.collect_all_And_tokens + and __diag__.warn_multiple_tokens_in_named_alternation): + if any(isinstance(e, And) for e in self.exprs): + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "may only return a single token for an And alternative, " + "in future will return the full list of tokens".format( + "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), + stacklevel=3) + + return super(Or, self)._setResultsName(name, listAllMatches) class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3566,19 +4233,25 @@ class MatchFirst(ParseExpression): number = Combine(Word(nums) + '.' + Word(nums)) | Word(nums) print(number.searchString("123 3.1416 789")) # Better -> [['123'], ['3.1416'], ['789']] """ - def __init__( self, exprs, savelist = False ): - super(MatchFirst,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=False): + super(MatchFirst, self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) else: self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(MatchFirst, self).streamline() + if __compat__.collect_all_And_tokens: + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): maxExcLoc = -1 maxException = None for e in self.exprs: try: - ret = e._parse( instring, loc, doActions ) + ret = e._parse(instring, loc, doActions) return ret except ParseException as err: if err.loc > maxExcLoc: @@ -3586,7 +4259,7 @@ class MatchFirst(ParseExpression): maxExcLoc = err.loc except IndexError: if len(instring) > maxExcLoc: - maxException = ParseException(instring,len(instring),e.errmsg,self) + maxException = ParseException(instring, len(instring), e.errmsg, self) maxExcLoc = len(instring) # only got here if no expression matched, raise exception for match that made it the furthest @@ -3597,13 +4270,13 @@ class MatchFirst(ParseExpression): else: raise ParseException(instring, loc, "no defined alternatives to match", self) - def __ior__(self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass( other ) - return self.append( other ) #MatchFirst( [ self, other ] ) + def __ior__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) + return self.append(other) # MatchFirst([self, other]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3611,19 +4284,32 @@ class MatchFirst(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) + + def _setResultsName(self, name, listAllMatches=False): + if (not __compat__.collect_all_And_tokens + and __diag__.warn_multiple_tokens_in_named_alternation): + if any(isinstance(e, And) for e in self.exprs): + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "may only return a single token for an And alternative, " + "in future will return the full list of tokens".format( + "warn_multiple_tokens_in_named_alternation", name, type(self).__name__), + stacklevel=3) + + return super(MatchFirst, self)._setResultsName(name, listAllMatches) class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3632,7 +4318,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3642,7 +4328,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3671,21 +4359,27 @@ class Each(ParseExpression): - shape: TRIANGLE - size: 20 """ - def __init__( self, exprs, savelist = True ): - super(Each,self).__init__(exprs, savelist) + def __init__(self, exprs, savelist=True): + super(Each, self).__init__(exprs, savelist) self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True - def parseImpl( self, instring, loc, doActions=True ): + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + + def parseImpl(self, instring, loc, doActions=True): if self.initExprGroups: - self.opt1map = dict((id(e.expr),e) for e in self.exprs if isinstance(e,Optional)) - opt1 = [ e.expr for e in self.exprs if isinstance(e,Optional) ] - opt2 = [ e for e in self.exprs if e.mayReturnEmpty and not isinstance(e,Optional)] + self.opt1map = dict((id(e.expr), e) for e in self.exprs if isinstance(e, Optional)) + opt1 = [e.expr for e in self.exprs if isinstance(e, Optional)] + opt2 = [e for e in self.exprs if e.mayReturnEmpty and not isinstance(e, (Optional, Regex))] self.optionals = opt1 + opt2 - self.multioptionals = [ e.expr for e in self.exprs if isinstance(e,ZeroOrMore) ] - self.multirequired = [ e.expr for e in self.exprs if isinstance(e,OneOrMore) ] - self.required = [ e for e in self.exprs if not isinstance(e,(Optional,ZeroOrMore,OneOrMore)) ] + self.multioptionals = [e.expr for e in self.exprs if isinstance(e, ZeroOrMore)] + self.multirequired = [e.expr for e in self.exprs if isinstance(e, OneOrMore)] + self.required = [e for e in self.exprs if not isinstance(e, (Optional, ZeroOrMore, OneOrMore))] self.required += self.multirequired self.initExprGroups = False tmpLoc = loc @@ -3699,11 +4393,11 @@ class Each(ParseExpression): failed = [] for e in tmpExprs: try: - tmpLoc = e.tryParse( instring, tmpLoc ) + tmpLoc = e.tryParse(instring, tmpLoc) except ParseException: failed.append(e) else: - matchOrder.append(self.opt1map.get(id(e),e)) + matchOrder.append(self.opt1map.get(id(e), e)) if e in tmpReqd: tmpReqd.remove(e) elif e in tmpOpt: @@ -3713,21 +4407,21 @@ class Each(ParseExpression): if tmpReqd: missing = ", ".join(_ustr(e) for e in tmpReqd) - raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing ) + raise ParseException(instring, loc, "Missing one or more required elements (%s)" % missing) # add any unmatched Optionals, in case they have default values defined - matchOrder += [e for e in self.exprs if isinstance(e,Optional) and e.expr in tmpOpt] + matchOrder += [e for e in self.exprs if isinstance(e, Optional) and e.expr in tmpOpt] resultlist = [] for e in matchOrder: - loc,results = e._parse(instring,loc,doActions) + loc, results = e._parse(instring, loc, doActions) resultlist.append(results) finalResults = sum(resultlist, ParseResults([])) return loc, finalResults - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3735,140 +4429,238 @@ class Each(ParseExpression): return self.strRepr - def checkRecursion( self, parseElementList ): - subRecCheckList = parseElementList[:] + [ self ] + def checkRecursion(self, parseElementList): + subRecCheckList = parseElementList[:] + [self] for e in self.exprs: - e.checkRecursion( subRecCheckList ) + e.checkRecursion(subRecCheckList) class ParseElementEnhance(ParserElement): + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. - """ - def __init__( self, expr, savelist=False ): - super(ParseElementEnhance,self).__init__(savelist) - if isinstance( expr, basestring ): - if issubclass(ParserElement._literalStringClass, Token): - expr = ParserElement._literalStringClass(expr) + def __init__(self, expr, savelist=False): + super(ParseElementEnhance, self).__init__(savelist) + if isinstance(expr, basestring): + if issubclass(self._literalStringClass, Token): + expr = self._literalStringClass(expr) else: - expr = ParserElement._literalStringClass(Literal(expr)) + expr = self._literalStringClass(Literal(expr)) self.expr = expr self.strRepr = None if expr is not None: self.mayIndexError = expr.mayIndexError self.mayReturnEmpty = expr.mayReturnEmpty - self.setWhitespaceChars( expr.whiteChars ) + self.setWhitespaceChars(expr.whiteChars) self.skipWhitespace = expr.skipWhitespace self.saveAsList = expr.saveAsList self.callPreparse = expr.callPreparse self.ignoreExprs.extend(expr.ignoreExprs) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.expr is not None: - return self.expr._parse( instring, loc, doActions, callPreParse=False ) + return self.expr._parse(instring, loc, doActions, callPreParse=False) else: - raise ParseException("",loc,self.errmsg,self) + raise ParseException("", loc, self.errmsg, self) - def leaveWhitespace( self ): + def leaveWhitespace(self): self.skipWhitespace = False self.expr = self.expr.copy() if self.expr is not None: self.expr.leaveWhitespace() return self - def ignore( self, other ): - if isinstance( other, Suppress ): + def ignore(self, other): + if isinstance(other, Suppress): if other not in self.ignoreExprs: - super( ParseElementEnhance, self).ignore( other ) + super(ParseElementEnhance, self).ignore(other) if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) + self.expr.ignore(self.ignoreExprs[-1]) else: - super( ParseElementEnhance, self).ignore( other ) + super(ParseElementEnhance, self).ignore(other) if self.expr is not None: - self.expr.ignore( self.ignoreExprs[-1] ) + self.expr.ignore(self.ignoreExprs[-1]) return self - def streamline( self ): - super(ParseElementEnhance,self).streamline() + def streamline(self): + super(ParseElementEnhance, self).streamline() if self.expr is not None: self.expr.streamline() return self - def checkRecursion( self, parseElementList ): + def checkRecursion(self, parseElementList): if self in parseElementList: - raise RecursiveGrammarException( parseElementList+[self] ) - subRecCheckList = parseElementList[:] + [ self ] + raise RecursiveGrammarException(parseElementList + [self]) + subRecCheckList = parseElementList[:] + [self] if self.expr is not None: - self.expr.checkRecursion( subRecCheckList ) + self.expr.checkRecursion(subRecCheckList) - def validate( self, validateTrace=[] ): - tmp = validateTrace[:]+[self] + def validate(self, validateTrace=None): + if validateTrace is None: + validateTrace = [] + tmp = validateTrace[:] + [self] if self.expr is not None: self.expr.validate(tmp) - self.checkRecursion( [] ) + self.checkRecursion([]) - def __str__( self ): + def __str__(self): try: - return super(ParseElementEnhance,self).__str__() + return super(ParseElementEnhance, self).__str__() except Exception: pass if self.strRepr is None and self.expr is not None: - self.strRepr = "%s:(%s)" % ( self.__class__.__name__, _ustr(self.expr) ) + self.strRepr = "%s:(%s)" % (self.__class__.__name__, _ustr(self.expr)) return self.strRepr class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ - def __init__( self, expr ): - super(FollowedBy,self).__init__(expr) + def __init__(self, expr): + super(FollowedBy, self).__init__(expr) self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + def parseImpl(self, instring, loc, doActions=True): + # by using self._expr.parse and deleting the contents of the returned ParseResults list + # we keep any named results that were defined in the FollowedBy expression + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + self.parseAction.append(lambda s, l, t: t.__delitem__(slice(None, None))) + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[max(0, loc - self.retreat):loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat + 1)+1): + try: + # print('trying', offset, instring_slice, repr(instring_slice[loc - offset:])) + _, ret = test_expr._parse(instring_slice, len(instring_slice) - offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ - def __init__( self, expr ): - super(NotAny,self).__init__(expr) - #~ self.leaveWhitespace() + def __init__(self, expr): + super(NotAny, self).__init__(expr) + # ~ self.leaveWhitespace() self.skipWhitespace = False # do NOT use self.leaveWhitespace(), don't want to propagate to exprs self.mayReturnEmpty = True - self.errmsg = "Found unwanted token, "+_ustr(self.expr) + self.errmsg = "Found unwanted token, " + _ustr(self.expr) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): if self.expr.canParseNext(instring, loc): raise ParseException(instring, loc, self.errmsg, self) return loc, [] - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3877,54 +4669,74 @@ class NotAny(ParseElementEnhance): return self.strRepr class _MultipleMatch(ParseElementEnhance): - def __init__( self, expr, stopOn=None): + def __init__(self, expr, stopOn=None): super(_MultipleMatch, self).__init__(expr) self.saveAsList = True ender = stopOn if isinstance(ender, basestring): - ender = ParserElement._literalStringClass(ender) - self.not_ender = ~ender if ender is not None else None + ender = self._literalStringClass(ender) + self.stopOn(ender) - def parseImpl( self, instring, loc, doActions=True ): + def stopOn(self, ender): + if isinstance(ender, basestring): + ender = self._literalStringClass(ender) + self.not_ender = ~ender if ender is not None else None + return self + + def parseImpl(self, instring, loc, doActions=True): self_expr_parse = self.expr._parse self_skip_ignorables = self._skipIgnorables check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: try_not_ender(instring, loc) - loc, tokens = self_expr_parse( instring, loc, doActions, callPreParse=False ) + loc, tokens = self_expr_parse(instring, loc, doActions, callPreParse=False) try: hasIgnoreExprs = (not not self.ignoreExprs) while 1: if check_ender: try_not_ender(instring, loc) if hasIgnoreExprs: - preloc = self_skip_ignorables( instring, loc ) + preloc = self_skip_ignorables(instring, loc) else: preloc = loc - loc, tmptokens = self_expr_parse( instring, preloc, doActions ) + loc, tmptokens = self_expr_parse(instring, preloc, doActions) if tmptokens or tmptokens.haskeys(): tokens += tmptokens - except (ParseException,IndexError): + except (ParseException, IndexError): pass return loc, tokens - + + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_ungrouped_named_tokens_in_collection: + for e in [self.expr] + getattr(self.expr, 'exprs', []): + if isinstance(e, ParserElement) and e.resultsName: + warnings.warn("{0}: setting results name {1!r} on {2} expression " + "collides with {3!r} on contained expression".format("warn_ungrouped_named_tokens_in_collection", + name, + type(self).__name__, + e.resultsName), + stacklevel=3) + + return super(_MultipleMatch, self)._setResultsName(name, listAllMatches) + + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3935,13 +4747,13 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3950,29 +4762,28 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ - def __init__( self, expr, stopOn=None): - super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) + def __init__(self, expr, stopOn=None): + super(ZeroOrMore, self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - - def parseImpl( self, instring, loc, doActions=True ): + + def parseImpl(self, instring, loc, doActions=True): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) - except (ParseException,IndexError): + except (ParseException, IndexError): return loc, [] - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -3980,6 +4791,7 @@ class ZeroOrMore(_MultipleMatch): return self.strRepr + class _NullToken(object): def __bool__(self): return False @@ -3987,29 +4799,30 @@ class _NullToken(object): def __str__(self): return "" -_optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -4023,28 +4836,30 @@ class Optional(ParseElementEnhance): ^ FAIL: Expected end of text (at char 5), (line:1, col:6) """ - def __init__( self, expr, default=_optionalNotMatched ): - super(Optional,self).__init__( expr, savelist=False ) + __optionalNotMatched = _NullToken() + + def __init__(self, expr, default=__optionalNotMatched): + super(Optional, self).__init__(expr, savelist=False) self.saveAsList = self.expr.saveAsList self.defaultValue = default self.mayReturnEmpty = True - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): try: - loc, tokens = self.expr._parse( instring, loc, doActions, callPreParse=False ) - except (ParseException,IndexError): - if self.defaultValue is not _optionalNotMatched: + loc, tokens = self.expr._parse(instring, loc, doActions, callPreParse=False) + except (ParseException, IndexError): + if self.defaultValue is not self.__optionalNotMatched: if self.expr.resultsName: - tokens = ParseResults([ self.defaultValue ]) + tokens = ParseResults([self.defaultValue]) tokens[self.expr.resultsName] = self.defaultValue else: - tokens = [ self.defaultValue ] + tokens = [self.defaultValue] else: tokens = [] return loc, tokens - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name if self.strRepr is None: @@ -4053,20 +4868,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4083,14 +4899,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4107,34 +4925,34 @@ class SkipTo(ParseElementEnhance): - issue_num: 79 - sev: Minor """ - def __init__( self, other, include=False, ignore=None, failOn=None ): - super( SkipTo, self ).__init__( other ) + def __init__(self, other, include=False, ignore=None, failOn=None): + super(SkipTo, self).__init__(other) self.ignoreExpr = ignore self.mayReturnEmpty = True self.mayIndexError = False self.includeMatch = include self.saveAsList = False if isinstance(failOn, basestring): - self.failOn = ParserElement._literalStringClass(failOn) + self.failOn = self._literalStringClass(failOn) else: self.failOn = failOn - self.errmsg = "No match found for "+_ustr(self.expr) + self.errmsg = "No match found for " + _ustr(self.expr) - def parseImpl( self, instring, loc, doActions=True ): + def parseImpl(self, instring, loc, doActions=True): startloc = loc instrlen = len(instring) expr = self.expr expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4142,7 +4960,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4160,113 +4978,135 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: - loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) + loc, mat = expr_parse(instring, loc, doActions, callPreParse=False) skipresult += mat return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ - def __init__( self, other=None ): - super(Forward,self).__init__( other, savelist=False ) + def __init__(self, other=None): + super(Forward, self).__init__(other, savelist=False) - def __lshift__( self, other ): - if isinstance( other, basestring ): - other = ParserElement._literalStringClass(other) + def __lshift__(self, other): + if isinstance(other, basestring): + other = self._literalStringClass(other) self.expr = other self.strRepr = None self.mayIndexError = self.expr.mayIndexError self.mayReturnEmpty = self.expr.mayReturnEmpty - self.setWhitespaceChars( self.expr.whiteChars ) + self.setWhitespaceChars(self.expr.whiteChars) self.skipWhitespace = self.expr.skipWhitespace self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - - def leaveWhitespace( self ): + + def leaveWhitespace(self): self.skipWhitespace = False return self - def streamline( self ): + def streamline(self): if not self.streamlined: self.streamlined = True if self.expr is not None: self.expr.streamline() return self - def validate( self, validateTrace=[] ): + def validate(self, validateTrace=None): + if validateTrace is None: + validateTrace = [] + if self not in validateTrace: - tmp = validateTrace[:]+[self] + tmp = validateTrace[:] + [self] if self.expr is not None: self.expr.validate(tmp) self.checkRecursion([]) - def __str__( self ): - if hasattr(self,"name"): + def __str__(self): + if hasattr(self, "name"): return self.name - return self.__class__.__name__ + ": ..." + if self.strRepr is not None: + return self.strRepr - # stubbed out for now - creates awful memory and perf issues - self._revertClass = self.__class__ - self.__class__ = _ForwardNoRecurse + # Avoid infinite recursion by setting a temporary strRepr + self.strRepr = ": ..." + + # Use the string representation of main expression. + retString = '...' try: if self.expr is not None: - retString = _ustr(self.expr) + retString = _ustr(self.expr)[:1000] else: retString = "None" finally: - self.__class__ = self._revertClass - return self.__class__.__name__ + ": " + retString + self.strRepr = self.__class__.__name__ + ": " + retString + return self.strRepr def copy(self): if self.expr is not None: - return super(Forward,self).copy() + return super(Forward, self).copy() else: ret = Forward() ret <<= self return ret -class _ForwardNoRecurse(Forward): - def __str__( self ): - return "..." + def _setResultsName(self, name, listAllMatches=False): + if __diag__.warn_name_set_on_empty_Forward: + if self.expr is None: + warnings.warn("{0}: setting results name {0!r} on {1} expression " + "that has no contained expression".format("warn_name_set_on_empty_Forward", + name, + type(self).__name__), + stacklevel=3) + + return super(Forward, self)._setResultsName(name, listAllMatches) class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ - def __init__( self, expr, savelist=False ): - super(TokenConverter,self).__init__( expr )#, savelist ) + def __init__(self, expr, savelist=False): + super(TokenConverter, self).__init__(expr) # , savelist) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4277,8 +5117,8 @@ class Combine(TokenConverter): # no match when there are internal spaces print(real.parseString('3. 1416')) # -> Exception: Expected W:(0123...) """ - def __init__( self, expr, joinString="", adjacent=True ): - super(Combine,self).__init__( expr ) + def __init__(self, expr, joinString="", adjacent=True): + super(Combine, self).__init__(expr) # suppress whitespace-stripping in contained parse expressions, but re-enable it on the Combine itself if adjacent: self.leaveWhitespace() @@ -4287,71 +5127,74 @@ class Combine(TokenConverter): self.joinString = joinString self.callPreparse = True - def ignore( self, other ): + def ignore(self, other): if self.adjacent: ParserElement.ignore(self, other) else: - super( Combine, self).ignore( other ) + super(Combine, self).ignore(other) return self - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): retToks = tokenlist.copy() del retToks[:] - retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults) + retToks += ParseResults(["".join(tokenlist._asStringList(self.joinString))], modal=self.modalResults) if self.resultsName and retToks.haskeys(): - return [ retToks ] + return [retToks] else: return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num func = ident + Optional(delimitedList(term)) - print(func.parseString("fn a,b,100")) # -> ['fn', 'a', 'b', '100'] + print(func.parseString("fn a, b, 100")) # -> ['fn', 'a', 'b', '100'] func = ident + Group(Optional(delimitedList(term))) - print(func.parseString("fn a,b,100")) # -> ['fn', ['a', 'b', '100']] + print(func.parseString("fn a, b, 100")) # -> ['fn', ['a', 'b', '100']] """ - def __init__( self, expr ): - super(Group,self).__init__( expr ) + def __init__(self, expr): + super(Group, self).__init__(expr) self.saveAsList = True - def postParse( self, instring, loc, tokenlist ): - return [ tokenlist ] + def postParse(self, instring, loc, tokenlist): + return [tokenlist] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4359,42 +5202,43 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ - def __init__( self, expr ): - super(Dict,self).__init__( expr ) + def __init__(self, expr): + super(Dict, self).__init__(expr) self.saveAsList = True - def postParse( self, instring, loc, tokenlist ): - for i,tok in enumerate(tokenlist): + def postParse(self, instring, loc, tokenlist): + for i, tok in enumerate(tokenlist): if len(tok) == 0: continue ikey = tok[0] - if isinstance(ikey,int): + if isinstance(ikey, int): ikey = _ustr(tok[0]).strip() - if len(tok)==1: - tokenlist[ikey] = _ParseResultsWithOffset("",i) - elif len(tok)==2 and not isinstance(tok[1],ParseResults): - tokenlist[ikey] = _ParseResultsWithOffset(tok[1],i) + if len(tok) == 1: + tokenlist[ikey] = _ParseResultsWithOffset("", i) + elif len(tok) == 2 and not isinstance(tok[1], ParseResults): + tokenlist[ikey] = _ParseResultsWithOffset(tok[1], i) else: - dictvalue = tok.copy() #ParseResults(i) + dictvalue = tok.copy() # ParseResults(i) del dictvalue[0] - if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()): - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i) + if len(dictvalue) != 1 or (isinstance(dictvalue, ParseResults) and dictvalue.haskeys()): + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue, i) else: - tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i) + tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0], i) if self.resultsName: - return [ tokenlist ] + return [tokenlist] else: return tokenlist class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4404,42 +5248,46 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ - def postParse( self, instring, loc, tokenlist ): + def postParse(self, instring, loc, tokenlist): return [] - def suppress( self ): + def suppress(self): return self class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) self.called = False - def __call__(self,s,l,t): + def __call__(self, s, l, t): if not self.called: - results = self.callable(s,l,t) + results = self.callable(s, l, t) self.called = True return results - raise ParseException(s,l,"") + raise ParseException(s, l, "") def reset(self): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:<current_source_line>, <parse_location>, <matched_tokens>)"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4448,7 +5296,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) <<leaving remove_duplicate_chars (ret: 'dfjkls') ['dfjkls'] @@ -4456,16 +5306,16 @@ def traceParseAction(f): f = _trim_arity(f) def z(*paArgs): thisFunc = f.__name__ - s,l,t = paArgs[-3:] - if len(paArgs)>3: + s, l, t = paArgs[-3:] + if len(paArgs) > 3: thisFunc = paArgs[0].__class__.__name__ + '.' + thisFunc - sys.stderr.write( ">>entering %s(line: '%s', %d, %r)\n" % (thisFunc,line(l,s),l,t) ) + sys.stderr.write(">>entering %s(line: '%s', %d, %r)\n" % (thisFunc, line(l, s), l, t)) try: ret = f(*paArgs) except Exception as exc: - sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) ) + sys.stderr.write("<<leaving %s (exception: %s)\n" % (thisFunc, exc)) raise - sys.stderr.write( "<<leaving %s (ret: %r)\n" % (thisFunc,ret) ) + sys.stderr.write("<<leaving %s (ret: %r)\n" % (thisFunc, ret)) return ret try: z.__name__ = f.__name__ @@ -4476,36 +5326,43 @@ def traceParseAction(f): # # global helpers # -def delimitedList( expr, delim=",", combine=False ): - """ - Helper to define a delimited list of expressions - the delimiter defaults to ','. - By default, the list elements and delimiters can have intervening whitespace, and - comments, but this can be overridden by passing C{combine=True} in the constructor. - If C{combine} is set to C{True}, the matching tokens are returned as a single token - string, with the delimiters included; otherwise, the matching tokens are returned - as a list of tokens, with the delimiters suppressed. +def delimitedList(expr, delim=",", combine=False): + """Helper to define a delimited list of expressions - the delimiter + defaults to ','. By default, the list elements and delimiters can + have intervening whitespace, and comments, but this can be + overridden by passing ``combine=True`` in the constructor. If + ``combine`` is set to ``True``, the matching tokens are + returned as a single token string, with the delimiters included; + otherwise, the matching tokens are returned as a list of tokens, + with the delimiters suppressed. Example:: + delimitedList(Word(alphas)).parseString("aa,bb,cc") # -> ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ - dlName = _ustr(expr)+" ["+_ustr(delim)+" "+_ustr(expr)+"]..." + dlName = _ustr(expr) + " [" + _ustr(delim) + " " + _ustr(expr) + "]..." if combine: - return Combine( expr + ZeroOrMore( delim + expr ) ).setName(dlName) + return Combine(expr + ZeroOrMore(delim + expr)).setName(dlName) else: - return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) + return (expr + ZeroOrMore(Suppress(delim) + expr)).setName(dlName) + +def countedArray(expr, intExpr=None): + """Helper to define a counted list of expressions. -def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4514,42 +5371,44 @@ def countedArray( expr, intExpr=None ): countedArray(Word(alphas), intExpr=binaryConstant).parseString('10 ab cd ef') # -> ['ab', 'cd'] """ arrayExpr = Forward() - def countFieldParseAction(s,l,t): + def countFieldParseAction(s, l, t): n = t[0] - arrayExpr << (n and Group(And([expr]*n)) or Group(empty)) + arrayExpr << (n and Group(And([expr] * n)) or Group(empty)) return [] if intExpr is None: - intExpr = Word(nums).setParseAction(lambda t:int(t[0])) + intExpr = Word(nums).setParseAction(lambda t: int(t[0])) else: intExpr = intExpr.copy() intExpr.setName("arrayLen") intExpr.addParseAction(countFieldParseAction, callDuringTry=True) - return ( intExpr + arrayExpr ).setName('(len) ' + _ustr(expr) + '...') + return (intExpr + arrayExpr).setName('(len) ' + _ustr(expr) + '...') def _flatten(L): ret = [] for i in L: - if isinstance(i,list): + if isinstance(i, list): ret.extend(_flatten(i)) else: ret.append(i) return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() - def copyTokenToRepeater(s,l,t): + def copyTokenToRepeater(s, l, t): if t: if len(t) == 1: rep << t[0] @@ -4564,128 +5423,145 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() rep <<= e2 - def copyTokenToRepeater(s,l,t): + def copyTokenToRepeater(s, l, t): matchTokens = _flatten(t.asList()) - def mustMatchTheseTokens(s,l,t): + def mustMatchTheseTokens(s, l, t): theseTokens = _flatten(t.asList()) - if theseTokens != matchTokens: - raise ParseException("",0,"") - rep.setParseAction( mustMatchTheseTokens, callDuringTry=True ) + if theseTokens != matchTokens: + raise ParseException('', 0, '') + rep.setParseAction(mustMatchTheseTokens, callDuringTry=True) expr.addParseAction(copyTokenToRepeater, callDuringTry=True) rep.setName('(prev) ' + _ustr(expr)) return rep def _escapeRegexRangeChars(s): - #~ escape these chars: ^-] - for c in r"\^-]": - s = s.replace(c,_bslash+c) - s = s.replace("\n",r"\n") - s = s.replace("\t",r"\t") + # ~ escape these chars: ^-[] + for c in r"\^-[]": + s = s.replace(c, _bslash + c) + s = s.replace("\n", r"\n") + s = s.replace("\t", r"\t") return _ustr(s) -def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. +def oneOf(strs, caseless=False, useRegex=True, asKeyword=False): + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True`` or ``asKeyword=True``, or if + creating a :class:`Regex` raises an exception) + - asKeyword - (default=``False``) - enforce Keyword-style matching on the + generated expressions Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ + if isinstance(caseless, basestring): + warnings.warn("More than one string argument passed to oneOf, pass " + "choices as a list or space-delimited string", stacklevel=2) + if caseless: - isequal = ( lambda a,b: a.upper() == b.upper() ) - masks = ( lambda a,b: b.upper().startswith(a.upper()) ) - parseElementClass = CaselessLiteral + isequal = (lambda a, b: a.upper() == b.upper()) + masks = (lambda a, b: b.upper().startswith(a.upper())) + parseElementClass = CaselessKeyword if asKeyword else CaselessLiteral else: - isequal = ( lambda a,b: a == b ) - masks = ( lambda a,b: b.startswith(a) ) - parseElementClass = Literal + isequal = (lambda a, b: a == b) + masks = (lambda a, b: b.startswith(a)) + parseElementClass = Keyword if asKeyword else Literal symbols = [] - if isinstance(strs,basestring): + if isinstance(strs, basestring): symbols = strs.split() elif isinstance(strs, Iterable): symbols = list(strs) else: warnings.warn("Invalid argument to oneOf, expected string or iterable", - SyntaxWarning, stacklevel=2) + SyntaxWarning, stacklevel=2) if not symbols: return NoMatch() - i = 0 - while i < len(symbols)-1: - cur = symbols[i] - for j,other in enumerate(symbols[i+1:]): - if ( isequal(other, cur) ): - del symbols[i+j+1] - break - elif ( masks(cur, other) ): - del symbols[i+j+1] - symbols.insert(i,other) - cur = other - break - else: - i += 1 - - if not caseless and useRegex: - #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] )) - try: - if len(symbols)==len("".join(symbols)): - return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) ).setName(' | '.join(symbols)) + if not asKeyword: + # if not producing keywords, need to reorder to take care to avoid masking + # longer choices with shorter ones + i = 0 + while i < len(symbols) - 1: + cur = symbols[i] + for j, other in enumerate(symbols[i + 1:]): + if isequal(other, cur): + del symbols[i + j + 1] + break + elif masks(cur, other): + del symbols[i + j + 1] + symbols.insert(i, other) + break else: - return Regex( "|".join(re.escape(sym) for sym in symbols) ).setName(' | '.join(symbols)) + i += 1 + + if not (caseless or asKeyword) and useRegex: + # ~ print (strs, "->", "|".join([_escapeRegexChars(sym) for sym in symbols])) + try: + if len(symbols) == len("".join(symbols)): + return Regex("[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols)).setName(' | '.join(symbols)) + else: + return Regex("|".join(re.escape(sym) for sym in symbols)).setName(' | '.join(symbols)) except Exception: warnings.warn("Exception creating Regex for oneOf, building MatchFirst", SyntaxWarning, stacklevel=2) - # last resort, just use MatchFirst return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) -def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. +def dictOf(key, value): + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4695,7 +5571,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4705,73 +5583,82 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test <b> bold <i>text</i> </b> normal text " - for tag in ("b","i"): - opener,closer = makeHTMLTags(tag) + for tag in ("b", "i"): + opener, closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + ['<b> bold <i>text</i> </b>'] ['<i>text</i>'] """ - locMarker = Empty().setParseAction(lambda s,loc,t: loc) + locMarker = Empty().setParseAction(lambda s, loc, t: loc) endlocMarker = locMarker.copy() endlocMarker.callPreparse = False matchExpr = locMarker("_original_start") + expr + endlocMarker("_original_end") if asString: - extractText = lambda s,l,t: s[t._original_start:t._original_end] + extractText = lambda s, l, t: s[t._original_start: t._original_end] else: - def extractText(s,l,t): + def extractText(s, l, t): t[:] = [s[t.pop('_original_start'):t.pop('_original_end')]] matchExpr.setParseAction(extractText) matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. - """ - return TokenConverter(expr).setParseAction(lambda t:t[0]) + return TokenConverter(expr).addParseAction(lambda t: t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{<TAB>} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ``<TAB>`` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] """ - locator = Empty().setParseAction(lambda s,l,t: l) + locator = Empty().setParseAction(lambda s, l, t: l) return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end")) @@ -4782,66 +5669,75 @@ lineEnd = LineEnd().setName("lineEnd") stringStart = StringStart().setName("stringStart") stringEnd = StringEnd().setName("stringEnd") -_escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1]) -_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16))) -_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8))) +_escapedPunc = Word(_bslash, r"\[]-*.$+^?()~ ", exact=2).setParseAction(lambda s, l, t: t[0][1]) +_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s, l, t: unichr(int(t[0].lstrip(r'\0x'), 16))) +_escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s, l, t: unichr(int(t[0][1:], 8))) _singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | CharsNotIn(r'\]', exact=1) _charRange = Group(_singleChar + Suppress("-") + _singleChar) -_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" +_reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group(OneOrMore(_charRange | _singleChar)).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ - _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) + _expanded = lambda p: p if not isinstance(p, ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]), ord(p[1]) + 1)) try: return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body) except Exception: return "" def matchOnlyAtCol(n): + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ - Helper method for defining parse actions that require matching at a specific - column in the input text. - """ - def verifyCol(strg,locn,toks): - if col(locn,strg) != n: - raise ParseException(strg,locn,"matched token not at column %d" % n) + def verifyCol(strg, locn, toks): + if col(locn, strg) != n: + raise ParseException(strg, locn, "matched token not at column %d" % n) return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString<ParserElement.transformString>}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString<ParserElement.transformString>` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ - return lambda s,l,t: [replStr] + return lambda s, l, t: [replStr] -def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. +def removeQuotes(s, l, t): + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4852,18 +5748,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4873,7 +5771,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4883,11 +5783,11 @@ def tokenMap(func, *args): now is the winter of our discontent made glorious summer by this sun of york ['Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York'] """ - def pa(s,l,t): + def pa(s, l, t): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4896,90 +5796,108 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - -def _makeTags(tagStr, xml): +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + +def _makeTags(tagStr, xml, + suppress_LT=Suppress("<"), + suppress_GT=Suppress(">")): """Internal helper to construct opening and closing tag expressions, given a tag name""" - if isinstance(tagStr,basestring): + if isinstance(tagStr, basestring): resname = tagStr tagStr = Keyword(tagStr, caseless=not xml) else: resname = tagStr.name - tagAttrName = Word(alphas,alphanums+"_-:") - if (xml): - tagAttrValue = dblQuotedString.copy().setParseAction( removeQuotes ) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") + tagAttrName = Word(alphas, alphanums + "_-:") + if xml: + tagAttrValue = dblQuotedString.copy().setParseAction(removeQuotes) + openTag = (suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName + Suppress("=") + tagAttrValue))) + + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') + + suppress_GT) else: - printablesLessRAbrack = "".join(c for c in printables if c not in ">") - tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack) - openTag = Suppress("<") + tagStr("tag") + \ - Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \ - Optional( Suppress("=") + tagAttrValue ) ))) + \ - Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">") - closeTag = Combine(_L("</") + tagStr + ">") + tagAttrValue = quotedString.copy().setParseAction(removeQuotes) | Word(printables, excludeChars=">") + openTag = (suppress_LT + + tagStr("tag") + + Dict(ZeroOrMore(Group(tagAttrName.setParseAction(downcaseTokens) + + Optional(Suppress("=") + tagAttrValue)))) + + Optional("/", default=[False])("empty").setParseAction(lambda s, l, t: t[0] == '/') + + suppress_GT) + closeTag = Combine(_L("</") + tagStr + ">", adjacent=False) - openTag = openTag.setResultsName("start"+"".join(resname.replace(":"," ").title().split())).setName("<%s>" % resname) - closeTag = closeTag.setResultsName("end"+"".join(resname.replace(":"," ").title().split())).setName("</%s>" % resname) + openTag.setName("<%s>" % resname) + # add start<tagname> results name in parse action now that ungrouped names are not reported at two levels + openTag.addParseAction(lambda t: t.__setitem__("start" + "".join(resname.replace(":", " ").title().split()), t.copy())) + closeTag = closeTag("end" + "".join(resname.replace(":", " ").title().split())).setName("</%s>" % resname) openTag.tag = resname closeTag.tag = resname + openTag.tag_body = SkipTo(closeTag()) return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple - a,a_end = makeHTMLTags("A") + + text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple + a, a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the <A> tag (like "href" shown here) are also accessible as named results + # attributes in the <A> tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ - return _makeTags( tagStr, False ) + return _makeTags(tagStr, False) def makeXMLTags(tagStr): + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. + + Example: similar to :class:`makeHTMLTags` """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + return _makeTags(tagStr, True) - Example: similar to L{makeHTMLTags} - """ - return _makeTags( tagStr, True ) +def withAttribute(*args, **attrDict): + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ``<TD>`` or ``<DIV>``. -def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{<TD>} or C{<DIV>}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align", "right"))`` - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = ''' <div> Some text @@ -4987,7 +5905,7 @@ def withAttribute(*args,**attrDict): <div type="graph">1,3 2,3 1,1</div> <div>this has no type</div> </div> - + ''' div,div_end = makeHTMLTags("div") @@ -4996,13 +5914,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -5012,23 +5932,24 @@ def withAttribute(*args,**attrDict): attrs = args[:] else: attrs = attrDict.items() - attrs = [(k,v) for k,v in attrs] - def pa(s,l,tokens): - for attrName,attrValue in attrs: + attrs = [(k, v) for k, v in attrs] + def pa(s, l, tokens): + for attrName, attrValue in attrs: if attrName not in tokens: - raise ParseException(s,l,"no matching attribute " + attrName) + raise ParseException(s, l, "no matching attribute " + attrName) if attrValue != withAttribute.ANY_VALUE and tokens[attrName] != attrValue: - raise ParseException(s,l,"attribute '%s' has value '%s', must be '%s'" % + raise ParseException(s, l, "attribute '%s' has value '%s', must be '%s'" % (attrName, tokens[attrName], attrValue)) return pa withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = ''' <div> Some text @@ -5036,84 +5957,96 @@ def withClass(classname, namespace=''): <div class="graph">1,3 2,3 1,1</div> <div>this <div> has no class</div> </div> - + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr: classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() -def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. +def infixNotation(baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')')): + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5123,27 +6056,34 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() - lastExpr = baseExpr | ( lpar + ret + rpar ) - for i,operDef in enumerate(opList): - opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4] + lastExpr = baseExpr | (lpar + ret + rpar) + for i, operDef in enumerate(opList): + opExpr, arity, rightLeftAssoc, pa = (operDef + (None, ))[:4] termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group(lastExpr + OneOrMore(opExpr)) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group(lastExpr + OneOrMore(opExpr + lastExpr)) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr + lastExpr) + Group(lastExpr + OneOrMore(lastExpr)) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ - Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) + matchExpr = (_FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + + Group(lastExpr + OneOrMore(opExpr1 + lastExpr + opExpr2 + lastExpr))) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") elif rightLeftAssoc == opAssoc.RIGHT: @@ -5151,15 +6091,15 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group(opExpr + thisExpr) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group(lastExpr + OneOrMore(opExpr + thisExpr)) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group(lastExpr + OneOrMore(thisExpr)) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ - Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) + matchExpr = (_FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + + Group(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr)) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") else: @@ -5169,128 +6109,144 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): matchExpr.setParseAction(*pa) else: matchExpr.setParseAction(pa) - thisExpr <<= ( matchExpr.setName(termName) | lastExpr ) + thisExpr <<= (matchExpr.setName(termName) | lastExpr) lastExpr = thisExpr ret <<= lastExpr return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" -dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") -sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") -quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"'| - Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("quotedString using single or double quotes") +dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"').setName("string enclosed in double quotes") +sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("string enclosed in single quotes") +quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*') + '"' + | Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*") + "'").setName("quotedString using single or double quotes") unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') number = pyparsing_common.number arg = Group(decl_data_type + ident) - LPAR,RPAR = map(Suppress, "()") + LPAR, RPAR = map(Suppress, "()") code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ if opener == closer: raise ValueError("opening and closing strings cannot be the same") if content is None: - if isinstance(opener,basestring) and isinstance(closer,basestring): - if len(opener) == 1 and len(closer)==1: + if isinstance(opener, basestring) and isinstance(closer, basestring): + if len(opener) == 1 and len(closer) == 1: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~ignoreExpr + + CharsNotIn(opener + + closer + + ParserElement.DEFAULT_WHITE_CHARS, exact=1) + ) + ).setParseAction(lambda t: t[0].strip())) else: - content = (empty.copy()+CharsNotIn(opener+closer+ParserElement.DEFAULT_WHITE_CHARS - ).setParseAction(lambda t:t[0].strip())) + content = (empty.copy() + CharsNotIn(opener + + closer + + ParserElement.DEFAULT_WHITE_CHARS + ).setParseAction(lambda t: t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + - ~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~ignoreExpr + + ~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) + ).setParseAction(lambda t: t[0].strip())) else: - content = (Combine(OneOrMore(~Literal(opener) + ~Literal(closer) + - CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) - ).setParseAction(lambda t:t[0].strip())) + content = (Combine(OneOrMore(~Literal(opener) + + ~Literal(closer) + + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS, exact=1)) + ).setParseAction(lambda t: t[0].strip())) else: raise ValueError("opening and closing arguments must be strings if no content expression is given") ret = Forward() if ignoreExpr is not None: - ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) ) + ret <<= Group(Suppress(opener) + ZeroOrMore(ignoreExpr | ret | content) + Suppress(closer)) else: - ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content ) + Suppress(closer) ) - ret.setName('nested %s%s expression' % (opener,closer)) + ret <<= Group(Suppress(opener) + ZeroOrMore(ret | content) + Suppress(closer)) + ret.setName('nested %s%s expression' % (opener, closer)) return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5317,21 +6273,23 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): stmt = Forward() identifier = Word(alphas, alphanums) - funcDecl = ("def" + identifier + Group( "(" + Optional( delimitedList(identifier) ) + ")" ) + ":") + funcDecl = ("def" + identifier + Group("(" + Optional(delimitedList(identifier)) + ")") + ":") func_body = indentedBlock(stmt, indentStack) - funcDef = Group( funcDecl + func_body ) + funcDef = Group(funcDecl + func_body) rvalue = Forward() funcCall = Group(identifier + "(" + Optional(delimitedList(rvalue)) + ")") rvalue << (funcCall | identifier | Word(nums)) assignment = Group(identifier + "=" + rvalue) - stmt << ( funcDef | assignment | identifier ) + stmt << (funcDef | assignment | identifier) module_body = OneOrMore(stmt) parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5349,49 +6307,58 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ - def checkPeerIndent(s,l,t): + backup_stack = indentStack[:] + + def reset_stack(): + indentStack[:] = backup_stack + + def checkPeerIndent(s, l, t): if l >= len(s): return - curCol = col(l,s) + curCol = col(l, s) if curCol != indentStack[-1]: if curCol > indentStack[-1]: - raise ParseFatalException(s,l,"illegal nesting") - raise ParseException(s,l,"not a peer entry") + raise ParseException(s, l, "illegal nesting") + raise ParseException(s, l, "not a peer entry") - def checkSubIndent(s,l,t): - curCol = col(l,s) + def checkSubIndent(s, l, t): + curCol = col(l, s) if curCol > indentStack[-1]: - indentStack.append( curCol ) + indentStack.append(curCol) else: - raise ParseException(s,l,"not a subentry") + raise ParseException(s, l, "not a subentry") - def checkUnindent(s,l,t): + def checkUnindent(s, l, t): if l >= len(s): return - curCol = col(l,s) - if not(indentStack and curCol < indentStack[-1] and curCol <= indentStack[-2]): - raise ParseException(s,l,"not an unindent") - indentStack.pop() + curCol = col(l, s) + if not(indentStack and curCol in indentStack): + raise ParseException(s, l, "not an unindent") + if curCol < indentStack[-1]: + indentStack.pop() - NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress()) + NL = OneOrMore(LineEnd().setWhitespaceChars("\t ").suppress(), stopOn=StringEnd()) INDENT = (Empty() + Empty().setParseAction(checkSubIndent)).setName('INDENT') PEER = Empty().setParseAction(checkPeerIndent).setName('') UNDENT = Empty().setParseAction(checkUnindent).setName('UNINDENT') if indent: - smExpr = Group( Optional(NL) + - #~ FollowedBy(blockStatementExpr) + - INDENT + (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) + UNDENT) + smExpr = Group(Optional(NL) + + INDENT + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL), stopOn=StringEnd()) + + UNDENT) else: - smExpr = Group( Optional(NL) + - (OneOrMore( PEER + Group(blockStatementExpr) + Optional(NL) )) ) + smExpr = Group(Optional(NL) + + OneOrMore(PEER + Group(blockStatementExpr) + Optional(NL), stopOn=StringEnd()) + + UNDENT) + smExpr.setFailAction(lambda a, b, c, d: reset_stack()) blockStatementExpr.ignore(_bslash + LineEnd()) return smExpr.setName('indented block') alphas8bit = srange(r"[\0xc0-\0xd6\0xd8-\0xf6\0xf8-\0xff]") punc8bit = srange(r"[\0xa1-\0xbf\0xd7\0xf7]") -anyOpenTag,anyCloseTag = makeHTMLTags(Word(alphas,alphanums+"_:").setName('any tag')) -_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(),'><& "\'')) +anyOpenTag, anyCloseTag = makeHTMLTags(Word(alphas, alphanums + "_:").setName('any tag')) +_htmlEntityMap = dict(zip("gt lt amp nbsp quot apos".split(), '><& "\'')) commonHTMLEntity = Regex('&(?P<entity>' + '|'.join(_htmlEntityMap.keys()) +");").setName("common HTML entity") def replaceHTMLEntity(t): """Helper parser action to replace common HTML entities with their special characters""" @@ -5399,51 +6366,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"<!--[\s\S]*?-->").setName("HTML comment") -"Comment of the form C{<!-- ... -->}" +"Comment of the form ``<!-- ... -->``" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" -cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/' | dblSlashComment).setName("C++ style comment") +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" -_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + - Optional( Word(" \t") + - ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") -commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + + Optional(Word(" \t") + + ~Literal(",") + ~LineEnd()))).streamline().setName("commaItem") +commaSeparatedList = delimitedList(Optional(quotedString.copy() | _commasepitem, default="")).setName("commaSeparatedList") +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers<integer>}, L{reals<real>}, L{scientific notation<sci_real>}) - - common L{programming identifiers<identifier>} - - network addresses (L{MAC<mac_address>}, L{IPv4<ipv4_address>}, L{IPv6<ipv6_address>}) - - ISO8601 L{dates<iso8601_date>} and L{datetime<iso8601_datetime>} - - L{UUID<uuid>} - - L{comma-separated list<comma_separated_list>} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers<integer>`, :class:`reals<real>`, + :class:`scientific notation<sci_real>`) + - common :class:`programming identifiers<identifier>` + - network addresses (:class:`MAC<mac_address>`, + :class:`IPv4<ipv4_address>`, :class:`IPv6<ipv6_address>`) + - ISO8601 :class:`dates<iso8601_date>` and + :class:`datetime<iso8601_datetime>` + - :class:`UUID<uuid>` + - :class:`comma-separated list<comma_separated_list>` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5490,7 +6467,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5574,7 +6553,7 @@ class pyparsing_common: integer = Word(nums).setName("integer").setParseAction(convertToInteger) """expression that parses an unsigned integer, returns an int""" - hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int,16)) + hex_integer = Word(hexnums).setName("hex integer").setParseAction(tokenMap(int, 16)) """expression that parses a hexadecimal integer, returns an int""" signed_integer = Regex(r'[+-]?\d+').setName("signed integer").setParseAction(convertToInteger) @@ -5588,11 +6567,12 @@ class pyparsing_common: """mixed integer of the form 'integer - fraction', with optional leading integer, returns float""" mixed_integer.addParseAction(sum) - real = Regex(r'[+-]?\d+\.\d*').setName("real number").setParseAction(convertToFloat) + real = Regex(r'[+-]?(?:\d+\.\d*|\.\d+)').setName("real number").setParseAction(convertToFloat) """expression that parses a floating point number and returns a float""" - sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + sci_real = Regex(r'[+-]?(?:\d+(?:[eE][+-]?\d+)|(?:\d+\.\d*|\.\d+)(?:[eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5600,21 +6580,24 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - - identifier = Word(alphas+'_', alphanums+'_').setName("identifier") + + identifier = Word(alphas + '_', alphanums + '_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") - _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") - _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part)*(0,6)) + "::" + Optional(_ipv6_part + (':' + _ipv6_part)*(0,6))).setName("short IPv6 address") + _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part) * 7).setName("full IPv6 address") + _short_ipv6_address = (Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) + + "::" + + Optional(_ipv6_part + (':' + _ipv6_part) * (0, 6)) + ).setName("short IPv6 address") _short_ipv6_address.addCondition(lambda t: sum(1 for tt in t if pyparsing_common._ipv6_part.matches(tt)) < 8) _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5624,16 +6607,19 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ - def cvt_fn(s,l,t): + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt).date() except ValueError as ve: @@ -5642,20 +6628,23 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ - def cvt_fn(s,l,t): + def cvt_fn(s, l, t): try: return datetime.strptime(t[0], fmt) except ValueError as ve: @@ -5663,33 +6652,40 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P<year>\d{4})(?:-(?P<month>\d\d)(?:-(?P<day>\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P<year>\d{4})-(?P<month>\d\d)-(?P<day>\d\d)[T ](?P<hour>\d\d):(?P<minute>\d\d)(:(?P<second>\d\d(\.\d*)?)?)?(?P<tz>Z|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = '<td>More info at the <a href="http://pyparsing.wikispaces.com">pyparsing</a> wiki page</td>' - td,td_end = makeHTMLTags("TD") + + # strip HTML links from normal text + text = '<td>More info at the <a href="https://github.com/pyparsing/pyparsing/wiki">pyparsing</a> wiki page</td>' + td, td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') - + Optional( White(" \t") ) ) ).streamline().setName("commaItem") - comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") + _commasepitem = Combine(OneOrMore(~Literal(",") + + ~LineEnd() + + Word(printables, excludeChars=',') + + Optional(White(" \t")))).streamline().setName("commaItem") + comma_separated_list = delimitedList(Optional(quotedString.copy() + | _commasepitem, default='') + ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" upcaseTokens = staticmethod(tokenMap(lambda t: _ustr(t).upper())) @@ -5699,6 +6695,346 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) + for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1] + 1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f),] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f),] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f),] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff),] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f),] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b),] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f),] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff),] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, u"العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, u"中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, u"кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, u"Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, u"עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, u"日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, u"漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, u"カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, u"ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, u"한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, u"ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, u"देवनागरी", pyparsing_unicode.Devanagari) + + +class pyparsing_test: + """ + namespace class for classes useful in writing unit tests + """ + + class reset_pyparsing_context: + """ + Context manager to be used when writing unit tests that modify pyparsing config values: + - packrat parsing + - default whitespace characters. + - default keyword characters + - literal string auto-conversion class + - __diag__ settings + + Example: + with reset_pyparsing_context(): + # test that literals used to construct a grammar are automatically suppressed + ParserElement.inlineLiteralsUsing(Suppress) + + term = Word(alphas) | Word(nums) + group = Group('(' + term[...] + ')') + + # assert that the '()' characters are not included in the parsed tokens + self.assertParseAndCheckLisst(group, "(abc 123 def)", ['abc', '123', 'def']) + + # after exiting context manager, literals are converted to Literal expressions again + """ + + def __init__(self): + self._save_context = {} + + def save(self): + self._save_context["default_whitespace"] = ParserElement.DEFAULT_WHITE_CHARS + self._save_context["default_keyword_chars"] = Keyword.DEFAULT_KEYWORD_CHARS + self._save_context[ + "literal_string_class" + ] = ParserElement._literalStringClass + self._save_context["packrat_enabled"] = ParserElement._packratEnabled + self._save_context["packrat_parse"] = ParserElement._parse + self._save_context["__diag__"] = { + name: getattr(__diag__, name) for name in __diag__._all_names + } + self._save_context["__compat__"] = { + "collect_all_And_tokens": __compat__.collect_all_And_tokens + } + return self + + def restore(self): + # reset pyparsing global state + if ( + ParserElement.DEFAULT_WHITE_CHARS + != self._save_context["default_whitespace"] + ): + ParserElement.setDefaultWhitespaceChars( + self._save_context["default_whitespace"] + ) + Keyword.DEFAULT_KEYWORD_CHARS = self._save_context["default_keyword_chars"] + ParserElement.inlineLiteralsUsing( + self._save_context["literal_string_class"] + ) + for name, value in self._save_context["__diag__"].items(): + setattr(__diag__, name, value) + ParserElement._packratEnabled = self._save_context["packrat_enabled"] + ParserElement._parse = self._save_context["packrat_parse"] + __compat__.collect_all_And_tokens = self._save_context["__compat__"] + + def __enter__(self): + return self.save() + + def __exit__(self, *args): + return self.restore() + + class TestParseResultsAsserts: + """ + A mixin class to add parse results assertion methods to normal unittest.TestCase classes. + """ + def assertParseResultsEquals( + self, result, expected_list=None, expected_dict=None, msg=None + ): + """ + Unit test assertion to compare a ParseResults object with an optional expected_list, + and compare any defined results names with an optional expected_dict. + """ + if expected_list is not None: + self.assertEqual(expected_list, result.asList(), msg=msg) + if expected_dict is not None: + self.assertEqual(expected_dict, result.asDict(), msg=msg) + + def assertParseAndCheckList( + self, expr, test_string, expected_list, msg=None, verbose=True + ): + """ + Convenience wrapper assert to test a parser element and input string, and assert that + the resulting ParseResults.asList() is equal to the expected_list. + """ + result = expr.parseString(test_string, parseAll=True) + if verbose: + print(result.dump()) + self.assertParseResultsEquals(result, expected_list=expected_list, msg=msg) + + def assertParseAndCheckDict( + self, expr, test_string, expected_dict, msg=None, verbose=True + ): + """ + Convenience wrapper assert to test a parser element and input string, and assert that + the resulting ParseResults.asDict() is equal to the expected_dict. + """ + result = expr.parseString(test_string, parseAll=True) + if verbose: + print(result.dump()) + self.assertParseResultsEquals(result, expected_dict=expected_dict, msg=msg) + + def assertRunTestResults( + self, run_tests_report, expected_parse_results=None, msg=None + ): + """ + Unit test assertion to evaluate output of ParserElement.runTests(). If a list of + list-dict tuples is given as the expected_parse_results argument, then these are zipped + with the report tuples returned by runTests and evaluated using assertParseResultsEquals. + Finally, asserts that the overall runTests() success value is True. + + :param run_tests_report: tuple(bool, [tuple(str, ParseResults or Exception)]) returned from runTests + :param expected_parse_results (optional): [tuple(str, list, dict, Exception)] + """ + run_test_success, run_test_results = run_tests_report + + if expected_parse_results is not None: + merged = [ + (rpt[0], rpt[1], expected) + for rpt, expected in zip(run_test_results, expected_parse_results) + ] + for test_string, result, expected in merged: + # expected should be a tuple containing a list and/or a dict or an exception, + # and optional failure message string + # an empty tuple will skip any result validation + fail_msg = next( + (exp for exp in expected if isinstance(exp, str)), None + ) + expected_exception = next( + ( + exp + for exp in expected + if isinstance(exp, type) and issubclass(exp, Exception) + ), + None, + ) + if expected_exception is not None: + with self.assertRaises( + expected_exception=expected_exception, msg=fail_msg or msg + ): + if isinstance(result, Exception): + raise result + else: + expected_list = next( + (exp for exp in expected if isinstance(exp, list)), None + ) + expected_dict = next( + (exp for exp in expected if isinstance(exp, dict)), None + ) + if (expected_list, expected_dict) != (None, None): + self.assertParseResultsEquals( + result, + expected_list=expected_list, + expected_dict=expected_dict, + msg=fail_msg or msg, + ) + else: + # warning here maybe? + print("no validation for {!r}".format(test_string)) + + # do this last, in case some specific test results can be reported instead + self.assertTrue( + run_test_success, msg=msg if msg is not None else "failed runTests" + ) + + @contextmanager + def assertRaisesParseException(self, exc_type=ParseException, msg=None): + with self.assertRaises(exc_type, msg=msg): + yield + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5712,7 +7048,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/vendor/blindspin/LICENSE b/pipenv/vendor/pythonfinder/LICENSE similarity index 95% rename from pipenv/vendor/blindspin/LICENSE rename to pipenv/vendor/pythonfinder/LICENSE index 00bf847d..c7ac395f 100644 --- a/pipenv/vendor/blindspin/LICENSE +++ b/pipenv/vendor/pythonfinder/LICENSE @@ -1,6 +1,6 @@ -The MIT License (MIT) +MIT License -Copyright 2018 Kenneth Reitz +Copyright (c) 2016 Steve Dower Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 8e12a198..6ba8a67d 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,16 +1,19 @@ -from __future__ import print_function, absolute_import - -__version__ = '1.1.10' +from __future__ import absolute_import, print_function # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: # # > No handlers could be found for logger "pythonfinder.models.pyenv" import logging + +from .exceptions import InvalidPythonVersion +from .models import SystemPath, WindowsFinder +from .pythonfinder import Finder + +__version__ = "1.2.2" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] -from .pythonfinder import Finder -from .models import SystemPath, WindowsFinder -from .exceptions import InvalidPythonVersion diff --git a/pipenv/vendor/pythonfinder/__main__.py b/pipenv/vendor/pythonfinder/__main__.py index c804d573..3083e72d 100644 --- a/pipenv/vendor/pythonfinder/__main__.py +++ b/pipenv/vendor/pythonfinder/__main__.py @@ -1,12 +1,17 @@ +#!env python +# -*- coding=utf-8 -*- + from __future__ import absolute_import import os import sys +from pythonfinder.cli import cli + + PYTHONFINDER_MAIN = os.path.dirname(os.path.abspath(__file__)) PYTHONFINDER_PACKAGE = os.path.dirname(PYTHONFINDER_MAIN) -from pythonfinder import cli as cli if __name__ == "__main__": sys.exit(cli()) diff --git a/pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE b/pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE new file mode 100644 index 00000000..c7ac395f --- /dev/null +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Steve Dower + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py b/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py index 2611f601..da72ecb5 100644 --- a/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/_registry.py @@ -172,7 +172,7 @@ class RegistryAccessor(object): items = info.items() else: raise TypeError('info must be a dictionary') - + self._set_all_values(self._root, self.subkey, items, errors) if len(errors) == 1: raise ValueError(errors[0]) diff --git a/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py b/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py index 2c09ccbc..f4104a45 100644 --- a/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py +++ b/pipenv/vendor/pythonfinder/_vendor/pep514tools/environment.py @@ -15,7 +15,8 @@ import sys # These tags are treated specially when the Company is 'PythonCore' _PYTHONCORE_COMPATIBILITY_TAGS = { '2.0', '2.1', '2.2', '2.3', '2.4', '2.5', '2.6', '2.7', - '3.0', '3.1', '3.2', '3.3', '3.4' + '3.0', '3.1', '3.2', '3.3', '3.4', '3.5', '3.6', '3.7', + '3.8', '3.9' } _IS_64BIT_OS = None diff --git a/pipenv/vendor/pythonfinder/_vendor/vendor.txt b/pipenv/vendor/pythonfinder/_vendor/vendor.txt index 88752498..e635a2a8 100644 --- a/pipenv/vendor/pythonfinder/_vendor/vendor.txt +++ b/pipenv/vendor/pythonfinder/_vendor/vendor.txt @@ -1 +1 @@ --e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools +git+https://github.com/zooba/pep514tools.git@master#egg=pep514tools diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 221cb2fd..fc86abd0 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -1,9 +1,8 @@ -#!/usr/bin/env python # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function, unicode_literals + import click -import crayons -import sys + from . import __version__ from .pythonfinder import Finder @@ -11,23 +10,30 @@ from .pythonfinder import Finder @click.command() @click.option("--find", default=False, nargs=1, help="Find a specific python version.") @click.option("--which", default=False, nargs=1, help="Run the which command.") -@click.option( - "--findall", is_flag=True, default=False, help="Find all python versions." -) +@click.option("--findall", is_flag=True, default=False, help="Find all python versions.") @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") -@click.version_option(prog_name='pyfinder', version=__version__) +@click.option( + "--ignore-unsupported/--no-unsupported", + is_flag=True, + default=True, + envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", + help="Ignore unsupported python versions.", +) +@click.version_option(prog_name="pyfinder", version=__version__) @click.pass_context -def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): +def cli( + ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True +): if version: click.echo( "{0} version {1}".format( - crayons.white("PythonFinder", bold=True), crayons.yellow(__version__) + click.style("PythonFinder", fg="white", bold=True), + click.style(str(__version__), fg="yellow") ) ) - sys.exit(0) + ctx.exit() finder = Finder(ignore_unsupported=ignore_unsupported) if findall: versions = [v for v in finder.find_all_python_versions()] @@ -46,7 +52,7 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup ), fg="yellow", ) - sys.exit(0) + ctx.exit() else: click.secho( "ERROR: No valid python versions found! Check your path and try again.", @@ -70,22 +76,22 @@ def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsup ), fg="yellow", ) - sys.exit(0) + ctx.exit() else: click.secho("Failed to find matching executable...", fg="yellow") - sys.exit(1) + ctx.exit(1) elif which: found = finder.system_path.which(which.strip()) if found: click.secho("Found Executable: {0}".format(found), fg="white") - sys.exit(0) + ctx.exit() else: click.secho("Failed to find matching executable...", fg="yellow") - sys.exit(1) + ctx.exit(1) else: click.echo("Please provide a command", color="red") - sys.exit(1) - sys.exit() + ctx.exit(1) + ctx.exit() if __name__ == "__main__": diff --git a/pipenv/vendor/pythonfinder/compat.py b/pipenv/vendor/pythonfinder/compat.py new file mode 100644 index 00000000..d76c4efc --- /dev/null +++ b/pipenv/vendor/pythonfinder/compat.py @@ -0,0 +1,42 @@ +# -*- coding=utf-8 -*- +import sys + +import six + +if sys.version_info[:2] <= (3, 4): + from pipenv.vendor.pathlib2 import Path # type: ignore # noqa +else: + from pathlib import Path + +if six.PY3: + from functools import lru_cache + from builtins import TimeoutError +else: + from backports.functools_lru_cache import lru_cache # type: ignore # noqa + + class TimeoutError(OSError): + pass + + +def getpreferredencoding(): + import locale + # Borrowed from Invoke + # (see https://github.com/pyinvoke/invoke/blob/93af29d/invoke/runners.py#L881) + _encoding = locale.getpreferredencoding(False) + if six.PY2 and not sys.platform == "win32": + _default_encoding = locale.getdefaultlocale()[1] + if _default_encoding is not None: + _encoding = _default_encoding + return _encoding + + +DEFAULT_ENCODING = getpreferredencoding() + + +def fs_str(string): + """Encodes a string into the proper filesystem encoding""" + + if isinstance(string, str): + return string + assert not isinstance(string, bytes) + return string.encode(DEFAULT_ENCODING) diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index e8878403..6b22fb08 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function + import os import platform import sys @@ -34,3 +35,20 @@ else: IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) +SUBPROCESS_TIMEOUT = os.environ.get("PYTHONFINDER_SUBPROCESS_TIMEOUT", 5) +"""The default subprocess timeout for determining python versions + +Set to **5** by default. +""" + + +def get_shim_paths(): + shim_paths = [] + if ASDF_INSTALLED: + shim_paths.append(os.path.join(ASDF_DATA_DIR, "shims")) + if PYENV_INSTALLED: + shim_paths.append(os.path.join(PYENV_ROOT, "shims")) + return [os.path.normpath(os.path.normcase(p)) for p in shim_paths] + + +SHIM_PATHS = get_shim_paths() diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index df381daf..adfac16b 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function class InvalidPythonVersion(Exception): diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 7d406548..76327115 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,16 +2,82 @@ from __future__ import absolute_import, unicode_literals import abc -import attr import operator +from collections import defaultdict + +import attr import six -from ..utils import ensure_path, KNOWN_EXTS, unnest +from ..compat import fs_str +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ( + KNOWN_EXTS, + Sequence, + expand_paths, + looks_like_python, + path_is_known_executable, +) + +if MYPY_RUNNING: + from .path import PathEntry + from .python import PythonVersion + from typing import ( + Optional, + Union, + Any, + Dict, + Iterator, + List, + DefaultDict, + Generator, + Tuple, + TypeVar, + Type, + ) + from ..compat import Path # noqa + + BaseFinderType = TypeVar("BaseFinderType") -@attr.s +@attr.s(slots=True) class BasePath(object): + path = attr.ib(default=None) # type: Path + _children = attr.ib( + default=attr.Factory(dict), cmp=False + ) # type: Dict[str, PathEntry] + only_python = attr.ib(default=False) # type: bool + name = attr.ib(type=str) + _py_version = attr.ib(default=None, cmp=False) # type: Optional[PythonVersion] + _pythons = attr.ib( + default=attr.Factory(defaultdict), cmp=False + ) # type: DefaultDict[str, PathEntry] + _is_dir = attr.ib(default=None, cmp=False) # type: Optional[bool] + _is_executable = attr.ib(default=None, cmp=False) # type: Optional[bool] + _is_python = attr.ib(default=None, cmp=False) # type: Optional[bool] + + def __str__(self): + # type: () -> str + return fs_str("{0}".format(self.path.as_posix())) + + def __lt__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() < other.path.as_posix() + + def __lte__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() <= other.path.as_posix() + + def __gt__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() > other.path.as_posix() + + def __gte__(self, other): + # type: ("BasePath") -> bool + return self.path.as_posix() >= other.path.as_posix() + def which(self, name): + # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. @@ -24,26 +90,223 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next( - ( - children[(self.path / child).as_posix()] - for child in valid_names - if (self.path / child).as_posix() in children - ), - None, - ) + found = None + if self.path is not None: + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found + def __del__(self): + for key in ["_is_dir", "_is_python", "_is_executable", "_py_version"]: + if getattr(self, key, None): + try: + delattr(self, key) + except Exception: + print("failed deleting key: {0}".format(key)) + self._children = {} + for key in list(self._pythons.keys()): + del self._pythons[key] + self._pythons = None + self._py_version = None + self.path = None + + @property + def children(self): + # type: () -> Dict[str, PathEntry] + if not self.is_dir: + return {} + return self._children + + @property + def as_python(self): + # type: () -> PythonVersion + py_version = None + if self.py_version: + return self.py_version + if not self.is_dir and self.is_python: + try: + from .python import PythonVersion + + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (ValueError, InvalidPythonVersion): + pass + if py_version is None: + pass + self.py_version = py_version + return py_version # type: ignore + + @name.default + def get_name(self): + # type: () -> Optional[str] + if self.path: + return self.path.name + return None + + @property + def is_dir(self): + # type: () -> bool + if self._is_dir is None: + if not self.path: + ret_val = False + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + self._is_dir = ret_val + return self._is_dir + + @is_dir.setter + def is_dir(self, val): + # type: (bool) -> None + self._is_dir = val + + @is_dir.deleter + def is_dir(self): + # type: () -> None + self._is_dir = None + + @property + def is_executable(self): + # type: () -> bool + if self._is_executable is None: + if not self.path: + self._is_executable = False + else: + self._is_executable = path_is_known_executable(self.path) + return self._is_executable + + @is_executable.setter + def is_executable(self, val): + # type: (bool) -> None + self._is_executable = val + + @is_executable.deleter + def is_executable(self): + # type: () -> None + self._is_executable = None + + @property + def is_python(self): + # type: () -> bool + if self._is_python is None: + if not self.path: + self._is_python = False + else: + self._is_python = self.is_executable and ( + looks_like_python(self.path.name) + ) + return self._is_python + + @is_python.setter + def is_python(self, val): + # type: (bool) -> None + self._is_python = val + + @is_python.deleter + def is_python(self): + # type: () -> None + self._is_python = None + + def get_py_version(self): + # type: () -> Optional[PythonVersion] + from ..environment import IGNORE_UNSUPPORTED + + if self.is_dir: + return None + if self.is_python: + py_version = None + from .python import PythonVersion + + try: + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (InvalidPythonVersion, ValueError): + py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise + return py_version + return None + + @property + def py_version(self): + # type: () -> Optional[PythonVersion] + if not self._py_version: + py_version = self.get_py_version() + self._py_version = py_version + else: + py_version = self._py_version + return py_version + + @py_version.setter + def py_version(self, val): + # type: (Optional[PythonVersion]) -> None + self._py_version = val + + @py_version.deleter + def py_version(self): + # type: () -> None + self._py_version = None + + def _iter_pythons(self): + # type: () -> Iterator + if self.is_dir: + for entry in self.children.values(): + if entry is None: + continue + elif entry.is_dir: + for python in entry._iter_pythons(): + yield python + elif entry.is_python and entry.as_python is not None: + yield entry + elif self.is_python and self.as_python is not None: + yield self # type: ignore + + @property + def pythons(self): + # type: () -> DefaultDict[Union[str, Path], PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + def __iter__(self): + # type: () -> Iterator + for entry in self.children.values(): + yield entry + + def __next__(self): + # type: () -> Generator + return next(iter(self)) + + def next(self): + # type: () -> Generator + return self.__next__() + def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -58,35 +321,29 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = ( - "find_all_python_versions" if self.is_dir else "find_python_version" - ) + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( - call_method, - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method, major, minor, patch, pre, dev, arch, name ) if not self.is_dir: return sub_finder(self) - path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + unnested = [sub_finder(path) for path in expand_paths(self)] version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + unnested = [p for p in unnested if p is not None and p.as_python is not None] + paths = sorted(unnested, key=version_sort, reverse=True) + return list(paths) def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -101,55 +358,60 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "matches", major, minor, patch, pre, dev, arch, python_name=name ) - is_py = operator.attrgetter("is_python") - py_version = operator.attrgetter("as_python") if not self.is_dir: if self.is_python and self.as_python and version_matcher(self.py_version): - return attr.evolve(self) - return - finder = ( - (child, child.as_python) - for child in unnest(self.pythons.values()) - if child.as_python - ) - py_filter = filter( - None, filter(lambda child: version_matcher(child[1]), finder) - ) - version_sort = operator.attrgetter("version_sort") - return next( - ( - c[0] - for c in sorted( - py_filter, key=lambda child: child[1].version_sort, reverse=True - ) - ), - None, - ) + return self # type: ignore + + matching_pythons = [ + [entry, entry.as_python.version_sort] + for entry in self._iter_pythons() + if ( + entry is not None + and entry.as_python is not None + and version_matcher(entry.py_version) + ) + ] + results = sorted(matching_pythons, key=operator.itemgetter(1, 0), reverse=True) + return next(iter(r[0] for r in results if r is not None), None) @six.add_metaclass(abc.ABCMeta) class BaseFinder(object): + def __init__(self): + #: Maps executable paths to PathEntries + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + self._versions = defaultdict(PathEntry) # type: Dict[Tuple, PathEntry] + def get_versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] """Return the available versions from the finder""" raise NotImplementedError @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Any, Any) -> BaseFinderType raise NotImplementedError @property def version_paths(self): - return self.versions.values() + # type: () -> Any + return self._versions.values() @property def expanded_paths(self): + # type: () -> Any return (p.paths.values() for p in self.version_paths) + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 221d892f..b855a05d 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -1,66 +1,160 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -import copy import operator import os import sys - from collections import defaultdict from itertools import chain import attr import six - from cached_property import cached_property +from ..compat import Path, fs_str -from vistir.compat import Path, fs_str - -from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR +from ..environment import ( + ASDF_DATA_DIR, + ASDF_INSTALLED, + MYPY_RUNNING, + PYENV_INSTALLED, + PYENV_ROOT, + SHIM_PATHS, + get_shim_paths, +) from ..exceptions import InvalidPythonVersion from ..utils import ( + Iterable, + Sequence, + dedup, ensure_path, + expand_paths, filter_pythons, - looks_like_python, - optional_instance_of, - path_is_known_executable, - unnest, + is_in_path, normalize_path, + optional_instance_of, + parse_asdf_version_order, parse_pyenv_version_order, - parse_asdf_version_order + path_is_known_executable, + split_version_and_name, + unnest, ) -from .python import PythonVersion +from .mixins import BaseFinder, BasePath +if MYPY_RUNNING: + from typing import ( + Optional, + Dict, + DefaultDict, + Iterator, + List, + Union, + Tuple, + Generator, + Callable, + Type, + Any, + TypeVar, + ) + from .python import PythonFinder, PythonVersion + from .windows import WindowsFinder -ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) -PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) -SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] + FinderType = TypeVar("FinderType", BaseFinder, PythonFinder, WindowsFinder) + ChildType = Union[PythonFinder, "PathEntry"] + PathType = Union[PythonFinder, "PathEntry"] @attr.s class SystemPath(object): global_search = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(defaultdict)) - _executables = attr.ib(default=attr.Factory(list)) - _python_executables = attr.ib(default=attr.Factory(list)) - path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib(default=attr.Factory(defaultdict)) - only_python = attr.ib(default=False) - pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) - asdf_finder = attr.ib(default=None) - system = attr.ib(default=False) - _version_dict = attr.ib(default=attr.Factory(defaultdict)) - ignore_unsupported = attr.ib(default=False) + paths = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] + _python_executables = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, PathEntry] + path_order = attr.ib(default=attr.Factory(list)) # type: List[str] + python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] + only_python = attr.ib(default=False, type=bool) + pyenv_finder = attr.ib(default=None) # type: Optional[PythonFinder] + asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] + windows_finder = attr.ib(default=None) # type: Optional[WindowsFinder] + system = attr.ib(default=False, type=bool) + _version_dict = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[Tuple, List[PathEntry]] + ignore_unsupported = attr.ib(default=False, type=bool) - __finders = attr.ib(default=attr.Factory(dict)) + __finders = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): + # type: (str, Union[WindowsFinder, PythonFinder]) -> "SystemPath" if finder_name not in self.__finders: self.__finders[finder_name] = finder + return self + + def clear_caches(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + if key in self.__dict__: + del self.__dict__[key] + for finder in list(self.__finders.keys()): + del self.__finders[finder] + self.__finders = {} + return attr.evolve( + self, + executables=[], + python_executables={}, + python_version_dict=defaultdict(list), + version_dict=defaultdict(list), + pyenv_finder=None, + windows_finder=None, + asdf_finder=None, + path_order=[], + paths=defaultdict(PathEntry), + ) + + def __del__(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + try: + del self.__dict__[key] + except KeyError: + pass + for finder in list(self.__finders.keys()): + del self.__finders[finder] + self.__finders = {} + self._python_executables = {} + self._executables = [] + self.python_version_dict = defaultdict(list) + self._version_dict = defaultdict(list) + self.path_order = [] + self.pyenv_finder = None + self.asdf_finder = None + self.paths = defaultdict(PathEntry) + self.__finders = {} + + @property + def finders(self): + # type: () -> List[str] + return [k for k in self.__finders.keys()] + + @staticmethod + def check_for_pyenv(): + return PYENV_INSTALLED or os.path.exists(normalize_path(PYENV_ROOT)) + + @staticmethod + def check_for_asdf(): + return ASDF_INSTALLED or os.path.exists(normalize_path(ASDF_DATA_DIR)) + + @python_version_dict.default + def create_python_version_dict(self): + # type: () -> DefaultDict[Tuple, List[PythonVersion]] + return defaultdict(list) @cached_property def executables(self): + # type: () -> List[PathEntry] self.executables = [ p for p in chain(*(child.children.values() for child in self.paths.values())) @@ -70,6 +164,7 @@ class SystemPath(object): @cached_property def python_executables(self): + # type: () -> Dict[str, PathEntry] python_executables = {} for child in self.paths.values(): if child.pythons: @@ -82,142 +177,255 @@ class SystemPath(object): @cached_property def version_dict(self): - self._version_dict = defaultdict(list) + # type: () -> DefaultDict[Tuple, List[PathEntry]] + self._version_dict = defaultdict( + list + ) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if type(entry).__name__ == "VersionPath": - for path in entry.paths.values(): - if path not in self._version_dict[version] and path.is_python: - self._version_dict[version].append(path) - continue - continue - elif entry not in self._version_dict[version] and entry.is_python: + if entry not in self._version_dict[version] and entry.is_python: self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): - version = entry.as_python + version = entry.as_python # type: PythonVersion if not version: continue - version = version.version_tuple + if not isinstance(version, tuple): + version = version.version_tuple if version and entry not in self._version_dict[version]: self._version_dict[version].append(entry) return self._version_dict - def __attrs_post_init__(self): - #: slice in pyenv + def _run_setup(self): + # type: () -> "SystemPath" if not self.__class__ == SystemPath: - return - if os.name == "nt": - self._setup_windows() - if PYENV_INSTALLED: - self._setup_pyenv() - if ASDF_INSTALLED: - self._setup_asdf() + return self + new_instance = self + path_order = new_instance.path_order[:] + path_entries = self.paths.copy() + if self.global_search and "PATH" in os.environ: + path_order = path_order + os.environ["PATH"].split(os.pathsep) + path_order = list(dedup(path_order)) + path_instances = [ + ensure_path(p.strip('"')) + for p in path_order + if not any( + is_in_path(normalize_path(str(p)), normalize_path(shim)) + for shim in SHIM_PATHS + ) + ] + path_entries.update( + { + p.as_posix(): PathEntry.create( + path=p.absolute(), is_root=True, only_python=self.only_python + ) + for p in path_instances + } + ) + new_instance = attr.evolve( + new_instance, + path_order=[p.as_posix() for p in path_instances], + paths=path_entries, + ) + if os.name == "nt" and "windows" not in self.finders: + new_instance = new_instance._setup_windows() + #: slice in pyenv + if self.check_for_pyenv() and "pyenv" not in self.finders: + new_instance = new_instance._setup_pyenv() + #: slice in asdf + if self.check_for_asdf() and "asdf" not in self.finders: + new_instance = new_instance._setup_asdf() venv = os.environ.get("VIRTUAL_ENV") if os.name == "nt": bin_dir = "Scripts" else: bin_dir = "bin" - if venv and (self.system or self.global_search): + if venv and (new_instance.system or new_instance.global_search): p = ensure_path(venv) - self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) - if self.system: + path_order = [(p / bin_dir).as_posix()] + new_instance.path_order + new_instance = attr.evolve(new_instance, path_order=path_order) + paths = new_instance.paths.copy() + paths[p] = new_instance.get_path(p.joinpath(bin_dir)) + new_instance = attr.evolve(new_instance, paths=paths) + if new_instance.system: syspath = Path(sys.executable) syspath_bin = syspath.parent if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir - self.path_order = [syspath_bin.as_posix()] + self.path_order - self.paths[syspath_bin] = PathEntry.create( + path_order = [syspath_bin.as_posix()] + new_instance.path_order + paths = new_instance.paths.copy() + paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) + new_instance = attr.evolve(new_instance, path_order=path_order, paths=paths) + return new_instance def _get_last_instance(self, path): + # type: (str) -> int reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) - last_instance = next( - iter(p for p in paths if normalized_target in p), None - ) - try: - path_index = self.path_order.index(last_instance) - except ValueError: - return + last_instance = next(iter(p for p in paths if normalized_target in p), None) + if last_instance is None: + raise ValueError("No instance found on path for target: {0!s}".format(path)) + path_index = self.path_order.index(last_instance) return path_index def _slice_in_paths(self, start_idx, paths): - before_path = self.path_order[: start_idx + 1] - after_path = self.path_order[start_idx + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in paths] + after_path - ) + # type: (int, List[Path]) -> "SystemPath" + before_path = [] # type: List[str] + after_path = [] # type: List[str] + if start_idx == 0: + after_path = self.path_order[:] + elif start_idx == -1: + before_path = self.path_order[:] + else: + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + path_order = before_path + [p.as_posix() for p in paths] + after_path + if path_order == self.path_order: + return self + return attr.evolve(self, path_order=path_order) def _remove_path(self, path): + # type: (str) -> "SystemPath" path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) - path_map = { - normalize_path(pth): pth - for pth in self.paths.keys() - } + path_map = {normalize_path(pth): pth for pth in self.paths.keys()} + new_paths = self.paths.copy() if target in path_map: - del self.paths[path_map.get(target)] + del new_paths[path_map[target]] for current_path in path_copy: normalized = normalize_path(current_path) if normalized != target: new_order.append(normalized) new_order = [p for p in reversed(new_order)] - self.path_order = new_order + return attr.evolve(self, path_order=new_order, paths=new_paths) def _setup_asdf(self): + # type: () -> "SystemPath" + if "asdf" in self.finders and self.asdf_finder is not None: + return self from .python import PythonFinder - self.asdf_finder = PythonFinder.create( - root=ASDF_DATA_DIR, ignore_unsupported=True, - sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") - asdf_index = self._get_last_instance(ASDF_DATA_DIR) - if not asdf_index: + + os_path = os.environ["PATH"].split(os.pathsep) + asdf_finder = PythonFinder.create( + root=ASDF_DATA_DIR, + ignore_unsupported=True, + sort_function=parse_asdf_version_order, + version_glob_path="installs/python/*", + ) + asdf_index = None + try: + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + except ValueError: + asdf_index = 0 if is_in_path(next(iter(os_path), ""), ASDF_DATA_DIR) else -1 + if asdf_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here - return - root_paths = [p for p in self.asdf_finder.roots] - self._slice_in_paths(asdf_index, root_paths) - self.paths.update(self.asdf_finder.roots) - self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) - self._register_finder("asdf", self.asdf_finder) + return self + # * These are the root paths for the finder + _ = [p for p in asdf_finder.roots] + new_instance = self._slice_in_paths(asdf_index, [asdf_finder.root]) + paths = self.paths.copy() + paths[asdf_finder.root] = asdf_finder + paths.update(asdf_finder.roots) + return ( + attr.evolve(new_instance, paths=paths, asdf_finder=asdf_finder) + ._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) + ._register_finder("asdf", asdf_finder) + ) + + def reload_finder(self, finder_name): + # type: (str) -> "SystemPath" + if finder_name is None: + raise TypeError("Must pass a string as the name of the target finder") + finder_attr = "{0}_finder".format(finder_name) + setup_attr = "_setup_{0}".format(finder_name) + try: + current_finder = getattr(self, finder_attr) # type: Any + except AttributeError: + raise ValueError("Must pass a valid finder to reload.") + try: + setup_fn = getattr(self, setup_attr) + except AttributeError: + raise ValueError("Finder has no valid setup function: %s" % finder_name) + if current_finder is None: + # TODO: This is called 'reload', should we load a new finder for the first + # time here? lets just skip that for now to avoid unallowed finders + pass + if (finder_name == "pyenv" and not PYENV_INSTALLED) or ( + finder_name == "asdf" and not ASDF_INSTALLED + ): + # Don't allow loading of finders that aren't explicitly 'installed' as it were + return self + setattr(self, finder_attr, None) + if finder_name in self.__finders: + del self.__finders[finder_name] + return setup_fn() def _setup_pyenv(self): + # type: () -> "SystemPath" + if "pyenv" in self.finders and self.pyenv_finder is not None: + return self from .python import PythonFinder - self.pyenv_finder = PythonFinder.create( - root=PYENV_ROOT, sort_function=parse_pyenv_version_order, - version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported + os_path = os.environ["PATH"].split(os.pathsep) + + pyenv_finder = PythonFinder.create( + root=PYENV_ROOT, + sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", + ignore_unsupported=self.ignore_unsupported, ) - pyenv_index = self._get_last_instance(PYENV_ROOT) - if not pyenv_index: + pyenv_index = None + try: + pyenv_index = self._get_last_instance(PYENV_ROOT) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if pyenv_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here - return - root_paths = [p for p in self.pyenv_finder.roots] - self._slice_in_paths(pyenv_index, root_paths) - - self.paths.update(self.pyenv_finder.roots) - self._remove_path(os.path.join(PYENV_ROOT, "shims")) - self._register_finder("pyenv", self.pyenv_finder) + return self + # * These are the root paths for the finder + _ = [p for p in pyenv_finder.roots] + new_instance = self._slice_in_paths(pyenv_index, [pyenv_finder.root]) + paths = new_instance.paths.copy() + paths[pyenv_finder.root] = pyenv_finder + paths.update(pyenv_finder.roots) + return ( + attr.evolve(new_instance, paths=paths, pyenv_finder=pyenv_finder) + ._remove_path(os.path.join(PYENV_ROOT, "shims")) + ._register_finder("pyenv", pyenv_finder) + ) def _setup_windows(self): + # type: () -> "SystemPath" + if "windows" in self.finders and self.windows_finder is not None: + return self from .windows import WindowsFinder - self.windows_finder = WindowsFinder.create() - root_paths = (p for p in self.windows_finder.paths if p.is_root) + windows_finder = WindowsFinder.create() + root_paths = (p for p in windows_finder.paths if p.is_root) path_addition = [p.path.as_posix() for p in root_paths] - self.path_order = self.path_order[:] + path_addition - self.paths.update({p.path: p for p in root_paths}) - self._register_finder("windows", self.windows_finder) + new_path_order = self.path_order[:] + path_addition + new_paths = self.paths.copy() + new_paths.update({p.path: p for p in root_paths}) + return attr.evolve( + self, + windows_finder=windows_finder, + path_order=new_path_order, + paths=new_paths, + )._register_finder("windows", windows_finder) def get_path(self, path): + # type: (Union[str, Path]) -> PathType + if path is None: + raise TypeError("A path must be provided in order to generate a path entry.") path = ensure_path(path) _path = self.paths.get(path) if not _path: @@ -227,69 +435,90 @@ class SystemPath(object): path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path + if not _path: + raise ValueError("Path not found or generated: {0!r}".format(path)) return _path def _get_paths(self): - return (self.get_path(k) for k in self.path_order) + # type: () -> Generator[Union[PathType, WindowsFinder], None, None] + for path in self.path_order: + try: + entry = self.get_path(path) + except ValueError: + continue + else: + yield entry @cached_property def path_entries(self): - paths = self._get_paths() + # type: () -> List[Union[PathType, WindowsFinder]] + paths = list(self._get_paths()) return paths def find_all(self, executable): - """Search the path for an executable. Return all copies. + # type: (str) -> List[Union[PathEntry, FinderType]] + """ + Search the path for an executable. Return all copies. :param executable: Name of the executable :type executable: str :returns: List[PathEntry] """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return list(filtered) def which(self, executable): - """Search for an executable on the path. + # type: (str) -> Union[PathEntry, None] + """ + Search for an executable on the path. :param executable: Name of the executable to be located. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` object. """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return next(iter(f for f in filtered if f is not None), None) def _filter_paths(self, finder): - return ( - pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) - if pth is not None - ) + # type: (Callable) -> Iterator + for path in self._get_paths(): + if path is None: + continue + python_versions = finder(path) + if python_versions is not None: + for python in python_versions: + if python is not None: + yield python def _get_all_pythons(self, finder): - paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} - paths.update(self.python_executables) - return (p for p in paths.values() if p is not None) + # type: (Callable) -> Iterator + for python in self._filter_paths(finder): + if python is not None and python.is_python: + yield python def get_pythons(self, finder): + # type: (Callable) -> Iterator sort_key = operator.attrgetter("as_python.version_sort") - return ( - k for k in sorted( - (p for p in self._filter_paths(finder) if p.is_python), - key=sort_key, - reverse=True - ) if k is not None - ) + pythons = [entry for entry in self._get_all_pythons(finder)] + for python in sorted(pythons, key=sort_key, reverse=True): + if python is not None: + yield python def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -305,21 +534,12 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_all_python_versions", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if major and not (minor or patch or pre or dev or arch or name): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, major ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -332,14 +552,16 @@ class SystemPath(object): def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] + sort_by_path=False, # type: bool ): + # type: (...) -> PathEntry """Search for a specific python version on the path. :param major: Major python version to search for. @@ -350,39 +572,19 @@ class SystemPath(object): :param bool dev: Search for devreleases (default None) - prioritize releases if None :param str arch: Architecture to include, e.g. '64bit', defaults to None :param str name: The name of a python version, e.g. ``anaconda3-5.3.0`` + :param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False) :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ - if isinstance(major, six.string_types) and not minor and not patch: - # Only proceed if this is in the format "x.y.z" or similar - if major.count(".") > 0 and major[0].isdigit(): - version = major.split(".", 2) - if len(version) > 3: - major, minor, patch, rest = version - elif len(version) == 3: - major, minor, patch = version - else: - major, minor = version - else: - name = "{0!s}".format(major) - major = None + major, minor, patch, name = split_version_and_name(major, minor, patch, name) sub_finder = operator.methodcaller( - "find_python_version", - major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_python_version", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None - if major and not (minor or patch or pre or dev or arch or name): + if name and not (minor or patch or pre or dev or arch or major): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, name ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -393,6 +595,18 @@ class SystemPath(object): windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version + if sort_by_path: + paths = [self.get_path(k) for k in self.path_order] + for path in paths: + found_version = sub_finder(path) + if found_version: + return found_version + if alternate_sub_finder: + for path in paths: + found_version = alternate_sub_finder(path) + if found_version: + return found_version + ver = next(iter(self.get_pythons(sub_finder)), None) if not ver and alternate_sub_finder is not None: ver = next(iter(self.get_pythons(alternate_sub_finder)), None) @@ -406,12 +620,13 @@ class SystemPath(object): @classmethod def create( cls, - path=None, - system=False, - only_python=False, - global_search=True, - ignore_unsupported=True, + path=None, # type: str + system=False, # type: bool + only_python=False, # type: bool + global_search=True, # type: bool + ignore_unsupported=True, # type: bool ): + # type: (...) -> SystemPath """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -423,14 +638,30 @@ class SystemPath(object): :rtype: :class:`pythonfinder.models.SystemPath` """ - path_entries = defaultdict(PathEntry) - paths = [] + path_entries = defaultdict( + PathEntry + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: - paths = os.environ.get("PATH").split(os.pathsep) + if "PATH" in os.environ: + paths = os.environ["PATH"].split(os.pathsep) + path_order = [] # type: List[str] if path: + path_order = [path] + path_instance = ensure_path(path) + path_entries.update( + { + path_instance.as_posix(): PathEntry.create( + path=path_instance.absolute(), + is_root=True, + only_python=only_python, + ) + } + ) paths = [path] + paths + paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)] _path_objects = [ensure_path(p.strip('"')) for p in paths] paths = [p.as_posix() for p in _path_objects] path_entries.update( @@ -439,33 +670,47 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects - if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) - return cls( + instance = cls( paths=path_entries, - path_order=paths, + path_order=path_order, only_python=only_python, system=system, global_search=global_search, ignore_unsupported=ignore_unsupported, ) + instance = instance._run_setup() + return instance @attr.s(slots=True) class PathEntry(BasePath): - path = attr.ib(default=None, validator=optional_instance_of(Path)) - _children = attr.ib(default=attr.Factory(dict)) - is_root = attr.ib(default=True) - only_python = attr.ib(default=False) - name = attr.ib() - py_version = attr.ib() - _pythons = attr.ib(default=attr.Factory(defaultdict)) + is_root = attr.ib(default=True, type=bool, cmp=False) - def __str__(self): - return fs_str("{0}".format(self.path.as_posix())) + def __lt__(self, other): + # type: (BasePath) -> bool + return self.path.as_posix() < other.path.as_posix() + + def __lte__(self, other): + # type: (BasePath) -> bool + return self.path.as_posix() <= other.path.as_posix() + + def __gt__(self, other): + # type: (BasePath) -> bool + return self.path.as_posix() > other.path.as_posix() + + def __gte__(self, other): + # type: (BasePath) -> bool + return self.path.as_posix() >= other.path.as_posix() + + def __del__(self): + if getattr(self, "_children"): + del self._children + BasePath.__del__(self) def _filter_children(self): + # type: () -> Iterator[Path] if self.only_python: children = filter_pythons(self.path) else: @@ -473,86 +718,55 @@ class PathEntry(BasePath): return children def _gen_children(self): + # type: () -> Iterator + shim_paths = get_shim_paths() pass_name = self.name != self.path.name pass_args = {"is_root": False, "only_python": self.only_python} if pass_name: - pass_args["name"] = self.name + if self.name is not None and isinstance(self.name, six.string_types): + pass_args["name"] = self.name # type: ignore + elif self.path is not None and isinstance(self.path.name, six.string_types): + pass_args["name"] = self.path.name # type: ignore if not self.is_dir: - yield (self.path.as_posix(), copy.deepcopy(self)) + yield (self.path.as_posix(), self) elif self.is_root: for child in self._filter_children(): - if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + if any(is_in_path(str(child), shim) for shim in shim_paths): continue if self.only_python: try: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore except (InvalidPythonVersion, ValueError): continue else: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore yield (child.as_posix(), entry) return - @cached_property + @property def children(self): - if not self._children: - children = {} + # type: () -> Dict[str, PathEntry] + children = getattr(self, "_children", {}) # type: Dict[str, PathEntry] + if not children: for child_key, child_val in self._gen_children(): children[child_key] = child_val - self._children = children + self.children = children return self._children - @name.default - def get_name(self): - return self.path.name + @children.setter + def children(self, val): + # type: (Dict[str, PathEntry]) -> None + self._children = val - @py_version.default - def get_py_version(self): - from ..environment import IGNORE_UNSUPPORTED - if self.is_dir: - return None - if self.is_python: - py_version = None - try: - py_version = PythonVersion.from_path(path=self, name=self.name) - except (InvalidPythonVersion, ValueError): - py_version = None - except Exception: - if not IGNORE_UNSUPPORTED: - raise - return py_version - return - - @property - def pythons(self): - if not self._pythons: - if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - self._pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - self._pythons[_path.as_posix()] = self - return self._pythons - - @cached_property - def as_python(self): - py_version = None - if self.py_version: - return self.py_version - if not self.is_dir and self.is_python: - try: - from .python import PythonVersion - py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) - except (ValueError, InvalidPythonVersion): - py_version = None - return py_version + @children.deleter + def children(self): + # type: () -> None + del self._children @classmethod def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): + # type: (Union[str, Path], bool, bool, Dict[str, PythonVersion], Optional[str]) -> PathEntry """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. :param str path: Path to the specified location. @@ -569,53 +783,35 @@ class PathEntry(BasePath): if not name: guessed_name = True name = target.name - creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python, + "name": name, + } if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} - child_creation_args = { - "is_root": False, - "only_python": only_python - } + child_creation_args = {"is_root": False, "only_python": only_python} if not guessed_name: - child_creation_args["name"] = name + child_creation_args["name"] = _new.name # type: ignore for pth, python in pythons.items(): if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): continue pth = ensure_path(pth) - children[pth.as_posix()] = PathEntry( - py_version=python, - path=pth, - **child_creation_args + children[pth.as_posix()] = PathEntry( # type: ignore + py_version=python, path=pth, **child_creation_args ) _new._children = children return _new - @cached_property - def is_dir(self): - try: - ret_val = self.path.is_dir() - except OSError: - ret_val = False - return ret_val - - @cached_property - def is_executable(self): - return path_is_known_executable(self.path) - - @cached_property - def is_python(self): - return self.is_executable and ( - looks_like_python(self.path.name) - ) - @attr.s class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - name = attr.ib(default=None) + base = attr.ib(default=None, validator=optional_instance_of(Path)) # type: Path + name = attr.ib(default=None) # type: str @classmethod def create(cls, path, only_python=True, pythons=None, name=None): @@ -623,6 +819,7 @@ class VersionPath(SystemPath): Generates the version listings for it""" from .path import PathEntry + path = ensure_path(path) path_entries = defaultdict(PathEntry) bin_ = "{base}/bin" diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4fcbbca6..619e7761 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -1,170 +1,266 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function -import copy -import platform -import operator import logging - +import operator +import platform +import sys from collections import defaultdict import attr +import six +from packaging.version import Version -from packaging.version import Version, LegacyVersion -from packaging.version import parse as parse_version -from vistir.compat import Path - -from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR +from ..compat import Path, lru_cache +from ..environment import ASDF_DATA_DIR, MYPY_RUNNING, PYENV_ROOT, SYSTEM_ARCH from ..exceptions import InvalidPythonVersion -from .mixins import BaseFinder, BasePath from ..utils import ( + RE_MATCHER, _filter_none, ensure_path, get_python_version, - optional_instance_of, - unnest, + guess_company, is_in_path, - parse_pyenv_version_order, + looks_like_python, + optional_instance_of, parse_asdf_version_order, + parse_pyenv_version_order, parse_python_version, + path_is_pythoncore, + unnest, ) +from .mixins import BaseFinder, BasePath + +if MYPY_RUNNING: + from typing import ( + DefaultDict, + Optional, + Callable, + Generator, + Any, + Union, + Tuple, + List, + Dict, + Type, + TypeVar, + Iterator, + overload, + ) + from .path import PathEntry + from .._vendor.pep514tools.environment import Environment +else: + + def overload(f): + return f + logger = logging.getLogger(__name__) @attr.s(slots=True) class PythonFinder(BaseFinder, BasePath): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - #: ignore_unsupported should come before versions, because its value is used - #: in versions's default initializer. - ignore_unsupported = attr.ib(default=True) - #: The function to use to sort version order when returning an ordered verion set - sort_function = attr.ib(default=None) - paths = attr.ib(default=attr.Factory(list)) - roots = attr.ib(default=attr.Factory(defaultdict)) + root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path) + # should come before versions, because its value is used in versions's default initializer. + #: Whether to ignore any paths which raise exceptions and are not actually python + ignore_unsupported = attr.ib(default=True, type=bool) #: Glob path for python versions off of the root directory - version_glob_path = attr.ib(default="versions/*") - versions = attr.ib() - pythons = attr.ib() + version_glob_path = attr.ib(default="versions/*", type=str) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) # type: Callable + #: The root locations used for discovery + roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + #: List of paths discovered during search + paths = attr.ib(type=list) + #: shim directory + shim_dir = attr.ib(default="shims", type=str) + #: Versions discovered in the specified paths + _versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + _pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + + def __del__(self): + # type: () -> None + self._versions = defaultdict() + self._pythons = defaultdict() + self.roots = defaultdict() + self.paths = [] @property def expanded_paths(self): + # type: () -> Generator return ( - path for path in unnest(p for p in self.versions.values()) - if path is not None + path for path in unnest(p for p in self.versions.values()) if path is not None ) @property def is_pyenv(self): + # type: () -> bool return is_in_path(str(self.root), PYENV_ROOT) @property def is_asdf(self): + # type: () -> bool return is_in_path(str(self.root), ASDF_DATA_DIR) def get_version_order(self): + # type: () -> List[Path] version_paths = [ - p for p in self.root.glob(self.version_glob_path) + p + for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} + version_order = [] # type: List[Path] if self.is_pyenv: - version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_pyenv_version_order() if v in versions + ] elif self.is_asdf: - version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_asdf_version_order() if v in versions + ] for version in version_order: - version_paths.remove(version) + if version in version_paths: + version_paths.remove(version) if version_order: version_order += version_paths else: version_order = version_paths return version_order + def get_bin_dir(self, base): + # type: (Union[Path, str]) -> Path + if isinstance(base, six.string_types): + base = Path(base) + return base / "bin" + @classmethod - def version_from_bin_dir(cls, base_dir, name=None): - from .path import PathEntry + def version_from_bin_dir(cls, entry): + # type: (PathEntry) -> Optional[PathEntry] py_version = None - version_path = PathEntry.create( - path=base_dir.absolute().as_posix(), - only_python=True, - name=base_dir.parent.name, - ) - py_version = next(iter(version_path.find_all_python_versions()), None) + py_version = next(iter(entry.find_all_python_versions()), None) return py_version - @versions.default - def get_versions(self): + def _iter_version_bases(self): + # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry - versions = defaultdict() - bin_ = "{base}/bin" + for p in self.get_version_order(): - bin_dir = Path(bin_.format(base=p.as_posix())) - version_path = None - if bin_dir.exists(): - version_path = PathEntry.create( - path=bin_dir.absolute().as_posix(), - only_python=False, - name=p.name, - is_root=True, + bin_dir = self.get_bin_dir(p) + if bin_dir.exists() and bin_dir.is_dir(): + entry = PathEntry.create( + path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True ) + self.roots[p] = entry + yield (p, entry) + + def _iter_versions(self): + # type: () -> Iterator[Tuple[Path, PathEntry, Tuple]] + for base_path, entry in self._iter_version_bases(): version = None + version_entry = None try: - version = PythonVersion.parse(p.name) + version = PythonVersion.parse(entry.name) except (ValueError, InvalidPythonVersion): - entry = next(iter(version_path.find_all_python_versions()), None) - if not entry: - if self.ignore_unsupported: - continue - raise - else: - version = entry.py_version.as_dict() + version_entry = next(iter(entry.find_all_python_versions()), None) + if version is None: + if not self.ignore_unsupported: + raise + continue + if version_entry is not None: + version = version_entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - "Unsupported Python version %r, ignoring...", p.name, exc_info=True + "Unsupported Python version %r, ignoring...", + base_path.name, + exc_info=True, ) continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug"), - ) - self.roots[p] = version_path - versions[version_tuple] = version_path - self.paths.append(version_path) - return versions + if version is not None: + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), + ) + yield (base_path, entry, version_tuple) + + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + for base_path, entry, version_tuple in self._iter_versions(): + self._versions[version_tuple] = entry + return self._versions + + def _iter_pythons(self): + # type: () -> Iterator + for path, entry, version_tuple in self._iter_versions(): + if path.as_posix() in self._pythons: + yield self._pythons[path.as_posix()] + elif version_tuple not in self.versions: + for python in entry.find_all_python_versions(): + yield python + else: + yield self.versions[version_tuple] + + @paths.default + def get_paths(self): + # type: () -> List[PathEntry] + _paths = [base for _, base in self._iter_version_bases()] + return _paths + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value - @pythons.default def get_pythons(self): - pythons = defaultdict() - for p in self.paths: - pythons.update(p.pythons) - return pythons + # type: () -> DefaultDict[str, PathEntry] + return self.pythons + @overload @classmethod - def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + def create(cls, root, sort_function, version_glob_path=None, ignore_unsupported=True): + # type: (str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" - return cls(root=root, ignore_unsupported=ignore_unsupported, - sort_function=sort_function, version_glob_path=version_glob_path) + return cls( + root=root, + path=root, + ignore_unsupported=ignore_unsupported, # type: ignore + sort_function=sort_function, + version_glob_path=version_glob_path, + ) def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -179,36 +275,37 @@ class PythonFinder(BaseFinder, BasePath): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + sub_finder = operator.methodcaller( + call_method, major, minor, patch, pre, dev, arch, name ) - py = operator.attrgetter("as_python") - pythons = ( - py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) - if py_ver is not None - ) - # pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return sorted(matching_versions, key=version_sort, reverse=True) + if not any([major, minor, patch, name]): + pythons = [ + next(iter(py for py in base.find_all_python_versions()), None) + for _, base in self._iter_version_bases() + ] + else: + pythons = [sub_finder(path) for path in self.paths] + pythons = [p for p in pythons if p and p.is_python and p.as_python is not None] + version_sort = operator.attrgetter("as_python.version_sort") + paths = [ + p + for p in sorted(list(pythons), key=version_sort, reverse=True) + if p is not None + ] + return paths def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -222,46 +319,85 @@ class PythonFinder(BaseFinder, BasePath): :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + sub_finder = operator.methodcaller( + "find_python_version", major, minor, patch, pre, dev, arch, name ) - pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + version_sort = operator.attrgetter("as_python.version_sort") + unnested = [sub_finder(self.roots[path]) for path in self.roots] + unnested = [ + p + for p in unnested + if p is not None and p.is_python and p.as_python is not None + ] + paths = sorted(list(unnested), key=version_sort, reverse=True) + return next(iter(p for p in paths if p is not None), None) + + def which(self, name): + # type: (str) -> Optional[PathEntry] + """Search in this path for an executable. + + :param executable: The name of an executable to search for. + :type executable: str + :returns: :class:`~pythonfinder.models.PathEntry` instance. + """ + + matches = (p.which(name) for p in self.paths) + non_empty_match = next(iter(m for m in matches if m is not None), None) + return non_empty_match @attr.s(slots=True) class PythonVersion(object): - major = attr.ib(default=0) - minor = attr.ib(default=None) - patch = attr.ib(default=0) - is_prerelease = attr.ib(default=False) - is_postrelease = attr.ib(default=False) - is_devrelease = attr.ib(default=False) - is_debug = attr.ib(default=False) - version = attr.ib(default=None) - architecture = attr.ib(default=None) - comes_from = attr.ib(default=None) - executable = attr.ib(default=None) - name = attr.ib(default=None) + major = attr.ib(default=0, type=int) + minor = attr.ib(default=None) # type: Optional[int] + patch = attr.ib(default=None) # type: Optional[int] + is_prerelease = attr.ib(default=False, type=bool) + is_postrelease = attr.ib(default=False, type=bool) + is_devrelease = attr.ib(default=False, type=bool) + is_debug = attr.ib(default=False, type=bool) + version = attr.ib(default=None) # type: Version + architecture = attr.ib(default=None) # type: Optional[str] + comes_from = attr.ib(default=None) # type: Optional[PathEntry] + executable = attr.ib(default=None) # type: Optional[str] + company = attr.ib(default=None) # type: Optional[str] + name = attr.ib(default=None, type=str) + + def __getattribute__(self, key): + result = super(PythonVersion, self).__getattribute__(key) + if key in ["minor", "patch"] and result is None: + executable = None # type: Optional[str] + if self.executable: + executable = self.executable + elif self.comes_from: + executable = self.comes_from.path.as_posix() + if executable is not None: + if not isinstance(executable, six.string_types): + executable = executable.as_posix() + instance_dict = self.parse_executable(executable) + for k in instance_dict.keys(): + try: + super(PythonVersion, self).__getattribute__(k) + except AttributeError: + continue + else: + setattr(self, k, instance_dict[k]) + result = instance_dict.get(key) + return result @property def version_sort(self): - """version_sort tuple for sorting against other instances of the same class. - - Returns a tuple of the python version but includes a point for non-dev, - and a point for non-prerelease versions. So released versions will have 2 points - for this value. E.g. `(3, 6, 6, 2)` is a release, `(3, 6, 6, 1)` is a prerelease, - `(3, 6, 6, 0)` is a dev release, and `(3, 6, 6, 3)` is a postrelease. + # type: () -> Tuple[int, int, Optional[int], int, int] """ + A tuple for sorting against other instances of the same class. + + Returns a tuple of the python version but includes points for core python, + non-dev, and non-prerelease versions. So released versions will have 2 points + for this value. E.g. ``(1, 3, 6, 6, 2)`` is a release, ``(1, 3, 6, 6, 1)`` is a + prerelease, ``(1, 3, 6, 6, 0)`` is a dev release, and ``(1, 3, 6, 6, 3)`` is a + postrelease. ``(0, 3, 7, 3, 2)`` represents a non-core python release, e.g. by + a repackager of python like Continuum. + """ + company_sort = 1 if (self.company and self.company == "PythonCore") else 0 release_sort = 2 if self.is_postrelease: release_sort = 3 @@ -271,11 +407,19 @@ class PythonVersion(object): release_sort = 0 elif self.is_debug: release_sort = 1 - return (self.major, self.minor, self.patch if self.patch else 0, release_sort) + return ( + company_sort, + self.major, + self.minor, + self.patch if self.patch else 0, + release_sort, + ) @property def version_tuple(self): - """Provides a version tuple for using as a dictionary key. + # type: () -> Tuple[int, Optional[int], Optional[int], bool, bool, bool] + """ + Provides a version tuple for using as a dictionary key. :return: A tuple describing the python version meetadata contained. :rtype: tuple @@ -292,45 +436,52 @@ class PythonVersion(object): def matches( self, - major=None, - minor=None, - patch=None, - pre=False, - dev=False, - arch=None, - debug=False, - name=None, + major=None, # type: Optional[int] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=False, # type: bool + dev=False, # type: bool + arch=None, # type: Optional[str] + debug=False, # type: bool + python_name=None, # type: Optional[str] ): + # type: (...) -> bool + result = False if arch: own_arch = self.get_architecture() if arch.isdigit(): arch = "{0}bit".format(arch) - return ( - (major is None or self.major == major) - and (minor is None or self.minor == minor) - and (patch is None or self.patch == patch) + if ( + (major is None or self.major and self.major == major) + and (minor is None or self.minor and self.minor == minor) + and (patch is None or self.patch and self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) and ( - name is None - or (name and self.name) - and (self.name == name or self.name.startswith(name)) + python_name is None + or (python_name and self.name) + and (self.name == python_name or self.name.startswith(python_name)) ) - ) + ): + result = True + return result def as_major(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"minor": None, "patch": None}) return self.create(**self_dict) def as_minor(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"patch": None}) return self.create(**self_dict) def as_dict(self): + # type: () -> Dict[str, Union[int, bool, Version, None]] return { "major": self.major, "minor": self.minor, @@ -340,37 +491,70 @@ class PythonVersion(object): "is_devrelease": self.is_devrelease, "is_debug": self.is_debug, "version": self.version, + "company": self.company, } + def update_metadata(self, metadata): + # type: (Dict[str, Union[str, int, Version]]) -> None + """ + Update the metadata on the current :class:`pythonfinder.models.python.PythonVersion` + + Given a parsed version dictionary from :func:`pythonfinder.utils.parse_python_version`, + update the instance variables of the current version instance to reflect the newly + supplied values. + """ + + for key in metadata: + try: + _ = getattr(self, key) + except AttributeError: + continue + else: + setattr(self, key, metadata[key]) + @classmethod + @lru_cache(maxsize=1024) def parse(cls, version): - """Parse a valid version string into a dictionary + # type: (str) -> Dict[str, Union[str, int, Version]] + """ + Parse a valid version string into a dictionary Raises: ValueError -- Unable to parse version string ValueError -- Not a valid python version + TypeError -- NoneType or unparseable type passed in - :param version: A valid version string - :type version: str + :param str version: A valid version string :return: A dictionary with metadata about the specified python version. - :rtype: dict. + :rtype: dict """ + if version is None: + raise TypeError("Must pass a value to parse!") version_dict = parse_python_version(str(version)) if not version_dict: raise ValueError("Not a valid python version: %r" % version) return version_dict def get_architecture(self): + # type: () -> str if self.architecture: return self.architecture - arch, _ = platform.architecture(self.comes_from.path.as_posix()) + arch = None + if self.comes_from is not None: + arch, _ = platform.architecture(self.comes_from.path.as_posix()) + elif self.executable is not None: + arch, _ = platform.architecture(self.executable) + if arch is None: + arch, _ = platform.architecture(sys.executable) self.architecture = arch return self.architecture @classmethod - def from_path(cls, path, name=None, ignore_unsupported=True): - """Parses a python version from a system path. + def from_path(cls, path, name=None, ignore_unsupported=True, company=None): + # type: (Union[str, PathEntry], Optional[str], bool, Optional[str]) -> PythonVersion + """ + Parses a python version from a system path. Raises: ValueError -- Not a valid python path @@ -379,6 +563,7 @@ class PythonVersion(object): :type path: str or :class:`~pythonfinder.models.path.PathEntry` instance :param str name: Name of the python distribution in question :param bool ignore_unsupported: Whether to ignore or error on unsupported paths. + :param Optional[str] company: The company or vendor packaging the distribution. :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ @@ -388,26 +573,65 @@ class PythonVersion(object): if not isinstance(path, PathEntry): path = PathEntry.create(path, is_root=False, only_python=True, name=name) from ..environment import IGNORE_UNSUPPORTED + ignore_unsupported = ignore_unsupported or IGNORE_UNSUPPORTED + path_name = getattr(path, "name", path.path.name) # str if not path.is_python: if not (ignore_unsupported or IGNORE_UNSUPPORTED): raise ValueError("Not a valid python path: %s" % path.path) - py_version = get_python_version(path.path.absolute().as_posix()) - instance_dict = cls.parse(py_version.strip()) - if not isinstance(instance_dict.get("version"), Version) and not ignore_unsupported: - raise ValueError("Not a valid python path: %s" % path.path) - if not name: - name = path.name + try: + instance_dict = cls.parse(path_name) + except Exception: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + else: + if instance_dict.get("minor") is None and looks_like_python(path.path.name): + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + + if ( + not isinstance(instance_dict.get("version"), Version) + and not ignore_unsupported + ): + raise ValueError("Not a valid python path: %s" % path) + if instance_dict.get("patch") is None: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + if name is None: + name = path_name + if company is None: + company = guess_company(path.path.as_posix()) instance_dict.update( - {"comes_from": path, "name": name} + {"comes_from": path, "name": name, "executable": path.path.as_posix()} ) - return cls(**instance_dict) + return cls(**instance_dict) # type: ignore @classmethod - def from_windows_launcher(cls, launcher_entry, name=None): + @lru_cache(maxsize=1024) + def parse_executable(cls, path): + # type: (str) -> Dict[str, Optional[Union[str, int, Version]]] + result_dict = {} # type: Dict[str, Optional[Union[str, int, Version]]] + result_version = None # type: Optional[str] + if path is None: + raise TypeError("Must pass a valid path to parse.") + if not isinstance(path, six.string_types): + path = path.as_posix() + # if not looks_like_python(path): + # raise ValueError("Path %r does not look like a valid python path" % path) + try: + result_version = get_python_version(path) + except Exception: + raise ValueError("Not a valid python path: %r" % path) + if result_version is None: + raise ValueError("Not a valid python path: %s" % path) + result_dict = cls.parse(result_version.strip()) + return result_dict + + @classmethod + def from_windows_launcher(cls, launcher_entry, name=None, company=None): + # type: (Environment, Optional[str], Optional[str]) -> PythonVersion """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. + :param Optional[str] name: The name of the distribution. + :param Optional[str] company: The name of the distributing company. :return: An instance of a PythonVersion. :rtype: :class:`~pythonfinder.models.python.PythonVersion` """ @@ -422,24 +646,26 @@ class PythonVersion(object): exe_path = ensure_path( getattr(launcher_entry.info.install_path, "executable_path", default_path) ) + company = getattr(launcher_entry, "company", guess_company(exe_path.as_posix())) creation_dict.update( { "architecture": getattr( launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, - "name": name + "name": name, + "company": company, } ) py_version = cls.create(**creation_dict) comes_from = PathEntry.create(exe_path, only_python=True, name=name) - comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from py_version.name = comes_from.name return py_version @classmethod def create(cls, **kwargs): + # type: (...) -> PythonVersion if "architecture" in kwargs: if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) @@ -448,24 +674,32 @@ class PythonVersion(object): @attr.s class VersionMap(object): - versions = attr.ib(default=attr.Factory(defaultdict(list))) + versions = attr.ib( + factory=defaultdict + ) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] def add_entry(self, entry): - version = entry.as_python + # type: (...) -> None + version = entry.as_python # type: PythonVersion if version: - entries = self.versions[version.version_tuple] + _ = self.versions[version.version_tuple] paths = {p.path for p in self.versions.get(version.version_tuple, [])} if entry.path not in paths: self.versions[version.version_tuple].append(entry) def merge(self, target): + # type: (VersionMap) -> None for version, entries in target.versions.items(): if version not in self.versions: self.versions[version] = entries else: - current_entries = {p.path for p in self.versions.get(version)} + current_entries = { + p.path + for p in self.versions[version] # type: ignore + if version in self.versions + } new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append( + self.versions[version].extend( [e for e in entries if e.path in new_entries] ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f985630f..a0e69b03 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -2,63 +2,61 @@ from __future__ import absolute_import, print_function import operator - from collections import defaultdict import attr +from ..environment import MYPY_RUNNING from ..exceptions import InvalidPythonVersion from ..utils import ensure_path from .mixins import BaseFinder from .path import PathEntry from .python import PythonVersion, VersionMap +if MYPY_RUNNING: + from typing import DefaultDict, Tuple, List, Optional, Union, TypeVar, Type, Any + + FinderType = TypeVar("FinderType") + @attr.s class WindowsFinder(BaseFinder): - paths = attr.ib(default=attr.Factory(list)) - version_list = attr.ib(default=attr.Factory(list)) - versions = attr.ib() - pythons = attr.ib() + paths = attr.ib(default=attr.Factory(list), type=list) + version_list = attr.ib(default=attr.Factory(list), type=list) + _versions = attr.ib() # type: DefaultDict[Tuple, PathEntry] + _pythons = attr.ib() # type: DefaultDict[str, PathEntry] def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - ) - py_filter = filter( - None, filter(lambda c: version_matcher(c), self.version_list) + "matches", major, minor, patch, pre, dev, arch, python_name=name ) + pythons = [py for py in self.version_list if version_matcher(py)] version_sort = operator.attrgetter("version_sort") - return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] + return [c.comes_from for c in sorted(pythons, key=version_sort, reverse=True)] def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] return next( - ( + iter( v for v in self.find_all_python_versions( major=major, @@ -67,21 +65,24 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, - name=None, + name=name, ) ), None, ) - @versions.default + @_versions.default def get_versions(self): - versions = defaultdict(PathEntry) + # type: () -> DefaultDict[Tuple, PathEntry] + versions = defaultdict(PathEntry) # type: DefaultDict[Tuple, PathEntry] from pythonfinder._vendor.pep514tools import environment as pep514env env_versions = pep514env.findall() path = None for version_object in env_versions: install_path = getattr(version_object.info, "install_path", None) + name = getattr(version_object, "tag", None) + company = getattr(version_object, "company", None) if install_path is None: continue try: @@ -89,28 +90,54 @@ class WindowsFinder(BaseFinder): except AttributeError: continue try: - py_version = PythonVersion.from_windows_launcher(version_object) + py_version = PythonVersion.from_windows_launcher( + version_object, name=name, company=company + ) except InvalidPythonVersion: continue + if py_version is None: + continue self.version_list.append(py_version) + python_path = ( + py_version.comes_from.path + if py_version.comes_from + else py_version.executable + ) + python_kwargs = {python_path: py_version} if python_path is not None else {} base_dir = PathEntry.create( - path, - is_root=True, - only_python=True, - pythons={py_version.comes_from.path: py_version}, + path, is_root=True, only_python=True, pythons=python_kwargs ) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) return versions - @pythons.default + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + self._versions = self.get_versions() + return self._versions + + @_pythons.default def get_pythons(self): - pythons = defaultdict() + # type: () -> DefaultDict[str, PathEntry] + pythons = defaultdict() # type: DefaultDict[str, PathEntry] for version in self.version_list: _path = ensure_path(version.comes_from.path) pythons[_path.as_posix()] = version.comes_from return pythons + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value + @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Type[FinderType], Any, Any) -> FinderType return cls() diff --git a/pipenv/vendor/pythonfinder/pep514tools.LICENSE b/pipenv/vendor/pythonfinder/pep514tools.LICENSE new file mode 100644 index 00000000..c7ac395f --- /dev/null +++ b/pipenv/vendor/pythonfinder/pep514tools.LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2016 Steve Dower + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 011754ea..96737355 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -1,62 +1,128 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import -import os -import six +from __future__ import absolute_import, print_function + +import importlib import operator -from .models import SystemPath -from vistir.compat import lru_cache +import os + +import six +from click import secho + +from . import environment +from .compat import lru_cache +from .exceptions import InvalidPythonVersion +from .utils import Iterable, filter_pythons, version_re + +if environment.MYPY_RUNNING: + from typing import Optional, Dict, Any, Union, List, Iterator, Text + from .models.path import Path, PathEntry + from .models.windows import WindowsFinder + from .models.path import SystemPath + + STRING_TYPE = Union[str, Text, bytes] class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): - """ - Finder A cross-platform Finder for locating python and other executables. - Searches for python and other specified binaries starting in `path`, if supplied, - but searching the bin path of `sys.executable` if `system=True`, and then - searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of - `path` and `system`. + """ + A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in *path*, if supplied, + but searching the bin path of ``sys.executable`` if *system* is ``True``, and then + searching in the ``os.environ['PATH']`` if *global_search* is ``True``. When *global_search* + is ``False``, this search operation is restricted to the allowed locations of + *path* and *system*. + """ + + def __init__( + self, + path=None, + system=False, + global_search=True, + ignore_unsupported=True, + sort_by_path=False, + ): + # type: (Optional[str], bool, bool, bool, bool) -> None + """Create a new :class:`~pythonfinder.pythonfinder.Finder` instance. :param path: A bin-directory search location, defaults to None :param path: str, optional - :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: Whether to include the bin-dir of ``sys.executable``, defaults to False :param system: bool, optional :param global_search: Whether to search the global path from os.environ, defaults to True :param global_search: bool, optional - :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an error is raised, defaults to True + :param ignore_unsupported: Whether to ignore unsupported python versions, if False, an + error is raised, defaults to True :param ignore_unsupported: bool, optional + :param bool sort_by_path: Whether to always sort by path :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ - self.path_prepend = path - self.global_search = global_search - self.system = system - self.ignore_unsupported = ignore_unsupported - self._system_path = None - self._windows_finder = None + self.path_prepend = path # type: Optional[str] + self.global_search = global_search # type: bool + self.system = system # type: bool + self.sort_by_path = sort_by_path # type: bool + self.ignore_unsupported = ignore_unsupported # type: bool + self._system_path = None # type: Optional[SystemPath] + self._windows_finder = None # type: Optional[WindowsFinder] def __hash__(self): + # type: () -> int return hash( (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) ) def __eq__(self, other): + # type: (Any) -> bool return self.__hash__() == other.__hash__() + def create_system_path(self): + # type: () -> SystemPath + pyfinder_path = importlib.import_module("pythonfinder.models.path") + return pyfinder_path.SystemPath.create( + path=self.path_prepend, + system=self.system, + global_search=self.global_search, + ignore_unsupported=self.ignore_unsupported, + ) + + def reload_system_path(self): + # type: () -> None + """ + Rebuilds the base system path and all of the contained finders within it. + + This will re-apply any changes to the environment or any version changes on the system. + """ + + if self._system_path is not None: + self._system_path = self._system_path.clear_caches() + self._system_path = None + pyfinder_path = importlib.import_module("pythonfinder.models.path") + six.moves.reload_module(pyfinder_path) + self._system_path = self.create_system_path() + + def rehash(self): + # type: () -> "Finder" + if not self._system_path: + self._system_path = self.create_system_path() + self.find_all_python_versions.cache_clear() + self.find_python_version.cache_clear() + if self._windows_finder is not None: + self._windows_finder = None + filter_pythons.cache_clear() + self.reload_system_path() + return self + @property def system_path(self): - if not self._system_path: - self._system_path = SystemPath.create( - path=self.path_prepend, - system=self.system, - global_search=self.global_search, - ignore_unsupported=self.ignore_unsupported, - ) + # type: () -> SystemPath + if self._system_path is None: + self._system_path = self.create_system_path() return self._system_path @property def windows_finder(self): + # type: () -> Optional[WindowsFinder] if os.name == "nt" and not self._windows_finder: from .models import WindowsFinder @@ -64,13 +130,127 @@ class Finder(object): return self._windows_finder def which(self, exe): + # type: (str) -> Optional[PathEntry] return self.system_path.which(exe) + @classmethod + def parse_major( + cls, + major, # type: Optional[str] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + ): + # type: (...) -> Dict[str, Union[int, str, bool, None]] + from .models import PythonVersion + + major_is_str = major and isinstance(major, six.string_types) + is_num = ( + major + and major_is_str + and all(part.isdigit() for part in major.split(".")[:2]) + ) + major_has_arch = ( + arch is None + and major + and major_is_str + and "-" in major + and major[0].isdigit() + ) + name = None + if major and major_has_arch: + orig_string = "{0!s}".format(major) + major, _, arch = major.rpartition("-") + if arch: + arch = arch.lower().lstrip("x").replace("bit", "") + if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): + major = orig_string + arch = None + else: + arch = "{0}bit".format(arch) + try: + version_dict = PythonVersion.parse(major) + except (ValueError, InvalidPythonVersion): + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} + elif major and major[0].isalpha(): + return {"major": None, "name": major, "arch": arch} + elif major and is_num: + match = version_re.match(major) + version_dict = match.groupdict() if match else {} # type: ignore + version_dict.update( + { + "is_prerelease": bool(version_dict.get("prerel", False)), + "is_devrelease": bool(version_dict.get("dev", False)), + } + ) + else: + version_dict = { + "major": major, + "minor": minor, + "patch": patch, + "pre": pre, + "dev": dev, + "arch": arch, + } + if not version_dict.get("arch") and arch: + version_dict["arch"] = arch + version_dict["minor"] = ( + int(version_dict["minor"]) if version_dict.get("minor") is not None else minor + ) + version_dict["patch"] = ( + int(version_dict["patch"]) if version_dict.get("patch") is not None else patch + ) + version_dict["major"] = ( + int(version_dict["major"]) if version_dict.get("major") is not None else major + ) + if not (version_dict["major"] or version_dict.get("name")): + version_dict["major"] = major + if name: + version_dict["name"] = name + return version_dict + @lru_cache(maxsize=1024) def find_python_version( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None + self, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] + sort_by_path=False, # type: bool ): - from .models import PythonVersion + # type: (...) -> Optional[PathEntry] + """ + Find the python version which corresponds most closely to the version requested. + + :param Union[str, int] major: The major version to look for, or the full version, or the name of the target version. + :param Optional[int] minor: The minor version. If provided, disables string-based lookups from the major version field. + :param Optional[int] patch: The patch version. + :param Optional[bool] pre: If provided, specifies whether to search pre-releases. + :param Optional[bool] dev: If provided, whether to search dev-releases. + :param Optional[str] arch: If provided, which architecture to search. + :param Optional[str] name: *Name* of the target python, e.g. ``anaconda3-5.3.0`` + :param bool sort_by_path: Whether to sort by path -- default sort is by version(default: False) + :return: A new *PathEntry* pointer at a matching python version, if one can be located. + :rtype: :class:`pythonfinder.models.path.PathEntry` + """ + + minor = int(minor) if minor is not None else minor + patch = int(patch) if patch is not None else patch + + version_dict = { + "minor": minor, + "patch": patch, + "name": name, + "arch": arch, + } # type: Dict[str, Union[str, int, Any]] if ( isinstance(major, six.string_types) @@ -79,71 +259,79 @@ class Finder(object): and dev is None and patch is None ): - if arch is None and "-" in major: - orig_string = "{0!s}".format(major) - major, _, arch = major.rpartition("-") - if arch.startswith("x"): - arch = arch.lstrip("x") - if arch.lower().endswith("bit"): - arch = arch.lower().replace("bit", "") - if not (arch.isdigit() and (int(arch) & int(arch) - 1) == 0): - major = orig_string - arch = None - else: - arch = "{0}bit".format(arch) - try: - version_dict = PythonVersion.parse(major) - except ValueError: - if name is None: - name = "{0!s}".format(major) - major = None - version_dict = {} - major = version_dict.get("major", major) - minor = version_dict.get("minor", minor) - patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is None else pre - dev = version_dict.get("is_devrelease", dev) if dev is None else dev - arch = version_dict.get("architecture", arch) if arch is None else arch - if os.name == "nt": - match = self.windows_finder.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + version_dict = self.parse_major(major, minor=minor, patch=patch, arch=arch) + major = version_dict["major"] + minor = version_dict.get("minor", minor) # type: ignore + patch = version_dict.get("patch", patch) # type: ignore + arch = version_dict.get("arch", arch) # type: ignore + name = version_dict.get("name", name) # type: ignore + _pre = version_dict.get("is_prerelease", pre) + pre = bool(_pre) if _pre is not None else pre + _dev = version_dict.get("is_devrelease", dev) + dev = bool(_dev) if _dev is not None else dev + if "architecture" in version_dict and isinstance( + version_dict["architecture"], six.string_types + ): + arch = version_dict["architecture"] # type: ignore + if os.name == "nt" and self.windows_finder is not None: + found = self.windows_finder.find_python_version( + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) - if match: - return match + if found: + return found return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, + sort_by_path=self.sort_by_path, ) @lru_cache(maxsize=1024) def find_all_python_versions( - self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None + self, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] version_sort = operator.attrgetter("as_python.version_sort") - python_version_dict = getattr(self.system_path, "python_version_dict") + python_version_dict = getattr(self.system_path, "python_version_dict", {}) if python_version_dict: - paths = filter( - None, - [ - path - for version in python_version_dict.values() - for path in version - if path.as_python - ], + paths = ( + path + for version in python_version_dict.values() + for path in version + if path is not None and path.as_python ) - paths = sorted(paths, key=version_sort, reverse=True) - return paths + path_list = sorted(paths, key=version_sort, reverse=True) + return path_list versions = self.system_path.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - if not isinstance(versions, list): + if not isinstance(versions, Iterable): versions = [versions] - paths = sorted(versions, key=version_sort, reverse=True) - path_map = {} - for path in paths: + # This list has already been mostly sorted on windows, we don't need to reverse it again + path_list = sorted(versions, key=version_sort, reverse=True) + path_map = {} # type: Dict[str, PathEntry] + for path in path_list: try: resolved_path = path.path.resolve() except OSError: resolved_path = path.path.absolute() if not path_map.get(resolved_path.as_posix()): path_map[resolved_path.as_posix()] = path - return list(path_map.values()) + return path_list diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 18441919..8150545c 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,60 +1,88 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import io import itertools import os - +import re +import subprocess +from collections import OrderedDict from fnmatch import fnmatch +from threading import Timer import attr -import io -import re import six - -import vistir - from packaging.version import LegacyVersion, Version -from .environment import PYENV_ROOT, ASDF_DATA_DIR, MYPY_RUNNING +from .compat import Path, lru_cache, TimeoutError # noqa +from .environment import MYPY_RUNNING, PYENV_ROOT, SUBPROCESS_TIMEOUT from .exceptions import InvalidPythonVersion -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable - -try: - from functools import lru_cache -except ImportError: - from backports.functools_lru_cache import lru_cache +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore # noqa +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa +# fmt: off +from six.moves import Iterable # type: ignore # noqa # isort:skip +from six.moves import Sequence # type: ignore # noqa # isort:skip +# fmt: on if MYPY_RUNNING: - from typing import Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional - from attr.validators import _OptionalValidator + from typing import Any, Union, List, Callable, Set, Tuple, Dict, Optional, Iterator + from attr.validators import _OptionalValidator # type: ignore + from .models.path import PathEntry -version_re = re.compile(r"(?P<major>\d+)(?:\.(?P<minor>\d+))?(?:\.(?P<patch>(?<=\.)[0-9]+))?\.?" - r"(?:(?P<prerel>[abc]|rc|dev)(?:(?P<prerelversion>\d+(?:\.\d+)*))?)" - r"?(?P<postdev>(\.post(?P<post>\d+))?(\.dev(?P<dev>\d+))?)?") +version_re_str = ( + r"(?P<major>\d+)(?:\.(?P<minor>\d+))?(?:\.(?P<patch>(?<=\.)[0-9]+))?\.?" + r"(?:(?P<prerel>[abc]|rc|dev)(?:(?P<prerelversion>\d+(?:\.\d+)*))?)" + r"?(?P<postdev>(\.post(?P<post>\d+))?(\.dev(?P<dev>\d+))?)?" +) +version_re = re.compile(version_re_str) PYTHON_IMPLEMENTATIONS = ( - "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython", "micropython" + "python", + "ironpython", + "jython", + "pypy", + "anaconda", + "miniconda", + "stackless", + "activepython", + "pyston", + "micropython", ) -RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] -RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] - -KNOWN_EXTS = {"exe", "py", "fish", "sh", ""} +if os.name == "nt": + KNOWN_EXTS = {"exe", "py", "bat", ""} +else: + KNOWN_EXTS = {"sh", "bash", "csh", "zsh", "fish", "py", ""} KNOWN_EXTS = KNOWN_EXTS | set( filter(None, os.environ.get("PATHEXT", "").split(os.pathsep)) ) +PY_MATCH_STR = r"((?P<implementation>{0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^zw])".format( + "|".join(PYTHON_IMPLEMENTATIONS) +) +EXE_MATCH_STR = r"{0}(?:\.(?P<ext>{1}))?".format(PY_MATCH_STR, "|".join(KNOWN_EXTS)) +RE_MATCHER = re.compile(r"({0}|{1})".format(version_re_str, PY_MATCH_STR)) +EXE_MATCHER = re.compile(EXE_MATCH_STR) +RULES_BASE = [ + "*{0}", + "*{0}?", + "*{0}?.?", + "*{0}?.?m", + "{0}?-?.?", + "{0}?-?.?.?", + "{0}?.?-?.?.?", +] +RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] MATCH_RULES = [] for rule in RULES: MATCH_RULES.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] + ["{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) for ext in KNOWN_EXTS] ) @@ -62,21 +90,38 @@ for rule in RULES: def get_python_version(path): # type: (str) -> str """Get python version string using subprocess from a given path.""" - version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] + version_cmd = [ + path, + "-c", + "import sys; print('.'.join([str(i) for i in sys.version_info[:3]]))", + ] + subprocess_kwargs = { + "env": os.environ.copy(), + "universal_newlines": True, + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "shell": False, + } + c = subprocess.Popen(version_cmd, **subprocess_kwargs) + timer = Timer(SUBPROCESS_TIMEOUT, c.kill) try: - c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False, write_to_stdout=False) + out, _ = c.communicate() + except (SystemExit, KeyboardInterrupt, TimeoutError): + c.terminate() + out, _ = c.communicate() + raise except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) - if not c.out: + if not out: raise InvalidPythonVersion("%s is not a valid python path" % path) - return c.out.strip() + return out.strip() @lru_cache(maxsize=1024) def parse_python_version(version_str): # type: (str) -> Dict[str, Union[str, int, Version]] from packaging.version import parse as parse_version + is_debug = False if version_str.endswith("-debug"): is_debug = True @@ -117,7 +162,7 @@ def parse_python_version(version_str): "is_prerelease": is_prerelease, "is_devrelease": is_devrelease, "is_debug": is_debug, - "version": version + "version": version, } @@ -147,13 +192,13 @@ def path_is_executable(path): @lru_cache(maxsize=1024) def path_is_known_executable(path): - # type: (vistir.compat.Path) -> bool + # type: (Path) -> bool """ Returns whether a given path is a known executable from known executable extensions or has the executable bit toggled. :param path: The path to the target executable. - :type path: :class:`~vistir.compat.Path` + :type path: :class:`~Path` :return: True if the path has chmod +x, or is a readable, known executable extension. :rtype: bool """ @@ -178,17 +223,20 @@ def looks_like_python(name): if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in MATCH_RULES) + match = RE_MATCHER.match(name) + if match: + return any(fnmatch(name, rule) for rule in MATCH_RULES) + return False @lru_cache(maxsize=1024) def path_is_python(path): - # type: (vistir.compat.Path) -> bool + # type: (Path) -> bool """ Determine whether the supplied path is executable and looks like a possible path to python. :param path: The path to an executable. - :type path: :class:`~vistir.compat.Path` + :type path: :class:`~Path` :return: Whether the provided path is an executable path to python. :rtype: bool """ @@ -196,9 +244,43 @@ def path_is_python(path): return path_is_executable(path) and looks_like_python(path.name) +@lru_cache(maxsize=1024) +def guess_company(path): + # type: (str) -> Optional[str] + """Given a path to python, guess the company who created it + + :param str path: The path to guess about + :return: The guessed company + :rtype: Optional[str] + """ + non_core_pythons = [impl for impl in PYTHON_IMPLEMENTATIONS if impl != "python"] + return next( + iter(impl for impl in non_core_pythons if impl in path.lower()), "PythonCore" + ) + + +@lru_cache(maxsize=1024) +def path_is_pythoncore(path): + # type: (str) -> bool + """Given a path, determine whether it appears to be pythoncore. + + Does not verify whether the path is in fact a path to python, but simply + does an exclusionary check on the possible known python implementations + to see if their names are present in the path (fairly dumb check). + + :param str path: The path to check + :return: Whether that path is a PythonCore path or not + :rtype: bool + """ + company = guess_company(path) + if company: + return company == "PythonCore" + return False + + @lru_cache(maxsize=1024) def ensure_path(path): - # type: (Union[vistir.compat.Path, str]) -> bool + # type: (Union[Path, str]) -> Path """ Given a path (either a string or a Path object), expand variables and return a Path object. @@ -208,9 +290,9 @@ def ensure_path(path): :rtype: :class:`~pathlib.Path` """ - if isinstance(path, vistir.compat.Path): + if isinstance(path, Path): return path - path = vistir.compat.Path(os.path.expandvars(path)) + path = Path(os.path.expandvars(path)) return path.absolute() @@ -224,17 +306,19 @@ def _filter_none(k, v): # TODO: Reimplement in vistir def normalize_path(path): # type: (str) -> str - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) @lru_cache(maxsize=1024) def filter_pythons(path): - # type: (Union[str, vistir.compat.Path]) -> Iterable + # type: (Union[str, Path]) -> Iterable """Return all valid pythons in a given path""" - if not isinstance(path, vistir.compat.Path): - path = vistir.compat.Path(str(path)) + if not isinstance(path, Path): + path = Path(str(path)) if not path.is_dir(): return path if path_is_python(path) else None return filter(path_is_python, path.iterdir()) @@ -248,13 +332,16 @@ def unnest(item): item, target = itertools.tee(item, 2) else: target = item - for el in target: - if isinstance(el, Iterable) and not isinstance(el, six.string_types): - el, el_copy = itertools.tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el + if getattr(target, "__iter__", None): + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el + else: + yield target def parse_pyenv_version_order(filename="version"): @@ -274,16 +361,89 @@ def parse_asdf_version_order(filename=".tool-versions"): if os.path.exists(version_order_file) and os.path.isfile(version_order_file): with io.open(version_order_file, encoding="utf-8") as fh: contents = fh.read() - python_section = next(iter( - line for line in contents.splitlines() if line.startswith("python") - ), None) + python_section = next( + iter(line for line in contents.splitlines() if line.startswith("python")), + None, + ) if python_section: - python_key, _, versions = python_section.partition(" ") + # python_key, _, versions + _, _, versions = python_section.partition(" ") if versions: return versions.split() return [] +def split_version_and_name( + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + name=None, # type: Optional[str] +): + # type: (...) -> Tuple[Optional[Union[str, int]], Optional[Union[str, int]], Optional[Union[str, int]], Optional[str]] # noqa + if isinstance(major, six.string_types) and not minor and not patch: + # Only proceed if this is in the format "x.y.z" or similar + if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()): + version = major.split(".", 2) + if isinstance(version, (tuple, list)): + if len(version) > 3: + major, minor, patch, _ = version + elif len(version) == 3: + major, minor, patch = version + elif len(version) == 2: + major, minor = version + else: + major = major[0] + else: + major = major + name = None + else: + name = "{0!s}".format(major) + major = None + return (major, minor, patch, name) + + # TODO: Reimplement in vistir def is_in_path(path, parent): return normalize_path(str(path)).startswith(normalize_path(str(parent))) + + +def expand_paths(path, only_python=True): + # type: (Union[Sequence, PathEntry], bool) -> Iterator + """ + Recursively expand a list or :class:`~pythonfinder.models.path.PathEntry` instance + + :param Union[Sequence, PathEntry] path: The path or list of paths to expand + :param bool only_python: Whether to filter to include only python paths, default True + :returns: An iterator over the expanded set of path entries + :rtype: Iterator[PathEntry] + """ + + if path is not None and ( + isinstance(path, Sequence) + and not getattr(path.__class__, "__name__", "") == "PathEntry" + ): + for p in unnest(path): + if p is None: + continue + for expanded in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield expanded + elif path is not None and path.is_dir: + for p in path.children.values(): + if p is not None and p.is_python and p.as_python is not None: + for sub_path in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield sub_path + else: + if path is not None and path.is_python and path.as_python is not None: + yield path + + +def dedup(iterable): + # type: (Iterable) -> Iterable + """Deduplicate an iterable object like iter(set(iterable)) but + order-reserved. + """ + return iter(OrderedDict.fromkeys(iterable)) diff --git a/pipenv/vendor/requests/LICENSE b/pipenv/vendor/requests/LICENSE index 841c6023..13d91ddc 100644 --- a/pipenv/vendor/requests/LICENSE +++ b/pipenv/vendor/requests/LICENSE @@ -1,4 +1,4 @@ -Copyright 2018 Kenneth Reitz +Copyright 2019 Kenneth Reitz Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/pipenv/vendor/requests/__init__.py b/pipenv/vendor/requests/__init__.py index bc168ee5..626247cb 100644 --- a/pipenv/vendor/requests/__init__.py +++ b/pipenv/vendor/requests/__init__.py @@ -9,14 +9,14 @@ Requests HTTP Library ~~~~~~~~~~~~~~~~~~~~~ -Requests is an HTTP library, written in Python, for human beings. Basic GET -usage: +Requests is an HTTP library, written in Python, for human beings. +Basic GET usage: >>> import requests >>> r = requests.get('https://www.python.org') >>> r.status_code 200 - >>> 'Python is a programming language' in r.content + >>> b'Python is a programming language' in r.content True ... or POST: @@ -27,14 +27,14 @@ usage: { ... "form": { - "key2": "value2", - "key1": "value1" + "key1": "value1", + "key2": "value2" }, ... } The other HTTP methods are supported - see `requests.api`. Full documentation -is at <http://python-requests.org>. +is at <https://requests.readthedocs.io>. :copyright: (c) 2017 by Kenneth Reitz. :license: Apache 2.0, see LICENSE for more details. @@ -57,10 +57,10 @@ def check_compatibility(urllib3_version, chardet_version): # Check urllib3 for compatibility. major, minor, patch = urllib3_version # noqa: F811 major, minor, patch = int(major), int(minor), int(patch) - # urllib3 >= 1.21.1, <= 1.24 + # urllib3 >= 1.21.1, <= 1.25 assert major == 1 assert minor >= 21 - assert minor <= 24 + assert minor <= 25 # Check chardet for compatibility. major, minor, patch = chardet_version.split('.')[:3] diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index 803773a0..b9e7df48 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -4,11 +4,11 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' -__url__ = 'http://python-requests.org' -__version__ = '2.20.1' -__build__ = 0x022001 +__url__ = 'https://requests.readthedocs.io' +__version__ = '2.23.0' +__build__ = 0x022300 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' -__copyright__ = 'Copyright 2018 Kenneth Reitz' +__copyright__ = 'Copyright 2020 Kenneth Reitz' __cake__ = u'\u2728 \U0001f370 \u2728' diff --git a/pipenv/vendor/requests/api.py b/pipenv/vendor/requests/api.py index abada96d..e978e203 100644 --- a/pipenv/vendor/requests/api.py +++ b/pipenv/vendor/requests/api.py @@ -16,10 +16,10 @@ from . import sessions def request(method, url, **kwargs): """Constructs and sends a :class:`Request <Request>`. - :param method: method for the new :class:`Request` object. + :param method: method for the new :class:`Request` object: ``GET``, ``OPTIONS``, ``HEAD``, ``POST``, ``PUT``, ``PATCH``, or ``DELETE``. :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send - in the body of the :class:`Request`. + in the query string for the :class:`Request`. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) A JSON serializable Python object to send in the body of the :class:`Request`. @@ -50,6 +50,7 @@ def request(method, url, **kwargs): >>> import requests >>> req = requests.request('GET', 'https://httpbin.org/get') + >>> req <Response [200]> """ @@ -65,7 +66,7 @@ def get(url, params=None, **kwargs): :param url: URL for the new :class:`Request` object. :param params: (optional) Dictionary, list of tuples or bytes to send - in the body of the :class:`Request`. + in the query string for the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :return: :class:`Response <Response>` object :rtype: requests.Response @@ -92,7 +93,9 @@ def head(url, **kwargs): r"""Sends a HEAD request. :param url: URL for the new :class:`Request` object. - :param \*\*kwargs: Optional arguments that ``request`` takes. + :param \*\*kwargs: Optional arguments that ``request`` takes. If + `allow_redirects` is not provided, it will be set to `False` (as + opposed to the default :meth:`request` behavior). :return: :class:`Response <Response>` object :rtype: requests.Response """ diff --git a/pipenv/vendor/requests/auth.py b/pipenv/vendor/requests/auth.py index bdde51c7..eeface39 100644 --- a/pipenv/vendor/requests/auth.py +++ b/pipenv/vendor/requests/auth.py @@ -50,7 +50,7 @@ def _basic_auth_str(username, password): "Non-string passwords will no longer be supported in Requests " "3.0.0. Please convert the object you've passed in ({!r}) to " "a string or bytes object in the near future to avoid " - "problems.".format(password), + "problems.".format(type(password)), category=DeprecationWarning, ) password = str(password) @@ -239,7 +239,7 @@ class HTTPDigestAuth(AuthBase): """ # If response is not 4xx, do not auth - # See https://github.com/requests/requests/issues/3772 + # See https://github.com/psf/requests/issues/3772 if not 400 <= r.status_code < 500: self._thread_local.num_401_calls = 1 return r diff --git a/pipenv/vendor/requests/compat.py b/pipenv/vendor/requests/compat.py index c44b35ef..5de0769f 100644 --- a/pipenv/vendor/requests/compat.py +++ b/pipenv/vendor/requests/compat.py @@ -43,6 +43,7 @@ if is_py2: import cookielib from Cookie import Morsel from StringIO import StringIO + # Keep OrderedDict for backwards compatibility. from collections import Callable, Mapping, MutableMapping, OrderedDict @@ -59,6 +60,7 @@ elif is_py3: from http import cookiejar as cookielib from http.cookies import Morsel from io import StringIO + # Keep OrderedDict for backwards compatibility. from collections import OrderedDict from collections.abc import Callable, Mapping, MutableMapping diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3dded57e..35798832 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -12,7 +12,7 @@ import sys # Import encoding now, to avoid implicit import later. # Implicit import within threads may cause LookupError when standard library is in a ZIP, -# such as in Embedded Python. See https://github.com/requests/requests/issues/3578. +# such as in Embedded Python. See https://github.com/psf/requests/issues/3578. import encodings.idna from urllib3.fields import RequestField @@ -280,6 +280,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): >>> import requests >>> req = requests.Request('GET', 'https://httpbin.org/get') >>> r = req.prepare() + >>> r <PreparedRequest [GET]> >>> s = requests.Session() @@ -358,7 +359,7 @@ class PreparedRequest(RequestEncodingMixin, RequestHooksMixin): #: We're unable to blindly call unicode/str functions #: as this will include the bytestring indicator (b'') #: on python 3.x. - #: https://github.com/requests/requests/pull/2238 + #: https://github.com/psf/requests/pull/2238 if isinstance(url, bytes): url = url.decode('utf8') else: @@ -608,7 +609,7 @@ class Response(object): #: File-like object representation of response (for advanced usage). #: Use of ``raw`` requires that ``stream=True`` be set on the request. - # This requirement does not apply for use internally to Requests. + #: This requirement does not apply for use internally to Requests. self.raw = None #: Final URL location of Response. @@ -781,7 +782,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/vendor/requests/sessions.py b/pipenv/vendor/requests/sessions.py index d73d700f..2845880b 100644 --- a/pipenv/vendor/requests/sessions.py +++ b/pipenv/vendor/requests/sessions.py @@ -11,9 +11,10 @@ import os import sys import time from datetime import timedelta +from collections import OrderedDict from .auth import _basic_auth_str -from .compat import cookielib, is_py3, OrderedDict, urljoin, urlparse, Mapping +from .compat import cookielib, is_py3, urljoin, urlparse, Mapping from .cookies import ( cookiejar_from_dict, extract_cookies_to_jar, RequestsCookieJar, merge_cookies) from .models import Request, PreparedRequest, DEFAULT_REDIRECT_LIMIT @@ -162,7 +163,7 @@ class SessionRedirectMixin(object): resp.raw.read(decode_content=False) if len(resp.history) >= self.max_redirects: - raise TooManyRedirects('Exceeded %s redirects.' % self.max_redirects, response=resp) + raise TooManyRedirects('Exceeded {} redirects.'.format(self.max_redirects), response=resp) # Release the connection back into the pool. resp.close() @@ -170,7 +171,7 @@ class SessionRedirectMixin(object): # Handle redirection without scheme (see: RFC 1808 Section 4) if url.startswith('//'): parsed_rurl = urlparse(resp.url) - url = '%s:%s' % (to_native_string(parsed_rurl.scheme), url) + url = ':'.join([to_native_string(parsed_rurl.scheme), url]) # Normalize url case and attach previous fragment if needed (RFC 7231 7.1.2) parsed = urlparse(url) @@ -192,19 +193,16 @@ class SessionRedirectMixin(object): self.rebuild_method(prepared_request, resp) - # https://github.com/requests/requests/issues/1084 + # https://github.com/psf/requests/issues/1084 if resp.status_code not in (codes.temporary_redirect, codes.permanent_redirect): - # https://github.com/requests/requests/issues/3490 + # https://github.com/psf/requests/issues/3490 purged_headers = ('Content-Length', 'Content-Type', 'Transfer-Encoding') for header in purged_headers: prepared_request.headers.pop(header, None) prepared_request.body = None headers = prepared_request.headers - try: - del headers['Cookie'] - except KeyError: - pass + headers.pop('Cookie', None) # Extract any cookies sent on the response to the cookiejar # in the new request. Because we've mutated our copied prepared @@ -271,7 +269,6 @@ class SessionRedirectMixin(object): if new_auth is not None: prepared_request.prepare_auth(new_auth) - return def rebuild_proxies(self, prepared_request, proxies): """This method re-evaluates the proxy configuration by considering the @@ -352,13 +349,13 @@ class Session(SessionRedirectMixin): Or as a context manager:: >>> with requests.Session() as s: - >>> s.get('https://httpbin.org/get') + ... s.get('https://httpbin.org/get') <Response [200]> """ __attrs__ = [ 'headers', 'cookies', 'auth', 'proxies', 'hooks', 'params', 'verify', - 'cert', 'prefetch', 'adapters', 'stream', 'trust_env', + 'cert', 'adapters', 'stream', 'trust_env', 'max_redirects', ] @@ -728,7 +725,7 @@ class Session(SessionRedirectMixin): return adapter # Nothing matches :-/ - raise InvalidSchema("No connection adapters were found for '%s'" % url) + raise InvalidSchema("No connection adapters were found for {!r}".format(url)) def close(self): """Closes all adapters and as such the session""" diff --git a/pipenv/vendor/requests/status_codes.py b/pipenv/vendor/requests/status_codes.py index 813e8c4e..d80a7cd4 100644 --- a/pipenv/vendor/requests/status_codes.py +++ b/pipenv/vendor/requests/status_codes.py @@ -5,12 +5,15 @@ The ``codes`` object defines a mapping from common names for HTTP statuses to their numerical codes, accessible either as attributes or as dictionary items. ->>> requests.codes['temporary_redirect'] -307 ->>> requests.codes.teapot -418 ->>> requests.codes['\o/'] -200 +Example:: + + >>> import requests + >>> requests.codes['temporary_redirect'] + 307 + >>> requests.codes.teapot + 418 + >>> requests.codes['\o/'] + 200 Some codes have multiple names, and both upper- and lower-case versions of the names are allowed. For example, ``codes.ok``, ``codes.OK``, and diff --git a/pipenv/vendor/requests/structures.py b/pipenv/vendor/requests/structures.py index da930e28..8ee0ba7a 100644 --- a/pipenv/vendor/requests/structures.py +++ b/pipenv/vendor/requests/structures.py @@ -7,7 +7,9 @@ requests.structures Data structures that power Requests. """ -from .compat import OrderedDict, Mapping, MutableMapping +from collections import OrderedDict + +from .compat import Mapping, MutableMapping class CaseInsensitiveDict(MutableMapping): diff --git a/pipenv/vendor/requests/utils.py b/pipenv/vendor/requests/utils.py index 8170a8d2..c1700d7f 100644 --- a/pipenv/vendor/requests/utils.py +++ b/pipenv/vendor/requests/utils.py @@ -19,6 +19,7 @@ import sys import tempfile import warnings import zipfile +from collections import OrderedDict from .__version__ import __version__ from . import certs @@ -26,7 +27,7 @@ from . import certs from ._internal_utils import to_native_string from .compat import parse_http_list as _parse_list_header from .compat import ( - quote, urlparse, bytes, str, OrderedDict, unquote, getproxies, + quote, urlparse, bytes, str, unquote, getproxies, proxy_bypass, urlunparse, basestring, integer_types, is_py3, proxy_bypass_environment, getproxies_environment, Mapping) from .cookies import cookiejar_from_dict @@ -179,7 +180,7 @@ def get_netrc_auth(url, raise_errors=False): except KeyError: # os.path.expanduser can fail when $HOME is undefined and # getpwuid fails. See https://bugs.python.org/issue20164 & - # https://github.com/requests/requests/issues/1846 + # https://github.com/psf/requests/issues/1846 return if os.path.exists(loc): @@ -266,6 +267,8 @@ def from_key_val_list(value): >>> from_key_val_list([('key', 'val')]) OrderedDict([('key', 'val')]) >>> from_key_val_list('string') + Traceback (most recent call last): + ... ValueError: cannot encode objects that are not 2-tuples >>> from_key_val_list({'key': 'val'}) OrderedDict([('key', 'val')]) @@ -292,7 +295,9 @@ def to_key_val_list(value): >>> to_key_val_list({'key': 'val'}) [('key', 'val')] >>> to_key_val_list('string') - ValueError: cannot encode objects that are not 2-tuples. + Traceback (most recent call last): + ... + ValueError: cannot encode objects that are not 2-tuples :rtype: list """ diff --git a/pipenv/vendor/requirementslib/LICENSE b/pipenv/vendor/requirementslib/LICENSE index 8c731e27..a6b8c96a 100644 --- a/pipenv/vendor/requirementslib/LICENSE +++ b/pipenv/vendor/requirementslib/LICENSE @@ -1,6 +1,6 @@ The MIT License (MIT) -Copyright 2018 Dan Ryan. +Copyright 2019 Dan Ryan. Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index aca59917..b8270bb7 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,16 +1,21 @@ # -*- coding=utf-8 -*- -__version__ = '1.3.2' +from __future__ import absolute_import, print_function import logging import warnings + from vistir.compat import ResourceWarning +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile +from .models.requirements import Requirement + +__version__ = "1.5.7" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) warnings.filterwarnings("ignore", category=ResourceWarning) -from .models.requirements import Requirement -from .models.lockfile import Lockfile -from .models.pipfile import Pipfile __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/vendor/requirementslib/exceptions.py index 17b884eb..d11dbce9 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/vendor/requirementslib/exceptions.py @@ -1,19 +1,21 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function + import errno import os -import six import sys - +import six from vistir.compat import FileNotFoundError - if six.PY2: + class FileExistsError(OSError): def __init__(self, *args, **kwargs): self.errno = errno.EEXIST super(FileExistsError, self).__init__(*args, **kwargs) + + else: from six.moves.builtins import FileExistsError @@ -24,8 +26,15 @@ class RequirementError(Exception): class MissingParameter(Exception): def __init__(self, param): - Exception.__init__(self) - print("Missing parameter: %s" % param, file=sys.stderr, flush=True) + self.message = self.get_message(param) + super(MissingParameter, self).__init__(self.message) + + @classmethod + def get_message(cls, param): + return "Missing Parameter: %s" % param + + def show(self, param): + print(self.message, file=sys.stderr, flush=True) class FileCorruptException(OSError): @@ -35,58 +44,67 @@ class FileCorruptException(OSError): if not backup_path and args: args = reversed(args) backup_path = args.pop() - if not isinstance(backup_path, six.string_types) or not os.path.exists(os.path.abspath(os.path.dirname(backup_path))): + if not isinstance(backup_path, six.string_types) or not os.path.exists( + os.path.abspath(os.path.dirname(backup_path)) + ): args.append(backup_path) backup_path = None if args: args = reversed(args) - self.path = path - self.backup_path = backup_path - self.show(self.path, self.backup_path) - OSError.__init__(self, path, *args, **kwargs) + self.message = self.get_message(path, backup_path=backup_path) + super(FileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load file at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load file at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: - msg = "it will be removed and replaced." - print("The file is corrupt, %s" % msg, file=sys.stderr, flush=True) + msg = "it will be removed and replaced on the next lock." + message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + return message + + def show(self): + print(self.message, file=sys.stderr, flush=True) class LockfileCorruptException(FileCorruptException): + def __init__(self, path, backup_path=None): + self.message = self.get_message(path, backup_path=backup_path) + super(LockfileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load lockfile at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load lockfile at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - print("Your lockfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + return message + + def show(self, path, backup_path=None): + print(self.message, file=sys.stderr, flush=True) class PipfileCorruptException(FileCorruptException): + def __init__(self, path, backup_path=None): + self.message = self.get_message(path, backup_path=backup_path) + super(PipfileCorruptException, self).__init__(self.message) - @classmethod - def show(cls, path, backup_path=None): - print("ERROR: Failed to load Pipfile at %s" % path, file=sys.stderr, flush=True) + def get_message(self, path, backup_path=None): + message = "ERROR: Failed to load Pipfile at %s" % path if backup_path: msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - print("Your Pipfile is corrupt, %s" % msg, file=sys.stderr, flush=True) + message = "{0}\nYour Pipfile is corrupt, {1}".format(message, msg) + return message + + def show(self, path, backup_path=None): + print(self.message, file=sys.stderr, flush=True) class PipfileNotFound(FileNotFoundError): def __init__(self, path, *args, **kwargs): self.errno = errno.ENOENT - self.path = path - self.show(path) - super(PipfileNotFound, self).__init__(*args, **kwargs) - - @classmethod - def show(cls, path): - print("ERROR: The file could not be found: %s" % path, file=sys.stderr, flush=True) - print("Aborting...", file=sys.stderr, flush=True) + self.filename = path + super(PipfileNotFound, self).__init__(self.filename) diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py index e819cdd5..40a96afc 100644 --- a/pipenv/vendor/requirementslib/models/__init__.py +++ b/pipenv/vendor/requirementslib/models/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -# This is intentionally left blank diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index f87fd585..78c78ace 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -9,25 +9,63 @@ import os import attr import packaging.markers import packaging.version -import requests - -from first import first -from packaging.utils import canonicalize_name - import pip_shims.shims -from vistir.compat import JSONDecodeError, fs_str, ResourceWarning +import requests +from packaging.utils import canonicalize_name +from vistir.compat import JSONDecodeError, fs_str from vistir.contextmanagers import cd, temp_environ -from vistir.misc import partialclass from vistir.path import create_tracked_tempdir -from ..utils import prepare_pip_source_args, _ensure_dir +from ..environment import MYPY_RUNNING +from ..utils import _ensure_dir, prepare_pip_source_args from .cache import CACHE_DIR, DependencyCache +from .setup_info import SetupInfo from .utils import ( - clean_requires_python, fix_requires_python_marker, format_requirement, - full_groupby, is_pinned_requirement, key_from_ireq, - make_install_requirement, name_from_req, version_from_ireq + clean_requires_python, + fix_requires_python_marker, + format_requirement, + full_groupby, + is_pinned_requirement, + key_from_ireq, + make_install_requirement, + name_from_req, + version_from_ireq, ) +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack + +if MYPY_RUNNING: + from typing import ( + Any, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + ) + from pip_shims.shims import ( + InstallRequirement, + InstallationCandidate, + PackageFinder, + Command, + ) + from packaging.requirements import Requirement as PackagingRequirement + + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels")) @@ -43,6 +81,7 @@ def _get_filtered_versions(ireq, versions, prereleases): def find_all_matches(finder, ireq, pre=False): + # type: (PackageFinder, InstallRequirement, bool) -> List[InstallationCandidate] """Find all matching dependencies using the supplied finder and the given ireq. @@ -54,7 +93,6 @@ def find_all_matches(finder, ireq, pre=False): :rtype: list[:class:`~pip._internal.index.InstallationCandidate`] """ - candidates = clean_requires_python(finder.find_all_candidates(ireq.name)) versions = {candidate.version for candidate in candidates} allowed_versions = _get_filtered_versions(ireq, versions, pre) @@ -65,31 +103,17 @@ def find_all_matches(finder, ireq, pre=False): def get_pip_command(): + # type: () -> Command # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are defered to pip. - import optparse - - class PipCommand(pip_shims.shims.Command): - name = "PipCommand" - - pip_command = PipCommand() - pip_command.parser.add_option(pip_shims.shims.cmdoptions.no_binary()) - pip_command.parser.add_option(pip_shims.shims.cmdoptions.only_binary()) - index_opts = pip_shims.shims.cmdoptions.make_option_group( - pip_shims.shims.cmdoptions.index_group, pip_command.parser - ) - pip_command.parser.insert_option_group(0, index_opts) - pip_command.parser.add_option( - optparse.Option("--pre", action="store_true", default=False) - ) - + pip_command = pip_shims.shims.InstallCommand() return pip_command @attr.s class AbstractDependency(object): - name = attr.ib() + name = attr.ib() # type: STRING_TYPE specifiers = attr.ib() markers = attr.ib() candidates = attr.ib() @@ -120,9 +144,9 @@ class AbstractDependency(object): :rtype: set(str) """ - if len(self.candidates) == 1 and first(self.candidates).editable: + if len(self.candidates) == 1 and next(iter(self.candidates)).editable: return self - elif len(other.candidates) == 1 and first(other.candidates).editable: + elif len(other.candidates) == 1 and next(iter(other.candidates)).editable: return other return self.version_set & other.version_set @@ -139,15 +163,19 @@ class AbstractDependency(object): from .requirements import Requirement - if len(self.candidates) == 1 and first(self.candidates).editable: + if len(self.candidates) == 1 and next(iter(self.candidates)).editable: return self - elif len(other.candidates) == 1 and first(other.candidates).editable: + elif len(other.candidates) == 1 and next(iter(other.candidates)).editable: return other new_specifiers = self.specifiers & other.specifiers - markers = set(self.markers,) if self.markers else set() + markers = set(self.markers) if self.markers else set() if other.markers: markers.add(other.markers) - new_markers = packaging.markers.Marker(" or ".join(str(m) for m in sorted(markers))) + new_markers = None + if markers: + new_markers = packaging.markers.Marker( + " or ".join(str(m) for m in sorted(markers)) + ) new_ireq = copy.deepcopy(self.requirement.ireq) new_ireq.req.specifier = new_specifiers new_ireq.req.marker = new_markers @@ -173,7 +201,7 @@ class AbstractDependency(object): requirement=new_requirement, parent=self.parent, dep_dict=dep_dict, - finder=self.finder + finder=self.finder, ) def get_deps(self, candidate): @@ -190,7 +218,7 @@ class AbstractDependency(object): from .requirements import Requirement req = Requirement.from_line(key) - req.merge_markers(self.markers) + req = req.merge_markers(self.markers) self.dep_dict[key] = req.get_abstract_dependencies() return self.dep_dict[key] @@ -211,18 +239,23 @@ class AbstractDependency(object): extras = requirement.ireq.extras is_pinned = is_pinned_requirement(requirement.ireq) is_constraint = bool(parent) - finder = get_finder(sources=None) + _, finder = get_finder(sources=None) candidates = [] if not is_pinned and not requirement.editable: for r in requirement.find_all_matches(finder=finder): req = make_install_requirement( - name, r.version, extras=extras, markers=markers, constraint=is_constraint, + name, + r.version, + extras=extras, + markers=markers, + constraint=is_constraint, ) - req.req.link = r.location + req.req.link = getattr(r, "location", getattr(r, "link", None)) req.parent = parent candidates.append(req) candidates = sorted( - set(candidates), key=lambda k: packaging.version.parse(version_from_ireq(k)), + set(candidates), + key=lambda k: packaging.version.parse(version_from_ireq(k)), ) else: candidates = [requirement.ireq] @@ -265,9 +298,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): for req in reqs: if isinstance(req, pip_shims.shims.InstallRequirement): - requirement = Requirement.from_line( - "{0}{1}".format(req.name, req.specifier) - ) + requirement = Requirement.from_line("{0}{1}".format(req.name, req.specifier)) if req.link: requirement.req.link = req.link requirement.markers = req.markers @@ -284,6 +315,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): def get_dependencies(ireq, sources=None, parent=None): + # type: (Union[InstallRequirement, InstallationCandidate], Optional[List[Dict[S, Union[S, bool]]]], Optional[AbstractDependency]) -> Set[S, ...] """Get all dependencies for a given install requirement. :param ireq: A single InstallRequirement @@ -296,27 +328,26 @@ def get_dependencies(ireq, sources=None, parent=None): :rtype: set(str) """ if not isinstance(ireq, pip_shims.shims.InstallRequirement): - name = getattr( - ireq, "project_name", - getattr(ireq, "project", ireq.name), - ) + name = getattr(ireq, "project_name", getattr(ireq, "project", ireq.name)) version = getattr(ireq, "version", None) if not version: ireq = pip_shims.shims.InstallRequirement.from_line("{0}".format(name)) else: - ireq = pip_shims.shims.InstallRequirement.from_line("{0}=={1}".format(name, version)) + ireq = pip_shims.shims.InstallRequirement.from_line( + "{0}=={1}".format(name, version) + ) pip_options = get_pip_options(sources=sources) getters = [ get_dependencies_from_cache, get_dependencies_from_wheel_cache, get_dependencies_from_json, - functools.partial(get_dependencies_from_index, pip_options=pip_options) + functools.partial(get_dependencies_from_index, pip_options=pip_options), ] for getter in getters: deps = getter(ireq) if deps is not None: return deps - raise RuntimeError('failed to get dependencies for {}'.format(ireq)) + raise RuntimeError("failed to get dependencies for {}".format(ireq)) def get_dependencies_from_wheel_cache(ireq): @@ -374,7 +405,7 @@ def get_dependencies_from_json(ireq): finally: session.close() requires_dist = info.get("requires_dist", info.get("requires")) - if not requires_dist: # The API can return None for this. + if not requires_dist: # The API can return None for this. return for requires in requires_dist: i = pip_shims.shims.InstallRequirement.from_line(requires) @@ -415,9 +446,9 @@ def get_dependencies_from_cache(ireq): dep_ireq = pip_shims.shims.InstallRequirement.from_line(line) name = canonicalize_name(dep_ireq.name) if _marker_contains_extra(dep_ireq): - broken = True # The "extra =" marker breaks everything. + broken = True # The "extra =" marker breaks everything. elif name == canonicalize_name(ireq.name): - broken = True # A package cannot depend on itself. + broken = True # A package cannot depend on itself. if broken: break except Exception: @@ -431,7 +462,7 @@ def get_dependencies_from_cache(ireq): def is_python(section): - return section.startswith('[') and ':' in section + return section.startswith("[") and ":" in section def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache=None): @@ -445,85 +476,23 @@ def get_dependencies_from_index(dep, sources=None, pip_options=None, wheel_cache :rtype: set(str) or None """ - finder = get_finder(sources=sources, pip_options=pip_options) + session, finder = get_finder(sources=sources, pip_options=pip_options) if not wheel_cache: wheel_cache = WHEEL_CACHE dep.is_direct = True - reqset = pip_shims.shims.RequirementSet() - reqset.add_requirement(dep) requirements = None setup_requires = {} - with temp_environ(), start_resolver(finder=finder, wheel_cache=wheel_cache) as resolver: - os.environ['PIP_EXISTS_ACTION'] = 'i' - dist = None + with temp_environ(): + os.environ["PIP_EXISTS_ACTION"] = "i" if dep.editable and not dep.prepared and not dep.req: - with cd(dep.setup_py_dir): - from setuptools.dist import distutils - try: - dist = distutils.core.run_setup(dep.setup_py) - except (ImportError, TypeError, AttributeError): - dist = None - else: - setup_requires[dist.get_name()] = dist.setup_requires - if not dist: - try: - dist = dep.get_dist() - except (TypeError, ValueError, AttributeError): - pass - else: - setup_requires[dist.get_name()] = dist.setup_requires - resolver.require_hashes = False - try: - results = resolver._resolve_one(reqset, dep) - except Exception: - # FIXME: Needs to bubble the exception somehow to the user. - results = [] - finally: - try: - wheel_cache.cleanup() - except AttributeError: - pass - resolver_requires_python = getattr(resolver, "requires_python", None) - requires_python = getattr(reqset, "requires_python", resolver_requires_python) - if requires_python: - add_marker = fix_requires_python_marker(requires_python) - reqset.remove(dep) - if dep.req.marker: - dep.req.marker._markers.extend(['and',].extend(add_marker._markers)) - else: - dep.req.marker = add_marker - reqset.add(dep) - requirements = set() - for r in results: - if requires_python: - if r.req.marker: - r.req.marker._markers.extend(['and',].extend(add_marker._markers)) - else: - r.req.marker = add_marker - requirements.add(format_requirement(r)) - for section in setup_requires: - python_version = section - not_python = not is_python(section) - - # This is for cleaning up :extras: formatted markers - # by adding them to the results of the resolver - # since any such extra would have been returned as a result anyway - for value in setup_requires[section]: - - # This is a marker. - if is_python(section): - python_version = value[1:-1] - else: - not_python = True - - if ':' not in value and not_python: - try: - requirement_str = "{0}{1}".format(value, python_version).replace(":", ";") - requirements.add(format_requirement(make_install_requirement(requirement_str).ireq)) - # Anything could go wrong here -- can't be too careful. - except Exception: - pass - + setup_info = SetupInfo.from_ireq(dep) + results = setup_info.get_info() + setup_requires.update(results["setup_requires"]) + requirements = set(results["requires"].values()) + else: + results = pip_shims.shims.resolve(dep) + requirements = [v for v in results.values() if v.name != dep.name] + requirements = set([format_requirement(r) for r in requirements]) if not dep.editable and is_pinned_requirement(dep) and requirements is not None: DEPENDENCY_CACHE[dep] = list(requirements) return requirements @@ -544,9 +513,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): if not pip_command: pip_command = get_pip_command() if not sources: - sources = [ - {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} - ] + sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] _ensure_dir(CACHE_DIR) pip_args = args pip_args = prepare_pip_source_args(sources, pip_args) @@ -556,6 +523,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): def get_finder(sources=None, pip_command=None, pip_options=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[Command], Any) -> PackageFinder """Get a package finder for looking up candidates to install :param sources: A list of pipfile-formatted sources, defaults to None @@ -569,40 +537,38 @@ def get_finder(sources=None, pip_command=None, pip_options=None): """ if not pip_command: - pip_command = get_pip_command() + pip_command = pip_shims.shims.InstallCommand() if not sources: - sources = [ - {"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True} - ] + sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] if not pip_options: pip_options = get_pip_options(sources=sources, pip_command=pip_command) session = pip_command._build_session(pip_options) atexit.register(session.close) - finder = pip_shims.shims.PackageFinder( - find_links=[], - index_urls=[s.get("url") for s in sources], - trusted_hosts=[], - allow_all_prereleases=pip_options.pre, - session=session, + finder = pip_shims.shims.get_package_finder( + pip_shims.shims.InstallCommand(), options=pip_options, session=session ) - return finder + return session, finder @contextlib.contextmanager -def start_resolver(finder=None, wheel_cache=None): +def start_resolver(finder=None, session=None, wheel_cache=None): """Context manager to produce a resolver. :param finder: A package finder to use for searching the index :type finder: :class:`~pip._internal.index.PackageFinder` + :param :class:`~requests.Session` session: A session instance + :param :class:`~pip._internal.cache.WheelCache` wheel_cache: A pip WheelCache instance :return: A 3-tuple of finder, preparer, resolver :rtype: (:class:`~pip._internal.operations.prepare.RequirementPreparer`, :class:`~pip._internal.resolve.Resolver`) """ pip_command = get_pip_command() pip_options = get_pip_options(pip_command=pip_command) - + session = None if not finder: - finder = get_finder(pip_command=pip_command, pip_options=pip_options) + session, finder = get_finder(pip_command=pip_command, pip_options=pip_options) + if not session: + session = pip_command._build_session(pip_options) if not wheel_cache: wheel_cache = WHEEL_CACHE @@ -613,38 +579,41 @@ def start_resolver(finder=None, wheel_cache=None): _build_dir = create_tracked_tempdir(fs_str("build")) _source_dir = create_tracked_tempdir(fs_str("source")) - preparer = partialclass( - pip_shims.shims.RequirementPreparer, - build_dir=_build_dir, - src_dir=_source_dir, - download_dir=download_dir, - wheel_download_dir=WHEEL_DOWNLOAD_DIR, - progress_bar="off", - build_isolation=False, - ) - resolver = partialclass( - pip_shims.shims.Resolver, - finder=finder, - session=finder.session, - upgrade_strategy="to-satisfy-only", - force_reinstall=True, - ignore_dependencies=False, - ignore_requires_python=True, - ignore_installed=True, - isolated=False, - wheel_cache=wheel_cache, - use_user_site=False, - ) try: - if packaging.version.parse(pip_shims.shims.pip_version) >= packaging.version.parse('18'): - with pip_shims.shims.RequirementTracker() as req_tracker: - preparer = preparer(req_tracker=req_tracker) - yield resolver(preparer=preparer) - else: - preparer = preparer() - yield resolver(preparer=preparer) + with ExitStack() as ctx: + ctx.enter_context(pip_shims.shims.global_tempdir_manager()) + preparer = ctx.enter_context( + pip_shims.shims.make_preparer( + options=pip_options, + finder=finder, + session=session, + build_dir=_build_dir, + src_dir=_source_dir, + download_dir=download_dir, + wheel_download_dir=WHEEL_DOWNLOAD_DIR, + progress_bar="off", + build_isolation=False, + install_cmd=pip_command, + ) + ) + resolver = pip_shims.shims.get_resolver( + finder=finder, + ignore_dependencies=False, + ignore_requires_python=True, + preparer=preparer, + session=session, + options=pip_options, + install_cmd=pip_command, + wheel_cache=wheel_cache, + force_reinstall=True, + ignore_installed=True, + upgrade_strategy="to-satisfy-only", + isolated=False, + use_user_site=False, + ) + yield resolver finally: - finder.session.close() + session.close() def get_grouped_dependencies(constraints): @@ -654,10 +623,10 @@ def get_grouped_dependencies(constraints): # then we take the loose match (which _is_ flexible) and start moving backwards in # versions by popping them off of a stack and checking for the conflicting package for _, ireqs in full_groupby(constraints, key=key_from_ireq): - ireqs = list(ireqs) - editable_ireq = first(ireqs, key=lambda ireq: ireq.editable) + ireqs = sorted(ireqs, key=lambda ireq: ireq.editable) + editable_ireq = next(iter(ireq for ireq in ireqs if ireq.editable), None) if editable_ireq: - yield editable_ireq # ignore all the other specs: the editable one is the one that counts + yield editable_ireq # only the editable match mattters, ignore all others continue ireqs = iter(ireqs) # deepcopy the accumulator so as to not modify the self.our_constraints invariant diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 54b2761c..3eabc504 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,24 +1,22 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import copy +import itertools import os import attr -import itertools import plette.lockfiles import six +from vistir.compat import FileNotFoundError, JSONDecodeError, Path -from vistir.compat import Path, FileNotFoundError, JSONDecodeError - +from ..exceptions import LockfileCorruptException, MissingParameter, PipfileNotFound +from ..utils import is_editable, is_vcs, merge_items from .project import ProjectFile from .requirements import Requirement - from .utils import optional_instance_of -from ..exceptions import LockfileCorruptException, PipfileNotFound, MissingParameter -from ..utils import is_vcs, is_editable, merge_items -DEFAULT_NEWLINES = u"\n" +DEFAULT_NEWLINES = six.text_type("\n") def preferred_newlines(f): @@ -42,7 +40,7 @@ class Lockfile(object): @path.default def _get_path(self): - return Path(os.curdir).absolute() + return Path(os.curdir).joinpath("Pipfile.lock").absolute() @projectfile.default def _get_projectfile(self): @@ -50,7 +48,7 @@ class Lockfile(object): @_lockfile.default def _get_lockfile(self): - return self.projectfile.lockfile + return self.projectfile.model @property def lockfile(self): @@ -127,16 +125,13 @@ class Lockfile(object): :rtype: :class:`~requirementslib.models.project.ProjectFile` """ - pf = ProjectFile.read( - path, - plette.lockfiles.Lockfile, - invalid_ok=True - ) + pf = ProjectFile.read(path, plette.lockfiles.Lockfile, invalid_ok=True) return pf @classmethod def lockfile_from_pipfile(cls, pipfile_path): from .pipfile import Pipfile + if os.path.isfile(pipfile_path): if not os.path.isabs(pipfile_path): pipfile_path = os.path.abspath(pipfile_path) @@ -164,7 +159,9 @@ class Lockfile(object): if not project_path.exists(): raise OSError("Project does not exist: %s" % project_path.as_posix()) elif not lockfile_path.exists() and not create: - raise FileNotFoundError("Lockfile does not exist: %s" % lockfile_path.as_posix()) + raise FileNotFoundError( + "Lockfile does not exist: %s" % lockfile_path.as_posix() + ) projectfile = cls.read_projectfile(lockfile_path.as_posix()) if not lockfile_path.exists(): if not data: @@ -207,10 +204,14 @@ class Lockfile(object): lockfile.update(data) else: lockfile = plette.lockfiles.Lockfile(data) - projectfile = ProjectFile(line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile) + projectfile = ProjectFile( + line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile + ) return cls( - projectfile=projectfile, lockfile=lockfile, - newlines=projectfile.line_ending, path=Path(projectfile.location) + projectfile=projectfile, + lockfile=lockfile, + newlines=projectfile.line_ending, + path=Path(projectfile.location), ) @classmethod @@ -243,7 +244,7 @@ class Lockfile(object): "projectfile": projectfile, "lockfile": projectfile.model, "newlines": projectfile.line_ending, - "path": lockfile_path + "path": lockfile_path, } return cls(**creation_args) @@ -300,9 +301,7 @@ class Lockfile(object): lines = [] section = self.dev_requirements if dev else self.requirements for req in section: - kwargs = { - "include_hashes": include_hashes, - } + kwargs = {"include_hashes": include_hashes} if req.editable: kwargs["include_markers"] = False r = req.as_line(**kwargs) diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/vendor/requirementslib/models/markers.py index 70fe3bc0..84637642 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/vendor/requirementslib/models/markers.py @@ -1,19 +1,46 @@ # -*- coding: utf-8 -*- +import itertools +import operator +import re + import attr - +import distlib.markers +import packaging.version +import six from packaging.markers import InvalidMarker, Marker +from packaging.specifiers import Specifier, SpecifierSet +from vistir.compat import Mapping, Set, lru_cache +from vistir.misc import dedup +from ..environment import MYPY_RUNNING from ..exceptions import RequirementError from .utils import filter_none, validate_markers +from six.moves import reduce # isort:skip + + +if MYPY_RUNNING: + from typing import Optional, List, Type, Any, Tuple, Union, AnyStr, Text, Iterator + + STRING_TYPE = Union[str, bytes, Text] + + +MAX_VERSIONS = {2: 7, 3: 11, 4: 0} +DEPRECATED_VERSIONS = ["3.0", "3.1", "3.2", "3.3"] + + +def is_instance(item, cls): + # type: (Any, Type) -> bool + if isinstance(item, cls) or item.__class__.__name__ == cls.__name__: + return True + return False + @attr.s class PipenvMarkers(object): """System-level requirements - see PEP508 for more detail""" - os_name = attr.ib( - default=None, validator=attr.validators.optional(validate_markers) - ) + os_name = attr.ib(default=None, validator=attr.validators.optional(validate_markers)) sys_platform = attr.ib( default=None, validator=attr.validators.optional(validate_markers) ) @@ -92,3 +119,617 @@ class PipenvMarkers(object): pass else: return combined_marker + + +@lru_cache(maxsize=1024) +def _tuplize_version(version): + # type: (STRING_TYPE) -> Tuple[int, ...] + return tuple(int(x) for x in filter(lambda i: i != "*", version.split("."))) + + +@lru_cache(maxsize=1024) +def _format_version(version): + # type: (Tuple[int, ...]) -> STRING_TYPE + if not isinstance(version, six.string_types): + return ".".join(str(i) for i in version) + return version + + +# Prefer [x,y) ranges. +REPLACE_RANGES = {">": ">=", "<=": "<"} + + +@lru_cache(maxsize=1024) +def _format_pyspec(specifier): + # type: (Union[STRING_TYPE, Specifier]) -> Specifier + if isinstance(specifier, str): + if not any(op in specifier for op in Specifier._operators.keys()): + specifier = "=={0}".format(specifier) + specifier = Specifier(specifier) + version = getattr(specifier, "version", specifier).rstrip() + if version and version.endswith("*"): + if version.endswith(".*"): + version = version[:-2] + version = version.rstrip("*") + specifier = Specifier("{0}{1}".format(specifier.operator, version)) + try: + op = REPLACE_RANGES[specifier.operator] + except KeyError: + return specifier + curr_tuple = _tuplize_version(version) + try: + next_tuple = (curr_tuple[0], curr_tuple[1] + 1) + except IndexError: + next_tuple = (curr_tuple[0], 1) + if not next_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: + if specifier.operator == "<" and curr_tuple[1] <= MAX_VERSIONS[next_tuple[0]]: + op = "<=" + next_tuple = (next_tuple[0], curr_tuple[1]) + else: + return specifier + specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) + return specifier + + +@lru_cache(maxsize=1024) +def _get_specs(specset): + if specset is None: + return + if is_instance(specset, Specifier): + new_specset = SpecifierSet() + specs = set() + specs.add(specset) + new_specset._specs = frozenset(specs) + specset = new_specset + if isinstance(specset, str): + specset = SpecifierSet(specset) + result = [] + for spec in set(specset): + version = spec.version + op = spec.operator + if op in ("in", "not in"): + versions = version.split(",") + op = "==" if op == "in" else "!=" + for ver in versions: + result.append((op, _tuplize_version(ver.strip()))) + else: + result.append((spec.operator, _tuplize_version(spec.version))) + return sorted(result, key=operator.itemgetter(1)) + + +# TODO: Rename this to something meaningful +def _group_by_op(specs): + # type: (Union[Set[Specifier], SpecifierSet]) -> Iterator + specs = [_get_specs(x) for x in list(specs)] + flattened = [ + ((op, len(version) > 2), version) for spec in specs for op, version in spec + ] + specs = sorted(flattened) + grouping = itertools.groupby(specs, key=operator.itemgetter(0)) + return grouping + + +# TODO: rename this to something meaningful +def normalize_specifier_set(specs): + # type: (Union[str, SpecifierSet]) -> Optional[Set[Specifier]] + """Given a specifier set, a string, or an iterable, normalize the specifiers + + .. note:: This function exists largely to deal with ``pyzmq`` which handles + the ``requires_python`` specifier incorrectly, using ``3.7*`` rather than + the correct form of ``3.7.*``. This workaround can likely go away if + we ever introduce enforcement for metadata standards on PyPI. + + :param Union[str, SpecifierSet] specs: Supplied specifiers to normalize + :return: A new set of specifiers or specifierset + :rtype: Union[Set[Specifier], :class:`~packaging.specifiers.SpecifierSet`] + """ + if not specs: + return None + if isinstance(specs, set): + return specs + # when we aren't dealing with a string at all, we can normalize this as usual + elif not isinstance(specs, six.string_types): + return {_format_pyspec(spec) for spec in specs} + spec_list = [] + for spec in specs.split(","): + spec = spec.strip() + if spec.endswith(".*"): + spec = spec[:-2] + spec = spec.rstrip("*") + spec_list.append(spec) + return normalize_specifier_set(SpecifierSet(",".join(spec_list))) + + +# TODO: Check if this is used by anything public otherwise make it private +# And rename it to something meaningful +def get_sorted_version_string(version_set): + # type: (Set[AnyStr]) -> AnyStr + version_list = sorted( + "{0}".format(_format_version(version)) for version in version_set + ) + version = ", ".join(version_list) + return version + + +# TODO: Rename this to something meaningful +# TODO: Add a deprecation decorator and deprecate this -- i'm sure it's used +# in other libraries +@lru_cache(maxsize=1024) +def cleanup_pyspecs(specs, joiner="or"): + specs = normalize_specifier_set(specs) + # for != operator we want to group by version + # if all are consecutive, join as a list + results = {} + translation_map = { + # if we are doing an or operation, we need to use the min for >= + # this way OR(>=2.6, >=2.7, >=3.6) picks >=2.6 + # if we do an AND operation we need to use MAX to be more selective + (">", ">="): { + "or": lambda x: _format_version(min(x)), + "and": lambda x: _format_version(max(x)), + }, + # we use inverse logic here so we will take the max value if we are + # using OR but the min value if we are using AND + ("<", "<="): { + "or": lambda x: _format_version(max(x)), + "and": lambda x: _format_version(min(x)), + }, + # leave these the same no matter what operator we use + ("!=", "==", "~=", "==="): { + "or": get_sorted_version_string, + "and": get_sorted_version_string, + }, + } + op_translations = { + "!=": lambda x: "not in" if len(x) > 1 else "!=", + "==": lambda x: "in" if len(x) > 1 else "==", + } + translation_keys = list(translation_map.keys()) + for op_and_version_type, versions in _group_by_op(tuple(specs)): + op = op_and_version_type[0] + versions = [version[1] for version in versions] + versions = sorted(dedup(versions)) + op_key = next(iter(k for k in translation_keys if op in k), None) + version_value = versions + if op_key is not None: + version_value = translation_map[op_key][joiner](versions) + if op in op_translations: + op = op_translations[op](versions) + results[(op, op_and_version_type[1])] = version_value + return sorted([(k[0], v) for k, v in results.items()], key=operator.itemgetter(1)) + + +# TODO: Rename this to something meaningful +@lru_cache(maxsize=1024) +def fix_version_tuple(version_tuple): + # type: (Tuple[AnyStr, AnyStr]) -> Tuple[AnyStr, AnyStr] + op, version = version_tuple + max_major = max(MAX_VERSIONS.keys()) + if version[0] > max_major: + return (op, (max_major, MAX_VERSIONS[max_major])) + max_allowed = MAX_VERSIONS[version[0]] + if op == "<" and version[1] > max_allowed and version[1] - 1 <= max_allowed: + op = "<=" + version = (version[0], version[1] - 1) + return (op, version) + + +# TODO: Rename this to something meaningful, deprecate it (See prior function) +@lru_cache(maxsize=128) +def get_versions(specset, group_by_operator=True): + # type: (Union[Set[Specifier], SpecifierSet], bool) -> List[Tuple[STRING_TYPE, STRING_TYPE]] + specs = [_get_specs(x) for x in list(tuple(specset))] + initial_sort_key = lambda k: (k[0], k[1]) + initial_grouping_key = operator.itemgetter(0) + if not group_by_operator: + initial_grouping_key = operator.itemgetter(1) + initial_sort_key = operator.itemgetter(1) + version_tuples = sorted( + set((op, version) for spec in specs for op, version in spec), key=initial_sort_key + ) + version_tuples = [fix_version_tuple(t) for t in version_tuples] + op_groups = [ + (grp, list(map(operator.itemgetter(1), keys))) + for grp, keys in itertools.groupby(version_tuples, key=initial_grouping_key) + ] + versions = [ + (op, packaging.version.parse(".".join(str(v) for v in val))) + for op, vals in op_groups + for val in vals + ] + return sorted(versions, key=operator.itemgetter(1)) + + +def _ensure_marker(marker): + # type: (Union[STRING_TYPE, Marker]) -> Marker + if not is_instance(marker, Marker): + return Marker(str(marker)) + return marker + + +def gen_marker(mkr): + # type: (List[STRING_TYPE]) -> Marker + m = Marker("python_version == '1'") + m._markers.pop() + m._markers.append(mkr) + return m + + +def _strip_extra(elements): + """Remove the "extra == ..." operands from the list.""" + + return _strip_marker_elem("extra", elements) + + +def _strip_pyversion(elements): + return _strip_marker_elem("python_version", elements) + + +def _strip_marker_elem(elem_name, elements): + """Remove the supplied element from the marker. + + This is not a comprehensive implementation, but relies on an important + characteristic of metadata generation: The element's operand is always + associated with an "and" operator. This means that we can simply remove the + operand and the "and" operator associated with it. + """ + + extra_indexes = [] + preceding_operators = ["and"] if elem_name == "extra" else ["and", "or"] + for i, element in enumerate(elements): + if isinstance(element, list): + cancelled = _strip_marker_elem(elem_name, element) + if cancelled: + extra_indexes.append(i) + elif isinstance(element, tuple) and element[0].value == elem_name: + extra_indexes.append(i) + for i in reversed(extra_indexes): + del elements[i] + if i > 0 and elements[i - 1] in preceding_operators: + # Remove the "and" before it. + del elements[i - 1] + elif elements: + # This shouldn't ever happen, but is included for completeness. + # If there is not an "and" before this element, try to remove the + # operator after it. + del elements[0] + return not elements + + +def _get_stripped_marker(marker, strip_func): + """Build a new marker which is cleaned according to `strip_func`""" + + if not marker: + return None + marker = _ensure_marker(marker) + elements = marker._markers + strip_func(elements) + if elements: + return marker + return None + + +def get_without_extra(marker): + """Build a new marker without the `extra == ...` part. + + The implementation relies very deep into packaging's internals, but I don't + have a better way now (except implementing the whole thing myself). + + This could return `None` if the `extra == ...` part is the only one in the + input marker. + """ + + return _get_stripped_marker(marker, _strip_extra) + + +def get_without_pyversion(marker): + """Built a new marker without the `python_version` part. + + This could return `None` if the `python_version` section is the only section in the + marker. + """ + + return _get_stripped_marker(marker, _strip_pyversion) + + +def _markers_collect_extras(markers, collection): + # Optimization: the marker element is usually appended at the end. + for el in reversed(markers): + if isinstance(el, tuple) and el[0].value == "extra" and el[1].value == "==": + collection.add(el[2].value) + elif isinstance(el, list): + _markers_collect_extras(el, collection) + + +def _markers_collect_pyversions(markers, collection): + local_collection = [] + marker_format_str = "{0}" + for i, el in enumerate(reversed(markers)): + if isinstance(el, tuple) and el[0].value == "python_version": + new_marker = str(gen_marker(el)) + local_collection.append(marker_format_str.format(new_marker)) + elif isinstance(el, list): + _markers_collect_pyversions(el, local_collection) + if local_collection: + # local_collection = "{0}".format(" ".join(local_collection)) + collection.extend(local_collection) + + +def _markers_contains_extra(markers): + # Optimization: the marker element is usually appended at the end. + return _markers_contains_key(markers, "extra") + + +def _markers_contains_pyversion(markers): + return _markers_contains_key(markers, "python_version") + + +def _markers_contains_key(markers, key): + for element in reversed(markers): + if isinstance(element, tuple) and element[0].value == key: + return True + elif isinstance(element, list): + if _markers_contains_key(element, key): + return True + return False + + +@lru_cache(maxsize=128) +def get_contained_extras(marker): + """Collect "extra == ..." operands from a marker. + + Returns a list of str. Each str is a speficied extra in this marker. + """ + if not marker: + return set() + extras = set() + marker = _ensure_marker(marker) + _markers_collect_extras(marker._markers, extras) + return extras + + +@lru_cache(maxsize=1024) +def get_contained_pyversions(marker): + """Collect all `python_version` operands from a marker. + """ + + collection = [] + if not marker: + return set() + marker = _ensure_marker(marker) + # Collect the (Variable, Op, Value) tuples and string joiners from the marker + _markers_collect_pyversions(marker._markers, collection) + marker_str = " and ".join(sorted(collection)) + if not marker_str: + return set() + # Use the distlib dictionary parser to create a dictionary 'trie' which is a bit + # easier to reason about + marker_dict = distlib.markers.parse_marker(marker_str)[0] + version_set = set() + pyversions, _ = parse_marker_dict(marker_dict) + if isinstance(pyversions, set): + version_set.update(pyversions) + elif pyversions is not None: + version_set.add(pyversions) + # Each distinct element in the set was separated by an "and" operator in the marker + # So we will need to reduce them with an intersection here rather than a union + # in order to find the boundaries + versions = set() + if version_set: + versions = reduce(lambda x, y: x & y, version_set) + return versions + + +@lru_cache(maxsize=128) +def contains_extra(marker): + """Check whehter a marker contains an "extra == ..." operand. + """ + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_extra(marker._markers) + + +@lru_cache(maxsize=128) +def contains_pyversion(marker): + """Check whether a marker contains a python_version operand. + """ + + if not marker: + return False + marker = _ensure_marker(marker) + return _markers_contains_pyversion(marker._markers) + + +def _split_specifierset_str(specset_str, prefix="=="): + # type: (str, str) -> Set[Specifier] + """ + Take a specifierset string and split it into a list to join for specifier sets + + :param str specset_str: A string containing python versions, often comma separated + :param str prefix: A prefix to use when generating the specifier set + :return: A list of :class:`Specifier` instances generated with the provided prefix + :rtype: Set[Specifier] + """ + specifiers = set() + if "," not in specset_str and " " in specset_str: + values = [v.strip() for v in specset_str.split()] + else: + values = [v.strip() for v in specset_str.split(",")] + if prefix == "!=" and any(v in values for v in DEPRECATED_VERSIONS): + values = DEPRECATED_VERSIONS[:] + for value in sorted(values): + specifiers.add(Specifier("{0}{1}".format(prefix, value))) + return specifiers + + +def _get_specifiers_from_markers(marker_item): + """ + Given a marker item, get specifiers from the version marker + + :param :class:`~packaging.markers.Marker` marker_sequence: A marker describing a version constraint + :return: A set of specifiers corresponding to the marker constraint + :rtype: Set[Specifier] + """ + specifiers = set() + if isinstance(marker_item, tuple): + variable, op, value = marker_item + if variable.value != "python_version": + return specifiers + if op.value == "in": + specifiers.update(_split_specifierset_str(value.value, prefix="==")) + elif op.value == "not in": + specifiers.update(_split_specifierset_str(value.value, prefix="!=")) + else: + specifiers.add(Specifier("{0}{1}".format(op.value, value.value))) + elif isinstance(marker_item, list): + parts = get_specset(marker_item) + if parts: + specifiers.update(parts) + return specifiers + + +def get_specset(marker_list): + # type: (List) -> Optional[SpecifierSet] + specset = set() + _last_str = "and" + for marker_parts in marker_list: + if isinstance(marker_parts, str): + _last_str = marker_parts # noqa + else: + specset.update(_get_specifiers_from_markers(marker_parts)) + specifiers = SpecifierSet() + specifiers._specs = frozenset(specset) + return specifiers + + +# TODO: Refactor this (reduce complexity) +def parse_marker_dict(marker_dict): + op = marker_dict["op"] + lhs = marker_dict["lhs"] + rhs = marker_dict["rhs"] + # This is where the spec sets for each side land if we have an "or" operator + side_spec_list = [] + side_markers_list = [] + finalized_marker = "" + # And if we hit the end of the parse tree we use this format string to make a marker + format_string = "{lhs} {op} {rhs}" + specset = SpecifierSet() + specs = set() + # Essentially we will iterate over each side of the parsed marker if either one is + # A mapping instance (i.e. a dictionary) and recursively parse and reduce the specset + # Union the "and" specs, intersect the "or"s to find the most appropriate range + if any(issubclass(type(side), Mapping) for side in (lhs, rhs)): + for side in (lhs, rhs): + side_specs = set() + side_markers = set() + if issubclass(type(side), Mapping): + merged_side_specs, merged_side_markers = parse_marker_dict(side) + side_specs.update(merged_side_specs) + side_markers.update(merged_side_markers) + else: + marker = _ensure_marker(side) + marker_parts = getattr(marker, "_markers", []) + if marker_parts[0][0].value == "python_version": + side_specs |= set(get_specset(marker_parts)) + else: + side_markers.add(str(marker)) + side_spec_list.append(side_specs) + side_markers_list.append(side_markers) + if op == "and": + # When we are "and"-ing things together, it probably makes the most sense + # to reduce them here into a single PySpec instance + specs = reduce(lambda x, y: set(x) | set(y), side_spec_list) + markers = reduce(lambda x, y: set(x) | set(y), side_markers_list) + if not specs and not markers: + return specset, finalized_marker + if markers and isinstance(markers, (tuple, list, Set)): + finalized_marker = Marker(" and ".join([m for m in markers if m])) + elif markers: + finalized_marker = str(markers) + specset._specs = frozenset(specs) + return specset, finalized_marker + # Actually when we "or" things as well we can also just turn them into a reduced + # set using this logic now + sides = reduce(lambda x, y: set(x) & set(y), side_spec_list) + finalized_marker = " or ".join( + [normalize_marker_str(m) for m in side_markers_list] + ) + specset._specs = frozenset(sorted(sides)) + return specset, finalized_marker + else: + # At the tip of the tree we are dealing with strings all around and they just need + # to be smashed together + specs = set() + if lhs == "python_version": + format_string = "{lhs}{op}{rhs}" + marker = Marker(format_string.format(**marker_dict)) + marker_parts = getattr(marker, "_markers", []) + _set = get_specset(marker_parts) + if _set: + specs |= set(_set) + specset._specs = frozenset(specs) + return specset, finalized_marker + + +def _contains_micro_version(version_string): + return re.search("\d+\.\d+\.\d+", version_string) is not None + + +def format_pyversion(parts): + op, val = parts + version_marker = ( + "python_full_version" if _contains_micro_version(val) else "python_version" + ) + return "{0} {1} '{2}'".format(version_marker, op, val) + + +def normalize_marker_str(marker): + # type: (Union[Marker, STRING_TYPE]) -> str + marker_str = "" + if not marker: + return None + if not is_instance(marker, Marker): + marker = _ensure_marker(marker) + pyversion = get_contained_pyversions(marker) + marker = get_without_pyversion(marker) + if pyversion: + parts = cleanup_pyspecs(pyversion) + marker_str = " and ".join([format_pyversion(pv) for pv in parts]) + if marker: + if marker_str: + marker_str = "{0!s} and {1!s}".format(marker_str, marker) + else: + marker_str = "{0!s}".format(marker) + return marker_str.replace('"', "'") + + +@lru_cache(maxsize=1024) +def marker_from_specifier(spec): + # type: (STRING_TYPE) -> Marker + if not any(spec.startswith(k) for k in Specifier._operators.keys()): + if spec.strip().lower() in ["any", "<any>", "*"]: + return None + spec = "=={0}".format(spec) + elif spec.startswith("==") and spec.count("=") > 3: + spec = "=={0}".format(spec.lstrip("=")) + if not spec: + return None + marker_segments = [] + for marker_segment in cleanup_pyspecs(spec): + marker_segments.append(format_pyversion(marker_segment)) + marker_str = " and ".join(marker_segments).replace('"', "'") + return Marker(marker_str) + + +def merge_markers(m1, m2): + # type: (Marker, Marker) -> Optional[Marker] + if not all((m1, m2)): + return next(iter(v for v in (m1, m2) if v), None) + m1 = _ensure_marker(m1) + m2 = _ensure_marker(m2) + _markers = [] # type: List[Marker] + for marker in (m1, m2): + _markers.append(str(marker)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + return _ensure_marker(normalize_marker_str(marker_str)) diff --git a/pipenv/vendor/requirementslib/models/metadata.py b/pipenv/vendor/requirementslib/models/metadata.py new file mode 100644 index 00000000..912b1b77 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/metadata.py @@ -0,0 +1,1240 @@ +# -*- coding=utf-8 -*- +import datetime +import functools +import io +import json +import logging +import operator +import os +import zipfile +from collections import defaultdict + +import attr +import dateutil.parser +import distlib.metadata +import distlib.wheel +import packaging.version +import requests +import six +import vistir +from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement +from packaging.specifiers import Specifier, SpecifierSet +from packaging.tags import Tag + +from ..environment import MYPY_RUNNING +from .markers import ( + get_contained_extras, + get_contained_pyversions, + get_without_extra, + get_without_pyversion, + marker_from_specifier, + merge_markers, + normalize_specifier_set, +) +from .requirements import Requirement +from .utils import filter_dict, get_pinned_version, is_pinned_requirement + +# fmt: off +from six.moves import Sequence # type: ignore # isort:skip +from six.moves import reduce # type: ignore # isort:skip +# fmt: on # isort:skip + + +ch = logging.StreamHandler() +formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") +ch.setFormatter(formatter) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +if MYPY_RUNNING: + from typing import ( + Any, + Callable, + Dict, + Generator, + Generic, + Iterator, + List, + Optional, + Set, + Tuple, + Type, + TypeVar, + Union, + ) + from attr import Attribute # noqa + from .setup_info import SetupInfo + + TAttrsClass = TypeVar("TAttrsClass") + AttrsClass = Generic[TAttrsClass] + TDigestDict = Dict[str, str] + TProjectUrls = Dict[str, str] + TReleaseUrlDict = Dict[str, Union[bool, int, str, TDigestDict]] + TReleasesList = List[TReleaseUrlDict] + TReleasesDict = Dict[str, TReleasesList] + TDownloads = Dict[str, int] + TPackageInfo = Dict[str, Optional[Union[str, List[str], TDownloads, TProjectUrls]]] + TPackage = Dict[str, Union[TPackageInfo, int, TReleasesDict, TReleasesList]] + + +VALID_ALGORITHMS = { + "sha1": 40, + "sha3_224": 56, + "sha512": 128, + "blake2b": 128, + "sha256": 64, + "sha384": 96, + "blake2s": 64, + "sha3_256": 64, + "sha3_512": 128, + "md5": 32, + "sha3_384": 96, + "sha224": 56, +} # type: Dict[str, int] + +PACKAGE_TYPES = { + "sdist", + "bdist_wheel", + "bdist_egg", + "bdist_dumb", + "bdist_wininst", + "bdist_rpm", + "bdist_msi", + "bdist_dmg", +} + + +class PackageEncoder(json.JSONEncoder): + def default(self, obj): # noqa:E0202 # noqa:W0221 + if isinstance(obj, datetime.datetime): + return obj.isoformat() + elif isinstance(obj, PackagingRequirement): + return obj.__dict__ + elif isinstance(obj, set): + return tuple(obj) + elif isinstance(obj, (Specifier, SpecifierSet, Marker)): + return str(obj) + else: + return json.JSONEncoder.default(self, obj) + + +def validate_extras(inst, attrib, value): + # type: ("Dependency", Attribute, Tuple[str, ...]) -> None + duplicates = [k for k in value if value.count(k) > 1] + if duplicates: + raise ValueError("Found duplicate keys: {0}".format(", ".join(duplicates))) + return None + + +def validate_digest(inst, attrib, value): + # type: ("Digest", Attribute, str) -> None + expected_length = VALID_ALGORITHMS[inst.algorithm.lower()] + if len(value) != expected_length: + raise ValueError( + "Expected a digest of length {0!s}, got one of length {1!s}".format( + expected_length, len(value) + ) + ) + return None + + +def get_local_wheel_metadata(wheel_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + with io.open(wheel_file, "rb") as fh: + with zipfile.ZipFile(fh, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(wheel_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def get_remote_sdist_metadata(line): + # type: (str) -> SetupInfo + req = Requirement.from_line(line) + try: + _ = req.run_requires() + except SystemExit: + raise RuntimeError("Failed to compute metadata for dependency {0}".format(line)) + else: + return req.line_instance.setup_info + + +def get_remote_wheel_metadata(whl_file): + # type: (str) -> Optional[distlib.metadata.Metadata] + parsed_metadata = None + data = io.BytesIO() + with vistir.contextmanagers.open_file(whl_file) as fp: + for chunk in iter(lambda: fp.read(8096), b""): + data.write(chunk) + with zipfile.ZipFile(data, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: + metadata = None + for fn in zf.namelist(): + if os.path.basename(fn) == "METADATA": + metadata = fn + break + if metadata is None: + raise RuntimeError("No metadata found in wheel: {0}".format(whl_file)) + with zf.open(metadata, "r") as metadata_fh: + parsed_metadata = distlib.metadata.Metadata(fileobj=metadata_fh) + return parsed_metadata + + +def create_specifierset(spec=None): + # type: (Optional[str]) -> SpecifierSet + if isinstance(spec, SpecifierSet): + return spec + elif isinstance(spec, (set, list, tuple)): + spec = " and ".join(spec) + if spec is None: + spec = "" + return SpecifierSet(spec) + + +@attr.s(frozen=True, eq=True) +class ExtrasCollection(object): + #: The name of the extras collection (e.g. 'security') + name = attr.ib(type=str) + #: The dependency the collection belongs to + parent = attr.ib(type="Dependency") + #: The members of the collection + dependencies = attr.ib(factory=set) # type: Set["Dependency"] + + def add_dependency(self, dependency): + # type: ("Dependency") -> "ExtrasCollection" + if not isinstance(dependency, Dependency): + raise TypeError( + "Expected a Dependency instance, received {0!r}".format(dependency) + ) + dependencies = self.dependencies.copy() + dependencies.add(dependency) + return attr.evolve(self, dependencies=dependencies) + + +@attr.s(frozen=True, eq=True) +class Dependency(object): + #: The name of the dependency + name = attr.ib(type=str) + #: A requirement instance + requirement = attr.ib(type=PackagingRequirement, eq=False) + #: The specifier defined in the dependency definition + specifier = attr.ib(type=SpecifierSet, converter=create_specifierset, eq=False) + #: Any extras this dependency declares + extras = attr.ib(factory=tuple, validator=validate_extras) # type: Tuple[str, ...] + #: The name of the extra meta-dependency this one came from (e.g. 'security') + from_extras = attr.ib(default=None, eq=False) # type: Optional[str] + #: The declared specifier set of allowable python versions for this dependency + python_version = attr.ib( + default="", type=SpecifierSet, converter=create_specifierset, eq=False + ) + #: The parent of this dependency (i.e. where it came from) + parent = attr.ib(default=None) # type: Optional[Dependency] + #: The markers for this dependency + markers = attr.ib(default=None, eq=False) # type: Optional[Marker] + _specset_str = attr.ib(default="", type=str) + _python_version_str = attr.ib(default="", type=str) + _marker_str = attr.ib(default="", type=str) + + def __str__(self): + # type: () -> str + return str(self.requirement) + + def as_line(self): + # type: () -> str + line_str = "{0}".format(self.name) + if self.extras: + line_str = "{0}[{1}]".format(line_str, ",".join(self.extras)) + if self.specifier: + line_str = "{0}{1!s}".format(line_str, self.specifier) + py_version_part = "" + if self.python_version: + specifiers = normalize_specifier_set(self.python_version) + markers = [] + if specifiers is not None: + markers = [marker_from_specifier(str(s)) for s in specifiers] + py_version_part = reduce(merge_markers, markers) + if self.markers: + line_str = "{0}; {1}".format(line_str, str(self.markers)) + if py_version_part: + line_str = "{0} and {1}".format(line_str, py_version_part) + elif py_version_part and not self.markers: + line_str = "{0}; {1}".format(line_str, py_version_part) + return line_str + + def pin(self): + # type: () -> "Package" + base_package = get_package(self.name) + sorted_releases = sorted( + base_package.releases.non_yanked_releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + version = next( + iter(self.specifier.filter((r.version for r in sorted_releases))), None + ) + if not version: + version = next( + iter( + self.specifier.filter( + (r.version for r in sorted_releases), prereleases=True + ) + ), + None, + ) + if not version: + raise RuntimeError( + "Failed to resolve {0} ({1!s})".format(self.name, self.specifier) + ) + match = get_package_version(self.name, str(version)) + return match + + @classmethod + def from_requirement(cls, req, parent=None): + # type: (PackagingRequirement, Optional["Dependency"]) -> "Dependency" + from_extras, marker, python_version = None, None, None + specset_str, py_version_str, marker_str = "", "", "" + if req.marker: + marker = Marker(str(req.marker)) + from_extras = next(iter(list(get_contained_extras(marker))), None) + python_version = get_contained_pyversions(marker) + marker = get_without_extra(get_without_pyversion(marker)) + if not str(marker) or not marker or not marker._markers: + marker = None + req.marker = marker + if marker is not None: + marker_str = str(marker) + if req.specifier: + specset_str = str(req.specifier) + if python_version: + py_version_str = str(python_version) + return cls( + name=req.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=from_extras, + python_version=python_version, + markers=marker, + parent=parent, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_info(cls, info): + # type: ("PackageInfo") -> "Dependency" + marker_str = "" + specset_str, py_version_str = "", "" + if info.requires_python: + # XXX: Some markers are improperly formatted -- we already handle most cases + # XXX: but learned about new broken formats, such as + # XXX: python_version in "2.6 2.7 3.2 3.3" (note the lack of commas) + # XXX: as a marker on a dependency of a library called 'pickleshare' + # XXX: Some packages also have invalid markers with stray characters, + # XXX: such as 'algoliasearch' + try: + marker = marker_from_specifier(info.requires_python) + except Exception: + marker_str = "" + else: + if not marker or not marker._markers: + marker_str = "" + else: + marker_str = "{0!s}".format(marker) + req_str = "{0}=={1}".format(info.name, info.version) + if marker_str: + req_str = "{0}; {1}".format(req_str, marker_str) + req = PackagingRequirement(req_str) + requires_python_str = ( + info.requires_python if info.requires_python is not None else "" + ) + if req.specifier: + specset_str = str(req.specifier) + if requires_python_str: + py_version_str = requires_python_str + return cls( + name=info.name, + specifier=req.specifier, + extras=tuple(sorted(set(req.extras))) if req.extras is not None else req.extras, + requirement=req, + from_extras=None, + python_version=SpecifierSet(requires_python_str), + markers=None, + parent=None, + specset_str=specset_str, + python_version_str=py_version_str, + marker_str=marker_str, + ) + + @classmethod + def from_str(cls, depstr, parent=None): + # type: (str, Optional["Dependency"]) -> "Dependency" + try: + req = PackagingRequirement(depstr) + except Exception: + raise + return cls.from_requirement(req, parent=parent) + + def add_parent(self, parent): + # type: ("Dependency") -> "Dependency" + return attr.evolve(self, parent=parent) + + +@attr.s(frozen=True, eq=True) +class Digest(object): + #: The algorithm declared for the digest, e.g. 'sha256' + algorithm = attr.ib( + type=str, validator=attr.validators.in_(VALID_ALGORITHMS.keys()), eq=True + ) + #: The digest value + value = attr.ib(type=str, validator=validate_digest, eq=True) + + def __str__(self): + # type: () -> str + return "{0}:{1}".format(self.algorithm, self.value) + + @classmethod + def create(cls, algorithm, value): + # type: (str, str) -> "Digest" + return cls(algorithm=algorithm, value=value) + + @classmethod + def collection_from_dict(cls, digest_dict): + # type: (TDigestDict) -> List["Digest"] + return [cls.create(k, v) for k, v in digest_dict.items()] + + +# XXX: This is necessary because attrs converters can only be functions, not classmethods +def create_digest_collection(digest_dict): + # type: (TDigestDict) -> List["Digest"] + return Digest.collection_from_dict(digest_dict) + + +def instance_check_converter(expected_type=None, converter=None): + # type: (Optional[Type], Optional[Callable]) -> Callable + def _converter(val): + if expected_type is not None and isinstance(val, expected_type): + return val + return converter(val) + + return _converter + + +@attr.s(frozen=True, eq=True) +class ParsedTag(object): + #: The marker string corresponding to the tag + marker_string = attr.ib(default=None) # type: Optional[str] + #: The python version represented by the tag + python_version = attr.ib(default=None) # type: Optional[str] + #: The platform represented by the tag + platform_system = attr.ib(default=None) # type: Optional[str] + #: the ABI represented by the tag + abi = attr.ib(default=None) # type: Optional[str] + + +def parse_tag(tag): + # type: (Tag) -> ParsedTag + """ + Parse a :class:`~packaging.tags.Tag` instance + + :param :class:`~packaging.tags.Tag` tag: A tag to parse + :return: A parsed tag with combined markers, supported platform and python version + :rtype: :class:`~ParsedTag` + """ + platform_system = None + python_version = None + version = None + marker_str = "" + if tag.platform.startswith("macos"): + platform_system = "Darwin" + elif tag.platform.startswith("manylinux") or tag.platform.startswith("linux"): + platform_system = "Linux" + elif tag.platform.startswith("win32"): + platform_system = "Windows" + if platform_system: + marker_str = 'platform_system == "{}"'.format(platform_system) + if tag.interpreter: + version = tag.interpreter[2:] + py_version_str = "" + if len(version) == 1: + py_version_str = ">={}.0,<{}".format(version, str(int(version) + 1)) + elif len(version) > 1 and len(version) <= 3: + # reverse the existing version so we can add 1 to the first element + # and re-reverse, generating the new version, e.g. [3, 2, 8] => + # [8, 2, 3] => [9, 2, 3] => [3, 2, 9] + next_version_list = list(reversed(version[:])) + next_version_list[0] = str(int(next_version_list[0]) + 1) + next_version = ".".join(list(reversed(next_version_list))) + version = ".".join(version) + py_version_str = ">={},<{}".format(version, next_version) + else: + py_version_str = "{0}".format(version) + python_version = marker_from_specifier(py_version_str) + if python_version: + if marker_str: + marker_str = "{0} and {1!s}".format(marker_str, python_version) + else: + marker_str = str(python_version) + return ParsedTag( + marker_string=marker_str, + python_version=version, + platform_system=platform_system, + abi=tag.abi, + ) + + +@attr.s(frozen=True, eq=True) +class ReleaseUrl(object): + #: The MD5 digest of the given release + md5_digest = attr.ib(type=Digest) + #: The package type of the url + packagetype = attr.ib(type=str, validator=attr.validators.in_(PACKAGE_TYPES)) + #: The upload timestamp from the package + upload_time = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The ISO8601 formatted upload timestamp of the package + upload_time_iso_8601 = attr.ib( + type=datetime.datetime, + converter=instance_check_converter(datetime.datetime, dateutil.parser.parse), # type: ignore + ) + #: The size in bytes of the package + size = attr.ib(type=int) + #: The URL of the package + url = attr.ib(type=str) + #: The digests of the package + digests = attr.ib( + converter=instance_check_converter(list, create_digest_collection) # type: ignore + ) # type: List[Digest] + #: The name of the package + name = attr.ib(type=str, default=None) + #: The available comments of the given upload + comment_text = attr.ib(type=str, default="") + #: The number of downloads (deprecated) + downloads = attr.ib(type=int, default=-1) + #: The filename of the current upload + filename = attr.ib(type=str, default="") + #: Whether the upload has a signature + has_sig = attr.ib(type=bool, default=False) + #: The python_version attribute of the upload (e.g. 'source', 'py27', etc) + python_version = attr.ib(type=str, default="source") + #: The 'requires_python' restriction on the package + requires_python = attr.ib(type=str, default=None) + #: A list of valid aprsed tags from the upload + tags = attr.ib(factory=list) # type: List[ParsedTag] + + @property + def is_wheel(self): + # type: () -> bool + return os.path.splitext(self.filename)[-1].lower() == ".whl" + + @property + def is_sdist(self): + # type: () -> bool + return self.python_version == "source" + + @property + def markers(self): + # type: () -> Optional[str] + # TODO: Compare dependencies in parent and add markers for python version + # TODO: Compare dependencies in parent and add markers for platform + # XXX: We can't use wheel-based markers until we do it via derived markers by + # XXX: comparing in the parent (i.e. 'Release' instance or so) and merging + # XXX: down to the common / minimal set of markers otherwise we wind up + # XXX: with an unmanageable set and combinatorial explosion + # if self.is_wheel: + # return self.get_markers_from_wheel() + if self.requires_python: + return marker_from_specifier(self.requires_python) + return None + + @property + def pep508_url(self): + # type: () -> str + markers = self.markers + req_str = "{0} @ {1}#egg={0}".format(self.name, self.url) + if markers: + req_str = "{0}; {1}".format(req_str, markers) + return req_str + + def get_markers_from_wheel(self): + # type: () -> str + supported_platforms = [] # type: List[str] + supported_pyversions = [] + supported_abis = [] + markers = [] + for parsed_tag in self.tags: + if parsed_tag.marker_string: + markers.append(Marker(parsed_tag.marker_string)) + if parsed_tag.python_version: + supported_pyversions.append(parsed_tag.python_version) + if parsed_tag.abi: + supported_abis.append(parsed_tag.abi) + if not (markers or supported_platforms): + return "" + if ( + all(pyversion in supported_pyversions for pyversion in ["2", "3"]) + and not supported_platforms + ): + marker_line = "" + else: + marker_line = " or ".join(["{}".format(str(marker)) for marker in markers]) + return marker_line + + def get_dependencies(self): + # type: () -> Tuple["ReleaseUrl", Dict[str, Union[List[str], str]]] + results = {"requires_python": None} + requires_dist = [] # type: List[str] + if self.is_wheel: + metadata = get_remote_wheel_metadata(self.url) + if metadata is not None: + requires_dist = metadata.run_requires + if not self.requires_python: + results["requires_python"] = metadata._legacy.get("Requires-Python") + else: + try: + metadata = get_remote_sdist_metadata(self.pep508_url) + except Exception: + requires_dist = [] + else: + requires_dist = [str(v) for v in metadata.requires.values()] + results["requires_dist"] = requires_dist + requires_python = getattr(self, "requires_python", results["requires_python"]) + return attr.evolve(self, requires_python=requires_python), results + + @property + def sha256(self): + # type: () -> str + return next( + iter(digest for digest in self.digests if digest.algorithm == "sha256") + ).value + + @classmethod + def create(cls, release_dict, name=None): + # type: (TReleaseUrlDict, Optional[str]) -> "ReleaseUrl" + valid_digest_keys = set("{0}_digest".format(k) for k in VALID_ALGORITHMS.keys()) + digest_keys = set(release_dict.keys()) & valid_digest_keys + creation_kwargs = {} # type: Dict[str, Union[bool, int, str, Digest, TDigestDict]] + creation_kwargs = { + k: v for k, v in release_dict.items() if k not in digest_keys + } + if name is not None: + creation_kwargs["name"] = name + for k in digest_keys: + digest = release_dict[k] + if not isinstance(digest, six.string_types): + raise TypeError("Digests must be strings, got {!r}".format(digest)) + creation_kwargs[k] = Digest.create(k.replace("_digest", ""), digest) + release_url = cls(**filter_dict(creation_kwargs)) # type: ignore + if release_url.is_wheel: + supported_tags = [ + parse_tag(Tag(*tag)) for tag in distlib.wheel.Wheel(release_url.url).tags + ] + release_url = attr.evolve(release_url, tags=supported_tags) + return release_url + + +def create_release_urls_from_list(urls, name=None): + # type: (Union[TReleasesList, List[ReleaseUrl]], Optional[str]) -> List[ReleaseUrl] + url_list = [] + for release_dict in urls: + if isinstance(release_dict, ReleaseUrl): + if name and not release_dict.name: + release_dict = attr.evolve(release_dict, name=name) + url_list.append(release_dict) + continue + url_list.append(ReleaseUrl.create(release_dict, name=name)) + return url_list + + +@attr.s(frozen=True, eq=True) +class ReleaseUrlCollection(Sequence): + #: A list of release URLs + urls = attr.ib(converter=create_release_urls_from_list) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + @classmethod + def create(cls, urls, name=None): + # type: (TReleasesList, Optional[str]) -> "ReleaseUrlCollection" + return cls(urls=urls, name=name) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_wheel: + continue + yield url + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for url in self.urls: + if not url.is_sdist: + continue + yield url + + def __iter__(self): + # type: () -> Iterator[ReleaseUrl] + return iter(self.urls) + + def __getitem__(self, key): + # type: (int) -> ReleaseUrl + return self.urls.__getitem__(key) + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def latest(self): + # type: () -> Optional[ReleaseUrl] + if not self.urls: + return None + return next( + iter(sorted(self.urls, key=operator.attrgetter("upload_time"), reverse=True)) + ) + + @property + def latest_timestamp(self): + # type: () -> Optional[datetime.datetime] + latest = self.latest + if latest is not None: + return latest.upload_time + return None + + def find_package_type(self, type_): + # type: (str) -> Optional[ReleaseUrl] + """ + Given a package type (e.g. sdist, bdist_wheel), find the matching release + + :param str type_: A package type from :const:`~PACKAGE_TYPES` + :return: The package from this collection matching that type, if available + :rtype: Optional[ReleaseUrl] + """ + if type_ not in PACKAGE_TYPES: + raise ValueError( + "Invalid package type: {0}. Expected one of {1}".format( + type_, " ".join(PACKAGE_TYPES) + ) + ) + return next(iter(url for url in self.urls if url.packagetype == type_), None) + + +def convert_release_urls_to_collection(urls=None, name=None): + # type: (Optional[TReleasesList], Optional[str]) -> ReleaseUrlCollection + if urls is None: + urls = [] + urls = create_release_urls_from_list(urls, name=name) + return ReleaseUrlCollection.create(urls, name=name) + + +@attr.s(frozen=True) +class Release(Sequence): + #: The version of the release + version = attr.ib(type=str) + #: The URL collection for the release + urls = attr.ib( + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + type=ReleaseUrlCollection, + ) + #: the name of the package + name = attr.ib(default=None) # type: Optional[str] + + def __iter__(self): + # type: () -> Iterator[ReleaseUrlCollection] + return iter(self.urls) + + def __getitem__(self, key): + return self.urls[key] + + def __len__(self): + # type: () -> int + return len(self.urls) + + @property + def yanked(self): + # type: () -> bool + if not self.urls: + return True + return False + + @property + def parsed_version(self): + # type: () -> packaging.version._BaseVersion + return packaging.version.parse(self.version) + + @property + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.wheels + + @property + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + return self.urls.sdists + + @property + def latest(self): + # type: () -> ReleaseUrl + return self.urls.latest + + @property + def latest_timestamp(self): + # type: () -> datetime.datetime + return self.urls.latest_timestamp + + def to_lockfile(self): + # type: () -> Dict[str, Union[List[str], str]] + return { + "hashes": [str(url.sha256) for url in self.urls if url.sha256 is not None], + "version": "=={0}".format(self.version), + } + + +def get_release(version, urls, name=None): + # type: (str, TReleasesList, Optional[str]) -> Release + release_kwargs = {"version": version, "name": name} + if not isinstance(urls, ReleaseUrlCollection): + release_kwargs["urls"] = convert_release_urls_to_collection(urls, name=name) + else: + release_kwargs["urls"] = urls + return Release(**release_kwargs) # type: ignore + + +def get_releases_from_package(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> List[Release] + release_list = [] + for version, urls in releases.items(): + release_list.append(get_release(version, urls, name=name)) + return release_list + + +@attr.s(frozen=True) +class ReleaseCollection(object): + releases = attr.ib( + factory=list, converter=instance_check_converter(list, get_releases_from_package), # type: ignore + ) # type: List[Release] + + def __iter__(self): + # type: () -> Iterator[Release] + return iter(self.releases) + + def __getitem__(self, key): + # type: (str) -> Release + result = next(iter(r for r in self.releases if r.version == key), None) + if result is None: + raise KeyError(key) + return result + + def __len__(self): + # type: () -> int + return len(self.releases) + + def get_latest_lockfile(self): + # type: () -> Dict[str, Union[str, List[str]]] + return self.latest.to_lockfile() + + def wheels(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for wheel in release.wheels: + yield wheel + + def sdists(self): + # type: () -> Iterator[ReleaseUrl] + for release in self.sort_releases(): + for sdist in release.sdists: + yield sdist + + @property + def non_yanked_releases(self): + # type: () -> List[Release] + return list(r for r in self.releases if not r.yanked) + + def sort_releases(self): + # type: () -> List[Release] + return sorted( + self.non_yanked_releases, + key=operator.attrgetter("latest_timestamp"), + reverse=True, + ) + + @property + def latest(self): + # type: () -> Optional[Release] + return next(iter(r for r in self.sort_releases() if not r.yanked)) + + @classmethod + def load(cls, releases, name=None): + # type: (Union[TReleasesDict, List[Release]], Optional[str]) -> "ReleaseCollection" + if not isinstance(releases, list): + releases = get_releases_from_package(releases, name=name) + return cls(releases) + + +def convert_releases_to_collection(releases, name=None): + # type: (TReleasesDict, Optional[str]) -> ReleaseCollection + return ReleaseCollection.load(releases, name=name) + + +def split_keywords(value): + # type: (Union[str, List]) -> List[str] + if value and isinstance(value, six.string_types): + return value.split(",") + elif isinstance(value, list): + return value + return [] + + +def create_dependencies( + requires_dist, # type: Optional[List[Dependency]] + parent=None, # type: Optional[Dependency] +): + # type: (...) -> Optional[Set[Dependency]] + if requires_dist is None: + return None + dependencies = set() + for req in requires_dist: + if not isinstance(req, Dependency): + dependencies.add(Dependency.from_str(req, parent=parent)) + else: + dependencies.add(req) + return dependencies + + +@attr.s(frozen=True) +class PackageInfo(object): + name = attr.ib(type=str) + version = attr.ib(type=str) + package_url = attr.ib(type=str) + summary = attr.ib(type=str, default=None) # type: Optional[str] + author = attr.ib(type=str, default=None) # type: Optional[str] + keywords = attr.ib(factory=list, converter=split_keywords) # type: List[str] + description = attr.ib(type=str, default="") + download_url = attr.ib(type=str, default="") + home_page = attr.ib(type=str, default="") + license = attr.ib(type=str, default="") + maintainer = attr.ib(type=str, default="") + maintainer_email = attr.ib(type=str, default="") + downloads = attr.ib(factory=dict) # type: Dict[str, int] + docs_url = attr.ib(default=None) # type: Optional[str] + platform = attr.ib(type=str, default="") + project_url = attr.ib(type=str, default="") + project_urls = attr.ib(factory=dict) # type: Dict[str, str] + requires_python = attr.ib(default=None) # type: Optional[str] + requires_dist = attr.ib(factory=list) # type: List[Dependency] + release_url = attr.ib(default=None) # type: Optional[str] + description_content_type = attr.ib(type=str, default="text/md") + bugtrack_url = attr.ib(default=None) # type: str + classifiers = attr.ib(factory=list) # type: List[str] + author_email = attr.ib(default=None) # type: Optional[str] + markers = attr.ib(default=None) # type: Optional[str] + dependencies = attr.ib(default=None) # type: Tuple[Dependency] + + @classmethod + def from_json(cls, info_json): + # type: (TPackageInfo) -> "PackageInfo" + return cls(**filter_dict(info_json)) # type: ignore + + def to_dependency(self): + # type: () -> Dependency + return Dependency.from_info(self) + + def create_dependencies(self, force=False): + # type: (bool) -> "PackageInfo" + """ + Create values for **self.dependencies**. + + :param bool force: Sets **self.dependencies** to an empty tuple if it would be + None, defaults to False. + :return: An updated instance of the current object with **self.dependencies** + updated accordingly. + :rtype: :class:`PackageInfo` + """ + if not self.dependencies and not self.requires_dist: + if force: + return attr.evolve(self, dependencies=tuple()) + return self + self_dependency = self.to_dependency() + deps = set() + self_dependencies = tuple() if not self.dependencies else self.dependencies + for dep in self_dependencies: + if dep is None: + continue + new_dep = dep.add_parent(self_dependency) + deps.add(new_dep) + created_deps = create_dependencies(self.requires_dist, parent=self_dependency) + if created_deps is not None: + for dep in created_deps: + if dep is None: + continue + deps.add(dep) + return attr.evolve(self, dependencies=tuple(sorted(deps))) + + +def convert_package_info(info_json): + # type: (Union[TPackageInfo, PackageInfo]) -> PackageInfo + if isinstance(info_json, PackageInfo): + return info_json + return PackageInfo.from_json(info_json) + + +def add_markers_to_dep(d, marker_str): + # type: (str, Union[str, Marker]) -> str + req = PackagingRequirement(d) + existing_marker = getattr(req, "marker", None) + if isinstance(marker_str, Marker): + marker_str = str(marker_str) + if existing_marker is not None: + marker_str = str(merge_markers(existing_marker, marker_str)) + if marker_str: + marker_str = marker_str.replace("'", '"') + req.marker = Marker(marker_str) + return str(req) + + +@attr.s +class Package(object): + info = attr.ib(type=PackageInfo, converter=convert_package_info) + last_serial = attr.ib(type=int) + releases = attr.ib( + type=ReleaseCollection, + converter=instance_check_converter( # type: ignore + ReleaseCollection, convert_releases_to_collection + ), + ) + # XXX: Note: sometimes releases have no urls at the top level (e.g. pyrouge) + urls = attr.ib( + type=ReleaseUrlCollection, + converter=instance_check_converter( # type: ignore + ReleaseUrlCollection, convert_release_urls_to_collection + ), + ) + + @urls.default + def _get_urls_collection(self): + return functools.partial(convert_release_urls_to_collection, urls=[], name=self.name) + + @property + def name(self): + # type: () -> str + return self.info.name + + @property + def version(self): + # type: () -> str + return self.info.version + + @property + def requirement(self): + # type: () -> PackagingRequirement + return self.info.to_dependency().requirement + + @property + def latest_sdist(self): + # type: () -> ReleaseUrl + return next(iter(self.urls.sdists)) + + @property + def latest_wheels(self): + # type: () -> Iterator[ReleaseUrl] + for wheel in self.urls.wheels: + yield wheel + + @property + def dependencies(self): + # type: () -> List[Dependency] + if self.info.dependencies is None and list(self.urls): + rval = self.get_dependencies() + return rval.dependencies + return list(self.info.dependencies) + + def get_dependencies(self): + # type: () -> "Package" + urls = [] # type: List[ReleaseUrl] + deps = set() # type: Set[str] + info = self.info + if info.dependencies is None: + for url in self.urls: + try: + url, dep_dict = url.get_dependencies() + except (RuntimeError, TypeError): + # This happens if we are parsing `setup.py` and we fail + if url.is_sdist: + continue + else: + raise + markers = url.markers + dep_list = dep_dict.get("requires_dist", []) + for dep in dep_list: + # XXX: We need to parse these as requirements and "and" the markers + # XXX: together because they may contain "extra" markers which we + # XXX: will need to parse and remove + deps.add(add_markers_to_dep(dep, markers)) + urls.append(url) + if None in deps: + deps.remove(None) + info = attr.evolve( + self.info, requires_dist=tuple(sorted(deps)) + ).create_dependencies(force=True) + return attr.evolve(self, info=info, urls=urls) + + @classmethod + def from_json(cls, package_json): + # type: (Dict[str, Any]) -> "Package" + info = convert_package_info(package_json["info"]).create_dependencies() + releases = convert_releases_to_collection( + package_json["releases"], name=info.name + ) + urls = convert_release_urls_to_collection(package_json["urls"], name=info.name) + return cls( + info=info, + releases=releases, + urls=urls, + last_serial=package_json["last_serial"], + ) + + def pin_dependencies(self, include_extras=None): + # type: (Optional[List[str]]) -> Tuple[List["Package"], Dict[str, List[SpecifierSet]]] + deps = [] + if include_extras: + include_extras = list(sorted(set(include_extras))) + else: + include_extras = [] + constraints = defaultdict(list) + for dep in self.dependencies: + if dep.from_extras and dep.from_extras not in include_extras: + continue + if dep.specifier: + constraints[dep.name].append(dep.specifier) + try: + pinned = dep.pin() + except requests.exceptions.HTTPError: + continue + deps.append(pinned) + return deps, constraints + + def get_latest_lockfile(self): + # type: () -> Dict[str, Dict[str, Union[List[str], str]]] + lockfile = {} + constraints = {dep.name: dep.specifier for dep in self.dependencies} + deps, _ = self.pin_dependencies() + for dep in deps: + dep = dep.get_dependencies() + for sub_dep in dep.dependencies: + if sub_dep.name not in constraints: + logger.info( + "Adding {0} (from {1}) {2!s}".format( + sub_dep.name, dep.name, sub_dep.specifier + ) + ) + constraints[sub_dep.name] = sub_dep.specifier + else: + existing = "{0} (from {1}): {2!s} + ".format( + sub_dep.name, dep.name, constraints[sub_dep.name] + ) + new_specifier = sub_dep.specifier + merged = constraints[sub_dep.name] & new_specifier + logger.info( + "Updating: {0}{1!s} = {2!s}".format( + existing, new_specifier, merged + ) + ) + constraints[sub_dep.name] = merged + + lockfile.update({dep.info.name: dep.releases.get_latest_lockfile()}) + for sub_dep_name, specset in constraints.items(): + try: + sub_dep_pkg = get_package(sub_dep_name) + except requests.exceptions.HTTPError: + continue + logger.info("Getting package: {0} ({1!s})".format(sub_dep, specset)) + sorted_releases = list( + sorted( + sub_dep_pkg.releases, + key=operator.attrgetter("parsed_version"), + reverse=True, + ) + ) + try: + version = next(iter(specset.filter((r.version for r in sorted_releases)))) + except StopIteration: + logger.info("No version of {0} matches specifier: {1}".format(sub_dep, specset)) + logger.info( + "Available versions: {0}".format( + " ".join([r.version for r in sorted_releases]) + ) + ) + raise + sub_dep_instance = get_package_version(sub_dep_name, version=str(version)) + if sub_dep_instance is None: + continue + lockfile.update( + { + sub_dep_instance.info.name: sub_dep_instance.releases.get_latest_lockfile() + } + ) + # lockfile.update(dep.get_latest_lockfile()) + lockfile.update({self.info.name: self.releases.get_latest_lockfile()}) + return lockfile + + def as_dict(self): + # type: () -> Dict[str, Any] + return json.loads(self.serialize()) + + def serialize(self): + # type: () -> str + return json.dumps(attr.asdict(self), cls=PackageEncoder, indent=4) + + +def get_package(name): + # type: (str) -> Package + url = "https://pypi.org/pypi/{}/json".format(name) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_version(name, version): + # type: (str, str) -> Package + url = "https://pypi.org/pypi/{0}/{1}/json".format(name, version) + with requests.get(url) as r: + r.raise_for_status() + result = r.json() + package = Package.from_json(result) + return package + + +def get_package_from_requirement(req): + # type: (PackagingRequirement) -> Tuple[Package, Set[str]] + versions = set() + if is_pinned_requirement(req): + version = get_pinned_version(req) + versions.add(version) + pkg = get_package_version(req.name, version) + else: + pkg = get_package(req.name) + sorted_releases = list( + sorted(pkg.releases, key=operator.attrgetter("parsed_version"), reverse=True) + ) + versions = set(req.specifier.filter((r.version for r in sorted_releases))) + version = next(iter(req.specifier.filter((r.version for r in sorted_releases)))) + if pkg.version not in versions: + pkg = get_package_version(pkg.name, version) + return pkg, versions diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 84a4a26d..9c0aea4e 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -3,78 +3,31 @@ from __future__ import absolute_import, print_function, unicode_literals import copy +import itertools import os import sys import attr -import tomlkit - import plette.models.base import plette.pipfiles - +import tomlkit from vistir.compat import FileNotFoundError, Path +from ..environment import MYPY_RUNNING from ..exceptions import RequirementError from ..utils import is_editable, is_vcs, merge_items from .project import ProjectFile from .requirements import Requirement -from .utils import optional_instance_of +from .utils import get_url_name, optional_instance_of, tomlkit_value_to_python -from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn - package_type = Dict[str, Dict[str, Union[List[str], str]]] - source_type = Dict[str, Union[str, bool]] + from typing import Union, Any, Dict, Iterable, Mapping, List, Text + + package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] + source_type = Dict[Text, Union[Text, bool]] sources_type = Iterable[source_type] - meta_type = Dict[str, Union[int, Dict[str, str], sources_type]] - lockfile_type = Dict[str, Union[package_type, meta_type]] - - -# Let's start by patching plette to make sure we can validate data without being broken -try: - import cerberus -except ImportError: - cerberus = None - -VALIDATORS = plette.models.base.VALIDATORS - - -def patch_plette(): - # type: () -> None - - global VALIDATORS - - def validate(cls, data): - # type: (Any, Dict[str, Any]) -> None - if not cerberus: # Skip validation if Cerberus is not available. - return - schema = cls.__SCHEMA__ - key = id(schema) - try: - v = VALIDATORS[key] - except KeyError: - v = VALIDATORS[key] = cerberus.Validator(schema, allow_unknown=True) - if v.validate(dict(data), normalize=False): - return - raise plette.models.base.ValidationError(data, v) - - names = ["plette.models.base", plette.models.base.__name__] - names = [name for name in names if name in sys.modules] - for name in names: - if name in sys.modules: - module = sys.modules[name] - else: - module = plette.models.base - original_fn = getattr(module, "validate") - for key in ["__qualname__", "__name__", "__module__"]: - original_val = getattr(original_fn, key, None) - if original_val is not None: - setattr(validate, key, original_val) - setattr(module, "validate", validate) - sys.modules[name] = module - - -patch_plette() + meta_type = Dict[Text, Union[int, Dict[Text, Text], sources_type]] + lockfile_type = Dict[Text, Union[package_type, meta_type]] is_pipfile = optional_instance_of(plette.pipfiles.Pipfile) @@ -83,41 +36,78 @@ is_projectfile = optional_instance_of(ProjectFile) def reorder_source_keys(data): - # type: ignore - sources = data["source"] # type: sources_type - for i, entry in enumerate(sources): - table = tomlkit.table() # type: Mapping - table["name"] = entry["name"] - table["url"] = entry["url"] - table["verify_ssl"] = entry["verify_ssl"] - data["source"][i] = table + # type: (tomlkit.toml_document.TOMLDocument) -> tomlkit.toml_document.TOMLDocument + sources = [] # type: sources_type + for source_key in ["source", "sources"]: + sources.extend(data.get(source_key, tomlkit.aot()).value) + new_source_aot = tomlkit.aot() + for entry in sources: + table = tomlkit.table() # type: tomlkit.items.Table + source_entry = PipfileLoader.populate_source(entry.copy()) + for key in ["name", "url", "verify_ssl"]: + table.update({key: source_entry[key]}) + new_source_aot.append(table) + data["source"] = new_source_aot + if data.get("sources", None): + del data["sources"] return data class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): - # type: (Dict[str, Any]) -> None + # type: (tomlkit.toml_document.TOMLDocument) -> None for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): - if key not in data or key == "source": + if key not in data or key == "sources": continue try: klass.validate(data[key]) except Exception: pass + @classmethod + def ensure_package_sections(cls, data): + # type: (tomlkit.toml_document.TOMLDocument[Text, Any]) -> tomlkit.toml_document.TOMLDocument[Text, Any] + """ + Ensure that all pipfile package sections are present in the given toml document + + :param :class:`~tomlkit.toml_document.TOMLDocument` data: The toml document to + ensure package sections are present on + :return: The updated toml document, ensuring ``packages`` and ``dev-packages`` + sections are present + :rtype: :class:`~tomlkit.toml_document.TOMLDocument` + """ + package_keys = ( + k for k in plette.pipfiles.PIPFILE_SECTIONS.keys() if k.endswith("packages") + ) + for key in package_keys: + if key not in data: + data.update({key: tomlkit.table()}) + return data + + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = str(source["verify_ssl"]).lower() == "true" + return source + @classmethod def load(cls, f, encoding=None): - # type: (Any, str) -> PipfileLoader + # type: (Any, Text) -> PipfileLoader content = f.read() if encoding is not None: content = content.decode(encoding) _data = tomlkit.loads(content) - _data["source"] = _data.get("source", []) + _data.get("sources", []) + should_reload = "source" not in _data _data = reorder_source_keys(_data) - if "source" not in _data: + if should_reload: if "sources" in _data: - _data["source"] = _data["sources"] content = tomlkit.dumps(_data) else: # HACK: There is no good way to prepend a section to an existing @@ -127,12 +117,21 @@ class PipfileLoader(plette.pipfiles.Pipfile): sep = "" if content.startswith("\n") else "\n" content = plette.pipfiles.DEFAULT_SOURCE_TOML + sep + content data = tomlkit.loads(content) + data = cls.ensure_package_sections(data) instance = cls(data) instance._data = dict(instance._data) return instance + def __contains__(self, key): + # type: (Text) -> bool + if key not in self._data: + package_keys = self._data.get("packages", {}).keys() + dev_package_keys = self._data.get("dev-packages", {}).keys() + return any(key in pkg_list for pkg_list in (package_keys, dev_package_keys)) + return True + def __getattribute__(self, key): - # type: (str) -> Any + # type: (Text) -> Any if key == "source": return self._data[key] return super(PipfileLoader, self).__getattribute__(key) @@ -143,10 +142,12 @@ class Pipfile(object): path = attr.ib(validator=is_path, type=Path) projectfile = attr.ib(validator=is_projectfile, type=ProjectFile) _pipfile = attr.ib(type=PipfileLoader) - _pyproject = attr.ib(default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument) + _pyproject = attr.ib( + default=attr.Factory(tomlkit.document), type=tomlkit.toml_document.TOMLDocument + ) build_system = attr.ib(default=attr.Factory(dict), type=dict) - requirements = attr.ib(default=attr.Factory(list), type=list) - dev_requirements = attr.ib(default=attr.Factory(list), type=list) + _requirements = attr.ib(default=attr.Factory(list), type=list) + _dev_requirements = attr.ib(default=attr.Factory(list), type=list) @path.default def _get_path(self): @@ -163,26 +164,41 @@ class Pipfile(object): # type: () -> Union[plette.pipfiles.Pipfile, PipfileLoader] return self.projectfile.model + @property + def root(self): + return self.path.parent + + @property + def extended_keys(self): + return [ + k + for k in itertools.product( + ("packages", "dev-packages"), ("", "vcs", "editable") + ) + ] + @property def pipfile(self): # type: () -> Union[PipfileLoader, plette.pipfiles.Pipfile] return self._pipfile def get_deps(self, dev=False, only=True): - # type: (bool, bool) -> Dict[str, Dict[str, Union[List[str], str]]] - deps = {} # type: Dict[str, Dict[str, Union[List[str], str]]] + # type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]] + deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] if dev: - deps.update(self.pipfile._data["dev-packages"]) + deps.update(dict(self.pipfile._data.get("dev-packages", {}))) if only: return deps - return merge_items([deps, self.pipfile._data["packages"]]) + return tomlkit_value_to_python( + merge_items([deps, dict(self.pipfile._data.get("packages", {}))]) + ) def get(self, k): - # type: (str) -> Any + # type: (Text) -> Any return self.__getitem__(k) def __contains__(self, k): - # type: (str) -> bool + # type: (Text) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True @@ -200,6 +216,7 @@ class Pipfile(object): if "-" in k: section, _, pkg_type = k.rpartition("-") vals = getattr(pipfile.get(section, {}), "_data", {}) + vals = tomlkit_value_to_python(vals) if pkg_type == "vcs": retval = {k: v for k, v in vals.items() if is_vcs(v)} elif pkg_type == "editable": @@ -225,7 +242,11 @@ class Pipfile(object): @property def requires_python(self): # type: () -> bool - return self._pipfile.requires.requires_python + return getattr( + self._pipfile.requires, + "python_version", + getattr(self._pipfile.requires, "python_full_version", None), + ) @property def allow_prereleases(self): @@ -234,26 +255,23 @@ class Pipfile(object): @classmethod def read_projectfile(cls, path): - # type: (str) -> ProjectFile + # type: (Text) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. - :param str path: Path to the target file. + :param Text path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ - pf = ProjectFile.read( - path, - PipfileLoader, - invalid_ok=True - ) + pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True) return pf @classmethod def load_projectfile(cls, path, create=False): - # type: (str, bool) -> ProjectFile - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> ProjectFile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -275,10 +293,11 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): - # type: (str, bool) -> Pipfile - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> Pipfile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -288,18 +307,10 @@ class Pipfile(object): projectfile = cls.load_projectfile(path, create=create) pipfile = projectfile.model - dev_requirements = [ - Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("dev-packages", {}).items() - ] - requirements = [ - Requirement.from_pipfile(k, getattr(v, "_data", v)) for k, v in pipfile.get("packages", {}).items() - ] creation_args = { "projectfile": projectfile, "pipfile": pipfile, - "dev_requirements": dev_requirements, - "requirements": requirements, - "path": Path(projectfile.location) + "path": Path(projectfile.location), } return cls(**creation_args) @@ -318,26 +329,56 @@ class Pipfile(object): # type: () -> List[Requirement] return self.requirements + @property + def dev_requirements(self): + # type: () -> List[Requirement] + if not self._dev_requirements: + packages = tomlkit_value_to_python(self.pipfile.get("dev-packages", {})) + self._dev_requirements = [ + Requirement.from_pipfile(k, v) + for k, v in packages.items() + if v is not None + ] + return self._dev_requirements + + @property + def requirements(self): + # type: () -> List[Requirement] + if not self._requirements: + packages = tomlkit_value_to_python(self.pipfile.get("packages", {})) + self._requirements = [ + Requirement.from_pipfile(k, v) + for k, v in packages.items() + if v is not None + ] + return self._requirements + def _read_pyproject(self): # type: () -> None pyproject = self.path.parent.joinpath("pyproject.toml") if pyproject.exists(): - self._pyproject = tomlkit.load(pyproject) + self._pyproject = tomlkit.loads(pyproject.read_text()) build_system = self._pyproject.get("build-system", None) - if not os.path.exists(self.path_to("setup.py")): - if not build_system or not build_system.get("requires"): - build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", - } - self._build_system = build_system + if build_system and not build_system.get("build_backend"): + build_system["build-backend"] = "setuptools.build_meta:__legacy__" + elif not build_system or not build_system.get("requires"): + build_system = { + "requires": ["setuptools>=40.8", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", + } + self.build_system = build_system @property def build_requires(self): - # type: () -> List[str] + # type: () -> List[Text] + if not self.build_system: + self._read_pyproject() return self.build_system.get("requires", []) @property def build_backend(self): - # type: () -> str + # type: () -> Text + pyproject = self.path.parent.joinpath("pyproject.toml") + if not self.build_system: + self._read_pyproject() return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py index f6e037d6..7c1b0e81 100644 --- a/pipenv/vendor/requirementslib/models/project.py +++ b/pipenv/vendor/requirementslib/models/project.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import collections import io @@ -13,14 +13,10 @@ import plette import plette.models import six import tomlkit +from vistir.compat import FileNotFoundError - -SectionDifference = collections.namedtuple("SectionDifference", [ - "inthis", "inthat", -]) -FileDifference = collections.namedtuple("FileDifference", [ - "default", "develop", -]) +SectionDifference = collections.namedtuple("SectionDifference", ["inthis", "inthat"]) +FileDifference = collections.namedtuple("FileDifference", ["default", "develop"]) def _are_pipfile_entries_equal(a, b): @@ -52,12 +48,15 @@ def preferred_newlines(f): class ProjectFile(object): """A file in the Pipfile project. """ + location = attr.ib() line_ending = attr.ib() model = attr.ib() @classmethod def read(cls, location, model_cls, invalid_ok=False): + if not os.path.exists(location) and not invalid_ok: + raise FileNotFoundError(location) try: with io.open(location, encoding="utf-8") as f: model = model_cls.load(f) @@ -89,14 +88,9 @@ class Project(object): def __attrs_post_init__(self): self.root = root = os.path.abspath(self.root) - self._p = ProjectFile.read( - os.path.join(root, "Pipfile"), - plette.Pipfile, - ) + self._p = ProjectFile.read(os.path.join(root, "Pipfile"), plette.Pipfile) self._l = ProjectFile.read( - os.path.join(root, "Pipfile.lock"), - plette.Lockfile, - invalid_ok=True, + os.path.join(root, "Pipfile.lock"), plette.Lockfile, invalid_ok=True ) @property @@ -138,14 +132,17 @@ class Project(object): self._get_pipfile_section(develop=True, insert=False), ] return any( - (packaging.utils.canonicalize_name(name) == - packaging.utils.canonicalize_name(key)) + ( + packaging.utils.canonicalize_name(name) + == packaging.utils.canonicalize_name(key) + ) for section in sections for name in section ) def add_line_to_pipfile(self, line, develop): from requirementslib import Requirement + requirement = Requirement.from_line(line) section = self._get_pipfile_section(develop=develop) key = requirement.normalized_name @@ -164,13 +161,9 @@ class Project(object): keys = {packaging.utils.canonicalize_name(key) for key in keys} sections = [] if default: - sections.append(self._get_pipfile_section( - develop=False, insert=False, - )) + sections.append(self._get_pipfile_section(develop=False, insert=False)) if develop: - sections.append(self._get_pipfile_section( - develop=True, insert=False, - )) + sections.append(self._get_pipfile_section(develop=True, insert=False)) for section in sections: removals = set() for name in section: diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index af3d07ed..0537ca08 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,85 +1,1292 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import collections import copy -import hashlib import os - +import sys from contextlib import contextmanager +from distutils.sysconfig import get_python_lib +from functools import partial import attr import pip_shims - -from first import first +import six +import vistir +from cached_property import cached_property from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier +from packaging.specifiers import ( + InvalidSpecifier, + LegacySpecifier, + Specifier, + SpecifierSet, +) from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path +from vistir.compat import FileNotFoundError, Path, lru_cache +from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, is_valid_url, + mkdir_p, + normalize_path, ) +from ..environment import MYPY_RUNNING from ..exceptions import RequirementError from ..utils import ( VCS_LIST, + add_ssh_scheme_to_git_uri, + get_setup_paths, + is_installable_dir, is_installable_file, is_vcs, - ensure_setup_py, - add_ssh_scheme_to_git_uri, strip_ssh_from_git_uri, ) -from .setup_info import SetupInfo +from .markers import ( + cleanup_pyspecs, + contains_pyversion, + format_pyversion, + get_contained_pyversions, + normalize_marker_str, +) +from .setup_info import ( + SetupInfo, + _prepare_wheel_building_kwargs, + ast_parse_setup_py, + get_metadata, + parse_setup_cfg, +) +from .url import URI from .utils import ( + DIRECT_URL_RE, HASH_STRING, - build_vcs_link, + URL_RE, + build_vcs_uri, + convert_direct_url_to_url, + create_link, extras_to_string, filter_none, format_requirement, + get_default_pyproject_backend, + get_pyproject, get_version, init_requirement, is_pinned_requirement, make_install_requirement, + normalize_name, parse_extras, + read_source, specs_to_string, split_markers_from_line, + split_ref_from_uri, split_vcs_method_from_uri, validate_path, validate_specifiers, validate_vcs, - normalize_name, - create_link, - get_pyproject ) +if MYPY_RUNNING: + from typing import ( + Optional, + TypeVar, + List, + Dict, + Union, + Any, + Tuple, + Sequence, + Set, + AnyStr, + Text, + Generator, + FrozenSet, + ) + from pip_shims.shims import ( + Link, + InstallRequirement, + PackageFinder, + InstallationCandidate, + ) -@attr.s(slots=True) + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + F = TypeVar("F", "FileRequirement", "VCSRequirement", covariant=True) + from six.moves.urllib.parse import SplitResult + from .vcs import VCSRepository + from .dependencies import AbstractDependency + + NON_STRING_ITERABLE = Union[List, Set, Tuple] + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + BASE_TYPES = Union[bool, STRING_TYPE, Tuple[STRING_TYPE, ...]] + CUSTOM_TYPES = Union[VCSRepository, RequirementType, SetupInfo, "Line"] + CREATION_ARG_TYPES = Union[BASE_TYPES, Link, CUSTOM_TYPES] + PIPFILE_ENTRY_TYPE = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PIPFILE_TYPE = Union[STRING_TYPE, Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE]] + TPIPFILE = Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE] + + +SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) + + +run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin=True) + + +class Line(object): + def __init__(self, line, extras=None): + # type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None + self.editable = False # type: bool + if line.startswith("-e "): + line = line[len("-e ") :] + self.editable = True + self.extras = () # type: Tuple[STRING_TYPE, ...] + if extras is not None: + self.extras = tuple(sorted(set(extras))) + self.line = line # type: STRING_TYPE + self.hashes = [] # type: List[STRING_TYPE] + self.markers = None # type: Optional[STRING_TYPE] + self.vcs = None # type: Optional[STRING_TYPE] + self.path = None # type: Optional[STRING_TYPE] + self.relpath = None # type: Optional[STRING_TYPE] + self.uri = None # type: Optional[STRING_TYPE] + self._link = None # type: Optional[Link] + self.is_local = False # type: bool + self._name = None # type: Optional[STRING_TYPE] + self._specifier = None # type: Optional[STRING_TYPE] + self.parsed_marker = None # type: Optional[Marker] + self.preferred_scheme = None # type: Optional[STRING_TYPE] + self._requirement = None # type: Optional[PackagingRequirement] + self.is_direct_url = False # type: bool + self._parsed_url = None # type: Optional[urllib_parse.ParseResult] + self._setup_cfg = None # type: Optional[STRING_TYPE] + self._setup_py = None # type: Optional[STRING_TYPE] + self._pyproject_toml = None # type: Optional[STRING_TYPE] + self._pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + self._pyproject_backend = None # type: Optional[STRING_TYPE] + self._wheel_kwargs = None # type: Optional[Dict[STRING_TYPE, STRING_TYPE]] + self._vcsrepo = None # type: Optional[VCSRepository] + self._setup_info = None # type: Optional[SetupInfo] + self._ref = None # type: Optional[STRING_TYPE] + self._ireq = None # type: Optional[InstallRequirement] + self._src_root = None # type: Optional[STRING_TYPE] + self.dist = None # type: Any + super(Line, self).__init__() + self.parse() + + def __hash__(self): + return hash( + ( + self.editable, + self.line, + self.markers, + tuple(self.extras), + tuple(self.hashes), + self.vcs, + self.uri, + self.path, + self.name, + self._requirement, + ) + ) + + def __repr__(self): + try: + return ( + "<Line (editable={self.editable}, name={self._name}, path={self.path}, " + "uri={self.uri}, extras={self.extras}, markers={self.markers}, vcs={self.vcs}" + ", specifier={self._specifier}, pyproject={self._pyproject_toml}, " + "pyproject_requires={self._pyproject_requires}, " + "pyproject_backend={self._pyproject_backend}, ireq={self._ireq})>".format( + self=self + ) + ) + except Exception: + return "<Line {0}>".format(self.__dict__.values()) + + def __str__(self): + # type: () -> str + if self.markers: + return "{0}; {1}".format(self.get_line(), self.markers) + return self.get_line() + + def get_line( + self, with_prefix=False, with_markers=False, with_hashes=True, as_list=False + ): + # type: (bool, bool, bool, bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + line = self.line + extras_str = extras_to_string(self.extras) + with_hashes = False if self.editable or self.is_vcs else with_hashes + hash_list = ["--hash={0}".format(h) for h in self.hashes] + if self.is_named: + line = self.name_and_specifier + elif self.is_direct_url: + line = self.link.url + elif extras_str: + if self.is_vcs: + line = self.link.url + if "git+file:/" in line and "git+file:///" not in line: + line = line.replace("git+file:/", "git+file:///") + elif extras_str not in line: + line = "{0}{1}".format(line, extras_str) + # XXX: For using markers on vcs or url requirements, they can be used + # as normal (i.e. no space between the requirement and the semicolon) + # and no additional quoting as long as they are not editable requirements + # HOWEVER, for editable requirements, the requirement+marker must be quoted + # We do this here for the line-formatted versions, but leave it up to the + # `Script.parse()` functionality in pipenv, for instance, to handle that + # in a cross-platform manner for the `as_list` approach since that is how + # we anticipate this will be used if passing directly to the command line + # for pip. + if with_markers and self.markers: + line = "{0}; {1}".format(line, self.markers) + if with_prefix and self.editable and not as_list: + line = '"{0}"'.format(line) + if as_list: + result_list = [] + if with_prefix and self.editable: + result_list.append("-e") + result_list.append(line) + if with_hashes: + result_list.extend(self.hashes) + return result_list + if with_prefix and self.editable: + line = "-e {0}".format(line) + if with_hashes and hash_list: + line = "{0} {1}".format(line, " ".join(hash_list)) + return line + + @property + def name_and_specifier(self): + name_str, spec_str = "", "" + if self.name: + name_str = "{0}".format(self.name.lower()) + extras_str = extras_to_string(self.extras) + if extras_str: + name_str = "{0}{1}".format(name_str, extras_str) + if self.specifier: + spec_str = "{0}".format(self.specifier) + return "{0}{1}".format(name_str, spec_str) + + @classmethod + def split_hashes(cls, line): + # type: (S) -> Tuple[S, List[S]] + if "--hash" not in line: + return line, [] + split_line = line.split() + line_parts = [] # type: List[S] + hashes = [] # type: List[S] + for part in split_line: + if part.startswith("--hash"): + param, _, value = part.partition("=") + hashes.append(value) + else: + line_parts.append(part) + line = " ".join(line_parts) + return line, hashes + + @property + def line_with_prefix(self): + # type: () -> STRING_TYPE + return self.get_line(with_prefix=True, with_hashes=False) + + @property + def line_for_ireq(self): + # type: () -> STRING_TYPE + line = "" # type: STRING_TYPE + if self.is_file or self.is_url and not self.is_vcs: + scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" + local_line = next( + iter( + [ + os.path.dirname(os.path.abspath(f)) + for f in [self.setup_py, self.setup_cfg, self.pyproject_toml] + if f is not None + ] + ), + None, + ) + if local_line and self.extras: + local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) + line = local_line if local_line is not None else self.line + if scheme == "path": + if not line and self.base_path is not None: + line = os.path.abspath(self.base_path) + else: + if DIRECT_URL_RE.match(self.line): + uri = URI.parse(self.line) + line = uri.full_url + self._requirement = init_requirement(self.line) + line = convert_direct_url_to_url(self.line) + else: + if self.link: + line = self.link.url + else: + try: + uri = URI.parse(line) + except ValueError: + line = line + else: + line = uri.base_url + self._link = uri.as_link + + if self.editable: + if not line: + if self.is_path or self.is_file: + if not self.path: + line = pip_shims.shims.url_to_path(self.url) + else: + line = self.path + if self.extras: + line = "{0}{1}".format(line, extras_to_string(self.extras)) + else: + line = self.link.url + elif self.is_vcs and not self.editable: + line = add_ssh_scheme_to_git_uri(self.line) + if not line: + line = self.line + return line + + @property + def base_path(self): + # type: () -> Optional[S] + if not self.link and not self.path: + self.parse_link() + if not self.path: + pass + path = normalize_path(self.path) + if os.path.exists(path) and os.path.isdir(path): + path = path + elif os.path.exists(path) and os.path.isfile(path): + path = os.path.dirname(path) + else: + path = None + return path + + @property + def setup_py(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_py is None: + self.populate_setup_paths() + return self._setup_py + + @property + def setup_cfg(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_cfg is None: + self.populate_setup_paths() + return self._setup_cfg + + @property + def pyproject_toml(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_toml is None: + self.populate_setup_paths() + return self._pyproject_toml + + @property + def specifier(self): + # type: () -> Optional[STRING_TYPE] + options = [self._specifier] + for req in (self.ireq, self.requirement): + if req is not None and getattr(req, "specifier", None): + options.append(req.specifier) + specifier = next( + iter(spec for spec in options if spec is not None), None + ) # type: Optional[Union[Specifier, SpecifierSet]] + spec_string = None # type: Optional[STRING_TYPE] + if specifier is not None: + spec_string = specs_to_string(specifier) + elif ( + specifier is None + and not self.is_named + and (self._setup_info is not None and self._setup_info.version) + ): + spec_string = "=={0}".format(self._setup_info.version) + if spec_string: + self._specifier = spec_string + return self._specifier + + @specifier.setter + def specifier(self, spec): + # type: (str) -> None + if not spec.startswith("=="): + spec = "=={0}".format(spec) + self._specifier = spec + self.specifiers = SpecifierSet(spec) + + @property + def specifiers(self): + # type: () -> Optional[SpecifierSet] + ireq_needs_specifier = False + req_needs_specifier = False + if self.ireq is None or self.ireq.req is None or not self.ireq.req.specifier: + ireq_needs_specifier = True + if self.requirement is None or not self.requirement.specifier: + req_needs_specifier = True + if any([ireq_needs_specifier, req_needs_specifier]): + # TODO: Should we include versions for VCS dependencies? IS there a reason not + # to? For now we are using hashes as the equivalent to pin + # note: we need versions for direct dependencies at the very least + if ( + self.is_file + or self.is_url + or self.is_path + or (self.is_vcs and not self.editable) + ): + if self.specifier is not None: + specifier = self.specifier + if not isinstance(specifier, SpecifierSet): + specifier = SpecifierSet(specifier) + self.specifiers = specifier + return specifier + if self.ireq is not None and self.ireq.req is not None: + return self.ireq.req.specifier + elif self.requirement is not None: + return self.requirement.specifier + return None + + @specifiers.setter + def specifiers(self, specifiers): + # type: (Union[Text, str, SpecifierSet]) -> None + if not isinstance(specifiers, SpecifierSet): + if isinstance(specifiers, six.string_types): + specifiers = SpecifierSet(specifiers) + else: + raise TypeError("Must pass a string or a SpecifierSet") + specs = self.get_requirement_specs(specifiers) + if self.ireq is not None and self._ireq and self._ireq.req is not None: + self._ireq.req.specifier = specifiers + self._ireq.req.specs = specs + if self.requirement is not None: + self.requirement.specifier = specifiers + self.requirement.specs = specs + + @classmethod + def get_requirement_specs(cls, specifierset): + # type: (SpecifierSet) -> List[Tuple[AnyStr, AnyStr]] + specs = [] + spec = next(iter(specifierset._specs), None) + if spec: + specs.append(spec._spec) + return specs + + @property + def requirement(self): + # type: () -> Optional[RequirementType] + if self._requirement is None: + self.parse_requirement() + if self._requirement is None and self._name is not None: + self._requirement = init_requirement(canonicalize_name(self.name)) + if self.is_file or self.is_url and self._requirement is not None: + self._requirement.url = self.url + if ( + self._requirement + and self._requirement.specifier + and not self._requirement.specs + ): + specs = self.get_requirement_specs(self._requirement.specifier) + self._requirement.specs = specs + return self._requirement + + def populate_setup_paths(self): + # type: () -> None + if not self.link and not self.path: + self.parse_link() + if not self.path: + return + base_path = self.base_path + if base_path is None: + return + setup_paths = get_setup_paths( + base_path, subdirectory=self.subdirectory + ) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] + self._setup_py = setup_paths.get("setup_py") + self._setup_cfg = setup_paths.get("setup_cfg") + self._pyproject_toml = setup_paths.get("pyproject_toml") + + @property + def pyproject_requires(self): + # type: () -> Optional[Tuple[STRING_TYPE, ...]] + if self._pyproject_requires is None and self.pyproject_toml is not None: + if self.path is not None: + pyproject_requires, pyproject_backend = None, None + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + self._pyproject_backend = pyproject_backend + return self._pyproject_requires + + @property + def pyproject_backend(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_requires is None and self.pyproject_toml is not None: + pyproject_requires = None # type: Optional[Sequence[STRING_TYPE]] + pyproject_backend = None # type: Optional[STRING_TYPE] + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if not pyproject_backend and self.setup_cfg is not None: + setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) + pyproject_backend = get_default_pyproject_backend() + pyproject_requires = setup_dict.get( + "build_requires", ["setuptools", "wheel"] + ) # type: ignore + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + if pyproject_backend: + self._pyproject_backend = pyproject_backend + return self._pyproject_backend + + def parse_hashes(self): + # type: () -> "Line" + """ + Parse hashes from *self.line* and set them on the current object. + :returns: Self + :rtype: `:class:~Line` + """ + line, hashes = self.split_hashes(self.line) + self.hashes = hashes + self.line = line + return self + + def parse_extras(self): + # type: () -> "Line" + """ + Parse extras from *self.line* and set them on the current object + :returns: self + :rtype: :class:`~Line` + """ + extras = None + if "@" in self.line or self.is_vcs or self.is_url: + line = "{0}".format(self.line) + uri = URI.parse(line) + name = uri.name + if name: + self._name = name + if uri.host and uri.path and uri.scheme: + self.line = uri.to_string( + escape_password=False, direct=False, strip_ssh=uri.is_implicit_ssh + ) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + extras_set = set() # type: Set[STRING_TYPE] + if extras is not None: + extras_set = set(parse_extras(extras)) + if self._name: + self._name, name_extras = pip_shims.shims._strip_extras(self._name) + if name_extras: + name_extras = set(parse_extras(name_extras)) + extras_set |= name_extras + if extras_set is not None: + self.extras = tuple(sorted(extras_set)) + return self + + def get_url(self): + # type: () -> STRING_TYPE + """Sets ``self.name`` if given a **PEP-508** style URL""" + line = self.line + try: + parsed = URI.parse(line) + line = parsed.to_string(escape_password=False, direct=False, strip_ref=True) + except ValueError: + pass + else: + self._parsed_url = parsed + return line + if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)): + _, _, _parseable = self.line.partition("+") + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable)) + line, _ = split_ref_from_uri(line) + else: + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line)) + if "@" in self.line and parsed.scheme == "": + name, _, url = self.line.partition("@") + if self._name is None: + url = url.strip() + self._name = name.strip() + if is_valid_url(url): + self.is_direct_url = True + line = url.strip() + parsed = urllib_parse.urlparse(line) + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + self._parsed_url = parsed + return line + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is None: + self.parse_name() + if self._name is None and not self.is_named and not self.is_wheel: + if self.setup_info: + self._name = self.setup_info.name + elif self.is_wheel: + self._name = self._parse_wheel() + if not self._name: + self._name = self.ireq.name + return self._name + + @name.setter + def name(self, name): + # type: (STRING_TYPE) -> None + self._name = name + if self._setup_info: + self._setup_info.name = name + if self.requirement and self._requirement: + self._requirement.name = name + if self.ireq and self._ireq and self._ireq.req: + self._ireq.req.name = name + + @property + def url(self): + # type: () -> Optional[STRING_TYPE] + if self.uri is not None: + url = add_ssh_scheme_to_git_uri(self.uri) + else: + url = getattr(self.link, "url_without_fragment", None) + if url is not None: + url = add_ssh_scheme_to_git_uri(unquote(url)) + if url is not None and self._parsed_url is None: + if self.vcs is not None: + _, _, _parseable = url.partition("+") + self._parsed_url = urllib_parse.urlparse(_parseable) + if self.is_vcs: + # strip the ref from the url + url, _ = split_ref_from_uri(url) + return url + + @property + def link(self): + # type: () -> Link + if self._link is None: + self.parse_link() + return self._link + + @property + def subdirectory(self): + # type: () -> Optional[STRING_TYPE] + if self.link is not None: + return self.link.subdirectory_fragment + return "" + + @property + def is_wheel(self): + # type: () -> bool + if self.link is None: + return False + return self.link.is_wheel + + @property + def is_artifact(self): + # type: () -> bool + + if self.link is None: + return False + return not self.link.is_vcs + + @property + def is_vcs(self): + # type: () -> bool + # Installable local files and installable non-vcs urls are handled + # as files, generally speaking + if is_vcs(self.line) or is_vcs(self.get_url()): + return True + return False + + @property + def is_url(self): + # type: () -> bool + url = self.get_url() + if is_valid_url(url) or is_file_url(url): + return True + return False + + @property + def is_path(self): + # type: () -> bool + if ( + self.path + and ( + self.path.startswith(".") + or os.path.isabs(self.path) + or os.path.exists(self.path) + ) + and is_installable_file(self.path) + ): + return True + elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( + os.path.exists(self.get_url()) and is_installable_file(self.get_url()) + ): + return True + return False + + @property + def is_file_url(self): + # type: () -> bool + url = self.get_url() + parsed_url_scheme = self._parsed_url.scheme if self._parsed_url else "" + if url and is_file_url(self.get_url()) or parsed_url_scheme == "file": + return True + return False + + @property + def is_file(self): + # type: () -> bool + if ( + self.is_path + or (is_file_url(self.get_url()) and is_installable_file(self.get_url())) + or ( + self._parsed_url + and self._parsed_url.scheme == "file" + and is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + ) + ): + return True + return False + + @property + def is_named(self): + # type: () -> bool + return not (self.is_file_url or self.is_url or self.is_file or self.is_vcs) + + @property + def ref(self): + # type: () -> Optional[STRING_TYPE] + if self._ref is None and self.relpath is not None: + self.relpath, self._ref = split_ref_from_uri(self.relpath) + return self._ref + + @property + def ireq(self): + # type: () -> Optional[pip_shims.InstallRequirement] + if self._ireq is None: + self.parse_ireq() + return self._ireq + + @property + def is_installable(self): + # type: () -> bool + possible_paths = (self.line, self.get_url(), self.path, self.base_path) + return any(is_installable_file(p) for p in possible_paths if p is not None) + + @property + def wheel_kwargs(self): + if not self._wheel_kwargs: + self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) + return self._wheel_kwargs + + def get_setup_info(self): + # type: () -> SetupInfo + setup_info = None + with pip_shims.shims.global_tempdir_manager(): + setup_info = SetupInfo.from_ireq(self.ireq) + if not setup_info.name: + setup_info.get_info() + return setup_info + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + if not self._setup_info and not self.is_named and not self.is_wheel: + # make two attempts at this before failing to allow for stale data + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + raise + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if setup_info.version: + self.specifier = setup_info.version + if setup_info.name and not self.name: + self.name = setup_info.name + + def _get_vcsrepo(self): + # type: () -> Optional[VCSRepository] + from .vcs import VCSRepository + + checkout_directory = self.wheel_kwargs["src_dir"] # type: ignore + if self.name is not None: + checkout_directory = os.path.join( + checkout_directory, self.name + ) # type: ignore + vcsrepo = VCSRepository( + url=self.link.url, + name=self.name, + ref=self.ref if self.ref else None, + checkout_directory=checkout_directory, + vcs_type=self.vcs, + subdirectory=self.subdirectory, + ) + if not (self.link.scheme.startswith("file") and self.editable): + vcsrepo.obtain() + return vcsrepo + + @property + def vcsrepo(self): + # type: () -> Optional[VCSRepository] + if self._vcsrepo is None and self.is_vcs: + self._vcsrepo = self._get_vcsrepo() + return self._vcsrepo + + @cached_property + def metadata(self): + # type: () -> Dict[Any, Any] + if self.is_local and self.path and is_installable_dir(self.path): + return get_metadata(self.path) + return {} + + @cached_property + def parsed_setup_cfg(self): + # type: () -> Dict[Any, Any] + if not ( + self.is_local + and self.path + and is_installable_dir(self.path) + and self.setup_cfg + ): + return {} + base_dir = os.path.dirname(os.path.abspath(self.setup_cfg)) + setup_content = read_source(self.setup_cfg) + return parse_setup_cfg(setup_content, base_dir) + + @cached_property + def parsed_setup_py(self): + # type: () -> Dict[Any, Any] + if self.is_local and self.path and is_installable_dir(self.path): + if self.setup_py: + return ast_parse_setup_py(self.setup_py) + return {} + + @vcsrepo.setter + def vcsrepo(self, repo): + # type (VCSRepository) -> None + self._vcsrepo = repo + ireq = self.ireq + wheel_kwargs = self.wheel_kwargs.copy() + wheel_kwargs["src_dir"] = repo.checkout_directory + ireq.ensure_has_source_dir(wheel_kwargs["src_dir"]) + with pip_shims.shims.global_tempdir_manager(), temp_path(): + sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] + setupinfo = SetupInfo.create( + repo.checkout_directory, + ireq=ireq, + subdirectory=self.subdirectory, + kwargs=wheel_kwargs, + ) + self._setup_info = setupinfo + self._setup_info.reload() + + def get_ireq(self): + # type: () -> InstallRequirement + line = self.line_for_ireq + if self.editable: + ireq = pip_shims.shims.install_req_from_editable(line) + else: + ireq = pip_shims.shims.install_req_from_line(line) + if self.is_named: + ireq = pip_shims.shims.install_req_from_line(self.line) + if self.is_file or self.is_url: + ireq.link = self.link + if self.extras and not ireq.extras: + ireq.extras = set(self.extras) + if self.parsed_marker is not None and not ireq.markers: + ireq.markers = self.parsed_marker + if not ireq.req and self._requirement is not None: + ireq.req = copy.deepcopy(self._requirement) + return ireq + + def parse_ireq(self): + # type: () -> None + if self._ireq is None: + self._ireq = self.get_ireq() + if self._ireq is not None: + if self.requirement is not None and self._ireq.req is None: + self._ireq.req = self.requirement + + def _parse_wheel(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_wheel: + pass + from pip_shims.shims import Wheel + + _wheel = Wheel(self.link.filename) + name = _wheel.name + version = _wheel.version + self._specifier = "=={0}".format(version) + return name + + def _parse_name_from_link(self): + # type: () -> Optional[STRING_TYPE] + if self.link is None: + return None + if getattr(self.link, "egg_fragment", None): + return self.link.egg_fragment + elif self.is_wheel: + return self._parse_wheel() + return None + + def _parse_name_from_line(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_named: + pass + try: + self._requirement = init_requirement(self.line) + except Exception: + raise RequirementError( + "Failed parsing requirement from {0!r}".format(self.line) + ) + name = self._requirement.name + if not self._specifier and self._requirement and self._requirement.specifier: + self._specifier = specs_to_string(self._requirement.specifier) + if self._requirement.extras and not self.extras: + self.extras = self._requirement.extras + if not name: + name = self.line + specifier_match = next( + iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None + ) + specifier = None # type: Optional[STRING_TYPE] + if specifier_match: + specifier = "{0!s}".format(specifier_match) + if specifier is not None and specifier in name: + version = None # type: Optional[STRING_TYPE] + name, specifier, version = name.partition(specifier) + self._specifier = "{0}{1}".format(specifier, version) + return name + + def _parse_name_from_path(self): + # type: () -> Optional[S] + if self.path and self.is_local and is_installable_dir(self.path): + metadata = get_metadata(self.path) + if metadata: + name = metadata.get("name", "") + if name and name != "wheel": + return name + parsed_setup_cfg = self.parsed_setup_cfg + if parsed_setup_cfg: + name = parsed_setup_cfg.get("name", "") + if name: + return name + + parsed_setup_py = self.parsed_setup_py + if parsed_setup_py: + name = parsed_setup_py.get("name", "") + if name: + return name + return None + + def parse_name(self): + # type: () -> "Line" + if self._name is None: + name = None + if self.link is not None: + name = self._parse_name_from_link() + if name is None and ( + (self.is_url or self.is_artifact or self.is_vcs) and self._parsed_url + ): + if self._parsed_url.fragment: + _, _, name = self._parsed_url.fragment.partition("egg=") + if "&" in name: + # subdirectory fragments might also be in here + name, _, _ = name.partition("&") + if name is None and self.is_named: + name = self._parse_name_from_line() + elif name is None and self.is_file or self.is_url or self.is_path: + if self.is_local: + name = self._parse_name_from_path() + if name is not None: + name, extras = pip_shims.shims._strip_extras(name) + if extras is not None and not self.extras: + self.extras = tuple(sorted(set(parse_extras(extras)))) + self._name = name + return self + + def _parse_requirement_from_vcs(self): + # type: () -> Optional[PackagingRequirement] + url = self.url if self.url else self.link.url + if url: + url = unquote(url) + if ( + url + and self.uri != url + and "git+ssh://" in url + and (self.uri is not None and "git+git@" in self.uri) + and self._requirement is not None + ): + self._requirement.line = self.uri + self._requirement.url = self.url + vcs_uri = build_vcs_uri( # type: ignore + vcs=self.vcs, + uri=self.url, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras, + name=self.name, + ) + if vcs_uri: + self._requirement.link = create_link(vcs_uri) + elif self.link: + self._requirement.link = self.link + # else: + # req.link = self.link + if self.ref and self._requirement is not None: + if self._vcsrepo is not None: + self._requirement.revision = self._vcsrepo.get_commit_hash() + else: + self._requirement.revision = self.ref + return self._requirement + + def parse_requirement(self): + # type: () -> "Line" + if self._name is None: + self.parse_name() + if not any([self._name, self.is_vcs, self.is_named]): + if self.setup_info and self.setup_info.name: + self._name = self.setup_info.name + name, extras, url = self.requirement_info + if name: + self._requirement = init_requirement(name) # type: PackagingRequirement + if extras: + self._requirement.extras = set(extras) + if url: + self._requirement.url = url + if self.is_direct_url: + url = self.link.url + if self.link: + self._requirement.link = self.link + self._requirement.editable = self.editable + if self.path and self.link and self.link.scheme.startswith("file"): + self._requirement.local_file = True + self._requirement.path = self.path + if self.is_vcs: + self._requirement.vcs = self.vcs + self._requirement.line = self.link.url + self._parse_requirement_from_vcs() + else: + self._requirement.line = self.line + if self.parsed_marker is not None: + self._requirement.marker = self.parsed_marker + if self.specifiers: + self._requirement.specifier = self.specifiers + specs = [] + spec = next(iter(s for s in self.specifiers._specs), None) + if spec: + specs.append(spec._spec) + self._requirement.spec = spec + else: + if self.is_vcs: + raise ValueError( + "pipenv requires an #egg fragment for version controlled " + "dependencies. Please install remote dependency " + "in the form {0}#egg=<package-name>.".format(url) + ) + return self + + def parse_link(self): + # type: () -> "Line" + parsed_url = None # type: Optional[URI] + if not is_valid_url(self.line) and ( + self.line.startswith("./") + or (os.path.exists(self.line) or os.path.isabs(self.line)) + ): + url = pip_shims.shims.path_to_url(os.path.abspath(self.line)) + parsed_url = URI.parse(url) + elif is_valid_url(self.line) or is_vcs(self.line) or is_file_url(self.line): + parsed_url = URI.parse(self.line) + if parsed_url is not None: + line = parsed_url.to_string( + escape_password=False, direct=False, strip_ref=True, strip_ssh=False + ) + if parsed_url.is_vcs: + self.vcs, _ = parsed_url.scheme.split("+") + if parsed_url.is_file_url: + self.is_local = True + parsed_link = parsed_url.as_link + self._ref = parsed_url.ref + self.uri = parsed_url.bare_url + if parsed_url.name: + self._name = parsed_url.name + if parsed_url.extras: + self.extras = tuple(sorted(set(parsed_url.extras))) + self._link = parsed_link + vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( + self.line + ) + ref = None + if link is not None and "@" in unquote(link.path) and uri is not None: + uri, _, ref = unquote(uri).rpartition("@") + if relpath is not None and "@" in relpath: + relpath, _, ref = relpath.rpartition("@") + if path is not None and "@" in path: + path, _ = split_ref_from_uri(path) + link_url = link.url_without_fragment + if "@" in link_url: + link_url, _ = split_ref_from_uri(link_url) + self.preferred_scheme = prefer + self.relpath = relpath + self.path = path + # self.uri = uri + if prefer in ("path", "relpath") or uri.startswith("file"): + self.is_local = True + if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link: + self._link = parsed_link + else: + self._link = link + return self + + def parse_markers(self): + # type: () -> None + if self.markers: + marker_str = self.markers.replace('"', "'") + markers = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker + self.parsed_marker = markers + + @property + def requirement_info(self): + # type: () -> Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a + :class:`~packaging.requirements.Requirement` out of. + + :return: A Tuple of an optional name, a Tuple of extras, and an optional URL. + :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + + # Direct URLs can be converted to packaging requirements directly, but + # only if they are `file://` (with only two slashes) + name = None # type: Optional[S] + extras = () # type: Tuple[Optional[S], ...] + url = None # type: Optional[STRING_TYPE] + # if self.is_direct_url: + if self._name: + name = canonicalize_name(self._name) + if self.is_file or self.is_url or self.is_path or self.is_file_url or self.is_vcs: + url = "" + if self.is_vcs: + url = self.url if self.url else self.uri + if self.is_direct_url: + url = self.link.url_without_fragment + else: + if self.link: + url = self.link.url_without_fragment + elif self.url: + url = self.url + if self.ref: + url = "{0}@{1}".format(url, self.ref) + else: + url = self.uri + if self.link and name is None: + self._name = self.link.egg_fragment + if self._name: + name = canonicalize_name(self._name) + return name, extras, url # type: ignore + + @property + def line_is_installable(self): + # type: () -> bool + """ + This is a safeguard against decoy requirements when a user installs a package + whose name coincides with the name of a folder in the cwd, e.g. install *alembic* + when there is a folder called *alembic* in the working directory. + + In this case we first need to check that the given requirement is a valid + URL, VCS requirement, or installable filesystem path before deciding to treat it + as a file requirement over a named requirement. + """ + line = self.line + if is_file_url(line): + link = create_link(line) + line = link.url_without_fragment + line, _ = split_ref_from_uri(line) + if ( + is_vcs(line) + or ( + is_valid_url(line) + and (not is_file_url(line) or is_installable_file(line)) + ) + or is_installable_file(line) + ): + return True + return False + + def parse(self): + # type: () -> None + self.line = self.line.strip() + if self.line.startswith('"'): + self.line = self.line.strip('"') + self.line, self.markers = split_markers_from_line(self.parse_hashes().line) + if self.markers: + self.markers = self.markers.replace('"', "'") + self.parse_extras() + self.line = self.line.strip('"').strip("'").strip() + if self.line.startswith("git+file:/") and not self.line.startswith( + "git+file:///" + ): + self.line = self.line.replace("git+file:/", "git+file:///") + self.parse_markers() + if self.is_file_url: + if self.line_is_installable: + self.populate_setup_paths() + else: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(self.line) + ) + self.parse_link() + # self.parse_requirement() + # self.parse_ireq() + + +@attr.s(slots=True, hash=True) class NamedRequirement(object): - name = attr.ib() - version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() - extras = attr.ib(default=attr.Factory(list)) - editable = attr.ib(default=False) + name = attr.ib() # type: STRING_TYPE + version = attr.ib() # type: Optional[STRING_TYPE] + req = attr.ib() # type: PackagingRequirement + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[STRING_TYPE, ...] + editable = attr.ib(default=False) # type: bool + _parsed_line = attr.ib(default=None) # type: Optional[Line] @req.default def get_requirement(self): + # type: () -> RequirementType req = init_requirement( "{0}{1}".format(canonicalize_name(self.name), self.version) ) return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + @classmethod - def from_line(cls, line): + def from_line(cls, line, parsed_line=None): + # type: (AnyStr, Optional[Line]) -> NamedRequirement req = init_requirement(line) - specifiers = None + specifiers = None # type: Optional[STRING_TYPE] if req.specifier: specifiers = specs_to_string(req.specifier) req.line = line @@ -90,39 +1297,57 @@ class NamedRequirement(object): if not name: name = getattr(req, "key", line) req.name = name - extras = None + creation_kwargs = { + "name": name, + "version": specifiers, + "req": req, + "parsed_line": parsed_line, + "extras": None, + } + extras = None # type: Optional[Tuple[STRING_TYPE, ...]] if req.extras: - extras = list(req.extras) - return cls(name=name, version=specifiers, req=req, extras=extras) + extras = tuple(req.extras) + creation_kwargs["extras"] = extras + return cls(**creation_kwargs) @classmethod def from_pipfile(cls, name, pipfile): - creation_args = {} + # type: (S, TPIPFILE) -> NamedRequirement + creation_args = {} # type: TPIPFILE if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] - creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} + creation_args = { + k: v for k, v in pipfile.items() if k in attr_fields + } # type: ignore creation_args["name"] = name - version = get_version(pipfile) + version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) - creation_args["version"] = version + creation_args["version"] = version # type: ignore req = init_requirement("{0}{1}".format(name, version)) - if extras: - req.extras += tuple(extras) + if req and extras and req.extras and isinstance(req.extras, tuple): + if isinstance(extras, six.string_types): + req.extras = (extras) + tuple(["{0}".format(xtra) for xtra in req.extras]) + elif isinstance(extras, (tuple, list)): + req.extras += tuple(extras) creation_args["req"] = req - return cls(**creation_args) + return cls(**creation_args) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(normalize_name(self.name)) + return normalize_name(self.name) @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + # type: () -> Dict[STRING_TYPE, Any] + pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" + if "_parsed_line" in pipfile_dict: + pipfile_dict.pop("_parsed_line") name = pipfile_dict.pop("name") return {name: pipfile_dict} @@ -132,40 +1357,46 @@ LinkInfo = collections.namedtuple( ) -@attr.s(slots=True) +@attr.s(slots=True, cmp=True, hash=True) class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None) + setup_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Whether the package is editable - editable = attr.ib(default=False) + editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(list)) - _uri_scheme = attr.ib(default=None) + extras = attr.ib( + default=attr.Factory(tuple), cmp=True + ) # type: Tuple[STRING_TYPE, ...] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: URI of the package - uri = attr.ib() + uri = attr.ib(cmp=True) # type: Optional[STRING_TYPE] #: Link object representing the package to clone - link = attr.ib() + link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(default=attr.Factory(list)) + pyproject_requires = attr.ib( + factory=tuple, cmp=True + ) # type: Optional[Tuple[STRING_TYPE, ...]] #: PyProject Build System - pyproject_backend = attr.ib(default=None) + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: PyProject Path - pyproject_path = attr.ib(default=None) - _has_hashed_name = attr.ib(default=False) - #: Package name - name = attr.ib() - #: A :class:`~pkg_resources.Requirement` isntance - req = attr.ib() + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Setup metadata e.g. dependencies - setup_info = attr.ib(default=None) + _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] + _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool + _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] + #: Package name + name = attr.ib(cmp=True) # type: Optional[STRING_TYPE] + #: A :class:`~pkg_resources.Requirement` instance + req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): + # type: (STRING_TYPE) -> LinkInfo """Parse link information from given requirement line. Return a 6-tuple: @@ -199,15 +1430,16 @@ class FileRequirement(object): # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. - fixed_line = add_ssh_scheme_to_git_uri(line) - added_ssh_scheme = fixed_line != line + fixed_line = add_ssh_scheme_to_git_uri(line) # type: STRING_TYPE + added_ssh_scheme = fixed_line != line # type: bool # We can assume a lot of things if this is a local filesystem path. if "://" not in fixed_line: - p = Path(fixed_line).absolute() - path = p.as_posix() - uri = p.as_uri() - link = create_link(uri) + p = Path(fixed_line).absolute() # type: Path + path = p.as_posix() # type: Optional[STRING_TYPE] + uri = p.as_uri() # type: STRING_TYPE + link = create_link(uri) # type: Link + relpath = None # type: Optional[STRING_TYPE] try: relpath = get_converted_relative_path(path) except ValueError: @@ -216,19 +1448,17 @@ class FileRequirement(object): # This is an URI. We'll need to perform some elaborated parsing. - parsed_url = urllib_parse.urlsplit(fixed_line) - original_url = parsed_url._replace() - if added_ssh_scheme and ":" in parsed_url.netloc: - original_netloc, original_path_start = parsed_url.netloc.rsplit(":", 1) - uri_path = "/{0}{1}".format(original_path_start, parsed_url.path) - parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) + parsed_url = urllib_parse.urlsplit(fixed_line) # type: SplitResult + original_url = parsed_url._replace() # type: SplitResult # Split the VCS part out if needed. - original_scheme = parsed_url.scheme + original_scheme = parsed_url.scheme # type: STRING_TYPE + vcs_type = None # type: Optional[STRING_TYPE] if "+" in original_scheme: - vcs_type, scheme = original_scheme.split("+", 1) - parsed_url = parsed_url._replace(scheme=scheme) - prefer = "uri" + scheme = None # type: Optional[STRING_TYPE] + vcs_type, _, scheme = original_scheme.partition("+") + parsed_url = parsed_url._replace(scheme=scheme) # type: ignore + prefer = "uri" # type: STRING_TYPE else: vcs_type = None prefer = "file" @@ -250,112 +1480,136 @@ class FileRequirement(object): relpath = None # Cut the fragment, but otherwise this is fixed_line. uri = urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme, fragment="") + parsed_url._replace(scheme=original_scheme, fragment="") # type: ignore ) if added_ssh_scheme: original_uri = urllib_parse.urlunsplit( - original_url._replace(scheme=original_scheme, fragment="") + original_url._replace(scheme=original_scheme, fragment="") # type: ignore ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = create_link( - urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) + urllib_parse.urlunsplit( + parsed_url._replace(scheme=original_scheme) + ) # type: ignore ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) @property def setup_py_dir(self): + # type: () -> Optional[STRING_TYPE] if self.setup_path: return os.path.dirname(os.path.abspath(self.setup_path)) + return None @property def dependencies(self): - build_deps = [] - setup_deps = [] - deps = {} + # type: () -> Tuple[Dict[S, PackagingRequirement], List[Union[S, PackagingRequirement]], List[S]] + build_deps = [] # type: List[Union[S, PackagingRequirement]] + setup_deps = [] # type: List[S] + deps = {} # type: Dict[S, PackagingRequirement] if self.setup_info: setup_info = self.setup_info.as_dict() deps.update(setup_info.get("requires", {})) setup_deps.extend(setup_info.get("setup_requires", [])) build_deps.extend(setup_info.get("build_requires", [])) + if self.extras and self.setup_info.extras: + for dep in self.extras: + if dep not in self.setup_info.extras: + continue + extras_list = self.setup_info.extras.get(dep, []) # type: ignore + for req_instance in extras_list: # type: ignore + deps[req_instance.key] = req_instance if self.pyproject_requires: - build_deps.extend(self.pyproject_requires) + build_deps.extend(list(self.pyproject_requires)) + setup_deps = list(set(setup_deps)) + build_deps = list(set(build_deps)) return deps, setup_deps, build_deps + def __attrs_post_init__(self): + # type: () -> None + if self.name is None and self.parsed_line: + if self.parsed_line.setup_info: + self._setup_info = self.parsed_line.setup_info + if self.parsed_line.setup_info.name: + self.name = self.parsed_line.setup_info.name + if self.req is None and ( + self._parsed_line is not None and self._parsed_line.requirement is not None + ): + self.req = self._parsed_line.requirement + if ( + self._parsed_line + and self._parsed_line.ireq + and not self._parsed_line.ireq.req + ): + if self.req is not None and self._parsed_line._ireq is not None: + self._parsed_line._ireq.req = self.req + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + from .setup_info import SetupInfo + + if self._setup_info is None and self.parsed_line: + if self.parsed_line and self._parsed_line and self.parsed_line.setup_info: + if ( + self._parsed_line._setup_info + and not self._parsed_line._setup_info.name + ): + with pip_shims.shims.global_tempdir_manager(): + self._parsed_line._setup_info.get_info() + self._setup_info = self.parsed_line._setup_info + elif self.parsed_line and ( + self.parsed_line.ireq and not self.parsed_line.is_wheel + ): + with pip_shims.shims.global_tempdir_manager(): + self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) + else: + if self.link and not self.link.is_wheel: + self._setup_info = Line(self.line_part).setup_info + with pip_shims.shims.global_tempdir_manager(): + self._setup_info.get_info() + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line._setup_info = setup_info + @uri.default def get_uri(self): + # type: () -> STRING_TYPE if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) - elif getattr(self, "req", None) and getattr(self.req, "url"): + elif ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "url") + ): return self.req.url + elif self.link is not None: + return self.link.url_without_fragment + return "" @name.default def get_name(self): - loc = self.path or self.uri - if loc and not self._uri_scheme: - self._uri_scheme = "path" if self.path else "file" - name = None - if getattr(self, "req", None) and getattr(self.req, "name") and self.req.name is not None: - if self.is_direct_url: - return self.req.name - if self.link and self.link.egg_fragment and not self._has_hashed_name: + # type: () -> STRING_TYPE + if self.parsed_line and self.parsed_line.name: + return self.parsed_line.name + elif self.link and self.link.egg_fragment: return self.link.egg_fragment - elif self.link and self.link.is_wheel: - from pip_shims import Wheel - self._has_hashed_name = False - return Wheel(self.link.filename).name - elif self.link and ((self.link.scheme == "file" or self.editable) or ( - self.path and self.setup_path and os.path.isfile(str(self.setup_path)) - )): - if self.editable: - line = pip_shims.shims.path_to_url(self.setup_py_dir) - if self.extras: - line = "{0}[{1}]".format(line, ",".join(self.extras)) - _ireq = pip_shims.shims.install_req_from_editable(line) - else: - line = Path(self.setup_py_dir).as_posix() - if self.extras: - line = "{0}[{1}]".format(line, ",".join(self.extras)) - _ireq = pip_shims.shims.install_req_from_line(line) - if getattr(self, "req", None): - _ireq.req = copy.deepcopy(self.req) - else: - if self.extras: - _ireq.extras = set(self.extras) - from .setup_info import SetupInfo - subdir = getattr(self, "subdirectory", None) - setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) - if setupinfo: - self.setup_info = setupinfo - setupinfo_dict = setupinfo.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - self._has_hashed_name = False - build_requires = setupinfo_dict.get("build_requires") - build_backend = setupinfo_dict.get("build_backend") - if build_requires and not self.pyproject_requires: - self.pyproject_requires = build_requires - if build_backend and not self.pyproject_backend: - self.pyproject_backend = build_backend - hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() - hashed_name = hashed_loc[-7:] - if not name or name.lower() == "unknown": - self._has_hashed_name = True - name = hashed_name - else: - self._has_hashed_name = False - name_in_link = getattr(self.link, "egg_fragment", "") if self.link else "" - if not self._has_hashed_name and name_in_link != name: - self.link = create_link("{0}#egg={1}".format(self.link.url, name)) - return name + elif self.setup_info and self.setup_info.name: + return self.setup_info.name @link.default def get_link(self): + # type: () -> pip_shims.shims.Link target = "{0}".format(self.uri) if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) @@ -364,191 +1618,88 @@ class FileRequirement(object): @req.default def get_requirement(self): - req = init_requirement(normalize_name(self.name)) - req.editable = False - req.line = self.link.url_without_fragment - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path - if self.editable: - req.url = None + # type: () -> RequirementType + if self.name is None: + if self._parsed_line is not None and self._parsed_line.name is not None: + self.name = self._parsed_line.name else: - req.url = self.link.url_without_fragment - else: - req.local_file = False - req.path = None - req.url = self.link.url_without_fragment - if self.editable: - req.editable = True - req.link = self.link - return req + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format( + self + ) + ) + if self._parsed_line: + try: + # initialize specifiers to make sure we capture them + self._parsed_line.specifiers + except Exception: + pass + req = copy.deepcopy(self._parsed_line.requirement) + if req: + return req + + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + + @property + def is_local(self): + # type: () -> bool + uri = getattr(self, "uri", None) + if uri is None: + if getattr(self, "path", None) and self.path is not None: + uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + elif ( + getattr(self, "req", None) + and self.req is not None + and (getattr(self.req, "url") and self.req.url is not None) + ): + uri = self.req.url + if uri and is_file_url(uri): + return True + return False @property def is_remote_artifact(self): + # type: () -> bool + if self.link is None: + return False return ( - any( - self.link.scheme.startswith(scheme) - for scheme in ("http", "https", "ftp", "ftps", "uri") - ) - and (self.link.is_artifact or self.link.is_wheel) + self._parsed_line + and not self._parsed_line.is_local + and (self._parsed_line.is_artifact or self._parsed_line.is_wheel) and not self.editable ) @property def is_direct_url(self): + # type: () -> bool + if self._parsed_line is not None and self._parsed_line.is_direct_url: + return True return self.is_remote_artifact @property def formatted_path(self): + # type: () -> Optional[STRING_TYPE] if self.path: path = self.path if not isinstance(path, Path): path = Path(path) return path.as_posix() - return + return None @classmethod - def create( - cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, - name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None - ): - if relpath and not path: - path = relpath - if not path and uri and link.scheme == "file": - path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) - try: - path = get_converted_relative_path(path) - except ValueError: # Vistir raises a ValueError if it can't make a relpath - path = path - if line and not (uri_scheme and uri and link): - vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) - if not uri_scheme: - uri_scheme = "path" if path else "file" - if path and not uri: - uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) - if not link: - link = create_link(uri) - if not uri: - uri = unquote(link.url_without_fragment) - if not extras: - extras = [] - pyproject_path = None - if path is not None: - pyproject_requires = get_pyproject(os.path.abspath(path)) - pyproject_backend = None - pyproject_requires = None - if pyproject_requires is not None: - pyproject_requires, pyproject_backend = pyproject_requires - if path: - pyproject_path = Path(path).joinpath("pyproject.toml") - if not pyproject_path.exists(): - pyproject_path = None - if not setup_path and path is not None: - setup_path = Path(path).joinpath("setup.py") - if setup_path and isinstance(setup_path, Path): - setup_path = setup_path.as_posix() - creation_kwargs = { - "editable": editable, - "extras": extras, - "pyproject_path": pyproject_path, - "setup_path": setup_path if setup_path else None, - "uri_scheme": uri_scheme, - "link": link, - "uri": uri, - "pyproject_requires": pyproject_requires, - "pyproject_backend": pyproject_backend - } - if vcs_type: - creation_kwargs["vcs_type"] = vcs_type - _line = None - if not name: - _line = unquote(link.url_without_fragment) if link.url else uri - if editable: - if extras: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) - ireq = pip_shims.shims.install_req_from_editable(_line) - else: - _line = path if (uri_scheme and uri_scheme == "path") else _line - if extras: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) - ireq = pip_shims.shims.install_req_from_line(_line) - if extras and not ireq.extras: - ireq.extras = set(extras) - setup_info = SetupInfo.from_ireq(ireq) - setupinfo_dict = setup_info.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - build_requires = setupinfo_dict.get("build_requires", []) - build_backend = setupinfo_dict.get("build_backend", []) - if not creation_kwargs.get("pyproject_requires") and build_requires: - creation_kwargs["pyproject_requires"] = build_requires - if not creation_kwargs.get("pyproject_backend") and build_backend: - creation_kwargs["pyproject_backend"] = build_backend - creation_kwargs["setup_info"] = setup_info - if path or relpath: - creation_kwargs["path"] = relpath if relpath else path - if req: - creation_kwargs["req"] = req - if creation_kwargs.get("req") and line and not getattr(creation_kwargs["req"], "line", None): - creation_kwargs["req"].line = line - if name: - creation_kwargs["name"] = name - cls_inst = cls(**creation_kwargs) - if not _line: - if editable and uri_scheme == "path": - _line = relpath if relpath else path - else: - _line = unquote(cls_inst.link.url_without_fragment) or cls_inst.uri - _line = "{0}#egg={1}".format(line, cls_inst.name) if not cls_inst._has_hashed_name else _line - cls_inst.req.line = line if line else _line - return cls_inst - - @classmethod - def from_line(cls, line, extras=None): - line = line.strip('"').strip("'") - link = None - path = None - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - setup_path = None - name = None - req = None - if not extras: - extras = [] - if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): - try: - req = init_requirement(line) - except Exception: - raise RequirementError( - "Supplied requirement is not installable: {0!r}".format(line) - ) - else: - name = getattr(req, "name", None) - line = getattr(req, "url", None) - vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - arg_dict = { - "path": relpath if relpath else path, - "uri": unquote(link.url_without_fragment), - "link": link, - "editable": editable, - "setup_path": setup_path, - "uri_scheme": prefer, - "line": line, - "extras": extras - } - if link and link.is_wheel: - from pip_shims import Wheel - - arg_dict["name"] = Wheel(link.filename).name - elif name: - arg_dict["name"] = name - elif link.egg_fragment: - arg_dict["name"] = link.egg_fragment - return cls.create(**arg_dict) + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F + parsed_line = Line(line) + file_req_from_parsed_line(parsed_line) @classmethod def from_pipfile(cls, name, pipfile): + # type: (STRING_TYPE, Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE, bool]]) -> F # Parse the values out. After this dance we should have two variables: # path - Local filesystem path. # uri - Absolute URI that is parsable with urlsplit. @@ -556,7 +1707,7 @@ class FileRequirement(object): uri = pipfile.get("uri") fil = pipfile.get("file") path = pipfile.get("path") - if path: + if path and isinstance(path, six.string_types): if isinstance(path, Path) and not path.is_absolute(): path = get_converted_relative_path(path.as_posix()) elif not os.path.isabs(path): @@ -580,124 +1731,201 @@ class FileRequirement(object): if not uri: uri = pip_shims.shims.path_to_url(path) - link = create_link(uri) + link_info = None # type: Optional[LinkInfo] + if uri and isinstance(uri, six.string_types): + link_info = cls.get_link_from_line(uri) + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + link = None # type: Optional[Link] + if link_info: + link = link_info.link + if link.url_without_fragment: + uri = unquote(link.url_without_fragment) + extras = () # type: Optional[Tuple[STRING_TYPE, ...]] + if "extras" in pipfile: + extras = tuple(pipfile["extras"]) # type: ignore + editable = pipfile["editable"] if "editable" in pipfile else False arg_dict = { "name": name, "path": path, - "uri": unquote(link.url_without_fragment), - "editable": pipfile.get("editable", False), + "uri": uri, + "editable": editable, "link": link, "uri_scheme": uri_scheme, + "extras": extras if extras else None, } - if link.scheme != "file" and not pipfile.get("editable", False): - arg_dict["line"] = "{0}@ {1}".format(name, link.url_without_fragment) - return cls.create(**arg_dict) + + line = "" # type: STRING_TYPE + extras_string = "" if not extras else extras_to_string(extras) + if editable and uri_scheme == "path": + line = "{0}{1}".format(path, extras_string) + else: + if name: + line_name = "{0}{1}".format(name, extras_string) + line = "{0}#egg={1}".format(unquote(link.url_without_fragment), line_name) + else: + if link: + line = unquote(link.url) + elif uri and isinstance(uri, six.string_types): + line = uri + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + line = "{0}{1}".format(line, extras_string) + if "subdirectory" in pipfile: + arg_dict["subdirectory"] = pipfile["subdirectory"] + line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) + if editable: + line = "-e {0}".format(line) + arg_dict["parsed_line"] = Line(line) + arg_dict["setup_info"] = arg_dict["parsed_line"].setup_info + return cls(**arg_dict) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE + link_url = None # type: Optional[STRING_TYPE] + seed = None # type: Optional[STRING_TYPE] + if self.link is not None: + link_url = unquote(self.link.url_without_fragment) + is_vcs = getattr(self.link, "is_vcs", not self.link.is_artifact) if self._uri_scheme and self._uri_scheme == "path": # We may need any one of these for passing to pip - seed = self.path or unquote(self.link.url_without_fragment) or self.uri + seed = self.path or link_url or self.uri elif (self._uri_scheme and self._uri_scheme == "file") or ( - (self.link.is_artifact or self.link.is_wheel) and self.link.url + (self.link.is_wheel or not is_vcs) and self.link.url ): - seed = unquote(self.link.url_without_fragment) or self.uri + seed = link_url or self.uri # add egg fragments to remote artifacts (valid urls only) - if not self._has_hashed_name and self.is_remote_artifact: + if not self._has_hashed_name and self.is_remote_artifact and seed is not None: seed += "#egg={0}".format(self.name) editable = "-e " if self.editable else "" + if seed is None: + raise ValueError("Could not calculate url for {0!r}".format(self)) return "{0}{1}".format(editable, seed) @property def pipfile_part(self): + # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] excludes = [ - "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_base_line", + "_has_hashed_name", + "setup_path", + "pyproject_path", + "_uri_scheme", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes - pipfile_dict = attr.asdict(self, filter=filter_func).copy() - name = pipfile_dict.pop("name") + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa + pipfile_dict = attr.asdict(self, filter=filter_func).copy() # type: Dict + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "_uri_scheme" in pipfile_dict: pipfile_dict.pop("_uri_scheme") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} + collision_order = ["file", "uri", "path"] # type: List[STRING_TYPE] + collisions = [] # type: List[STRING_TYPE] + key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) + is_vcs = None + if self.link is not None: + is_vcs = getattr(self.link, "is_vcs", not self.link.is_artifact) if self._uri_scheme: dict_key = self._uri_scheme - target_key = ( - dict_key - if dict_key in pipfile_dict - else next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - ) - if target_key: + target_key = dict_key if dict_key in pipfile_dict else key_match + if target_key is not None: winning_value = pipfile_dict.pop(target_key) - collisions = (k for k in collision_keys if k in pipfile_dict) + collisions = [k for k in collision_keys if k in pipfile_dict] for key in collisions: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value elif ( self.is_remote_artifact - or self.link.is_artifact + or (is_vcs is not None and not is_vcs) and (self._uri_scheme and self._uri_scheme == "file") ): dict_key = "file" # Look for uri first because file is a uri format and this is designed # to make sure we add file keys to the pipfile as a replacement of uri - target_key = next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - winning_value = pipfile_dict.pop(target_key) + if key_match is not None: + winning_value = pipfile_dict.pop(key_match) key_to_remove = (k for k in collision_keys if k in pipfile_dict) for key in key_to_remove: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value else: - collisions = [key for key in ["path", "file", "uri"] if key in pipfile_dict] + collisions = [key for key in collision_order if key in pipfile_dict.keys()] if len(collisions) > 1: for k in collisions[1:]: pipfile_dict.pop(k) return {name: pipfile_dict} -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class VCSRequirement(FileRequirement): #: Whether the repository is editable - editable = attr.ib(default=None) + editable = attr.ib(default=None) # type: Optional[bool] #: URI for the repository - uri = attr.ib(default=None) + uri = attr.ib(default=None) # type: Optional[STRING_TYPE] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib( + default=None, validator=attr.validators.optional(validate_path) + ) # type: Optional[STRING_TYPE] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) + vcs = attr.ib( + validator=attr.validators.optional(validate_vcs), default=None + ) # type: Optional[STRING_TYPE] #: vcs reference name (branch / commit / tag) - ref = attr.ib(default=None) + ref = attr.ib(default=None) # type: Optional[STRING_TYPE] #: Subdirectory to use for installation if applicable - subdirectory = attr.ib(default=None) - _repo = attr.ib(default=None) - _base_line = attr.ib(default=None) - name = attr.ib() - link = attr.ib() - req = attr.ib() + subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE] + _repo = attr.ib(default=None) # type: Optional[VCSRepository] + _base_line = attr.ib(default=None) # type: Optional[STRING_TYPE] + name = attr.ib() # type: STRING_TYPE + link = attr.ib() # type: Optional[pip_shims.shims.Link] + req = attr.ib() # type: Optional[RequirementType] def __attrs_post_init__(self): + # type: () -> None if not self.uri: if self.path: self.uri = pip_shims.shims.path_to_url(self.path) - split = urllib_parse.urlsplit(self.uri) - scheme, rest = split[0], split[1:] - vcs_type = "" - if "+" in scheme: - vcs_type, scheme = scheme.split("+", 1) - vcs_type = "{0}+".format(vcs_type) - new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) - new_uri = "{0}{1}".format(vcs_type, new_uri) - self.uri = new_uri + if self.uri is not None: + split = urllib_parse.urlsplit(self.uri) + scheme, rest = split[0], split[1:] + vcs_type = "" + if "+" in scheme: + vcs_type, scheme = scheme.split("+", 1) + vcs_type = "{0}+".format(vcs_type) + new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) + new_uri = "{0}{1}".format(vcs_type, new_uri) + self.uri = new_uri + + @property + def url(self): + # type: () -> STRING_TYPE + if self.link and self.link.url: + return self.link.url + elif self.uri: + return self.uri + raise ValueError("No valid url found for requirement {0!r}".format(self)) @link.default def get_link(self): + # type: () -> pip_shims.shims.Link uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) - return build_vcs_link( + vcs_uri = build_vcs_uri( self.vcs, add_ssh_scheme_to_git_uri(uri), name=self.name, @@ -705,26 +1933,68 @@ class VCSRequirement(FileRequirement): subdirectory=self.subdirectory, extras=self.extras, ) + return self.get_link_from_line(vcs_uri).link @name.default def get_name(self): - return ( - self.link.egg_fragment or self.req.name - if getattr(self, "req", None) - else super(VCSRequirement, self).get_name() - ) + # type: () -> STRING_TYPE + if self.link and self.link.egg_fragment: + return self.link.egg_fragment + if self.req and self.req.name: + return self.req.name + return super(VCSRequirement, self).get_name() @property def vcs_uri(self): + # type: () -> Optional[STRING_TYPE] uri = self.uri - if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): - uri = "{0}+{1}".format(self.vcs, uri) + if uri and not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): + if self.vcs: + uri = "{0}+{1}".format(self.vcs, uri) return uri + @property + def setup_info(self): + if self._parsed_line and self._parsed_line.setup_info: + if not self._parsed_line.setup_info.name: + with pip_shims.shims.global_tempdir_manager(): + self._parsed_line._setup_info.get_info() + return self._parsed_line.setup_info + if self._repo: + from .setup_info import SetupInfo + + with pip_shims.shims.global_tempdir_manager(): + self._setup_info = SetupInfo.from_ireq( + Line(self._repo.checkout_directory).ireq + ) + self._setup_info.get_info() + return self._setup_info + ireq = self.parsed_line.ireq + from .setup_info import SetupInfo + + with pip_shims.shims.global_tempdir_manager(): + self._setup_info = SetupInfo.from_ireq(ireq) + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line.setup_info = setup_info + @req.default def get_requirement(self): - name = self.name or self.link.egg_fragment - url = self.uri or self.link.url_without_fragment + # type: () -> PackagingRequirement + name = None # type: Optional[STRING_TYPE] + if self.name: + name = self.name + elif self.link and self.link.egg_fragment: + name = self.link.egg_fragment + url = None + if self.uri: + url = self.uri + elif self.link is not None: + url = self.link.url_without_fragment if not name: raise ValueError( "pipenv requires an #egg fragment for version controlled " @@ -733,9 +2003,25 @@ class VCSRequirement(FileRequirement): ) req = init_requirement(canonicalize_name(self.name)) req.editable = self.editable - if not getattr(req, "url") and self.uri: - req.url = self.uri - req.line = self.link.url + if not getattr(req, "url"): + if url is not None: + url = add_ssh_scheme_to_git_uri(url) + elif self.uri is not None: + link = self.get_link_from_line(self.uri).link + if link: + url = link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + if url: + req.url = url + line = url if url else self.vcs_uri + if self.editable: + line = "-e {0}".format(line) + req.line = line if self.ref: req.revision = self.ref if self.extras: @@ -746,27 +2032,38 @@ class VCSRequirement(FileRequirement): req.path = self.path req.link = self.link if ( - self.uri != unquote(self.link.url_without_fragment) + self.link + and self.link.url_without_fragment + and self.uri + and self.uri != unquote(self.link.url_without_fragment) and "git+ssh://" in self.link.url and "git+git@" in self.uri ): req.line = self.uri - req.url = self.uri + url = self.link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + req.url = url return req - @property - def is_local(self): - if is_file_url(self.uri): - return True - return False - @property def repo(self): + # type: () -> VCSRepository if self._repo is None: - self._repo = self.get_vcs_repo() + if self._parsed_line and self._parsed_line.vcsrepo: + self._repo = self._parsed_line.vcsrepo + else: + self._repo = self.get_vcs_repo() + if self._parsed_line: + self._parsed_line.vcsrepo = self._repo return self._repo def get_checkout_dir(self, src_dir=None): + # type: (Optional[S]) -> STRING_TYPE src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: @@ -776,22 +2073,20 @@ class VCSRequirement(FileRequirement): if path and os.path.exists(path): checkout_dir = os.path.abspath(path) return checkout_dir + if src_dir is not None: + checkout_dir = os.path.join(os.path.abspath(src_dir), self.name) + mkdir_p(src_dir) + return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - def get_vcs_repo(self, src_dir=None): + def get_vcs_repo(self, src_dir=None, checkout_dir=None): + # type: (Optional[STRING_TYPE], STRING_TYPE) -> VCSRepository from .vcs import VCSRepository - checkout_dir = self.get_checkout_dir(src_dir=src_dir) - link = build_vcs_link( - self.vcs, - self.uri, - name=self.name, - ref=self.ref, - subdirectory=self.subdirectory, - extras=self.extras, - ) + if checkout_dir is None: + checkout_dir = self.get_checkout_dir(src_dir=src_dir) vcsrepo = VCSRepository( - url=link.url, + url=self.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, @@ -803,7 +2098,9 @@ class VCSRequirement(FileRequirement): pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") - self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + self.pyproject_path = os.path.join( + checkout_dir, self.subdirectory, "pyproject.toml" + ) pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") @@ -811,16 +2108,18 @@ class VCSRequirement(FileRequirement): pyproject_info = get_pyproject(checkout_dir) if pyproject_info is not None: pyproject_requires, pyproject_backend = pyproject_info - self.pyproject_requires = pyproject_requires + self.pyproject_requires = tuple(pyproject_requires) self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): + # type: () -> STRING_TYPE hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): + # type: (Optional[STRING_TYPE], Optional[STRING_TYPE]) -> STRING_TYPE if ref: self.ref = ref else: @@ -830,29 +2129,57 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() - self.req.revision = repo_hash + if self.req: + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + # type: (Optional[AnyStr]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - self.req.revision = vcsrepo.get_commit_hash() + if not self.req: + if self.parsed_line is not None: + self.req = self.parsed_line.requirement + else: + self.req = self.get_requirement() + revision = self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed - if "@" in self.link.show_url and "@" in self.uri: - uri, ref = self.uri.rsplit("@", 1) - checkout = self.req.revision - if checkout and ref in checkout: + if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri: + uri, ref = split_ref_from_uri(self.uri) + checkout = revision + if checkout and ref and ref in checkout: self.uri = uri - - yield vcsrepo + orig_repo = self._repo self._repo = vcsrepo + if self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self._setup_info: + self._setup_info = attr.evolve( + self._setup_info, + requirements=(), + _extras_requirements=(), + build_requires=(), + setup_requires=(), + version=None, + metadata=None, + ) + if self.parsed_line and self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self.req and not self.editable: + self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) + try: + yield self._repo + except Exception: + self._repo = orig_repo + raise @classmethod def from_pipfile(cls, name, pipfile): - creation_args = {} + # type: (STRING_TYPE, Dict[S, Union[Tuple[S, ...], S, bool]]) -> F + creation_args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] pipfile_keys = [ k for k in ( @@ -868,153 +2195,221 @@ class VCSRequirement(FileRequirement): + VCS_LIST if k in pipfile ] + # extras = None # type: Optional[Tuple[STRING_TYPE, ...]] for key in pipfile_keys: - if key == "extras": - extras = pipfile.get(key, None) - if extras: - pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) - if key in VCS_LIST: - creation_args["vcs"] = key - target = pipfile.get(key) - drive, path = os.path.splitdrive(target) - if ( - not drive - and not os.path.exists(target) - and ( - is_valid_url(target) - or is_file_url(target) - or target.startswith("git@") - ) - ): - creation_args["uri"] = target + if key == "extras" and key in pipfile: + extras = pipfile[key] + if isinstance(extras, (list, tuple)): + pipfile[key] = tuple(sorted({extra.lower() for extra in extras})) else: - creation_args["path"] = target - if os.path.isabs(target): - creation_args["uri"] = pip_shims.shims.path_to_url(target) - else: - creation_args[key] = pipfile.get(key) + pipfile[key] = extras + if key in VCS_LIST and key in pipfile_keys: + creation_args["vcs"] = key + target = pipfile[key] + if isinstance(target, six.string_types): + drive, path = os.path.splitdrive(target) + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): + creation_args["uri"] = target + else: + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = pip_shims.shims.path_to_url(target) + elif key in pipfile_keys: + creation_args[key] = pipfile[key] creation_args["name"] = name - return cls(**creation_args) + cls_inst = cls(**creation_args) # type: ignore + return cls_inst @classmethod - def from_line(cls, line, editable=None, extras=None): - relpath = None - if line.startswith("-e "): - editable = True - line = line.split(" ", 1)[1] - vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) - if not extras and link.egg_fragment: - name, extras = pip_shims.shims._strip_extras(link.egg_fragment) - if extras: - extras = parse_extras(extras) - else: - name = link.egg_fragment - subdirectory = link.subdirectory_fragment - ref = None - if "@" in link.path and "@" in uri: - uri, _, ref = uri.rpartition("@") - if relpath and "@" in relpath: - relpath, ref = relpath.rsplit("@", 1) - return cls( - name=name, - ref=ref, - vcs=vcs_type, - subdirectory=subdirectory, - link=link, - path=relpath or path, - editable=editable, - uri=uri, - extras=extras, - base_line=line, - ) + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F + parsed_line = Line(line) + return vcs_req_from_parsed_line(parsed_line) @property def line_part(self): + # type: () -> STRING_TYPE """requirements.txt compatible line part sans-extras""" + base = "" # type: STRING_TYPE if self.is_local: base_link = self.link if not self.link: base_link = self.get_link() - final_format = ( - "{{0}}#egg={0}".format(base_link.egg_fragment) - if base_link.egg_fragment - else "{0}" - ) + if base_link and base_link.egg_fragment: + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) + else: + final_format = "{0}" base = final_format.format(self.vcs_uri) - elif self._base_line: + elif self._parsed_line is not None and ( + self._parsed_line.is_direct_url and self._parsed_line.line_with_prefix + ): + return self._parsed_line.line_with_prefix + elif getattr(self, "_base_line", None) and ( + isinstance(self._base_line, six.string_types) + ): base = self._base_line else: - base = self.link.url - if base and self.extras and not extras_to_string(self.extras) in base: + base = getattr(self, "link", self.get_link()).url + if base and self.extras and extras_to_string(self.extras) not in base: if self.subdirectory: base = "{0}".format(self.get_link().url) else: base = "{0}{1}".format(base, extras_to_string(sorted(self.extras))) - if self.editable: + if "git+file:/" in base and "git+file:///" not in base: + base = base.replace("git+file:/", "git+file:///") + if self.editable and not base.startswith("-e "): base = "-e {0}".format(base) return base @staticmethod def _choose_vcs_source(pipfile): + # type: (Dict[S, Union[S, Any]]) -> Dict[S, Union[S, Any]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] + vcs_type = "" # type: Optional[STRING_TYPE] + alt_type = "" # type: Optional[STRING_TYPE] + vcs_value = "" # type: STRING_TYPE if src_keys: - chosen_key = first(src_keys) + chosen_key = next(iter(src_keys)) vcs_type = pipfile.pop("vcs") - _, pipfile_url = split_vcs_method_from_uri(pipfile.get(chosen_key)) - pipfile[vcs_type] = pipfile_url + if chosen_key in pipfile: + vcs_value = pipfile[chosen_key] + alt_type, pipfile_url = split_vcs_method_from_uri(vcs_value) + if vcs_type is None: + vcs_type = alt_type + if vcs_type and pipfile_url: + pipfile[vcs_type] = pipfile_url for removed in src_keys: pipfile.pop(removed) return pipfile @property def pipfile_part(self): + # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, RequirementType, pip_shims.shims.Link]]] excludes = [ - "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_repo", + "_base_line", + "setup_path", + "_has_hashed_name", + "pyproject_path", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", + "_uri_scheme", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = pip_shims.shims._strip_extras(pipfile_dict.pop("name")) - return {name: pipfile_dict} + name, _ = pip_shims.shims._strip_extras(name) + return {name: pipfile_dict} # type: ignore -@attr.s +@attr.s(cmp=True, hash=True) class Requirement(object): - name = attr.ib() - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) - req = attr.ib(default=None) - markers = attr.ib(default=None) - specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) - index = attr.ib(default=None) - editable = attr.ib(default=None) - hashes = attr.ib(default=attr.Factory(list), converter=list) - extras = attr.ib(default=attr.Factory(list)) - abstract_dep = attr.ib(default=None) - _ireq = None + _name = attr.ib(cmp=True) # type: STRING_TYPE + vcs = attr.ib( + default=None, validator=attr.validators.optional(validate_vcs), cmp=True + ) # type: Optional[STRING_TYPE] + req = attr.ib( + default=None, cmp=True + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + markers = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + _specifiers = attr.ib( + validator=attr.validators.optional(validate_specifiers), cmp=True + ) # type: Optional[STRING_TYPE] + index = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + editable = attr.ib(default=None, cmp=True) # type: Optional[bool] + hashes = attr.ib( + factory=frozenset, converter=frozenset, cmp=True + ) # type: FrozenSet[STRING_TYPE] + extras = attr.ib(factory=tuple, cmp=True) # type: Tuple[STRING_TYPE, ...] + abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] + _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] + _ireq = attr.ib( + default=None, cmp=False + ) # type: Optional[pip_shims.InstallRequirement] - @name.default + def __hash__(self): + return hash(self.as_line()) + + @_name.default def get_name(self): - return self.req.name + # type: () -> Optional[STRING_TYPE] + if self.req is not None: + return self.req.name + return None + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is not None: + return self._name + name = None + if self.req and self.req.name: + name = self.req.name + elif self.req and self.is_file_or_url and self.req.setup_info: + name = self.req.setup_info.name + self._name = name + return name @property def requirement(self): - return self.req.req + # type: () -> Optional[PackagingRequirement] + if self.req: + return self.req.req + return None + + def add_hashes(self, hashes): + # type: (Union[S, List[S], Set[S], Tuple[S, ...]]) -> Requirement + new_hashes = set() # type: Set[STRING_TYPE] + if self.hashes is not None: + new_hashes |= set(self.hashes) + if isinstance(hashes, six.string_types): + new_hashes.add(hashes) + else: + new_hashes |= set(hashes) + return attr.evolve(self, hashes=tuple(new_hashes)) def get_hashes_as_pip(self, as_list=False): - if self.hashes: - if as_list: - return [HASH_STRING.format(h) for h in self.hashes] - return "".join([HASH_STRING.format(h) for h in self.hashes]) - return "" if not as_list else [] + # type: (bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + hashes = "" # type: Union[STRING_TYPE, List[STRING_TYPE]] + if as_list: + hashes = [] + if self.hashes: + hashes = [HASH_STRING.format(h) for h in self.hashes] + else: + hashes = "" + if self.hashes: + hashes = "".join([HASH_STRING.format(h) for h in self.hashes]) + return hashes @property def hashes_as_pip(self): - self.get_hashes_as_pip() + # type: () -> STRING_TYPE + hashes = self.get_hashes_as_pip() + assert isinstance(hashes, six.string_types) + return hashes @property def markers_as_pip(self): + # type: () -> S if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -1022,141 +2417,249 @@ class Requirement(object): @property def extras_as_pip(self): + # type: () -> STRING_TYPE if self.extras: return "[{0}]".format( - ",".join(sorted([extra.lower() for extra in self.extras])) + ",".join(sorted([extra.lower() for extra in self.extras])) # type: ignore ) return "" - @property + @cached_property def commit_hash(self): - if not self.is_vcs: + # type: () -> Optional[S] + if self.req is None or not isinstance(self.req, VCSRequirement): return None commit_hash = None - with self.req.locked_vcs_repo() as repo: - commit_hash = repo.get_commit_hash() + if self.req is not None: + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() return commit_hash - @specifiers.default + @_specifiers.default def get_specifiers(self): - if self.req and self.req.req.specifier: + # type: () -> S + if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) - return + return "" + + def update_name_from_path(self, path): + metadata = get_metadata(path) + name = self.name + if metadata is not None: + metadata_name = metadata.get("name") + if metadata_name and metadata_name != "wheel": + name = metadata_name + if name is not None: + if self.req.name is None: + self.req.name = name + if self.req.req and self.req.req.name is None: + self.req.req.name = name + if self._line_instance._name is None: + self._line_instance.name = name + if self.req._parsed_line._name is None: + self.req._parsed_line.name = name + if self.req._setup_info and self.req._setup_info.name is None: + self.req._setup_info.name = name + + def get_line_instance(self): + # type: () -> Line + line_parts = [] + if self.req: + if self.req.line_part.startswith("-e "): + line_parts.extend(self.req.line_part.split(" ", 1)) + else: + line_parts.append(self.req.line_part) + if not self.is_vcs and not self.vcs and self.extras_as_pip: + line_parts.append(self.extras_as_pip) + if self._specifiers and not (self.is_file_or_url or self.is_vcs): + line_parts.append(self._specifiers) + if self.markers: + line_parts.append("; {0}".format(self.markers.replace('"', "'"))) + if self.hashes_as_pip and not (self.editable or self.vcs or self.is_vcs): + line_parts.append(self.hashes_as_pip) + if self.editable: + if line_parts[0] == "-e": + line = "".join(line_parts[1:]) + else: + line = "".join(line_parts) + if self.markers: + line = '"{0}"'.format(line) + line = "-e {0}".format(line) + else: + line = "".join(line_parts) + return Line(line) + + @property + def line_instance(self): + # type: () -> Optional[Line] + if self._line_instance is None: + self.line_instance = self.get_line_instance() + return self._line_instance + + @line_instance.setter + def line_instance(self, line_instance): + # type: (Line) -> None + if self.req: + self.req._parsed_line = line_instance + self._line_instance = line_instance + + @property + def specifiers(self): + # type: () -> Optional[STRING_TYPE] + if self._specifiers: + return self._specifiers + else: + specs = self.get_specifiers() + if specs: + self._specifiers = specs + return specs + if not self._specifiers and ( + self.req is not None + and isinstance(self.req, NamedRequirement) + and self.req.version + ): + self._specifiers = self.req.version + elif ( + not self.editable + and self.req + and (not isinstance(self.req, NamedRequirement) and self.req.setup_info) + ): + if ( + self.line_instance + and self.line_instance.setup_info + and self.line_instance.setup_info.version + ): + self._specifiers = "=={0}".format(self.req.setup_info.version) + elif not self._specifiers: + if self.req and self.req.parsed_line and self.req.parsed_line.specifiers: + self._specifiers = specs_to_string(self.req.parsed_line.specifiers) + elif self.line_instance and self.line_instance.specifiers: + self._specifiers = specs_to_string(self.line_instance.specifiers) + elif self.is_file_or_url or self.is_vcs: + try: + setupinfo_dict = self.run_requires() + except Exception: + setupinfo_dict = None + if setupinfo_dict is not None: + self._specifiers = "=={0}".format(setupinfo_dict.get("version")) + if self._specifiers: + specset = SpecifierSet(self._specifiers) + if self.line_instance and not self.line_instance.specifiers: + self.line_instance.specifiers = specset + if self.req: + if self.req._parsed_line and not self.req._parsed_line.specifiers: + self.req._parsed_line.specifiers = specset + elif not self.req._parsed_line and self.line_instance: + self.req._parsed_line = self.line_instance + if self.req and self.req.req and not self.req.req.specifier: + self.req.req.specifier = specset + return self._specifiers @property def is_vcs(self): + # type: () -> bool return isinstance(self.req, VCSRequirement) + @property + def build_backend(self): + # type: () -> Optional[STRING_TYPE] + if self.req is not None and ( + not isinstance(self.req, NamedRequirement) and self.req.is_local + ): + with pip_shims.shims.global_tempdir_manager(): + setup_info = self.run_requires() + build_backend = setup_info.get("build_backend") + return build_backend + return "setuptools.build_meta" + + @property + def uses_pep517(self): + # type: () -> bool + if self.build_backend: + return True + return False + @property def is_file_or_url(self): + # type: () -> bool return isinstance(self.req, FileRequirement) @property def is_named(self): + # type: () -> bool return isinstance(self.req, NamedRequirement) + @property + def is_wheel(self): + # type: () -> bool + if ( + self.req + and not isinstance(self.req, NamedRequirement) + and (self.req.link is not None and self.req.link.is_wheel) + ): + return True + return False + @property def normalized_name(self): + # type: () -> S return canonicalize_name(self.name) def copy(self): return attr.evolve(self) @classmethod + @lru_cache() def from_line(cls, line): + # type: (AnyStr) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) - hashes = None - if "--hash=" in line: - hashes = line.split(" --hash=") - line, hashes = hashes[0], hashes[1:] - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - line, markers = split_markers_from_line(line) - line, extras = pip_shims.shims._strip_extras(line) - if extras: - extras = parse_extras(extras) - line = line.strip('"').strip("'").strip() - line_with_prefix = "-e {0}".format(line) if editable else line - vcs = None - # Installable local files and installable non-vcs urls are handled - # as files, generally speaking - line_is_vcs = is_vcs(line) - # check for pep-508 compatible requirements - name, _, possible_url = line.partition("@") - if is_installable_file(line) or ( - (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and - not (line_is_vcs or is_vcs(possible_url)) - ): - r = FileRequirement.from_line(line_with_prefix, extras=extras) - elif line_is_vcs: - r = VCSRequirement.from_line(line_with_prefix, extras=extras) - vcs = r.vcs + parsed_line = Line(line) + r = ( + None + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + if ( + (parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url + ) and not parsed_line.is_vcs: + r = file_req_from_parsed_line(parsed_line) + elif parsed_line.is_vcs: + r = vcs_req_from_parsed_line(parsed_line) elif line == "." and not is_installable_file(line): raise RequirementError( "Error parsing requirement %s -- are you sure it is installable?" % line ) else: - specs = "!=<>~" - spec_matches = set(specs) & set(line) - version = None - name = line - if spec_matches: - spec_idx = min((line.index(match) for match in spec_matches)) - name = line[:spec_idx] - version = line[spec_idx:] - if not extras: - name, extras = pip_shims.shims._strip_extras(name) - if extras: - extras = parse_extras(extras) - if version: - name = "{0}{1}".format(name, version) - r = NamedRequirement.from_line(line) + r = named_req_from_parsed_line(parsed_line) req_markers = None - if markers: - req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - r.req.local_file = getattr(r.req, "local_file", False) - name = getattr(r.req, "name", None) - if not name: - name = getattr(r.req, "project_name", None) - r.req.name = name - if not name: - name = getattr(r.req, "key", None) - if name: - r.req.name = name + if parsed_line.markers: + req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) + if r is not None and r.req is not None: + r.req.marker = getattr(req_markers, "marker", None) if req_markers else None + args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] args = { "name": r.name, - "vcs": vcs, + "vcs": parsed_line.vcs, "req": r, - "markers": markers, - "editable": editable, + "markers": parsed_line.markers, + "editable": parsed_line.editable, + "line_instance": parsed_line, } - if extras: - extras = sorted(dedup([extra.lower() for extra in extras])) + if parsed_line.extras: + extras = () # type: Tuple[STRING_TYPE, ...] + extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) args["extras"] = extras - r.req.extras = extras - r.extras = extras - elif r.extras: - args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) - if hashes: - args["hashes"] = hashes - cls_inst = cls(**args) - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) + if r is not None: + r.extras = extras + elif r is not None and r.extras is not None: + args["extras"] = tuple( + sorted(dedup([extra.lower() for extra in r.extras])) + ) # type: ignore + if r.req is not None: + r.req.extras = args["extras"] + if parsed_line.hashes: + args["hashes"] = tuple(parsed_line.hashes) # type: ignore + cls_inst = cls(**args) # type: ignore return cls_inst @classmethod @@ -1177,7 +2680,7 @@ class Requirement(object): if hasattr(pipfile, "keys"): _pipfile = dict(pipfile).copy() _pipfile["version"] = get_version(pipfile) - vcs = first([vcs for vcs in VCS_LIST if vcs in _pipfile]) + vcs = next(iter([vcs for vcs in VCS_LIST if vcs in _pipfile]), None) if vcs: _pipfile["vcs"] = vcs r = VCSRequirement.from_pipfile(name, pipfile) @@ -1190,41 +2693,29 @@ class Requirement(object): if markers: markers = str(markers) req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) - r.req.specifier = SpecifierSet(_pipfile["version"]) + if r.req is not None: + r.req.marker = req_markers.marker extras = _pipfile.get("extras") - r.req.extras = ( - sorted(dedup([extra.lower() for extra in extras])) if extras else [] - ) + if r.req: + if r.req.specifier: + r.req.specifier = SpecifierSet(_pipfile["version"]) + r.req.extras = ( + tuple(sorted(dedup([extra.lower() for extra in extras]))) + if extras + else () + ) args = { "name": r.name, "vcs": vcs, "req": r, "markers": markers, - "extras": _pipfile.get("extras"), + "extras": tuple(_pipfile.get("extras", ())), "editable": _pipfile.get("editable", False), "index": _pipfile.get("index"), } if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) - if cls_inst.is_named: - cls_inst.req.req.line = cls_inst.as_line() - old_name = cls_inst.req.req.name or cls_inst.req.name - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) return cls_inst def as_line( @@ -1244,27 +2735,14 @@ class Requirement(object): in the requirement line. """ - include_specifiers = True if self.specifiers else False - if self.is_vcs: - include_extras = False - if self.is_file_or_url or self.is_vcs: - include_specifiers = False - parts = [ - self.req.line_part, - self.extras_as_pip if include_extras else "", - self.specifiers if include_specifiers else "", - self.markers_as_pip if include_markers else "", - ] - if as_list: - # This is used for passing to a subprocess call - parts = ["".join(parts)] - if include_hashes: - hashes = self.get_hashes_as_pip(as_list=as_list) - if as_list: - parts.extend(hashes) - else: - parts.append(hashes) - if sources and not (self.requirement.local_file or self.vcs): + assert self.line_instance is not None + parts = self.line_instance.get_line( + with_prefix=True, + with_hashes=include_hashes, + with_markers=include_markers, + as_list=as_list, + ) + if sources and self.requirement and not (self.line_instance.is_local or self.vcs): from ..utils import prepare_pip_source_args if self.index: @@ -1274,20 +2752,19 @@ class Requirement(object): parts.extend(sources) else: index_string = " ".join(source_list) - parts.extend([" ", index_string]) - if as_list: - return parts - line = "".join(parts) - return line + parts = "{0} {1}".format(parts, index_string) + return parts def get_markers(self): + # type: () -> Marker markers = self.markers if markers: fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) - markers = fake_pkg.markers + markers = fake_pkg.marker return markers def get_specifier(self): + # type: () -> Union[SpecifierSet, LegacySpecifier] try: return Specifier(self.specifiers) except InvalidSpecifier: @@ -1315,7 +2792,11 @@ class Requirement(object): @property def is_direct_url(self): - return self.is_file_or_url and self.req.is_direct_url + return ( + self.is_file_or_url + and self.req.is_direct_url + or (self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url) + ) def as_pipfile(self): good_keys = ( @@ -1334,10 +2815,20 @@ class Requirement(object): name = self.name if "markers" in req_dict and req_dict["markers"]: req_dict["markers"] = req_dict["markers"].replace('"', "'") + if not self.req.name: + name_carriers = (self.req, self, self.line_instance, self.req.parsed_line) + name_options = [ + getattr(carrier, "name", None) + for carrier in name_carriers + if carrier is not None + ] + req_name = next(iter(n for n in name_options if n is not None), None) + self.req.name = req_name + req_name, dict_from_subreq = self.req.pipfile_part.popitem() base_dict = { k: v - for k, v in self.req.pipfile_part[name].items() - if k not in ["req", "link"] + for k, v in dict_from_subreq.items() + if k not in ["req", "link", "_setup_info"] } base_dict.update(req_dict) conflicting_keys = ("file", "path", "uri") @@ -1354,24 +2845,31 @@ class Requirement(object): except AttributeError: hashes.append(_hash) base_dict["hashes"] = sorted(hashes) + if "extras" in base_dict: + base_dict["extras"] = list(base_dict["extras"]) if len(base_dict.keys()) == 1 and "version" in base_dict: base_dict = base_dict.get("version") return {name: base_dict} def as_ireq(self): - ireq_line = self.as_line(include_hashes=False) - if self.editable or self.req.editable: - if ireq_line.startswith("-e "): - ireq_line = ireq_line[len("-e ") :] - with ensure_setup_py(self.req.setup_path): - ireq = pip_shims.shims.install_req_from_editable(ireq_line) - else: - ireq = pip_shims.shims.install_req_from_line(ireq_line) + if self.line_instance and self.line_instance.ireq: + return self.line_instance.ireq + elif getattr(self.req, "_parsed_line", None) and self.req._parsed_line.ireq: + return self.req._parsed_line.ireq + kwargs = {"include_hashes": False} + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + kwargs["include_markers"] = False + ireq_line = self.as_line(**kwargs) + ireq = Line(ireq_line).ireq if not getattr(ireq, "req", None): ireq.req = self.req.req + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + if getattr(ireq, "req", None) and getattr(ireq.req, "marker", None): + ireq.req.marker = None else: ireq.req.extras = self.req.req.extras - ireq.req.marker = self.req.req.marker + if not ((self.is_file_or_url and self.req.is_local) or self.is_vcs): + ireq.req.marker = self.req.req.marker return ireq @property @@ -1431,17 +2929,17 @@ class Requirement(object): else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies( - deps, sources=sources, parent=self.abstract_dep - ) + return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) def find_all_matches(self, sources=None, finder=None): + # type: (Optional[List[Dict[S, Union[S, bool]]]], Optional[PackageFinder]) -> List[InstallationCandidate] """Find all matching candidates for the current requirement. Consults a finder to find all matching candidates. :param sources: Pipfile-formatted sources, defaults to None :param sources: list[dict], optional + :param PackageFinder finder: A **PackageFinder** instance from pip's repository implementation :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ @@ -1449,23 +2947,28 @@ class Requirement(object): from .dependencies import get_finder, find_all_matches if not finder: - finder = get_finder(sources=sources) + _, finder = get_finder(sources=sources) return find_all_matches(finder, self.as_ireq()) def run_requires(self, sources=None, finder=None): if self.req and self.req.setup_info is not None: info_dict = self.req.setup_info.as_dict() + elif self.line_instance and self.line_instance.setup_info is not None: + info_dict = self.line_instance.setup_info.as_dict() else: from .setup_info import SetupInfo + if not finder: from .dependencies import get_finder + finder = get_finder(sources=sources) - info = SetupInfo.from_requirement(self, finder=finder) - if info is None: - return {} - info_dict = info.get_info() + with pip_shims.shims.global_tempdir_manager(): + info = SetupInfo.from_requirement(self, finder=finder) + if info is None: + return {} + info_dict = info.get_info() if self.req and not self.req.setup_info: - self.req.setup_info = info + self.req._setup_info = info if self.req._has_hashed_name and info_dict.get("name"): self.req.name = self.name = info_dict["name"] if self.req.req.name != info_dict["name"]: @@ -1473,10 +2976,122 @@ class Requirement(object): return info_dict def merge_markers(self, markers): + # type: (Union[AnyStr, Marker]) -> None + if not markers: + return self if not isinstance(markers, Marker): markers = Marker(markers) - _markers = set(Marker(self.ireq.markers)) if self.ireq.markers else set(markers) - _markers.add(markers) - new_markers = Marker(" or ".join([str(m) for m in sorted(_markers)])) - self.markers = str(new_markers) - self.req.req.marker = new_markers + _markers = [] # type: List[Marker] + ireq = self.as_ireq() + if ireq and ireq.markers: + ireq_marker = ireq.markers + _markers.append(str(ireq_marker)) + _markers.append(str(markers)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + new_marker = Marker(marker_str) + line = copy.deepcopy(self._line_instance) + line.markers = marker_str + line.parsed_marker = new_marker + if getattr(line, "_requirement", None) is not None: + line._requirement.marker = new_marker + if getattr(line, "_ireq", None) is not None and line._ireq.req: + line._ireq.req.marker = new_marker + new_ireq = getattr(self, "ireq", None) + if new_ireq and new_ireq.req: + new_ireq.req.marker = new_marker + req = self.req + if req.req: + req_requirement = req.req + req_requirement.marker = new_marker + req = attr.evolve(req, req=req_requirement, parsed_line=line) + return attr.evolve( + self, markers=str(new_marker), ireq=new_ireq, req=req, line_instance=line + ) + + +def file_req_from_parsed_line(parsed_line): + # type: (Line) -> FileRequirement + path = parsed_line.relpath if parsed_line.relpath else parsed_line.path + pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + pyproject_path = ( + Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None + ) + req_dict = { + "setup_path": parsed_line.setup_py, + "path": path, + "editable": parsed_line.editable, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": parsed_line.link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": pyproject_path, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + } + if parsed_line.name is not None: + req_dict["name"] = parsed_line.name + return FileRequirement(**req_dict) # type: ignore + + +def vcs_req_from_parsed_line(parsed_line): + # type: (Line) -> VCSRequirement + line = "{0}".format(parsed_line.line) + if parsed_line.editable: + line = "-e {0}".format(line) + if parsed_line.url is not None: + link = create_link( + build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=list(parsed_line.extras), + ) + ) + else: + link = parsed_line.link + pyproject_requires = () # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + vcs_dict = { + "setup_path": parsed_line.setup_py, + "path": parsed_line.path, + "editable": parsed_line.editable, + "vcs": parsed_line.vcs, + "ref": parsed_line.ref, + "subdirectory": parsed_line.subdirectory, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + "base_line": line, + } + if parsed_line.name: + vcs_dict["name"] = parsed_line.name + return VCSRequirement(**vcs_dict) # type: ignore + + +def named_req_from_parsed_line(parsed_line): + # type: (Line) -> NamedRequirement + if parsed_line.name is not None: + return NamedRequirement( + name=parsed_line.name, + version=parsed_line.specifier, + req=parsed_line.requirement, + extras=parsed_line.extras, + editable=parsed_line.editable, + parsed_line=parsed_line, + ) + return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/resolvers.py b/pipenv/vendor/requirementslib/models/resolvers.py index bd773ba6..43590523 100644 --- a/pipenv/vendor/requirementslib/models/resolvers.py +++ b/pipenv/vendor/requirementslib/models/resolvers.py @@ -3,7 +3,6 @@ from contextlib import contextmanager import attr import six - from pip_shims.shims import Wheel from .cache import HashCache @@ -40,15 +39,16 @@ class DependencyResolver(object): def create(cls, finder=None, allow_prereleases=False, get_all_hashes=True): if not finder: from .dependencies import get_finder + finder_args = [] if allow_prereleases: - finder_args.append('--pre') + finder_args.append("--pre") finder = get_finder(*finder_args) creation_kwargs = { - 'allow_prereleases': allow_prereleases, - 'include_incompatible_hashes': get_all_hashes, - 'finder': finder, - 'hash_cache': HashCache(), + "allow_prereleases": allow_prereleases, + "include_incompatible_hashes": get_all_hashes, + "finder": finder, + "hash_cache": HashCache(), } resolver = cls(**creation_kwargs) return resolver @@ -75,9 +75,9 @@ class DependencyResolver(object): compatible_versions = self.dep_dict[dep.name].compatible_versions(dep) if compatible_versions: self.candidate_dict[dep.name] = compatible_versions - self.dep_dict[dep.name] = self.dep_dict[ - dep.name - ].compatible_abstract_dep(dep) + self.dep_dict[dep.name] = self.dep_dict[dep.name].compatible_abstract_dep( + dep + ) else: raise ResolutionError else: @@ -103,8 +103,10 @@ class DependencyResolver(object): old_version = version_from_ireq(self.pinned_deps[name]) if not pin.editable: new_version = version_from_ireq(pin) - if (new_version != old_version and - new_version not in self.candidate_dict[name]): + if ( + new_version != old_version + and new_version not in self.candidate_dict[name] + ): continue pin.parent = abs_dep.parent pin_subdeps = self.dep_dict[name].get_deps(pin) @@ -141,6 +143,7 @@ class DependencyResolver(object): # We accept str, Requirement, and AbstractDependency as input. from .dependencies import AbstractDependency from ..utils import log + for dep in root_nodes: if isinstance(dep, six.string_types): dep = AbstractDependency.from_string(dep) @@ -185,32 +188,40 @@ class DependencyResolver(object): def get_hashes_for_one(self, ireq): if not self.finder: from .dependencies import get_finder + finder_args = [] if self.allow_prereleases: - finder_args.append('--pre') + finder_args.append("--pre") self.finder = get_finder(*finder_args) if ireq.editable: return set() from pip_shims import VcsSupport + vcs = VcsSupport() - if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + if ( + ireq.link + and ireq.link.scheme in vcs.all_schemes + and "ssh" in ireq.link.scheme + ): return set() if not is_pinned_requirement(ireq): - raise TypeError( - "Expected pinned requirement, got {}".format(ireq)) + raise TypeError("Expected pinned requirement, got {}".format(ireq)) matching_candidates = set() with self.allow_all_wheels(): from .dependencies import find_all_matches - matching_candidates = ( - find_all_matches(self.finder, ireq, pre=self.allow_prereleases) + + matching_candidates = find_all_matches( + self.finder, ireq, pre=self.allow_prereleases ) return { - self.hash_cache.get_hash(candidate.location) + self.hash_cache.get_hash( + getattr(candidate, "location", getattr(candidate, "link", None)) + ) for candidate in matching_candidates } @@ -222,6 +233,7 @@ class DependencyResolver(object): This also saves the candidate cache and set a new one, or else the results from the previous non-patched calls will interfere. """ + def _wheel_supported(self, tags=None): # Ignore current platform. Support everything. return True diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index ec631e67..610eb68b 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,28 +1,62 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import ast +import atexit import contextlib +import importlib +import io +import operator import os +import shutil import sys +from functools import partial import attr -import packaging.version +import chardet import packaging.specifiers import packaging.utils +import packaging.version +import pep517.envbuild +import pep517.wrappers import six +from appdirs import user_cache_dir +from distlib.wheel import Wheel +from packaging.markers import Marker +from six.moves import configparser +from six.moves.urllib.parse import unquote, urlparse, urlunparse +from vistir.compat import FileNotFoundError, Iterable, Mapping, Path, lru_cache +from vistir.contextmanagers import cd, temp_path +from vistir.misc import run +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree + +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from .utils import ( + get_default_pyproject_backend, + get_name_variants, + get_pyproject, + init_requirement, + read_source, + split_vcs_method_from_uri, + strip_extras_markers_from_requirement, +) try: - from setuptools.dist import distutils + import pkg_resources.extern.packaging.requirements as pkg_resources_requirements +except ImportError: + pkg_resources_requirements = None + +try: + from setuptools.dist import distutils, Distribution except ImportError: import distutils + from distutils.core import Distribution -from appdirs import user_cache_dir -from six.moves import configparser -from six.moves.urllib.parse import unquote -from vistir.compat import Path, Iterable -from vistir.contextmanagers import cd -from vistir.misc import run -from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p - -from .utils import init_requirement, get_pyproject +try: + from contextlib import ExitStack +except ImportError: + from contextlib2 import ExitStack try: from os import scandir @@ -30,6 +64,42 @@ except ImportError: from scandir import scandir +if MYPY_RUNNING: + from typing import ( + Any, + Callable, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + AnyStr, + Sequence, + ) + import requests + from pip_shims.shims import InstallRequirement, PackageFinder + from pkg_resources import ( + PathMetadata, + DistInfoDistribution, + EggInfoDistribution, + Requirement as PkgResourcesRequirement, + ) + from packaging.requirements import Requirement as PackagingRequirement + + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + AST_SEQ = TypeVar("AST_SEQ", ast.Tuple, ast.List) + + CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) # The following are necessary for people who like to use "if __name__" conditionals @@ -38,8 +108,238 @@ _setup_stop_after = None _setup_distribution = None +def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): + # type: (List[AnyStr], Optional[AnyStr], Optional[Mapping[S, S]]) -> None + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + run( + cmd, + cwd=cwd, + env=env, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class BuildEnv(pep517.envbuild.BuildEnvironment): + def pip_install(self, reqs): + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--ignore-installed", + "--prefix", + self.path, + ] + list(reqs) + run( + cmd, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class HookCaller(pep517.wrappers.Pep517HookCaller): + def __init__(self, source_dir, build_backend, backend_path=None): + self.source_dir = os.path.abspath(source_dir) + self.build_backend = build_backend + self._subprocess_runner = pep517_subprocess_runner + if backend_path: + backend_path = [ + pep517.wrappers.norm_and_check(self.source_dir, p) for p in backend_path + ] + self.backend_path = backend_path + + +def parse_special_directives(setup_entry, package_dir=None): + # type: (S, Optional[STRING_TYPE]) -> S + rv = setup_entry + if not package_dir: + package_dir = os.getcwd() + if setup_entry.startswith("file:"): + _, path = setup_entry.split("file:") + path = path.strip() + if os.path.exists(path): + rv = read_source(path) + elif setup_entry.startswith("attr:"): + _, resource = setup_entry.split("attr:") + resource = resource.strip() + with temp_path(): + sys.path.insert(0, package_dir) + if "." in resource: + resource, _, attribute = resource.rpartition(".") + package, _, path = resource.partition(".") + base_path = os.path.join(package_dir, package) + if path: + path = os.path.join(base_path, os.path.join(*path.split("."))) + else: + path = base_path + if not os.path.exists(path) and os.path.exists("{0}.py".format(path)): + path = "{0}.py".format(path) + elif os.path.isdir(path): + path = os.path.join(path, "__init__.py") + rv = ast_parse_attribute_from_file(path, attribute) + if rv: + return str(rv) + module = importlib.import_module(resource) + rv = getattr(module, attribute) + if not isinstance(rv, six.string_types): + rv = str(rv) + return rv + + +def make_base_requirements(reqs): + # type: (Sequence[STRING_TYPE]) -> Set[BaseRequirement] + requirements = set() + if not isinstance(reqs, (list, tuple, set)): + reqs = [reqs] + for req in reqs: + if isinstance(req, BaseRequirement): + requirements.add(req) + elif pkg_resources_requirements is not None and isinstance( + req, pkg_resources_requirements.Requirement + ): + requirements.add(BaseRequirement.from_req(req)) + elif req and isinstance(req, six.string_types) and not req.startswith("#"): + requirements.add(BaseRequirement.from_string(req)) + return requirements + + +def setuptools_parse_setup_cfg(path): + from setuptools.config import read_configuration + + parsed = read_configuration(path) + results = parsed.get("metadata", {}) + results.update(parsed.get("options", {})) + results["install_requires"] = make_base_requirements( + results.get("install_requires", []) + ) + extras = {} + for extras_section, extras_reqs in results.get("extras_require", {}).items(): + new_reqs = tuple(make_base_requirements(extras_reqs)) + if new_reqs: + extras[extras_section] = new_reqs + results["extras_require"] = extras + results["setup_requires"] = make_base_requirements(results.get("setup_requires", [])) + return results + + +def get_package_dir_from_setupcfg(parser, base_dir=None): + # type: (configparser.ConfigParser, STRING_TYPE) -> Text + if base_dir is not None: + package_dir = base_dir + else: + package_dir = os.getcwd() + if parser.has_option("options.packages.find", "where"): + pkg_dir = parser.get("options.packages.find", "where") + if isinstance(pkg_dir, Mapping): + pkg_dir = pkg_dir.get("where") + package_dir = os.path.join(package_dir, pkg_dir) + elif parser.has_option("options", "packages"): + pkg_dir = parser.get("options", "packages") + if "find:" in pkg_dir: + _, pkg_dir = pkg_dir.split("find:") + pkg_dir = pkg_dir.strip() + package_dir = os.path.join(package_dir, pkg_dir) + elif os.path.exists(os.path.join(package_dir, "setup.py")): + setup_py = ast_parse_setup_py(os.path.join(package_dir, "setup.py")) + if "package_dir" in setup_py: + package_lookup = setup_py["package_dir"] + if not isinstance(package_lookup, Mapping): + package_dir = package_lookup + package_dir = package_lookup.get( + next(iter(list(package_lookup.keys()))), package_dir + ) + if not os.path.isabs(package_dir): + if not base_dir: + package_dir = os.path.join(os.path.getcwd(), package_dir) + else: + package_dir = os.path.join(base_dir, package_dir) + return package_dir + + +def get_name_and_version_from_setupcfg(parser, package_dir): + # type: (configparser.ConfigParser, STRING_TYPE) -> Tuple[Optional[S], Optional[S]] + name, version = None, None + if parser.has_option("metadata", "name"): + name = parse_special_directives(parser.get("metadata", "name"), package_dir) + if parser.has_option("metadata", "version"): + version = parse_special_directives(parser.get("metadata", "version"), package_dir) + return name, version + + +def get_extras_from_setupcfg(parser): + # type: (configparser.ConfigParser) -> Dict[STRING_TYPE, Tuple[BaseRequirement, ...]] + extras = {} # type: Dict[STRING_TYPE, Tuple[BaseRequirement, ...]] + if "options.extras_require" not in parser.sections(): + return extras + extras_require_section = parser.options("options.extras_require") + for section in extras_require_section: + if section in ["options", "metadata"]: + continue + section_contents = parser.get("options.extras_require", section) + section_list = section_contents.split("\n") + section_extras = tuple(make_base_requirements(section_list)) + if section_extras: + extras[section] = section_extras + return extras + + +def parse_setup_cfg( + setup_cfg_contents, # type: S + base_dir, # type: S +): + # type: (...) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Dict[STRING_TYPE, Tuple[BaseRequirement]]]] + default_opts = { + "metadata": {"name": "", "version": ""}, + "options": { + "install_requires": "", + "python_requires": "", + "build_requires": "", + "setup_requires": "", + "extras": "", + "packages.find": {"where": "."}, + }, + } + parser = configparser.ConfigParser(default_opts) + if six.PY2: + buff = io.BytesIO(setup_cfg_contents) + parser.readfp(buff) + else: + parser.read_string(setup_cfg_contents) + results = {} + package_dir = get_package_dir_from_setupcfg(parser, base_dir=base_dir) + name, version = get_name_and_version_from_setupcfg(parser, package_dir) + results["name"] = name + results["version"] = version + install_requires = set() # type: Set[BaseRequirement] + if parser.has_option("options", "install_requires"): + install_requires = make_base_requirements( + parser.get("options", "install_requires").split("\n") + ) + results["install_requires"] = install_requires + if parser.has_option("options", "python_requires"): + results["python_requires"] = parse_special_directives( + parser.get("options", "python_requires"), package_dir + ) + if parser.has_option("options", "build_requires"): + results["build_requires"] = parser.get("options", "build_requires") + results["extras_require"] = get_extras_from_setupcfg(parser) + return results + + @contextlib.contextmanager def _suppress_distutils_logs(): + # type: () -> Generator[None, None, None] """Hack to hide noise generated by `setup.py develop`. There isn't a good way to suppress them now, so let's monky-patch. @@ -57,19 +357,47 @@ def _suppress_distutils_logs(): distutils.log.Log._log = f +def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel"): + if config_settings is None: + config_settings = {} + requires, backend = get_pyproject(source_dir) + hookcaller = HookCaller(source_dir, backend) + if dist_type == "sdist": + get_requires_fn = hookcaller.get_requires_for_build_sdist + build_fn = hookcaller.build_sdist + else: + get_requires_fn = hookcaller.get_requires_for_build_wheel + build_fn = hookcaller.build_wheel + + with BuildEnv() as env: + env.pip_install(requires) + reqs = get_requires_fn(config_settings) + env.pip_install(reqs) + return build_fn(build_dir, config_settings) + + @ensure_mkdir_p(mode=0o775) -def _get_src_dir(): +def _get_src_dir(root): + # type: (AnyStr) -> AnyStr src = os.environ.get("PIP_SRC") if src: return src virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env: + if virtual_env is not None: return os.path.join(virtual_env, "src") - return os.path.join(os.getcwd(), "src") # Match pip's behavior. + if root is not None: + # Intentionally don't match pip's behavior here -- this is a temporary copy + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") + else: + src_dir = os.path.join(root, "src") + return src_dir +@lru_cache() def ensure_reqs(reqs): + # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] import pkg_resources + if not isinstance(reqs, Iterable): raise TypeError("Expecting an Iterable, got %r" % reqs) new_reqs = [] @@ -78,31 +406,35 @@ def ensure_reqs(reqs): continue if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse("{0}".format(str(req))) + # req = strip_extras_markers_from_requirement(req) new_reqs.append(req) return new_reqs -def _prepare_wheel_building_kwargs(ireq): - download_dir = os.path.join(CACHE_DIR, "pkgs") +def _prepare_wheel_building_kwargs( + ireq=None, # type: Optional[InstallRequirement] + src_root=None, # type: Optional[STRING_TYPE] + src_dir=None, # type: Optional[STRING_TYPE] + editable=False, # type: bool +): + # type: (...) -> Dict[STRING_TYPE, STRING_TYPE] + download_dir = os.path.join(CACHE_DIR, "pkgs") # type: STRING_TYPE mkdir_p(download_dir) - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: STRING_TYPE mkdir_p(wheel_download_dir) + if src_dir is None: + if editable and src_root is not None: + src_dir = src_root + elif src_root is not None: + src_dir = _get_src_dir(root=src_root) # type: STRING_TYPE + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") - if ireq.source_dir is not None: - src_dir = ireq.source_dir - elif ireq.editable: - src_dir = _get_src_dir() - else: + # Let's always resolve in isolation + if src_dir is None: src_dir = create_tracked_tempdir(prefix="reqlib-src") - - # This logic matches pip's behavior, although I don't fully understand the - # intention. I guess the idea is to build editables in-place, otherwise out - # of the source tree? - if ireq.editable: - build_dir = src_dir - else: - build_dir = create_tracked_tempdir(prefix="reqlib-build") + build_dir = create_tracked_tempdir(prefix="reqlib-build") return { "build_dir": build_dir, @@ -112,263 +444,1124 @@ def _prepare_wheel_building_kwargs(ireq): } -def iter_egginfos(path, pkg_name=None): - for entry in scandir(path): - if entry.is_dir(): - if not entry.name.endswith("egg-info"): - for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): - yield dir_entry - elif pkg_name is None or entry.name.startswith(pkg_name.replace("-", "_")): - yield entry +class ScandirCloser(object): + def __init__(self, path): + self.iterator = scandir(path) + + def __next__(self): + return next(iter(self.iterator)) + + def __iter__(self): + return self + + def next(self): + return self.__next__() + + def close(self): + if getattr(self.iterator, "close", None): + self.iterator.close() + else: + pass + + +def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): + # type: (AnyStr, Optional[AnyStr], AnyStr) -> Generator + if pkg_name is not None: + pkg_variants = get_name_variants(pkg_name) + non_matching_dirs = [] + with contextlib.closing(ScandirCloser(path)) as path_iterator: + for entry in path_iterator: + if entry.is_dir(): + entry_name, ext = os.path.splitext(entry.name) + if ext.endswith(metadata_type): + if pkg_name is None or entry_name.lower() in pkg_variants: + yield entry + elif not entry.name.endswith(metadata_type): + non_matching_dirs.append(entry) + for entry in non_matching_dirs: + for dir_entry in iter_metadata( + entry.path, pkg_name=pkg_name, metadata_type=metadata_type + ): + yield dir_entry def find_egginfo(target, pkg_name=None): - egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + # type: (AnyStr, Optional[AnyStr]) -> Generator + egg_dirs = ( + egg_dir + for egg_dir in iter_metadata(target, pkg_name=pkg_name) + if egg_dir is not None + ) if pkg_name: - yield next(iter(egg_dirs), None) + yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None) else: for egg_dir in egg_dirs: yield egg_dir -def get_metadata(path, pkg_name=None): +def find_distinfo(target, pkg_name=None): + # type: (AnyStr, Optional[AnyStr]) -> Generator + dist_dirs = ( + dist_dir + for dist_dir in iter_metadata( + target, pkg_name=pkg_name, metadata_type="dist-info" + ) + if dist_dir is not None + ) if pkg_name: - pkg_name = packaging.utils.canonicalize_name(pkg_name) + yield next(iter(dist for dist in dist_dirs if dist is not None), None) + else: + for dist_dir in dist_dirs: + yield dist_dir + + +def get_distinfo_dist(path, pkg_name=None): + # type: (S, Optional[S]) -> Optional[DistInfoDistribution] + import pkg_resources + + dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) + if dist_dir is not None: + metadata_dir = dist_dir.path + base_dir = os.path.dirname(metadata_dir) + dist = next(iter(pkg_resources.find_distributions(base_dir)), None) + if dist is not None: + return dist + return None + + +def get_egginfo_dist(path, pkg_name=None): + # type: (S, Optional[S]) -> Optional[EggInfoDistribution] + import pkg_resources + egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) if egg_dir is not None: - import pkg_resources + metadata_dir = egg_dir.path + base_dir = os.path.dirname(metadata_dir) + path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir) + dist_iter = pkg_resources.distributions_from_metadata(path_metadata.egg_info) + dist = next(iter(dist_iter), None) + if dist is not None: + return dist + return None - egg_dir = os.path.abspath(egg_dir.path) - base_dir = os.path.dirname(egg_dir) - path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) - dist = next( - iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), - None, + +def get_metadata(path, pkg_name=None, metadata_type=None): + # type: (S, Optional[S], Optional[S]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + wheel_allowed = metadata_type == "wheel" or metadata_type is None + egg_allowed = metadata_type == "egg" or metadata_type is None + dist = None # type: Optional[Union[DistInfoDistribution, EggInfoDistribution]] + if wheel_allowed: + dist = get_distinfo_dist(path, pkg_name=pkg_name) + if egg_allowed and dist is None: + dist = get_egginfo_dist(path, pkg_name=pkg_name) + if dist is not None: + return get_metadata_from_dist(dist) + return {} + + +@lru_cache() +def get_extra_name_from_marker(marker): + # type: (MarkerType) -> Optional[S] + if not marker: + raise ValueError("Invalid value for marker: {0!r}".format(marker)) + if not getattr(marker, "_markers", None): + raise TypeError("Expecting a marker instance, received {0!r}".format(marker)) + for elem in marker._markers: + if isinstance(elem, tuple) and elem[0].value == "extra": + return elem[2].value + return None + + +def get_metadata_from_wheel(wheel_path): + # type: (S) -> Dict[Any, Any] + if not isinstance(wheel_path, six.string_types): + raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) + try: + dist = Wheel(wheel_path) + except Exception: + pass + metadata = dist.metadata + name = metadata.name + version = metadata.version + requires = [] + extras_keys = getattr(metadata, "extras", []) # type: List[STRING_TYPE] + extras = { + k: [] for k in extras_keys + } # type: Dict[STRING_TYPE, List[RequirementType]] + for req in getattr(metadata, "run_requires", []): + parsed_req = init_requirement(req) + parsed_marker = parsed_req.marker + if parsed_marker: + extra = get_extra_name_from_marker(parsed_marker) + if extra is None: + requires.append(parsed_req) + continue + if extra not in extras: + extras[extra] = [] + parsed_req = strip_extras_markers_from_requirement(parsed_req) + extras[extra].append(parsed_req) + else: + requires.append(parsed_req) + return {"name": name, "version": version, "requires": requires, "extras": extras} + + +def get_metadata_from_dist(dist): + # type: (Union[PathMetadata, EggInfoDistribution, DistInfoDistribution]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + try: + requires = dist.requires() + except Exception: + requires = [] + try: + dep_map = dist._build_dep_map() + except Exception: + dep_map = {} + deps = [] # type: List[PkgResourcesRequirement] + extras = {} + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + extra = None + _deps = dep_map.get(k) + if k.startswith(":python_version"): + marker = k.replace(":", "; ") + else: + if ":python_version" in k: + extra, _, marker = k.partition(":") + marker = "; {0}".format(marker) + else: + marker = "" + extra = "{0}".format(k) + _deps = ensure_reqs( + tuple(["{0}{1}".format(str(req), marker) for req in _deps]) + ) + if extra: + extras[extra] = _deps + else: + deps.extend(_deps) + requires.extend(deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + "extras": extras, + } + + +AST_BINOP_MAP = dict( + ( + (ast.Add, operator.add), + (ast.Sub, operator.sub), + (ast.Mult, operator.mul), + (ast.Div, operator.floordiv), + (ast.Mod, operator.mod), + (ast.Pow, operator.pow), + (ast.LShift, operator.lshift), + (ast.RShift, operator.rshift), + (ast.BitAnd, operator.and_), + (ast.BitOr, operator.or_), + (ast.BitXor, operator.xor), + ) +) + + +AST_COMPARATORS = dict( + ( + (ast.Lt, operator.lt), + (ast.LtE, operator.le), + (ast.Eq, operator.eq), + (ast.Gt, operator.gt), + (ast.GtE, operator.ge), + (ast.NotEq, operator.ne), + (ast.Is, operator.is_), + (ast.IsNot, operator.is_not), + (ast.And, operator.and_), + (ast.Or, operator.or_), + (ast.Not, operator.not_), + (ast.In, lambda a, b: operator.contains(b, a)), + ) +) + + +class Analyzer(ast.NodeVisitor): + def __init__(self): + self.name_types = [] + self.function_map = {} # type: Dict[Any, Any] + self.function_names = {} + self.resolved_function_names = {} + self.functions = [] + self.strings = [] + self.assignments = {} + self.binOps = [] + self.binOps_map = {} + super(Analyzer, self).__init__() + + def generic_visit(self, node): + if isinstance(node, ast.Call): + self.functions.append(node) + self.function_map.update(ast_unparse(node, initial_mapping=True)) + if isinstance(node, ast.Name): + self.name_types.append(node) + if isinstance(node, ast.Str): + self.strings.append(node) + if isinstance(node, ast.Assign): + self.assignments.update(ast_unparse(node, initial_mapping=True)) + super(Analyzer, self).generic_visit(node) + + def visit_BinOp(self, node): + node = ast_unparse(node, initial_mapping=True) + self.binOps.append(node) + + def unmap_binops(self): + for binop in self.binOps: + self.binOps_map[binop] = ast_unparse(binop, analyzer=self) + + def match_assignment_str(self, match): + return next( + iter(k for k in self.assignments if getattr(k, "id", "") == match), None ) - if dist: - try: - requires = dist.requires() - except exception: - requires = [] - try: - dep_map = dist._build_dep_map() - except Exception: - dep_map = {} - deps = [] - extras = {} - for k in dep_map.keys(): - if k is None: - deps.extend(dep_map.get(k)) + + def match_assignment_name(self, match): + return next( + iter(k for k in self.assignments if getattr(k, "id", "") == match.id), None + ) + + def parse_function_names(self, should_retry=True, function_map=None): + if function_map is None: + function_map = {} + retries = [] + for k, v in function_map.items(): + fn_name = "" + if k in self.function_names: + fn_name = self.function_names[k] + elif isinstance(k, ast.Name): + fn_name = k.id + elif isinstance(k, ast.Attribute): + try: + fn = ast_unparse(k, analyzer=self) + except Exception: + if should_retry: + retries.append((k, v)) continue else: - extra = None - _deps = dep_map.get(k) - if k.startswith(":python_version"): - marker = k.replace(":", "; ") - else: - marker = "" - extra = "{0}".format(k) - _deps = ["{0}{1}".format(str(req), marker) for req in _deps] - _deps = ensure_reqs(_deps) - if extra: - extras[extra] = _deps - else: - deps.extend(_deps) - return { - "name": dist.project_name, - "version": dist.version, - "requires": requires, - "extras": extras - } + if isinstance(fn, six.string_types): + _, _, fn_name = fn.rpartition(".") + if fn_name: + self.resolved_function_names[fn_name] = ast_unparse(v, analyzer=self) + return retries + + def parse_functions(self): + retries = self.parse_function_names(function_map=self.function_map) + if retries: + failures = self.parse_function_names( + should_retry=False, function_map=dict(retries) + ) + return self.resolved_function_names -@attr.s(slots=True) +def ast_unparse(item, initial_mapping=False, analyzer=None, recurse=True): # noqa:C901 + # type: (Any, bool, Optional[Analyzer], bool) -> Union[List[Any], Dict[Any, Any], Tuple[Any, ...], STRING_TYPE] + unparse = partial( + ast_unparse, initial_mapping=initial_mapping, analyzer=analyzer, recurse=recurse + ) + if getattr(ast, "Constant", None): + constant = (ast.Constant, ast.Ellipsis) + else: + constant = ast.Ellipsis + unparsed = item + if isinstance(item, ast.Dict): + unparsed = dict(zip(unparse(item.keys), unparse(item.values))) + elif isinstance(item, ast.List): + unparsed = [unparse(el) for el in item.elts] + elif isinstance(item, ast.Tuple): + unparsed = tuple([unparse(el) for el in item.elts]) + elif isinstance(item, ast.Str): + unparsed = item.s + elif isinstance(item, ast.Subscript): + unparsed = unparse(item.value) + if not initial_mapping: + if isinstance(item.slice, ast.Index): + try: + unparsed = unparsed[unparse(item.slice.value)] + except KeyError: + # not everything can be looked up before runtime + unparsed = item + elif any(isinstance(item, k) for k in AST_BINOP_MAP.keys()): + unparsed = AST_BINOP_MAP[type(item)] + elif isinstance(item, ast.Num): + unparsed = item.n + elif isinstance(item, ast.BinOp): + if analyzer and item in analyzer.binOps_map: + unparsed = analyzer.binOps_map[item] + else: + right_item = unparse(item.right) + left_item = unparse(item.left) + op = getattr(item, "op", None) + op_func = unparse(op) if op is not None else op + if not initial_mapping: + try: + unparsed = op_func(left_item, right_item) + except Exception: + unparsed = (left_item, op_func, right_item) + else: + item.left = left_item + item.right = right_item + item.op = op_func + unparsed = item + elif isinstance(item, ast.Name): + if not initial_mapping: + unparsed = item.id + if analyzer and recurse: + if item in analyzer.assignments: + items = unparse(analyzer.assignments[item]) + unparsed = items.get(item.id, item.id) + else: + assignment = analyzer.match_assignment_name(item) + if assignment is not None: + items = unparse(analyzer.assignments[assignment]) + unparsed = items.get(item.id, item.id) + else: + unparsed = item + elif six.PY3 and isinstance(item, ast.NameConstant): + unparsed = item.value + elif any(isinstance(item, k) for k in AST_COMPARATORS.keys()): + unparsed = AST_COMPARATORS[type(item)] + elif isinstance(item, constant): + unparsed = item.value + elif isinstance(item, ast.Compare): + if isinstance(item.left, ast.Attribute) or isinstance(item.left, ast.Str): + import importlib + + left = unparse(item.left) + if "." in left: + name, _, val = left.rpartition(".") + left = getattr(importlib.import_module(name), val, left) + comparators = [] + for comparator in item.comparators: + right = unparse(comparator) + if isinstance(comparator, ast.Attribute) and "." in right: + name, _, val = right.rpartition(".") + right = getattr(importlib.import_module(name), val, right) + comparators.append(right) + unparsed = (left, unparse(item.ops), comparators) + elif isinstance(item, ast.IfExp): + if initial_mapping: + unparsed = item + else: + ops, truth_vals = [], [] + if isinstance(item.test, ast.Compare): + left, ops, right = unparse(item.test) + else: + result = ast_unparse(item.test) + if isinstance(result, dict): + k, v = result.popitem() + if not v: + truth_vals = [False] + for i, op in enumerate(ops): + if i == 0: + truth_vals.append(op(left, right[i])) + else: + truth_vals.append(op(right[i - 1], right[i])) + if all(truth_vals): + unparsed = unparse(item.body) + else: + unparsed = unparse(item.orelse) + elif isinstance(item, ast.Attribute): + attr_name = getattr(item, "value", None) + attr_attr = getattr(item, "attr", None) + name = None + if initial_mapping: + unparsed = item + elif attr_name and not recurse: + name = attr_name + else: + name = unparse(attr_name) if attr_name is not None else attr_attr + if name and attr_attr: + if not initial_mapping and isinstance(name, six.string_types): + unparsed = ".".join([item for item in (name, attr_attr) if item]) + else: + unparsed = item + elif attr_attr and not name and not initial_mapping: + unparsed = attr_attr + else: + unparsed = name if not unparsed else unparsed + elif isinstance(item, ast.Call): + unparsed = {} + if isinstance(item.func, (ast.Name, ast.Attribute)): + func_name = unparse(item.func) + else: + try: + func_name = unparse(item.func) + except Exception: + func_name = None + if isinstance(func_name, dict): + unparsed.update(func_name) + func_name = next(iter(func_name.keys())) + for keyword in getattr(item, "keywords", []): + unparsed[func_name].update(unparse(keyword)) + elif func_name: + unparsed[func_name] = {} + for keyword in getattr(item, "keywords", []): + unparsed[func_name].update(unparse(keyword)) + elif isinstance(item, ast.keyword): + unparsed = {unparse(item.arg): unparse(item.value)} + elif isinstance(item, ast.Assign): + # XXX: DO NOT UNPARSE THIS + # XXX: If we unparse this it becomes impossible to map it back + # XXX: To the original node in the AST so we can find the + # XXX: Original reference + if not initial_mapping: + target = unparse(next(iter(item.targets)), recurse=False) + val = unparse(item.value, recurse=False) + if isinstance(target, (tuple, set, list)): + unparsed = dict(zip(target, val)) + else: + unparsed = {target: val} + else: + unparsed = {next(iter(item.targets)): item} + elif isinstance(item, Mapping): + unparsed = {} + for k, v in item.items(): + try: + unparsed[unparse(k)] = unparse(v) + except TypeError: + unparsed[k] = unparse(v) + elif isinstance(item, (list, tuple)): + unparsed = type(item)([unparse(el) for el in item]) + elif isinstance(item, six.string_types): + unparsed = item + return unparsed + + +def ast_parse_attribute_from_file(path, attribute): + # type: (S) -> Any + analyzer = ast_parse_file(path) + target_value = None + for k, v in analyzer.assignments.items(): + name = "" + if isinstance(k, ast.Name): + name = k.id + elif isinstance(k, ast.Attribute): + fn = ast_unparse(k) + if isinstance(fn, six.string_types): + _, _, name = fn.rpartition(".") + if name == attribute: + target_value = ast_unparse(v, analyzer=analyzer) + break + if isinstance(target_value, Mapping) and attribute in target_value: + return target_value[attribute] + return target_value + + +def ast_parse_file(path): + # type: (S) -> Analyzer + try: + tree = ast.parse(read_source(path)) + except SyntaxError: + # The source may be encoded strangely, e.g. azure-storage + # which has a setup.py encoded with utf-8-sig + with open(path, "rb") as fh: + contents = fh.read() + encoding = chardet.detect(contents)["encoding"] + tree = ast.parse(contents.decode(encoding)) + ast_analyzer = Analyzer() + ast_analyzer.visit(tree) + return ast_analyzer + + +def ast_parse_setup_py(path): + # type: (S) -> Dict[Any, Any] + ast_analyzer = ast_parse_file(path) + setup = {} # type: Dict[Any, Any] + ast_analyzer.unmap_binops() + function_names = ast_analyzer.parse_functions() + if "setup" in function_names: + setup = ast_unparse(function_names["setup"], analyzer=ast_analyzer) + return setup + + +def run_setup(script_path, egg_base=None): + # type: (str, Optional[str]) -> Distribution + """Run a `setup.py` script with a target **egg_base** if provided. + + :param S script_path: The path to the `setup.py` script to run + :param Optional[S] egg_base: The metadata directory to build in + :raises FileNotFoundError: If the provided `script_path` does not exist + :return: The metadata dictionary + :rtype: Dict[Any, Any] + """ + + if not os.path.exists(script_path): + raise FileNotFoundError(script_path) + target_cwd = os.path.dirname(os.path.abspath(script_path)) + if egg_base is None: + egg_base = os.path.join(target_cwd, "reqlib-metadata") + with temp_path(), cd(target_cwd), _suppress_distutils_logs(): + # This is for you, Hynek + # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py + args = ["egg_info"] + if egg_base: + args += ["--egg-base", egg_base] + script_name = os.path.basename(script_path) + g = {"__file__": script_name, "__name__": "__main__"} + sys.path.insert(0, target_cwd) + local_dict = {} + if sys.version_info < (3, 5): + save_argv = sys.argv + else: + save_argv = sys.argv.copy() + try: + global _setup_distribution, _setup_stop_after + _setup_stop_after = "run" + sys.argv[0] = script_name + sys.argv[1:] = args + with open(script_name, "rb") as f: + contents = f.read() + if six.PY3: + contents.replace(br"\r\n", br"\n") + else: + contents.replace(r"\r\n", r"\n") + if sys.version_info < (3, 5): + exec(contents, g, local_dict) + else: + exec(contents, g) + # We couldn't import everything needed to run setup + except Exception: + python = os.environ.get("PIP_PYTHON_PATH", sys.executable) + out, _ = run( + [python, "setup.py"] + args, + cwd=target_cwd, + block=True, + combine_stderr=False, + return_object=False, + nospin=True, + ) + finally: + _setup_stop_after = None + sys.argv = save_argv + _setup_distribution = get_metadata(egg_base, metadata_type="egg") + dist = _setup_distribution + return dist + + +@attr.s(slots=True, frozen=True) +class BaseRequirement(object): + name = attr.ib(default="", cmp=True) # type: STRING_TYPE + requirement = attr.ib( + default=None, cmp=True + ) # type: Optional[PkgResourcesRequirement] + + def __str__(self): + # type: () -> S + return "{0}".format(str(self.requirement)) + + def as_dict(self): + # type: () -> Dict[STRING_TYPE, Optional[PkgResourcesRequirement]] + return {self.name: self.requirement} + + def as_tuple(self): + # type: () -> Tuple[STRING_TYPE, Optional[PkgResourcesRequirement]] + return (self.name, self.requirement) + + @classmethod + @lru_cache() + def from_string(cls, line): + # type: (S) -> BaseRequirement + line = line.strip() + req = init_requirement(line) + return cls.from_req(req) + + @classmethod + @lru_cache() + def from_req(cls, req): + # type: (PkgResourcesRequirement) -> BaseRequirement + name = None + key = getattr(req, "key", None) + name = getattr(req, "name", None) + project_name = getattr(req, "project_name", None) + if key is not None: + name = key + if name is None: + name = project_name + return cls(name=name, requirement=req) + + +@attr.s(slots=True, frozen=True) +class Extra(object): + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) + + def __str__(self): + # type: () -> S + return "{0}: {{{1}}}".format( + self.name, ", ".join([r.name for r in self.requirements]) + ) + + def add(self, req): + # type: (BaseRequirement) -> "Extra" + if req not in self.requirements: + current_set = set(self.requirements) + current_set.add(req) + return attr.evolve(self, requirements=frozenset(current_set)) + return self + + def as_dict(self): + # type: () -> Dict[STRING_TYPE, Tuple[RequirementType, ...]] + return {self.name: tuple([r.requirement for r in self.requirements])} + + +@attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(type=str, default=None) - base_dir = attr.ib(type=Path, default=None) - version = attr.ib(type=packaging.version.Version, default=None) - requires = attr.ib(type=dict, default=attr.Factory(dict)) - build_requires = attr.ib(type=list, default=attr.Factory(list)) - build_backend = attr.ib(type=list, default=attr.Factory(list)) - setup_requires = attr.ib(type=dict, default=attr.Factory(list)) - python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) - extras = attr.ib(type=dict, default=attr.Factory(dict)) - setup_cfg = attr.ib(type=Path, default=None) - setup_py = attr.ib(type=Path, default=None) - pyproject = attr.ib(type=Path, default=None) - ireq = attr.ib(default=None) - extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + base_dir = attr.ib(default=None, cmp=True, hash=False) # type: STRING_TYPE + _version = attr.ib(default=None, cmp=True) # type: STRING_TYPE + _requirements = attr.ib( + type=frozenset, factory=frozenset, cmp=True, hash=True + ) # type: Optional[frozenset] + build_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + build_backend = attr.ib(cmp=True) # type: STRING_TYPE + setup_requires = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + python_requires = attr.ib( + default=None, cmp=True + ) # type: Optional[packaging.specifiers.SpecifierSet] + _extras_requirements = attr.ib(default=None, cmp=True) # type: Optional[Tuple] + setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) + setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) + pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) + ireq = attr.ib( + default=None, cmp=True, hash=False + ) # type: Optional[InstallRequirement] + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) + metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]] + + @build_backend.default + def get_build_backend(self): + # type: () -> STRING_TYPE + return get_default_pyproject_backend() + + @property + def requires(self): + # type: () -> Dict[S, RequirementType] + if self._requirements is None: + self._requirements = frozenset() + self.get_info() + return {req.name: req.requirement for req in self._requirements} + + @property + def extras(self): + # type: () -> Dict[S, Optional[Any]] + if self._extras_requirements is None: + self._extras_requirements = () + self.get_info() + extras_dict = {} + extras = set(self._extras_requirements) + for section, deps in extras: + if isinstance(deps, BaseRequirement): + extras_dict[section] = deps.requirement + elif isinstance(deps, (list, tuple)): + extras_dict[section] = [d.requirement for d in deps] + return extras_dict + + @property + def version(self): + # type: () -> Optional[str] + if not self._version: + info = self.get_info() + self._version = info.get("version", None) + return self._version + + @property + def egg_base(self): + # type: () -> S + base = None # type: Optional[STRING_TYPE] + if self.setup_py.exists(): + base = self.setup_py.parent + elif self.pyproject.exists(): + base = self.pyproject.parent + elif self.setup_cfg.exists(): + base = self.setup_cfg.parent + if base is None: + base = Path(self.base_dir) + if base is None: + base = Path(self.extra_kwargs["src_dir"]) + egg_base = base.joinpath("reqlib-metadata") + if not egg_base.exists(): + atexit.register(rmtree, egg_base.as_posix()) + egg_base.mkdir(parents=True, exist_ok=True) + return egg_base.as_posix() + + def update_from_dict(self, metadata): + name = metadata.get("name", self.name) + if isinstance(name, six.string_types): + self.name = self.name if self.name else name + version = metadata.get("version", None) + if version: + try: + packaging.version.parse(version) + except TypeError: + version = self.version if self.version else None + else: + version = version + if version: + self._version = version + build_requires = metadata.get("build_requires", []) + if self.build_requires is None: + self.build_requires = () + self.build_requires = tuple(set(self.build_requires) | set(build_requires)) + self._requirements = ( + frozenset() if self._requirements is None else self._requirements + ) + requirements = set(self._requirements) + install_requires = make_base_requirements(metadata.get("install_requires", [])) + requirements |= install_requires + setup_requires = make_base_requirements(metadata.get("setup_requires", [])) + if self.setup_requires is None: + self.setup_requires = () + self.setup_requires = tuple(set(self.setup_requires) | setup_requires) + if self.ireq.editable: + requirements |= setup_requires + # TODO: Should this be a specifierset? + self.python_requires = metadata.get("python_requires", self.python_requires) + extras_require = metadata.get("extras_require", {}) + extras_tuples = [] + for section in set(list(extras_require.keys())) - set(list(self.extras.keys())): + extras = extras_require[section] + extras_set = make_base_requirements(extras) + if self.ireq and self.ireq.extras and section in self.ireq.extras: + requirements |= extras_set + extras_tuples.append((section, tuple(extras_set))) + if self._extras_requirements is None: + self._extras_requirements = () + self._extras_requirements += tuple(extras_tuples) + build_backend = metadata.get("build_backend", "setuptools.build_meta:__legacy__") + if not self.build_backend: + self.build_backend = build_backend + self._requirements = frozenset(requirements) + + def get_extras_from_ireq(self): + # type: () -> None + if self.ireq and self.ireq.extras: + for extra in self.ireq.extras: + if extra in self.extras: + extras = make_base_requirements(self.extras[extra]) + self._requirements = frozenset(set(self._requirements) | extras) + else: + extras = tuple(make_base_requirements(extra)) + self._extras_requirements += (extra, extras) def parse_setup_cfg(self): + # type: () -> Dict[STRING_TYPE, Any] if self.setup_cfg is not None and self.setup_cfg.exists(): - default_opts = { - "metadata": {"name": "", "version": ""}, - "options": { - "install_requires": "", - "python_requires": "", - "build_requires": "", - "setup_requires": "", - "extras": "", - }, - } - parser = configparser.ConfigParser(default_opts) - parser.read(self.setup_cfg.as_posix()) - if parser.has_option("metadata", "name"): - name = parser.get("metadata", "name") - if not self.name and name is not None: - self.name = name - if parser.has_option("metadata", "version"): - version = parser.get("metadata", "version") - if not self.version and version is not None: - self.version = version - if parser.has_option("options", "install_requires"): - self.requires.update( - { - dep.strip(): init_requirement(dep.strip()) - for dep in parser.get("options", "install_requires").split("\n") - if dep - } - ) - if parser.has_option("options", "python_requires"): - python_requires = parser.get("options", "python_requires") - if python_requires and not self.python_requires: - self.python_requires = python_requires - if "options.extras_require" in parser.sections(): - self.extras.update( - { - section: [ - init_requirement(dep.strip()) - for dep in parser.get( - "options.extras_require", section - ).split("\n") - if dep - ] - for section in parser.options("options.extras_require") - if section not in ["options", "metadata"] - } - ) - if self.ireq.extras: - self.requires.update({ - extra: self.extras[extra] - for extra in self.ireq.extras if extra in self.extras - }) + contents = self.setup_cfg.read_text() + base_dir = self.setup_cfg.absolute().parent.as_posix() + try: + parsed = setuptools_parse_setup_cfg(self.setup_cfg.as_posix()) + except Exception: + if six.PY2: + contents = self.setup_cfg.read_bytes() + parsed = parse_setup_cfg(contents, base_dir) + if not parsed: + return {} + return parsed + return {} + + def parse_setup_py(self): + # type: () -> Dict[STRING_TYPE, Any] + if self.setup_py is not None and self.setup_py.exists(): + parsed = ast_parse_setup_py(self.setup_py.as_posix()) + if not parsed: + return {} + return parsed + return {} def run_setup(self): + # type: () -> "SetupInfo" if self.setup_py is not None and self.setup_py.exists(): + dist = run_setup(self.setup_py.as_posix(), egg_base=self.egg_base) target_cwd = self.setup_py.parent.as_posix() - with cd(target_cwd), _suppress_distutils_logs(): - # This is for you, Hynek - # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py - script_name = self.setup_py.as_posix() - args = ["egg_info"] - g = {"__file__": script_name, "__name__": "__main__"} - local_dict = {} - if sys.version_info < (3, 5): - save_argv = sys.argv - else: - save_argv = sys.argv.copy() - try: - global _setup_distribution, _setup_stop_after - _setup_stop_after = "run" - sys.argv[0] = script_name - sys.argv[1:] = args - with open(script_name, 'rb') as f: - if sys.version_info < (3, 5): - exec(f.read(), g, local_dict) - else: - exec(f.read(), g) - # We couldn't import everything needed to run setup - except NameError: - python = os.environ.get('PIP_PYTHON_PATH', sys.executable) - out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, - combine_stderr=False, return_object=False, nospin=True) - finally: - _setup_stop_after = None - sys.argv = save_argv - dist = _setup_distribution + with temp_path(), cd(target_cwd): if not dist: - self.get_egg_metadata() - return + metadata = self.get_egg_metadata() + if metadata: + return self.populate_metadata(metadata) + if isinstance(dist, Mapping): + self.populate_metadata(dist) + return name = dist.get_name() if name: self.name = name - if dist.python_requires and not self.python_requires: - self.python_requires = packaging.specifiers.SpecifierSet( - dist.python_requires - ) - if dist.extras_require and not self.extras: - self.extras = dist.extras_require - install_requires = dist.get_requires() - if not install_requires: - install_requires = dist.install_requires - if install_requires and not self.requires: - requirements = [init_requirement(req) for req in install_requires] - self.requires.update({req.key: req for req in requirements}) - if dist.setup_requires and not self.setup_requires: - self.setup_requires = dist.setup_requires - if not self.version: - self.version = dist.get_version() - - def get_egg_metadata(self): - if self.setup_py is not None and self.setup_py.exists(): - metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) - if metadata: - if not self.name: - self.name = metadata.get("name", self.name) - if not self.version: - self.version = metadata.get("version", self.version) - self.requires.update( - {req.key: req for req in metadata.get("requires", {})} + update_dict = {} + if dist.python_requires: + update_dict["python_requires"] = dist.python_requires + update_dict["extras_require"] = {} + if dist.extras_require: + for extra, extra_requires in dist.extras_require: + extras_tuple = make_base_requirements(extra_requires) + update_dict["extras_require"][extra] = extras_tuple + update_dict["install_requires"] = make_base_requirements( + dist.get_requires() ) - if getattr(self.ireq, "extras", None): - for extra in self.ireq.extras: - extras = metadata.get("extras", {}).get(extra, []) - if extras: - extras = ensure_reqs(extras) - self.extras[extra] = set(extras) - self.requires.update( - {req.key: req for req in extras if req is not None} - ) + if dist.setup_requires: + update_dict["setup_requires"] = make_base_requirements( + dist.setup_requires + ) + version = dist.get_version() + if version: + update_dict["version"] = version + return self.update_from_dict(update_dict) + + @property + def pep517_config(self): + config = {} + config.setdefault("--global-option", []) + return config + + def build_wheel(self): + # type: () -> S + if not self.pyproject.exists(): + build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) + self.pyproject.write_text( + six.text_type( + """ +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="wheel", + ) + + # noinspection PyPackageRequirements + def build_sdist(self): + # type: () -> S + if not self.pyproject.exists(): + if not self.build_requires: + build_requires = '"setuptools", "wheel"' + else: + build_requires = ", ".join( + ['"{0}"'.format(r) for r in self.build_requires] + ) + self.pyproject.write_text( + six.text_type( + """ +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="sdist", + ) + + def build(self): + # type: () -> "SetupInfo" + dist_path = None + metadata = None + try: + dist_path = self.build_wheel() + metadata = self.get_metadata_from_wheel( + os.path.join(self.extra_kwargs["build_dir"], dist_path) + ) + except Exception: + try: + dist_path = self.build_sdist() + metadata = self.get_egg_metadata(metadata_type="egg") + if metadata: + self.populate_metadata(metadata) + except Exception: + pass + if metadata: + self.populate_metadata(metadata) + if not self.metadata or not self.name: + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) + if not self.metadata or not self.name: + return self.run_setup() + return self + + def reload(self): + # type: () -> Dict[S, Any] + """Wipe existing distribution info metadata for rebuilding. + + Erases metadata from **self.egg_base** and unsets **self.requirements** + and **self.extras**. + """ + for metadata_dir in os.listdir(self.egg_base): + shutil.rmtree(metadata_dir, ignore_errors=True) + self.metadata = None + self._requirements = frozenset() + self._extras_requirements = () + self.get_info() + + def get_metadata_from_wheel(self, wheel_path): + # type: (S) -> Dict[Any, Any] + """Given a path to a wheel, return the metadata from that wheel. + + :return: A dictionary of metadata from the provided wheel + :rtype: Dict[Any, Any] + """ + + metadata_dict = get_metadata_from_wheel(wheel_path) + return metadata_dict + + def get_egg_metadata(self, metadata_dir=None, metadata_type=None): + # type: (Optional[AnyStr], Optional[AnyStr]) -> Dict[Any, Any] + """Given a metadata directory, return the corresponding metadata dictionary. + + :param Optional[str] metadata_dir: Root metadata path, default: `os.getcwd()` + :param Optional[str] metadata_type: Type of metadata to search for, default None + :return: A metadata dictionary built from the metadata in the given location + :rtype: Dict[Any, Any] + """ + + package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] + metadata_dirs = [] # type: List[STRING_TYPE] + if any([fn is not None and fn.exists() for fn in package_indicators]): + metadata_dirs = [ + self.extra_kwargs["build_dir"], + self.egg_base, + self.extra_kwargs["src_dir"], + ] + if metadata_dir is not None: + metadata_dirs = [metadata_dir] + metadata_dirs + metadata = [ + get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) + for d in metadata_dirs + if os.path.exists(d) + ] + metadata = next(iter(d for d in metadata if d), None) + return metadata + + def populate_metadata(self, metadata): + # type: (Dict[Any, Any]) -> "SetupInfo" + """Populates the metadata dictionary from the supplied metadata. + + :return: The current instance. + :rtype: `SetupInfo` + """ + + _metadata = () + for k, v in metadata.items(): + if k == "extras" and isinstance(v, dict): + extras = () + for extra, reqs in v.items(): + extras += ((extra, tuple(reqs)),) + _metadata += extras + elif isinstance(v, (list, tuple)): + _metadata += (k, tuple(v)) + else: + _metadata += (k, v) + self.metadata = _metadata + cleaned = metadata.copy() + cleaned.update({"install_requires": metadata.get("requires", [])}) + if cleaned: + self.update_from_dict(cleaned.copy()) + else: + self.update_from_dict(metadata) + return self def run_pyproject(self): + # type: () -> "SetupInfo" + """Populates the **pyproject.toml** metadata if available. + + :return: The current instance + :rtype: `SetupInfo` + """ + if self.pyproject and self.pyproject.exists(): result = get_pyproject(self.pyproject.parent) if result is not None: requires, backend = result + if self.build_requires is None: + self.build_requires = () if backend: self.build_backend = backend - if requires and not self.build_requires: - self.build_requires = requires + else: + self.build_backend = get_default_pyproject_backend() + if requires: + self.build_requires = tuple(set(requires) | set(self.build_requires)) + else: + self.build_requires = ("setuptools", "wheel") + return self + + def get_initial_info(self): + # type: () -> Dict[S, Any] + parse_setupcfg = False + parse_setuppy = False + if self.setup_cfg and self.setup_cfg.exists(): + parse_setupcfg = True + if self.setup_py and self.setup_py.exists(): + parse_setuppy = True + if parse_setuppy or parse_setupcfg: + with cd(self.base_dir): + if parse_setuppy: + self.update_from_dict(self.parse_setup_py()) + if parse_setupcfg: + self.update_from_dict(self.parse_setup_cfg()) + if self.name is not None and any( + [ + self.requires, + self.setup_requires, + self._extras_requirements, + self.build_backend, + ] + ): + return self.as_dict() + return self.get_info() def get_info(self): - if self.setup_cfg and self.setup_cfg.exists(): - self.parse_setup_cfg() - if self.setup_py and self.setup_py.exists(): + # type: () -> Dict[S, Any] + with cd(self.base_dir): + self.run_pyproject() + self.build() + + if self.setup_py and self.setup_py.exists() and self.metadata is None: if not self.requires or not self.name: try: - self.run_setup() + with cd(self.base_dir): + self.run_setup() except Exception: - self.get_egg_metadata() - if not self.requires or not self.name: - self.get_egg_metadata() + with cd(self.base_dir): + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) + if self.metadata is None or not self.name: + with cd(self.base_dir): + metadata = self.get_egg_metadata() + if metadata: + self.populate_metadata(metadata) - if self.pyproject and self.pyproject.exists(): - self.run_pyproject() return self.as_dict() def as_dict(self): + # type: () -> Dict[STRING_TYPE, Any] prop_dict = { "name": self.name, - "version": self.version, + "version": self.version if self._version else None, "base_dir": self.base_dir, "ireq": self.ireq, "build_backend": self.build_backend, "build_requires": self.build_requires, - "requires": self.requires, + "requires": self.requires if self._requirements else None, "setup_requires": self.setup_requires, "python_requires": self.python_requires, - "extras": self.extras, + "extras": self.extras if self._extras_requirements else None, "extra_kwargs": self.extra_kwargs, "setup_cfg": self.setup_cfg, "setup_py": self.setup_py, @@ -378,66 +1571,82 @@ class SetupInfo(object): @classmethod def from_requirement(cls, requirement, finder=None): + # type: (TRequirement, Optional[PackageFinder]) -> Optional[SetupInfo] ireq = requirement.as_ireq() subdir = getattr(requirement.req, "subdirectory", None) return cls.from_ireq(ireq, subdir=subdir, finder=finder) @classmethod - def from_ireq(cls, ireq, subdir=None, finder=None): + @lru_cache() + def from_ireq(cls, ireq, subdir=None, finder=None, session=None): + # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder], Optional[requests.Session]) -> Optional[SetupInfo] import pip_shims.shims + if not ireq.link: + return None if ireq.link.is_wheel: - return + return None if not finder: from .dependencies import get_finder - finder = get_finder() + session, finder = get_finder() + vcs, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + parsed = urlparse(uri) + if "file" in parsed.scheme: + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + uri = urlunparse(parsed) + path = None + if ireq.link.scheme == "file" or uri.startswith("file://"): + if "file:/" in uri and "file:///" not in uri: + uri = uri.replace("file:/", "file:///") + path = pip_shims.shims.url_to_path(uri) kwargs = _prepare_wheel_building_kwargs(ireq) - ireq.populate_link(finder, False, False) - ireq.ensure_has_source_dir(kwargs["build_dir"]) - if not ( - ireq.editable - and pip_shims.shims.is_file_url(ireq.link) - and not ireq.link.is_artifact - ): + is_artifact_or_vcs = getattr( + ireq.link, "is_vcs", getattr(ireq.link, "is_artifact", False) + ) + is_vcs = True if vcs else is_artifact_or_vcs + if not (ireq.editable and pip_shims.shims.is_file_url(ireq.link) and is_vcs): if ireq.is_wheel: only_download = True download_dir = kwargs["wheel_download_dir"] else: only_download = False download_dir = kwargs["download_dir"] - ireq_src_dir = None - if ireq.link.scheme == "file": - path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) - if pip_shims.shims.is_installable_dir(path): - ireq_src_dir = path - if not ireq.editable or not (pip_shims.is_file_url(ireq.link) and ireq_src_dir): - pip_shims.shims.unpack_url( - ireq.link, - ireq.source_dir, - download_dir, + elif path is not None and os.path.isdir(path): + raise RequirementError( + "The file URL points to a directory not installable: {}".format(ireq.link) + ) + # this ensures the build dir is treated as the temporary build location + # and the source dir is treated as permanent / not deleted by pip + build_location_func = getattr(ireq, "build_location", None) + if build_location_func is None: + build_location_func = getattr(ireq, "ensure_build_location", None) + build_location_func(kwargs["build_dir"]) + ireq.ensure_has_source_dir(kwargs["src_dir"]) + src_dir = ireq.source_dir + with pip_shims.shims.global_tempdir_manager(): + pip_shims.shims.shim_unpack( + link=ireq.link, + location=kwargs["src_dir"], + download_dir=download_dir, only_download=only_download, - session=finder.session, + session=session, hashes=ireq.hashes(False), progress_bar="off", ) - if ireq.editable: - created = cls.create( - ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - else: - build_dir = ireq.build_location(kwargs["build_dir"]) - ireq._temp_build_dir.path = kwargs["build_dir"] - created = cls.create( - build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - created.get_info() + created = cls.create( + kwargs["src_dir"], subdirectory=subdir, ireq=ireq, kwargs=kwargs + ) return created @classmethod def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + # type: (AnyStr, Optional[AnyStr], Optional[InstallRequirement], Optional[Dict[AnyStr, AnyStr]]) -> Optional[SetupInfo] if not base_dir or base_dir is None: - return + return None creation_kwargs = {"extra_kwargs": kwargs} if not isinstance(base_dir, Path): @@ -454,4 +1663,6 @@ class SetupInfo(object): creation_kwargs["setup_cfg"] = setup_cfg if ireq: creation_kwargs["ireq"] = ireq - return cls(**creation_kwargs) + created = cls(**creation_kwargs) + created.get_initial_info() + return created diff --git a/pipenv/vendor/requirementslib/models/url.py b/pipenv/vendor/requirementslib/models/url.py new file mode 100644 index 00000000..200eba69 --- /dev/null +++ b/pipenv/vendor/requirementslib/models/url.py @@ -0,0 +1,492 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import attr +import pip_shims.shims +from orderedmultidict import omdict +from six.moves.urllib.parse import quote_plus, unquote_plus +from urllib3 import util as urllib3_util +from urllib3.util import parse_url as urllib3_parse +from urllib3.util.url import Url + +from ..environment import MYPY_RUNNING +from .utils import extras_to_string, parse_extras + +if MYPY_RUNNING: + from typing import Dict, List, Optional, Text, Tuple, TypeVar, Union + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + + +def _get_parsed_url(url): + # type: (S) -> Url + """ + This is a stand-in function for `urllib3.util.parse_url` + + The orignal function doesn't handle special characters very well, this simply splits + out the authentication section, creates the parsed url, then puts the authentication + section back in, bypassing validation. + + :return: The new, parsed URL object + :rtype: :class:`~urllib3.util.url.Url` + """ + + try: + parsed = urllib3_parse(url) + except ValueError: + scheme, _, url = url.partition("://") + auth, _, url = url.rpartition("@") + url = "{scheme}://{url}".format(scheme=scheme, url=url) + parsed = urllib3_parse(url)._replace(auth=auth) + if parsed.auth and unquote_plus(parsed.auth) != parsed.auth: + return parsed._replace(auth=unquote_plus(parsed.auth)) + return parsed + + +def remove_password_from_url(url): + # type: (S) -> S + """ + Given a url, remove the password and insert 4 dashes + + :param url: The url to replace the authentication in + :type url: S + :return: The new URL without authentication + :rtype: S + """ + + parsed = _get_parsed_url(url) + if parsed.auth: + auth, _, _ = parsed.auth.partition(":") + return parsed._replace(auth="{auth}:----".format(auth=auth)).url + return parsed.url + + +@attr.s(hash=True) +class URI(object): + #: The target hostname, e.g. `amazon.com` + host = attr.ib(type=str) + #: The URI Scheme, e.g. `salesforce` + scheme = attr.ib(default="https", type=str) + #: The numeric port of the url if specified + port = attr.ib(default=None, type=int) + #: The url path, e.g. `/path/to/endpoint` + path = attr.ib(default="", type=str) + #: Query parameters, e.g. `?variable=value...` + query = attr.ib(default="", type=str) + #: URL Fragments, e.g. `#fragment=value` + fragment = attr.ib(default="", type=str) + #: Subdirectory fragment, e.g. `&subdirectory=blah...` + subdirectory = attr.ib(default="", type=str) + #: VCS ref this URI points at, if available + ref = attr.ib(default="", type=str) + #: The username if provided, parsed from `user:password@hostname` + username = attr.ib(default="", type=str) + #: Password parsed from `user:password@hostname` + password = attr.ib(default="", type=str, repr=False) + #: An orderedmultidict representing query fragments + query_dict = attr.ib(factory=omdict, type=omdict) + #: The name of the specified package in case it is a VCS URI with an egg fragment + name = attr.ib(default="", type=str) + #: Any extras requested from the requirement + extras = attr.ib(factory=tuple, type=tuple) + #: Whether the url was parsed as a direct pep508-style URL + is_direct_url = attr.ib(default=False, type=bool) + #: Whether the url was an implicit `git+ssh` url (passed as `git+git@`) + is_implicit_ssh = attr.ib(default=False, type=bool) + _auth = attr.ib(default=None, type=str, repr=False) + _fragment_dict = attr.ib(factory=dict, type=dict) + _username_is_quoted = attr.ib(type=bool, default=False) + _password_is_quoted = attr.ib(type=bool, default=False) + + def _parse_query(self): + # type: () -> URI + query = self.query if self.query is not None else "" + query_dict = omdict() + queries = query.split("&") + query_items = [] + for q in queries: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + query_items.append((key, val)) + query_dict.load(query_items) + return attr.evolve(self, query_dict=query_dict, query=query) + + def _parse_fragment(self): + # type: () -> URI + subdirectory = self.subdirectory if self.subdirectory else "" + fragment = self.fragment if self.fragment else "" + if self.fragment is None: + return self + fragments = self.fragment.split("&") + fragment_items = {} + name = self.name if self.name else "" + extras = self.extras + for q in fragments: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + fragment_items[key] = val + if key == "egg": + from .utils import parse_extras + + name, stripped_extras = pip_shims.shims._strip_extras(val) + if stripped_extras: + extras = tuple(parse_extras(stripped_extras)) + elif key == "subdirectory": + subdirectory = val + return attr.evolve( + self, + fragment_dict=fragment_items, + subdirectory=subdirectory, + fragment=fragment, + extras=extras, + name=name, + ) + + def _parse_auth(self): + # type: () -> URI + if self._auth: + username, _, password = self._auth.partition(":") + username_is_quoted, password_is_quoted = False, False + quoted_username, quoted_password = "", "" + if password: + quoted_password = quote_plus(password) + password_is_quoted = quoted_password != password + if username: + quoted_username = quote_plus(username) + username_is_quoted = quoted_username != username + return attr.evolve( + self, + username=quoted_username, + password=quoted_password, + username_is_quoted=username_is_quoted, + password_is_quoted=password_is_quoted, + ) + return self + + def get_password(self, unquote=False, include_token=True): + # type: (bool, bool) -> str + password = self.password if self.password else "" + if password and unquote and self._password_is_quoted: + password = unquote_plus(password) + return password + + def get_username(self, unquote=False): + # type: (bool) -> str + username = self.username if self.username else "" + if username and unquote and self._username_is_quoted: + username = unquote_plus(username) + return username + + @staticmethod + def parse_subdirectory(url_part): + # type: (str) -> Tuple[str, Optional[str]] + subdir = None + if "&subdirectory" in url_part: + url_part, _, subdir = url_part.rpartition("&") + subdir = "&{0}".format(subdir.strip()) + return url_part.strip(), subdir + + @classmethod + def get_parsed_url(cls, url): + # if there is a "#" in the auth section, this could break url parsing + parsed_url = _get_parsed_url(url) + if "@" in url and "#" in url: + scheme = "{0}://".format(parsed_url.scheme) + if parsed_url.scheme == "file": + scheme = "{0}/".format(scheme) + url_without_scheme = url.replace(scheme, "") + maybe_auth, _, maybe_url = url_without_scheme.partition("@") + if "#" in maybe_auth and (not parsed_url.host or "." not in parsed_url.host): + new_parsed_url = _get_parsed_url("{0}{1}".format(scheme, maybe_url)) + new_parsed_url = new_parsed_url._replace(auth=maybe_auth) + return new_parsed_url + return parsed_url + + @classmethod + def parse(cls, url): + # type: (S) -> URI + from .utils import DIRECT_URL_RE, split_ref_from_uri + + is_direct_url = False + name_with_extras = None + is_implicit_ssh = url.strip().startswith("git+git@") + if is_implicit_ssh: + from ..utils import add_ssh_scheme_to_git_uri + + url = add_ssh_scheme_to_git_uri(url) + direct_match = DIRECT_URL_RE.match(url) + if direct_match is not None: + is_direct_url = True + name_with_extras, _, url = url.partition("@") + name_with_extras = name_with_extras.strip() + url, ref = split_ref_from_uri(url.strip()) + if "file:/" in url and "file:///" not in url: + url = url.replace("file:/", "file:///") + parsed = cls.get_parsed_url(url) + # if there is a "#" in the auth section, this could break url parsing + if not (parsed.scheme and parsed.host): + # check if this is a file uri + if not ( + parsed.scheme + and parsed.path + and (parsed.scheme == "file" or parsed.scheme.endswith("+file")) + ): + raise ValueError("Failed parsing URL {0!r} - Not a valid url".format(url)) + parsed_dict = dict(parsed._asdict()).copy() + parsed_dict["is_direct_url"] = is_direct_url + parsed_dict["is_implicit_ssh"] = is_implicit_ssh + parsed_dict.update( + **update_url_name_and_fragment(name_with_extras, ref, parsed_dict) + ) # type: ignore + return cls(**parsed_dict)._parse_auth()._parse_query()._parse_fragment() + + def to_string( + self, + escape_password=True, # type: bool + unquote=True, # type: bool + direct=None, # type: Optional[bool] + strip_ssh=False, # type: bool + strip_ref=False, # type: bool + strip_name=False, # type: bool + strip_subdir=False, # type: bool + ): + # type: (...) -> str + """ + Converts the current URI to a string, unquoting or escaping the password as needed + + :param escape_password: Whether to replace password with ``----``, default True + :param escape_password: bool, optional + :param unquote: Whether to unquote url-escapes in the password, default False + :param unquote: bool, optional + :param bool direct: Whether to format as a direct URL + :param bool strip_ssh: Whether to strip the SSH scheme from the url (git only) + :param bool strip_ref: Whether to drop the VCS ref (if present) + :param bool strip_name: Whether to drop the name and extras (if present) + :param bool strip_subdir: Whether to drop the subdirectory (if present) + :return: The reconstructed string representing the URI + :rtype: str + """ + + if direct is None: + direct = self.is_direct_url + if escape_password: + password = "----" if self.password else "" + if password: + username = self.get_username(unquote=unquote) + elif self.username: + username = "----" + else: + username = "" + else: + password = self.get_password(unquote=unquote) + username = self.get_username(unquote=unquote) + auth = "" + if username: + if password: + auth = "{username}:{password}@".format( + password=password, username=username + ) + else: + auth = "{username}@".format(username=username) + query = "" + if self.query: + query = "{query}?{self.query}".format(query=query, self=self) + if not direct: + if self.name and not strip_name: + fragment = "#egg={self.name_with_extras}".format(self=self) + elif not strip_name and ( + self.extras and self.scheme and self.scheme.startswith("file") + ): + from .utils import extras_to_string + + fragment = extras_to_string(self.extras) + else: + fragment = "" + query = "{query}{fragment}".format(query=query, fragment=fragment) + if self.subdirectory and not strip_subdir: + query = "{query}&subdirectory={self.subdirectory}".format( + query=query, self=self + ) + host_port_path = self.get_host_port_path(strip_ref=strip_ref) + url = "{self.scheme}://{auth}{host_port_path}{query}".format( + self=self, auth=auth, host_port_path=host_port_path, query=query + ) + if strip_ssh: + from ..utils import strip_ssh_from_git_uri + + url = strip_ssh_from_git_uri(url) + if self.name and direct and not strip_name: + return "{self.name_with_extras}@ {url}".format(self=self, url=url) + return url + + def get_host_port_path(self, strip_ref=False): + # type: (bool) -> str + host = self.host if self.host else "" + if self.port is not None: + host = "{host}:{self.port!s}".format(host=host, self=self) + path = "{self.path}".format(self=self) if self.path else "" + if self.ref and not strip_ref: + path = "{path}@{self.ref}".format(path=path, self=self) + return "{host}{path}".format(host=host, path=path) + + @property + def hidden_auth(self): + # type: () -> str + auth = "" + if self.username and self.password: + password = "****" + username = self.get_username(unquote=True) + auth = "{username}:{password}".format(username=username, password=password) + elif self.username and not self.password: + auth = "****" + return auth + + @property + def name_with_extras(self): + # type: () -> str + from .utils import extras_to_string + + if not self.name: + return "" + extras = extras_to_string(self.extras) + return "{self.name}{extras}".format(self=self, extras=extras) + + @property + def as_link(self): + # type: () -> Link + link = pip_shims.shims.Link( + self.to_string(escape_password=False, strip_ssh=False, direct=False) + ) + return link + + @property + def bare_url(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + strip_subdir=True, + ) + + @property + def url_without_fragment_or_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + ) + + @property + def url_without_fragment(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + ) + + @property + def url_without_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_ref=True, + ) + + @property + def base_url(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + unquote=False, + ) + + @property + def full_url(self): + # type: () -> str + return self.to_string(escape_password=False, strip_ssh=False, direct=False) + + @property + def secret(self): + # type: () -> str + return self.full_url + + @property + def safe_string(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) + + @property + def unsafe_string(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=True) + + @property + def uri_escape(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=False) + + @property + def is_vcs(self): + # type: () -> bool + from ..utils import VCS_SCHEMES + + return self.scheme in VCS_SCHEMES + + @property + def is_file_url(self): + # type: () -> bool + return all([self.scheme, self.scheme == "file"]) + + def __str__(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) + + +def update_url_name_and_fragment(name_with_extras, ref, parsed_dict): + # type: (Optional[str], Optional[str], Dict[str, Optional[str]]) -> Dict[str, Optional[str]] + if name_with_extras: + fragment = "" # type: Optional[str] + parsed_extras = () + name, extras = pip_shims.shims._strip_extras(name_with_extras) + if extras: + parsed_extras = parsed_extras + tuple(parse_extras(extras)) + if parsed_dict["fragment"] is not None: + fragment = "{0}".format(parsed_dict["fragment"]) + if fragment.startswith("egg="): + _, _, fragment_part = fragment.partition("=") + fragment_name, fragment_extras = pip_shims.shims._strip_extras( + fragment_part + ) + name = name if name else fragment_name + if fragment_extras: + parsed_extras = parsed_extras + tuple(parse_extras(fragment_extras)) + name_with_extras = "{0}{1}".format(name, extras_to_string(parsed_extras)) + parsed_dict["fragment"] = "egg={0}".format(name_with_extras) + elif ( + parsed_dict.get("path") is not None and "&subdirectory" in parsed_dict["path"] + ): + path, fragment = URI.parse_subdirectory(parsed_dict["path"]) # type: ignore + parsed_dict["path"] = path + elif ref is not None and "&subdirectory" in ref: + ref, fragment = URI.parse_subdirectory(ref) + parsed_dict["name"] = name + parsed_dict["extras"] = parsed_extras + if ref: + parsed_dict["ref"] = ref.strip() + return parsed_dict diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 0fac2aa3..6c3b7de8 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -1,47 +1,202 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import io import os +import re +import string import sys - from collections import defaultdict from itertools import chain, groupby -from operator import attrgetter import six import tomlkit - from attr import validators -from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet +from packaging.version import parse as parse_version +from plette.models import Package, PackageCollection +from tomlkit.container import Container +from tomlkit.items import AoT, Array, Bool, InlineTable, Item, String, Table +from urllib3 import util as urllib3_util +from urllib3.util import parse_url as urllib3_parse +from vistir.compat import lru_cache from vistir.misc import dedup +from vistir.path import is_valid_url + +from ..environment import MYPY_RUNNING +from ..utils import SCHEME_LIST, VCS_LIST, is_star + +if MYPY_RUNNING: + from attr import _ValidatorType # noqa + from packaging.requirements import Requirement as PackagingRequirement + from pip_shims.shims import Link + from pkg_resources import Requirement as PkgResourcesRequirement + from pkg_resources.extern.packaging.markers import ( + Op as PkgResourcesOp, + Variable as PkgResourcesVariable, + Value as PkgResourcesValue, + Marker as PkgResourcesMarker, + ) + from typing import ( + Union, + Optional, + List, + Set, + Any, + TypeVar, + Tuple, + Sequence, + Dict, + Text, + AnyStr, + Match, + Iterable, # noqa + ) + from urllib3.util.url import Url + from vistir.compat import Path + + _T = TypeVar("_T") + TMarker = Union[Marker, PkgResourcesMarker] + TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) + TValue = TypeVar("TValue", PkgResourcesValue, Value) + TOp = TypeVar("TOp", PkgResourcesOp, Op) + MarkerTuple = Tuple[TVariable, TOp, TValue] + TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] + STRING_TYPE = Union[bytes, str, Text] + TOML_DICT_TYPES = Union[Container, Package, PackageCollection, Table, InlineTable] + S = TypeVar("S", bytes, str, Text) -from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri - +TOML_DICT_OBJECTS = (Container, Package, Table, InlineTable, PackageCollection) +TOML_DICT_NAMES = [o.__class__.__name__ for o in TOML_DICT_OBJECTS] HASH_STRING = " --hash={0}" +ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits) +PUNCTUATION = r"[\-_\.]" +ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits) +NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC) +REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits) +EXTRAS = r"(?P<extras>\[{0}(?:,{0})*\])".format(NAME) +NAME_WITH_EXTRAS = r"(?P<name>{0}){1}?".format(NAME, EXTRAS) +NAME_RE = re.compile(NAME_WITH_EXTRAS) +SUBDIR_RE = r"(?:[&#]subdirectory=(?P<subdirectory>.*))" +URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) +REF_RE = r"(?:@(?P<ref>{0}+)?)".format(REF) +PATH_RE = r"(?P<pathsep>[:/])(?P<path>[^ @]+){0}?".format(REF_RE) +PASS_RE = r"(?:(?<=:)(?P<password>[^ ]+))" +AUTH_RE = r"(?:(?P<username>[^ ]+)[:@]{0}?@)".format(PASS_RE) +HOST_RE = r"(?:{0}?(?P<host>[^ ]+?\.?{1}+(?P<port>:\d+)?))?".format( + AUTH_RE, ALPHA_NUMERIC +) +URL = r"(?P<scheme>[^ ]+://){0}{1}".format(HOST_RE, PATH_RE) +URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) +DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) + def filter_none(k, v): + # type: (AnyStr, Any) -> bool if v: return True return False +def filter_dict(dict_): + # type: (Dict[AnyStr, Any]) -> Dict[AnyStr, Any] + return {k: v for k, v in dict_.items() if filter_none(k, v)} + + def optional_instance_of(cls): + # type: (Any) -> _ValidatorType[Optional[_T]] return validators.optional(validators.instance_of(cls)) def create_link(link): - from pip_shims import Link + # type: (AnyStr) -> Link + + if not isinstance(link, six.string_types): + raise TypeError("must provide a string to instantiate a new link") + from pip_shims.shims import Link + return Link(link) +def tomlkit_value_to_python(toml_value): + # type: (Union[Array, AoT, TOML_DICT_TYPES, Item]) -> Union[List, Dict] + value_type = type(toml_value).__name__ + if ( + isinstance(toml_value, TOML_DICT_OBJECTS + (dict,)) + or value_type in TOML_DICT_NAMES + ): + return tomlkit_dict_to_python(toml_value) + elif isinstance(toml_value, AoT) or value_type == "AoT": + return [tomlkit_value_to_python(val) for val in toml_value._body] + elif isinstance(toml_value, Array) or value_type == "Array": + return [tomlkit_value_to_python(val) for val in list(toml_value)] + elif isinstance(toml_value, String) or value_type == "String": + return "{0!s}".format(toml_value) + elif isinstance(toml_value, Bool) or value_type == "Bool": + return toml_value.value + elif isinstance(toml_value, Item): + return toml_value.value + return toml_value + + +def tomlkit_dict_to_python(toml_dict): + # type: (TOML_DICT_TYPES) -> Dict + value_type = type(toml_dict).__name__ + if toml_dict is None: + raise TypeError("Invalid type NoneType when converting toml dict to python") + converted = None # type: Optional[Dict] + if isinstance(toml_dict, (InlineTable, Table)) or value_type in ( + "InlineTable", + "Table", + ): + converted = toml_dict.value + elif isinstance(toml_dict, (Package, PackageCollection)) or value_type in ( + "Package, PackageCollection" + ): + converted = toml_dict._data + if isinstance(converted, Container) or type(converted).__name__ == "Container": + converted = converted.value + elif isinstance(toml_dict, Container) or value_type == "Container": + converted = toml_dict.value + elif isinstance(toml_dict, dict): + converted = toml_dict.copy() + else: + raise TypeError( + "Invalid type for conversion: expected Container, Dict, or Table, " + "got {0!r}".format(toml_dict) + ) + if isinstance(converted, dict): + return {k: tomlkit_value_to_python(v) for k, v in converted.items()} + elif isinstance(converted, (TOML_DICT_OBJECTS)) or value_type in TOML_DICT_NAMES: + return tomlkit_dict_to_python(converted) + return converted + + +def get_url_name(url): + # type: (AnyStr) -> AnyStr + """ + Given a url, derive an appropriate name to use in a pipfile. + + :param str url: A url to derive a string from + :returns: The name of the corresponding pipfile entry + :rtype: Text + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected a string, got {0!r}".format(url)) + return urllib3_util.parse_url(url).host + + def init_requirement(name): + # type: (AnyStr) -> TRequirement + + if not isinstance(name, six.string_types): + raise TypeError("must supply a name to generate a requirement") from pkg_resources import Requirement + req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -51,73 +206,297 @@ def init_requirement(name): def extras_to_string(extras): - """Turn a list of extras into a string""" + # type: (Iterable[S]) -> S + """Turn a list of extras into a string + + :param List[str]] extras: a list of extras to format + :return: A string of extras + :rtype: str + """ if isinstance(extras, six.string_types): if extras.startswith("["): return extras - else: extras = [extras] - return "[{0}]".format(",".join(sorted(extras))) + if not extras: + return "" + return "[{0}]".format(",".join(sorted(set(extras)))) # type: ignore def parse_extras(extras_str): - """Turn a string of extras into a parsed extras list""" + # type: (AnyStr) -> List[AnyStr] + """Turn a string of extras into a parsed extras list + + :param str extras_str: An extras string + :return: A sorted list of extras + :rtype: List[str] + """ + from pkg_resources import Requirement + extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) def specs_to_string(specs): - """Turn a list of specifier tuples into a string""" + # type: (List[Union[STRING_TYPE, Specifier]]) -> AnyStr + """Turn a list of specifier tuples into a string + + :param List[Union[Specifier, str]] specs: a list of specifiers to format + :return: A string of specifiers + :rtype: str + """ + if specs: if isinstance(specs, six.string_types): return specs try: extras = ",".join(["".join(spec) for spec in specs]) except TypeError: - extras = ",".join(["".join(spec._spec) for spec in specs]) + extras = ",".join(["".join(spec._spec) for spec in specs]) # type: ignore return extras return "" -def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None): +def build_vcs_uri( + vcs, # type: Optional[S] + uri, # type: S + name=None, # type: Optional[S] + ref=None, # type: Optional[S] + subdirectory=None, # type: Optional[S] + extras=None, # type: Optional[Iterable[S]] +): + # type: (...) -> STRING_TYPE if extras is None: extras = [] - vcs_start = "{0}+".format(vcs) - if not uri.startswith(vcs_start): - uri = "{0}{1}".format(vcs_start, uri) - uri = add_ssh_scheme_to_git_uri(uri) + vcs_start = "" + if vcs is not None: + vcs_start = "{0}+".format(vcs) + if not uri.startswith(vcs_start): + uri = "{0}{1}".format(vcs_start, uri) if ref: uri = "{0}@{1}".format(uri, ref) if name: uri = "{0}#egg={1}".format(uri, name) if extras: - extras = extras_to_string(extras) - uri = "{0}{1}".format(uri, extras) + extras_string = extras_to_string(extras) + uri = "{0}{1}".format(uri, extras_string) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return create_link(uri) + return uri + + +def _get_parsed_url(url): + # type: (S) -> Url + """ + This is a stand-in function for `urllib3.util.parse_url` + + The orignal function doesn't handle special characters very well, this simply splits + out the authentication section, creates the parsed url, then puts the authentication + section back in, bypassing validation. + + :return: The new, parsed URL object + :rtype: :class:`~urllib3.util.url.Url` + """ + + try: + parsed = urllib3_parse(url) + except ValueError: + scheme, _, url = url.partition("://") + auth, _, url = url.rpartition("@") + url = "{scheme}://{url}".format(scheme=scheme, url=url) + parsed = urllib3_parse(url)._replace(auth=auth) + return parsed + + +def convert_direct_url_to_url(direct_url): + # type: (AnyStr) -> AnyStr + """Converts direct URLs to standard, link-style URLs + + Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` + compatible URL by moving the name and extras into an **egg_fragment**. + + :param str direct_url: A pep-508 compliant direct url. + :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. + :rtype: AnyStr + """ + direct_match = DIRECT_URL_RE.match(direct_url) # type: Optional[Match] + if direct_match is None: + url_match = URL_RE.match(direct_url) + if url_match or is_valid_url(direct_url): + return direct_url + match_dict = ( + {} + ) # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] + if direct_match is not None: + match_dict = direct_match.groupdict() # type: ignore + if not match_dict: + raise ValueError( + "Failed converting value to normal URL, is it a direct URL? {0!r}".format( + direct_url + ) + ) + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + url = "" # type: STRING_TYPE + url = "".join([s for s in url_segments if s is not None]) # type: ignore + new_url = build_vcs_uri( + None, + url, + ref=match_dict.get("ref"), + name=match_dict.get("name"), + extras=match_dict.get("extras"), + subdirectory=match_dict.get("subdirectory"), + ) + return new_url + + +def convert_url_to_direct_url(url, name=None): + # type: (AnyStr, Optional[AnyStr]) -> AnyStr + """ + Converts normal link-style URLs to direct urls. + + Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as + defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. + + :param AnyStr url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[AnyStr] name: A name to use in case the supplied URL doesn't provide one. + :return: A pep-508 compliant direct url. + :rtype: AnyStr + + :raises ValueError: Raised when the URL can't be parsed or a name can't be found. + :raises TypeError: When a non-string input is provided. + """ + if not isinstance(url, six.string_types): + raise TypeError( + "Expected a string to convert to a direct url, got {0!r}".format(url) + ) + direct_match = DIRECT_URL_RE.match(url) + if direct_match: + return url + url_match = URL_RE.match(url) + if url_match is None or not url_match.groupdict(): + raise ValueError("Failed parse a valid URL from {0!r}".format(url)) + match_dict = url_match.groupdict() + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + name = match_dict.get("name", name) + extras = match_dict.get("extras") + new_url = "" + if extras and not name: + url_segments.append(extras) + elif extras and name: + new_url = "{0}{1}@ ".format(name, extras) + else: + if name is not None: + new_url = "{0}@ ".format(name) + else: + raise ValueError( + "Failed to construct direct url: " + "No name could be parsed from {0!r}".format(url) + ) + if match_dict.get("ref"): + url_segments.append("@{0}".format(match_dict.get("ref"))) + url = "".join([s for s in url if s is not None]) + url = "{0}{1}".format(new_url, url) + return url def get_version(pipfile_entry): + # type: (Union[STRING_TYPE, Dict[STRING_TYPE, Union[STRING_TYPE, bool, Iterable[STRING_TYPE]]]]) -> STRING_TYPE if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" - elif hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: + if hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: if is_star(pipfile_entry.get("version")): return "" - return pipfile_entry.get("version", "") + return pipfile_entry.get("version", "").strip().lstrip("(").rstrip(")") if isinstance(pipfile_entry, six.string_types): - return pipfile_entry + return pipfile_entry.strip().lstrip("(").rstrip(")") return "" +def strip_extras_markers_from_requirement(req): + # type: (TRequirement) -> TRequirement + """ + Strips extras markers from requirement instances. + + Given a :class:`~packaging.requirements.Requirement` instance with markers defining + *extra == 'name'*, strip out the extras from the markers and return the cleaned + requirement + + :param PackagingRequirement req: A packaging requirement to clean + :return: A cleaned requirement + :rtype: PackagingRequirement + """ + if req is None: + raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) + if getattr(req, "marker", None) is not None: + marker = req.marker # type: TMarker + marker._markers = _strip_extras_markers(marker._markers) + if not marker._markers: + req.marker = None + else: + req.marker = marker + return req + + +def _strip_extras_markers(marker): + # type: (Union[MarkerTuple, List[Union[MarkerTuple, str]]]) -> List[Union[MarkerTuple, str]] + if marker is None or not isinstance(marker, (list, tuple)): + raise TypeError("Expecting a marker type, received {0!r}".format(marker)) + markers_to_remove = [] + # iterate forwards and generate a list of indexes to remove first, then reverse the + # list so we can remove the text that normally occurs after (but we will already + # be past it in the loop) + for i, marker_list in enumerate(marker): + if isinstance(marker_list, list): + cleaned = _strip_extras_markers(marker_list) + if not cleaned: + markers_to_remove.append(i) + elif isinstance(marker_list, tuple) and marker_list[0].value == "extra": + markers_to_remove.append(i) + for i in reversed(markers_to_remove): + del marker[i] + if i > 0 and marker[i - 1] == "and": + del marker[i - 1] + return marker + + +@lru_cache() +def get_setuptools_version(): + # type: () -> Optional[STRING_TYPE] + import pkg_resources + + setuptools_dist = pkg_resources.get_distribution( + pkg_resources.Requirement("setuptools") + ) + return getattr(setuptools_dist, "version", None) + + +def get_default_pyproject_backend(): + # type: () -> STRING_TYPE + st_version = get_setuptools_version() + if st_version is not None: + parsed_st_version = parse_version(st_version) + if parsed_st_version >= parse_version("40.8.0"): + return "setuptools.build_meta:__legacy__" + return "setuptools.build_meta" + + def get_pyproject(path): - from vistir.compat import Path + # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] + """ + Given a base path, look for the corresponding ``pyproject.toml`` file and return its + build_requires and build_backend. + + :param AnyStr path: The root path of the project, should be a directory (will be truncated) + :return: A 2 tuple of build requirements and the build backend + :rtype: Optional[Tuple[List[AnyStr], AnyStr]] + """ if not path: return + from vistir.compat import Path + if not isinstance(path, Path): path = Path(path) if not path.is_dir(): @@ -125,8 +504,10 @@ def get_pyproject(path): pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): - if setup_py.exists(): + if not setup_py.exists(): return None + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: @@ -134,24 +515,28 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools", "wheel"] - backend = "setuptools.build_meta" + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: - requires = ["setuptools>=38.2.5", "wheel"] - backend = "setuptools.build_meta" - build_system = { - "requires": requires, - "build-backend": backend - } + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() + build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: - requires = build_system.get("requires") - backend = build_system.get("build-backend") - return (requires, backend) + requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) + backend = build_system.get("build-backend", get_default_pyproject_backend()) + return requires, backend def split_markers_from_line(line): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] """Split markers from a dependency""" + quote_chars = ["'", '"'] + line_quote = next( + iter(quote for quote in quote_chars if line.startswith(quote)), None + ) + if line_quote and line.endswith(line_quote): + line = line.strip(line_quote) if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" else: @@ -164,14 +549,39 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): + # type: (AnyStr) -> Tuple[Optional[STRING_TYPE], STRING_TYPE] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" - vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) + vcs = next( + iter([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]), None + ) if vcs: vcs, uri = uri.split("+", 1) return vcs, uri +def split_ref_from_uri(uri): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] + """ + Given a path or URI, check for a ref and split it from the path if it is present, + returning a tuple of the original input and the ref or None. + + :param AnyStr uri: The path or URI to split + :returns: A 2-tuple of the path or URI and the ref + :rtype: Tuple[AnyStr, Optional[AnyStr]] + """ + if not isinstance(uri, six.string_types): + raise TypeError("Expected a string, received {0!r}".format(uri)) + parsed = _get_parsed_url(uri) + path = parsed.path if parsed.path else "" + scheme = parsed.scheme if parsed.scheme else "" + ref = None + if scheme != "file" and "@" in path: + path, _, ref = path.rpartition("@") + parsed = parsed._replace(path=path) + return (parsed.url, ref) + + def validate_vcs(instance, attr_, value): if value not in VCS_LIST: raise ValueError("Invalid vcs {0!r}".format(value)) @@ -208,28 +618,27 @@ def key_from_ireq(ireq): def key_from_req(req): """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, 'key'): + if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync key = req.key else: # from packaging, such as install requirements from requirements.txt key = req.name - key = key.replace('_', '-').lower() + key = key.replace("_", "-").lower() return key def _requirement_to_str_lowercase_name(requirement): - """ - Formats a packaging.requirements.Requirement with a lowercase name. + """Formats a packaging.requirements.Requirement with a lowercase name. This is simply a copy of https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124 modified to lowercase the dependency name. Previously, we were invoking the original Requirement.__str__ method and - lowercasing the entire result, which would lowercase the name, *and* other, - important stuff that should not be lowercased (such as the marker). See + lower-casing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lower-cased (such as the marker). See this issue for more information: https://github.com/pypa/pipenv/issues/2113. """ parts = [requirement.name.lower()] @@ -250,35 +659,44 @@ def _requirement_to_str_lowercase_name(requirement): def format_requirement(ireq): - """ + """Formats an `InstallRequirement` instance as a string. + Generic formatter for pretty printing InstallRequirements to the terminal in a less verbose way than using its `__str__` method. + + :param :class:`InstallRequirement` ireq: A pip **InstallRequirement** instance. + :return: A formatted string for prettyprinting + :rtype: str """ if ireq.editable: - line = '-e {}'.format(ireq.link) + line = "-e {}".format(ireq.link) else: line = _requirement_to_str_lowercase_name(ireq.req) if str(ireq.req.marker) != str(ireq.markers): if not ireq.req.marker: - line = '{}; {}'.format(line, ireq.markers) + line = "{}; {}".format(line, ireq.markers) else: name, markers = line.split(";", 1) markers = markers.strip() - line = '{}; ({}) and ({})'.format(name, markers, ireq.markers) + line = "{}; ({}) and ({})".format(name, markers, ireq.markers) return line def format_specifier(ireq): - """ - Generic formatter for pretty printing the specifier part of - InstallRequirements to the terminal. + """Generic formatter for pretty printing specifiers. + + Pretty-prints specifiers from InstallRequirements for output to terminal. + + :param :class:`InstallRequirement` ireq: A pip **InstallRequirement** instance. + :return: A string of specifiers in the given install requirement or <any> + :rtype: str """ # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier._specs if ireq.req is not None else [] specs = sorted(specs, key=lambda x: x._spec[1]) - return ','.join(str(s) for s in specs) or '<any>' + return ",".join(str(s) for s in specs) or "<any>" def get_pinned_version(ireq): @@ -303,11 +721,9 @@ def get_pinned_version(ireq): try: specifier = ireq.specifier except AttributeError: - raise TypeError("Expected InstallRequirement, not {}".format( - type(ireq).__name__, - )) + raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) - if ireq.editable: + if getattr(ireq, "editable", False): raise ValueError("InstallRequirement is editable") if not specifier: raise ValueError("InstallRequirement has no version specification") @@ -315,16 +731,15 @@ def get_pinned_version(ireq): raise ValueError("InstallRequirement has multiple specifications") op, version = next(iter(specifier._specs))._spec - if op not in ('==', '===') or version.endswith('.*'): - raise ValueError("InstallRequirement not pinned (is {0!r})".format( - op + version, - )) + if op not in ("==", "===") or version.endswith(".*"): + raise ValueError("InstallRequirement not pinned (is {0!r})".format(op + version)) return version def is_pinned_requirement(ireq): - """Returns whether an InstallRequirement is a "pinned" requirement. + """ + Returns whether an InstallRequirement is a "pinned" requirement. An InstallRequirement is considered pinned if: @@ -339,6 +754,7 @@ def is_pinned_requirement(ireq): django~=1.8 # NOT pinned django==1.* # NOT pinned """ + try: get_pinned_version(ireq) except (TypeError, ValueError): @@ -350,22 +766,29 @@ def as_tuple(ireq): """ Pulls out the (name: str, version:str, extras:(str)) tuple from the pinned InstallRequirement. """ + if not is_pinned_requirement(ireq): - raise TypeError('Expected a pinned InstallRequirement, got {}'.format(ireq)) + raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) name = key_from_req(ireq.req) - version = first(ireq.specifier._specs)._spec[1] + version = next(iter(ireq.specifier._specs))._spec[1] extras = tuple(sorted(ireq.extras)) return name, version, extras def full_groupby(iterable, key=None): - """Like groupby(), but sorts the input on the group key first.""" + """ + Like groupby(), but sorts the input on the group key first. + """ + return groupby(sorted(iterable, key=key), key=key) def flat_map(fn, collection): - """Map a function over a collection and flatten the result by one-level""" + """ + Map a function over a collection and flatten the result by one-level + """ + return chain.from_iterable(map(fn, collection)) @@ -385,8 +808,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): For key functions that uniquely identify values, set unique=True: >>> assert lookup_table( - ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], - ... unique=True) == { + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], unique=True) == { ... 'b': 'baz', ... 'f': 'foo', ... 'q': 'quux' @@ -404,13 +826,13 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): ... 'f': {'oo'}, ... 'q': {'uux', 'ux'} ... } - """ + if keyval is None: if key is None: - keyval = (lambda v: v) + keyval = lambda v: v else: - keyval = (lambda v: (key(v), v)) + keyval = lambda v: (key(v), v) if unique: return dict(keyval(v) for v in values) @@ -434,7 +856,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): def name_from_req(req): """Get the name of the requirement""" - if hasattr(req, 'project_name'): + if hasattr(req, "project_name"): # from pkg_resources, such as installed dists for pip-sync return req.project_name else: @@ -442,8 +864,11 @@ def name_from_req(req): return req.name -def make_install_requirement(name, version, extras, markers, constraint=False): - """make_install_requirement Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. +def make_install_requirement( + name, version=None, extras=None, markers=None, constraint=False +): + """ + Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. Create an InstallRequirement from the supplied metadata. @@ -463,19 +888,18 @@ def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank from pip_shims.shims import install_req_from_line + extras_string = "" + requirement_string = "{0}".format(name) if extras: # Sort extras for stability extras_string = "[{}]".format(",".join(sorted(extras))) - - if not markers: - return install_req_from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) - else: - return install_req_from_line( - str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), - constraint=constraint) + requirement_string = "{0}{1}".format(requirement_string, extras_string) + if version: + requirement_string = "{0}=={1}".format(requirement_string, str(version)) + if markers: + requirement_string = "{0}; {1}".format(requirement_string, str(markers)) + return install_req_from_line(requirement_string, constraint=constraint) def version_from_ireq(ireq): @@ -487,23 +911,34 @@ def version_from_ireq(ireq): :rtype: str """ - return first(ireq.specifier._specs).version + return next(iter(ireq.specifier._specs)).version + + +def _get_requires_python(candidate): + # type: (Any) -> str + requires_python = getattr(candidate, "requires_python", None) + if requires_python is not None: + link = getattr(candidate, "location", getattr(candidate, "link", None)) + requires_python = getattr(link, "requires_python", None) + return requires_python def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] - sys_version = '.'.join(map(str, sys.version_info[:3])) + sys_version = ".".join(map(str, sys.version_info[:3])) from packaging.version import parse as parse_version - py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) + + py_version = parse_version(os.environ.get("PIP_PYTHON_VERSION", sys_version)) for c in candidates: - from_location = attrgetter("location.requires_python") - requires_python = getattr(c, "requires_python", from_location(c)) + requires_python = _get_requires_python(c) if requires_python: # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,<digit+1' if requires_python.isdigit(): - requires_python = '>={0},<{1}'.format(requires_python, int(requires_python) + 1) + requires_python = ">={0},<{1}".format( + requires_python, int(requires_python) + 1 + ) try: specifierset = SpecifierSet(requires_python) except InvalidSpecifier: @@ -517,7 +952,8 @@ def clean_requires_python(candidates): def fix_requires_python_marker(requires_python): from packaging.requirements import Requirement as PackagingRequirement - marker_str = '' + + marker_str = "" if any(requires_python.startswith(op) for op in Specifier._operators.keys()): spec_dict = defaultdict(set) # We are checking first if we have leading specifier operator @@ -525,26 +961,77 @@ def fix_requires_python_marker(requires_python): specifierset = list(SpecifierSet(requires_python)) # for multiple specifiers, the correct way to represent that in # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` - marker_key = Variable('python_version') + marker_key = Variable("python_version") for spec in specifierset: operator, val = spec._spec cleaned_val = Value(val).serialize().replace('"', "") spec_dict[Op(operator).serialize()].add(cleaned_val) - marker_str = ' and '.join([ - "{0}{1}'{2}'".format(marker_key.serialize(), op, ','.join(vals)) - for op, vals in spec_dict.items() - ]) - marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker + marker_str = " and ".join( + [ + "{0}{1}'{2}'".format(marker_key.serialize(), op, ",".join(vals)) + for op, vals in spec_dict.items() + ] + ) + marker_to_add = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker return marker_to_add def normalize_name(pkg): + # type: (AnyStr) -> AnyStr """Given a package name, return its normalized, non-canonicalized form. - :param str pkg: The name of a package + :param AnyStr pkg: The name of a package :return: A normalized package name - :rtype: str + :rtype: AnyStr """ assert isinstance(pkg, six.string_types) return pkg.replace("_", "-").lower() + + +def get_name_variants(pkg): + # type: (STRING_TYPE) -> Set[STRING_TYPE] + """ + Given a packager name, get the variants of its name for both the canonicalized + and "safe" forms. + + :param AnyStr pkg: The package to lookup + :returns: A list of names. + :rtype: Set + """ + + if not isinstance(pkg, six.string_types): + raise TypeError("must provide a string to derive package names") + from pkg_resources import safe_name + from packaging.utils import canonicalize_name + + pkg = pkg.lower() + names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} + return names + + +def read_source(path, encoding="utf-8"): + # type: (S, S) -> S + """ + Read a source file and get the contents with proper encoding for Python 2/3. + + :param AnyStr path: the file path + :param AnyStr encoding: the encoding that defaults to UTF-8 + :returns: The contents of the source file + :rtype: AnyStr + """ + if six.PY3: + with open(path, "r", encoding=encoding) as fp: + return fp.read() + else: + with open(path, "r") as fp: + return fp.read() + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 6a15db3f..2cd62249 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,40 +1,80 @@ # -*- coding=utf-8 -*- -import attr +from __future__ import absolute_import, print_function + +import importlib import os +import sys + +import attr import pip_shims +import six + +from ..environment import MYPY_RUNNING +from .url import URI + +if MYPY_RUNNING: + from typing import Any, Optional, Tuple -@attr.s +@attr.s(hash=True) class VCSRepository(object): - url = attr.ib() - name = attr.ib() - checkout_directory = attr.ib() - vcs_type = attr.ib() - subdirectory = attr.ib(default=None) - commit_sha = attr.ib(default=None) - ref = attr.ib(default=None) - repo_instance = attr.ib() + DEFAULT_RUN_ARGS = None + + url = attr.ib() # type: str + name = attr.ib() # type: str + checkout_directory = attr.ib() # type: str + vcs_type = attr.ib() # type: str + parsed_url = attr.ib() # type: URI + subdirectory = attr.ib(default=None) # type: Optional[str] + commit_sha = attr.ib(default=None) # type: Optional[str] + ref = attr.ib(default=None) # type: Optional[str] + repo_backend = attr.ib() # type: Any + clone_log = attr.ib(default=None) # type: Optional[str] + + @parsed_url.default + def get_parsed_url(self): + # type: () -> URI + return URI.parse(self.url) + + @repo_backend.default + def get_repo_backend(self): + if self.DEFAULT_RUN_ARGS is None: + default_run_args = self.monkeypatch_pip() + else: + default_run_args = self.DEFAULT_RUN_ARGS + from pip_shims.shims import VcsSupport - @repo_instance.default - def get_repo_instance(self): - from pip_shims import VcsSupport VCS_SUPPORT = VcsSupport() - backend = VCS_SUPPORT._registry.get(self.vcs_type) - return backend(url=self.url) + backend = VCS_SUPPORT.get_backend(self.vcs_type) + # repo = backend(url=self.url) + if backend.run_command.__func__.__defaults__ != default_run_args: + backend.run_command.__func__.__defaults__ = default_run_args + return backend @property def is_local(self): + # type: () -> bool url = self.url - if '+' in url: - url = url.split('+')[1] + if "+" in url: + url = url.split("+")[1] return url.startswith("file") def obtain(self): - if (os.path.exists(self.checkout_directory) and not - self.repo_instance.is_repository_directory(self.checkout_directory)): - self.repo_instance.unpack(self.checkout_directory) + # type: () -> None + lt_pip_19_2 = ( + pip_shims.parsed_pip_version.parsed_version < pip_shims.parse_version("19.2") + ) + if lt_pip_19_2: + self.repo_backend = self.repo_backend(self.url) + if os.path.exists( + self.checkout_directory + ) and not self.repo_backend.is_repository_directory(self.checkout_directory): + self.repo_backend.unpack(self.checkout_directory) elif not os.path.exists(self.checkout_directory): - self.repo_instance.obtain(self.checkout_directory) + if lt_pip_19_2: + self.repo_backend.obtain(self.checkout_directory) + else: + self.repo_backend.obtain(self.checkout_directory, self.parsed_url) else: if self.ref: self.checkout_ref(self.ref) @@ -42,19 +82,48 @@ class VCSRepository(object): self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): - if not self.repo_instance.is_commit_id_equal( - self.checkout_directory, self.get_commit_hash() - ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): - if not self.is_local: - self.update(ref) + # type: (str) -> None + rev_opts = self.repo_backend.make_rev_options(ref) + if not any( + [ + self.repo_backend.is_commit_id_equal(self.checkout_directory, ref), + self.repo_backend.is_commit_id_equal(self.checkout_directory, rev_opts), + self.is_local, + ] + ): + self.update(ref) def update(self, ref): - target_ref = self.repo_instance.make_rev_options(ref) - if pip_shims.parse_version(pip_shims.pip_version) > pip_shims.parse_version("18.0"): - self.repo_instance.update(self.checkout_directory, self.url, target_ref) + # type: (str) -> None + target_ref = self.repo_backend.make_rev_options(ref) + if pip_shims.parse_version(pip_shims.pip_version) > pip_shims.parse_version( + "18.0" + ): + self.repo_backend.update(self.checkout_directory, self.url, target_ref) else: - self.repo_instance.update(self.checkout_directory, target_ref) + self.repo_backend.update(self.checkout_directory, target_ref) self.commit_sha = self.get_commit_hash() def get_commit_hash(self, ref=None): - return self.repo_instance.get_revision(self.checkout_directory) + # type: (Optional[str]) -> str + return self.repo_backend.get_revision(self.checkout_directory) + + @classmethod + def monkeypatch_pip(cls): + # type: () -> Tuple[Any, ...] + target_module = pip_shims.shims.VcsSupport.__module__ + pip_vcs = importlib.import_module(target_module) + run_command_defaults = pip_vcs.VersionControl.run_command.__defaults__ + # set the default to not write stdout, the first option sets this value + new_defaults = [False] + list(run_command_defaults)[1:] + new_defaults = tuple(new_defaults) + if six.PY3: + try: + pip_vcs.VersionControl.run_command.__defaults__ = new_defaults + except AttributeError: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + else: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + sys.modules[target_module] = pip_vcs + cls.DEFAULT_RUN_ARGS = new_defaults + return new_defaults diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index f3653e32..d76f82e9 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -1,24 +1,45 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function -import contextlib import logging import os - -import six import sys -import tomlkit - -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) -from six.moves import Mapping, Sequence, Set, ItemsView -from six.moves.urllib.parse import urlparse, urlsplit import pip_shims.shims -from vistir.compat import Path -from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir +import six +import six.moves +import tomlkit +import vistir +from six.moves.urllib.parse import urlparse, urlsplit, urlunparse +from vistir.compat import Path, fs_decode +from vistir.path import ensure_mkdir_p, is_valid_url + +from .environment import MYPY_RUNNING + +# fmt: off +six.add_move( # type: ignore + six.MovedAttribute("Mapping", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("Sequence", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("Set", "collections", "collections.abc") # type: ignore +) # noqa # isort:skip +six.add_move( # type: ignore + six.MovedAttribute("ItemsView", "collections", "collections.abc") # type: ignore +) # noqa +from six.moves import ItemsView, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on + + +if MYPY_RUNNING: + from typing import Dict, Any, Optional, Union, Tuple, List, Iterable, Text, TypeVar + + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] VCS_LIST = ("git", "svn", "hg", "bzr") @@ -68,11 +89,12 @@ VCS_SCHEMES = [ def is_installable_dir(path): + # type: (STRING_TYPE) -> bool if pip_shims.shims.is_installable_dir(path): return True - path = Path(path) - pyproject = path.joinpath("pyproject.toml") - if pyproject.exists(): + pyproject_path = os.path.join(path, "pyproject.toml") + if os.path.exists(pyproject_path): + pyproject = Path(pyproject_path) pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: @@ -81,22 +103,39 @@ def is_installable_dir(path): def strip_ssh_from_git_uri(uri): + # type: (S) -> S """Return git+ssh:// formatted URI to git+git@ format""" if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+", 1) + if "git+ssh://" in uri: + parsed = urlparse(uri) + # split the path on the first separating / so we can put the first segment + # into the 'netloc' section with a : separator + path_part, _, path = parsed.path.lstrip("/").partition("/") + path = "/{0}".format(path) + parsed = parsed._replace( + netloc="{0}:{1}".format(parsed.netloc, path_part), path=path + ) + uri = urlunparse(parsed).replace("git+ssh://", "git+", 1) return uri def add_ssh_scheme_to_git_uri(uri): + # type: (S) -> S """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: uri = uri.replace("git+", "git+ssh://", 1) + parsed = urlparse(uri) + if ":" in parsed.netloc: + netloc, _, path_start = parsed.netloc.rpartition(":") + path = "/{0}{1}".format(path_start, parsed.path) + uri = urlunparse(parsed._replace(netloc=netloc, path=path)) return uri def is_vcs(pipfile_entry): + # type: (PipfileType) -> bool """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) @@ -111,34 +150,48 @@ def is_vcs(pipfile_entry): def is_editable(pipfile_entry): + # type: (PipfileType) -> bool if isinstance(pipfile_entry, Mapping): return pipfile_entry.get("editable", False) is True + if isinstance(pipfile_entry, six.string_types): + return pipfile_entry.startswith("-e ") return False -def multi_split(s, split): - """Splits on multiple given separators.""" - for r in split: - s = s.replace(r, "|") - return [i for i in s.split("|") if len(i) > 0] - - def is_star(val): + # type: (PipfileType) -> bool return (isinstance(val, six.string_types) and val == "*") or ( isinstance(val, Mapping) and val.get("version", "") == "*" ) +def convert_entry_to_path(path): + # type: (Dict[S, Union[S, bool, Tuple[S], List[S]]]) -> S + """Convert a pipfile entry to a string""" + + if not isinstance(path, Mapping): + raise TypeError("expecting a mapping, received {0!r}".format(path)) + + if not any(key in path for key in ["file", "path"]): + raise ValueError("missing path-like entry in supplied mapping {0!r}".format(path)) + + if "file" in path: + path = vistir.path.url_to_path(path["file"]) + + elif "path" in path: + path = path["path"] + if not os.name == "nt": + return fs_decode(path) + return Path(fs_decode(path)).as_posix() + + def is_installable_file(path): + # type: (PipfileType) -> bool """Determine if a path can potentially be installed""" from packaging import specifiers - if hasattr(path, "keys") and any( - key for key in path.keys() if key in ["file", "path"] - ): - path = urlparse(path["file"]).path if "file" in path else path["path"] - if not isinstance(path, six.string_types) or path == "*": - return False + if isinstance(path, Mapping): + path = convert_entry_to_path(path) # If the string starts with a valid specifier operator, test if it is a valid # specifier set before making a path object (to avoid breaking windows) @@ -152,41 +205,94 @@ def is_installable_file(path): return False parsed = urlparse(path) - if parsed.scheme == "file": - path = parsed.path - - if not os.path.exists(os.path.abspath(path)): + is_local = ( + not parsed.scheme + or parsed.scheme == "file" + or (len(parsed.scheme) == 1 and os.name == "nt") + ) + if parsed.scheme and parsed.scheme == "file": + path = vistir.compat.fs_decode(vistir.path.url_to_path(path)) + normalized_path = vistir.path.normalize_path(path) + if is_local and not os.path.exists(normalized_path): return False - lookup_path = Path(path) - absolute_path = "{0}".format(lookup_path.absolute()) - if lookup_path.is_dir() and is_installable_dir(absolute_path): + is_archive = pip_shims.shims.is_archive_file(normalized_path) + is_local_project = os.path.isdir(normalized_path) and is_installable_dir( + normalized_path + ) + if is_local and is_local_project or is_archive: return True - elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): + if not is_local and pip_shims.shims.is_archive_file(parsed.path): return True return False +def get_dist_metadata(dist): + import pkg_resources + from email.parser import FeedParser + + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata( + "METADATA" + ): + metadata = dist.get_metadata("METADATA") + elif dist.has_metadata("PKG-INFO"): + metadata = dist.get_metadata("PKG-INFO") + else: + metadata = "" + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() + + +def get_setup_paths(base_path, subdirectory=None): + # type: (S, Optional[S]) -> Dict[S, Optional[S]] + if base_path is None: + raise TypeError("must provide a path to derive setup paths from") + setup_py = os.path.join(base_path, "setup.py") + setup_cfg = os.path.join(base_path, "setup.cfg") + pyproject_toml = os.path.join(base_path, "pyproject.toml") + if subdirectory is not None: + base_path = os.path.join(base_path, subdirectory) + subdir_setup_py = os.path.join(subdirectory, "setup.py") + subdir_setup_cfg = os.path.join(subdirectory, "setup.cfg") + subdir_pyproject_toml = os.path.join(subdirectory, "pyproject.toml") + if subdirectory and os.path.exists(subdir_setup_py): + setup_py = subdir_setup_py + if subdirectory and os.path.exists(subdir_setup_cfg): + setup_cfg = subdir_setup_cfg + if subdirectory and os.path.exists(subdir_pyproject_toml): + pyproject_toml = subdir_pyproject_toml + return { + "setup_py": setup_py if os.path.exists(setup_py) else None, + "setup_cfg": setup_cfg if os.path.exists(setup_cfg) else None, + "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None, + } + + def prepare_pip_source_args(sources, pip_args=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[List[S]]) -> List[S] if pip_args is None: pip_args = [] if sources: # Add the source to pip9. - pip_args.extend(["-i", sources[0]["url"]]) + pip_args.extend(["-i", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) + pip_args.extend( + ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ) # type: ignore # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + pip_args.extend(["--extra-index-url", source["url"]]) # type: ignore # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( ["--trusted-host", urlparse(source["url"]).hostname] - ) + ) # type: ignore return pip_args @@ -195,29 +301,6 @@ def _ensure_dir(path): return path -@contextlib.contextmanager -def ensure_setup_py(base_dir): - if not base_dir: - base_dir = create_tracked_tempdir(prefix="requirementslib-setup") - base_dir = Path(base_dir) - if base_dir.exists() and base_dir.name == "setup.py": - base_dir = base_dir.parent - elif not (base_dir.exists() and base_dir.is_dir()): - base_dir = base_dir.parent - if not (base_dir.exists() and base_dir.is_dir()): - base_dir = base_dir.parent - setup_py = base_dir.joinpath("setup.py") - - is_new = False if setup_py.exists() else True - if not setup_py.exists(): - setup_py.write_text(u"") - try: - yield - finally: - if is_new: - setup_py.unlink() - - _UNSET = object() _REMAP_EXIT = object() @@ -325,9 +408,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError( - "%r object is not indexable" % type(cur).__name__ - ) + exc = TypeError("%r object is not indexable" % type(cur).__name__) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: diff --git a/pipenv/vendor/resolvelib/__init__.py b/pipenv/vendor/resolvelib/__init__.py index e0e37434..aaba5b3a 100644 --- a/pipenv/vendor/resolvelib/__init__.py +++ b/pipenv/vendor/resolvelib/__init__.py @@ -1,16 +1,26 @@ __all__ = [ - '__version__', - 'AbstractProvider', 'BaseReporter', 'Resolver', - 'NoVersionsAvailable', 'RequirementsConflicted', - 'ResolutionError', 'ResolutionImpossible', 'ResolutionTooDeep', + "__version__", + "AbstractProvider", + "AbstractResolver", + "BaseReporter", + "InconsistentCandidate", + "Resolver", + "RequirementsConflicted", + "ResolutionError", + "ResolutionImpossible", + "ResolutionTooDeep", ] -__version__ = '0.2.2' +__version__ = "0.3.0" -from .providers import AbstractProvider +from .providers import AbstractProvider, AbstractResolver from .reporters import BaseReporter from .resolvers import ( - NoVersionsAvailable, RequirementsConflicted, - Resolver, ResolutionError, ResolutionImpossible, ResolutionTooDeep, + InconsistentCandidate, + RequirementsConflicted, + Resolver, + ResolutionError, + ResolutionImpossible, + ResolutionTooDeep, ) diff --git a/pipenv/vendor/resolvelib/providers.py b/pipenv/vendor/resolvelib/providers.py index 515c0db4..db168219 100644 --- a/pipenv/vendor/resolvelib/providers.py +++ b/pipenv/vendor/resolvelib/providers.py @@ -1,6 +1,7 @@ class AbstractProvider(object): - """Delegate class to provide requirment interface for the resolver. + """Delegate class to provide requirement interface for the resolver. """ + def identify(self, dependency): """Given a dependency, return an identifier for it. @@ -56,15 +57,15 @@ class AbstractProvider(object): consulted to find concrete candidates for this requirement. The returned candidates should be sorted by reversed preference, e.g. - the latest should be LAST. This is done so list-popping can be as - efficient as possible. + the most preferred should be LAST. This is done so list-popping can be + as efficient as possible. """ raise NotImplementedError def is_satisfied_by(self, requirement, candidate): """Whether the given requirement can be satisfied by a candidate. - A boolean should be retuened to indicate whether `candidate` is a + A boolean should be returned to indicate whether `candidate` is a viable solution to the requirement. """ raise NotImplementedError @@ -76,3 +77,45 @@ class AbstractProvider(object): specifies as its dependencies. """ raise NotImplementedError + + +class AbstractResolver(object): + """The thing that performs the actual resolution work. + """ + + base_exception = Exception + + def __init__(self, provider, reporter): + self.provider = provider + self.reporter = reporter + + def resolve(self, requirements, **kwargs): + """Take a collection of constraints, spit out the resolution result. + + Parameters + ---------- + requirements : Collection + A collection of constraints + kwargs : optional + Additional keyword arguments that subclasses may accept. + + Raises + ------ + self.base_exception + Any raised exception is guaranteed to be a subclass of + self.base_exception. The string representation of an exception + should be human readable and provide context for why it occurred. + + Returns + ------- + retval : object + A representation of the final resolution state. It can be any object + with a `mapping` attribute that is a Mapping. Other attributes can + be used to provide resolver-specific information. + + The `mapping` attribute MUST be key-value pair is an identifier of a + requirement (as returned by the provider's `identify` method) mapped + to the resolved candidate (chosen from the return value of the + provider's `find_matches` method). + """ + raise NotImplementedError diff --git a/pipenv/vendor/resolvelib/reporters.py b/pipenv/vendor/resolvelib/reporters.py index c723031f..c7e9e88b 100644 --- a/pipenv/vendor/resolvelib/reporters.py +++ b/pipenv/vendor/resolvelib/reporters.py @@ -1,6 +1,7 @@ class BaseReporter(object): """Delegate class to provider progress reporting for the resolver. """ + def starting(self): """Called before the resolution actually starts. """ @@ -21,3 +22,15 @@ class BaseReporter(object): def ending(self, state): """Called before the resolution ends successfully. """ + + def adding_requirement(self, requirement): + """Called when the resolver adds a new requirement into the resolve criteria. + """ + + def backtracking(self, candidate): + """Called when the resolver rejects a candidate during backtracking. + """ + + def pinning(self, candidate): + """Called when adding a candidate to the potential solution. + """ diff --git a/pipenv/vendor/resolvelib/resolvers.py b/pipenv/vendor/resolvelib/resolvers.py index 9c69628e..b51d337d 100644 --- a/pipenv/vendor/resolvelib/resolvers.py +++ b/pipenv/vendor/resolvelib/resolvers.py @@ -1,52 +1,91 @@ import collections +from .providers import AbstractResolver from .structs import DirectedGraph -RequirementInformation = collections.namedtuple('RequirementInformation', [ - 'requirement', 'parent', -]) +RequirementInformation = collections.namedtuple( + "RequirementInformation", ["requirement", "parent"] +) -class NoVersionsAvailable(Exception): - def __init__(self, requirement, parent): - super(NoVersionsAvailable, self).__init__() - self.requirement = requirement - self.parent = parent +class ResolverException(Exception): + """A base class for all exceptions raised by this module. + + Exceptions derived by this class should all be handled in this module. Any + bubbling pass the resolver should be treated as a bug. + """ -class RequirementsConflicted(Exception): +class RequirementsConflicted(ResolverException): def __init__(self, criterion): - super(RequirementsConflicted, self).__init__() + super(RequirementsConflicted, self).__init__(criterion) self.criterion = criterion + def __str__(self): + return "Requirements conflict: {}".format( + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + + +class InconsistentCandidate(ResolverException): + def __init__(self, candidate, criterion): + super(InconsistentCandidate, self).__init__(candidate, criterion) + self.candidate = candidate + self.criterion = criterion + + def __str__(self): + return "Provided candidate {!r} does not satisfy {}".format( + self.candidate, + ", ".join(repr(r) for r in self.criterion.iter_requirement()), + ) + class Criterion(object): - """Internal representation of possible resolution results of a package. + """Representation of possible resolution results of a package. - This holds two attributes: + This holds three attributes: - * `information` is a collection of `RequirementInformation` pairs. Each - pair is a requirement contributing to this criterion, and the candidate - that provides the requirement. + * `information` is a collection of `RequirementInformation` pairs. + Each pair is a requirement contributing to this criterion, and the + candidate that provides the requirement. + * `incompatibilities` is a collection of all known not-to-work candidates + to exclude from consideration. * `candidates` is a collection containing all possible candidates deducted - from the union of contributing requirements. It should never be empty. + from the union of contributing requirements and known incompatibilities. + It should never be empty, except when the criterion is an attribute of a + raised `RequirementsConflicted` (in which case it is always empty). + + .. note:: + This class is intended to be externally immutable. **Do not** mutate + any of its attribute containers. """ - def __init__(self, candidates, information): + + def __init__(self, candidates, information, incompatibilities): self.candidates = candidates self.information = information + self.incompatibilities = incompatibilities + + def __repr__(self): + requirements = ", ".join( + "{!r} from {!r}".format(req, parent) + for req, parent in self.information + ) + return "<Criterion {}>".format(requirements) @classmethod def from_requirement(cls, provider, requirement, parent): """Build an instance from a requirement. """ candidates = provider.find_matches(requirement) - if not candidates: - raise NoVersionsAvailable(requirement, parent) - return cls( + criterion = cls( candidates=candidates, information=[RequirementInformation(requirement, parent)], + incompatibilities=[], ) + if not candidates: + raise RequirementsConflicted(criterion) + return criterion def iter_requirement(self): return (i.requirement for i in self.information) @@ -60,22 +99,38 @@ class Criterion(object): infos = list(self.information) infos.append(RequirementInformation(requirement, parent)) candidates = [ - c for c in self.candidates + c + for c in self.candidates if provider.is_satisfied_by(requirement, c) ] + criterion = type(self)(candidates, infos, list(self.incompatibilities)) if not candidates: - raise RequirementsConflicted(self) - return type(self)(candidates, infos) + raise RequirementsConflicted(criterion) + return criterion + + def excluded_of(self, candidate): + """Build a new instance from this, but excluding specified candidate. + + Returns the new instance, or None if we still have no valid candidates. + """ + incompats = list(self.incompatibilities) + incompats.append(candidate) + candidates = [c for c in self.candidates if c != candidate] + if not candidates: + return None + criterion = type(self)(candidates, list(self.information), incompats) + return criterion -class ResolutionError(Exception): +class ResolutionError(ResolverException): pass class ResolutionImpossible(ResolutionError): - def __init__(self, requirements): - super(ResolutionImpossible, self).__init__() - self.requirements = requirements + def __init__(self, causes): + super(ResolutionImpossible, self).__init__(causes) + # causes is a list of RequirementInformation objects + self.causes = causes class ResolutionTooDeep(ResolutionError): @@ -85,7 +140,7 @@ class ResolutionTooDeep(ResolutionError): # Resolution state in a round. -State = collections.namedtuple('State', 'mapping graph') +State = collections.namedtuple("State", "mapping criteria") class Resolution(object): @@ -94,10 +149,10 @@ class Resolution(object): This is designed as a one-off object that holds information to kick start the resolution process, and holds the results afterwards. """ + def __init__(self, provider, reporter): self._p = provider self._r = reporter - self._criteria = {} self._states = [] @property @@ -105,7 +160,7 @@ class Resolution(object): try: return self._states[-1] except IndexError: - raise AttributeError('state') + raise AttributeError("state") def _push_new_state(self): """Push a new state into history. @@ -116,30 +171,29 @@ class Resolution(object): try: base = self._states[-1] except IndexError: - graph = DirectedGraph() - graph.add(None) # Sentinel as root dependencies' parent. - state = State(mapping={}, graph=graph) + state = State(mapping=collections.OrderedDict(), criteria={}) else: state = State( - mapping=base.mapping.copy(), - graph=base.graph.copy(), + mapping=base.mapping.copy(), criteria=base.criteria.copy(), ) self._states.append(state) - def _contribute_to_criteria(self, name, requirement, parent): + def _merge_into_criterion(self, requirement, parent): + self._r.adding_requirement(requirement) + name = self._p.identify(requirement) try: - crit = self._criteria[name] + crit = self.state.criteria[name] except KeyError: crit = Criterion.from_requirement(self._p, requirement, parent) else: crit = crit.merged_with(self._p, requirement, parent) - self._criteria[name] = crit + return name, crit def _get_criterion_item_preference(self, item): name, criterion = item try: pinned = self.state.mapping[name] - except (IndexError, KeyError): + except KeyError: pinned = None return self._p.get_preference( pinned, criterion.candidates, criterion.information, @@ -155,114 +209,183 @@ class Resolution(object): for r in criterion.iter_requirement() ) - def _check_pinnability(self, candidate, dependencies): - backup = self._criteria.copy() - contributed = set() - try: - for subdep in dependencies: - key = self._p.identify(subdep) - self._contribute_to_criteria(key, subdep, parent=candidate) - contributed.add(key) - except RequirementsConflicted: - self._criteria = backup - return None - return contributed + def _get_criteria_to_update(self, candidate): + criteria = {} + for r in self._p.get_dependencies(candidate): + name, crit = self._merge_into_criterion(r, parent=candidate) + criteria[name] = crit + return criteria - def _pin_candidate(self, name, criterion, candidate, child_names): - try: - self.state.graph.remove(name) - except KeyError: - pass - self.state.mapping[name] = candidate - self.state.graph.add(name) - for parent in criterion.iter_parent(): - parent_name = None if parent is None else self._p.identify(parent) + def _attempt_to_pin_criterion(self, name, criterion): + causes = [] + for candidate in reversed(criterion.candidates): try: - self.state.graph.connect(parent_name, name) - except KeyError: - # Parent is not yet pinned. Skip now; this edge will be - # connected when the parent is being pinned. - pass - for child_name in child_names: - try: - self.state.graph.connect(name, child_name) - except KeyError: - # Child is not yet pinned. Skip now; this edge will be - # connected when the child is being pinned. - pass - - def _pin_criteria(self): - criterion_names = [name for name, _ in sorted( - self._criteria.items(), - key=self._get_criterion_item_preference, - )] - for name in criterion_names: - # Any pin may modify any criterion during the loop. Criteria are - # replaced, not updated in-place, so we need to read this value - # in the loop instead of outside. (sarugaku/resolvelib#5) - criterion = self._criteria[name] - - if self._is_current_pin_satisfying(name, criterion): - # If the current pin already works, just use it. + criteria = self._get_criteria_to_update(candidate) + except RequirementsConflicted as e: + causes.append(e.criterion) continue - candidates = list(criterion.candidates) - while candidates: - candidate = candidates.pop() - dependencies = self._p.get_dependencies(candidate) - child_names = self._check_pinnability(candidate, dependencies) - if child_names is None: - continue - self._pin_candidate(name, criterion, candidate, child_names) - break - else: # All candidates tried, nothing works. Give up. (?) - raise ResolutionImpossible(list(criterion.iter_requirement())) + + # Put newly-pinned candidate at the end. This is essential because + # backtracking looks at this mapping to get the last pin. + self._r.pinning(candidate) + self.state.mapping.pop(name, None) + self.state.mapping[name] = candidate + self.state.criteria.update(criteria) + + # Check the newly-pinned candidate actually works. This should + # always pass under normal circumstances, but in the case of a + # faulty provider, we will raise an error to notify the implementer + # to fix find_matches() and/or is_satisfied_by(). + if not self._is_current_pin_satisfying(name, criterion): + raise InconsistentCandidate(candidate, criterion) + + return [] + + # All candidates tried, nothing works. This criterion is a dead + # end, signal for backtracking. + return causes + + def _backtrack(self): + # We need at least 3 states here: + # (a) One known not working, to drop. + # (b) One to backtrack to. + # (c) One to restore state (b) to its state prior to candidate-pinning, + # so we can pin another one instead. + while len(self._states) >= 3: + del self._states[-1] + + # Retract the last candidate pin, and create a new (b). + name, candidate = self._states.pop().mapping.popitem() + self._r.backtracking(candidate) + self._push_new_state() + + # Mark the retracted candidate as incompatible. + criterion = self.state.criteria[name].excluded_of(candidate) + if criterion is None: + # This state still does not work. Try the still previous state. + continue + self.state.criteria[name] = criterion + + return True + + return False def resolve(self, requirements, max_rounds): if self._states: - raise RuntimeError('already resolved') + raise RuntimeError("already resolved") - for requirement in requirements: + self._push_new_state() + for r in requirements: try: - name = self._p.identify(requirement) - self._contribute_to_criteria(name, requirement, parent=None) + name, crit = self._merge_into_criterion(r, parent=None) except RequirementsConflicted as e: - # If initial requirements conflict, nothing would ever work. - raise ResolutionImpossible(e.requirements + [requirement]) + raise ResolutionImpossible(e.criterion.information) + self.state.criteria[name] = crit - last = None self._r.starting() for round_index in range(max_rounds): self._r.starting_round(round_index) self._push_new_state() - self._pin_criteria() - curr = self.state - if last is not None and len(curr.mapping) == len(last.mapping): - # Nothing new added. Done! Remove the duplicated entry. + + unsatisfied_criterion_items = [ + item + for item in self.state.criteria.items() + if not self._is_current_pin_satisfying(*item) + ] + + # All criteria are accounted for. Nothing more to pin, we are done! + if not unsatisfied_criterion_items: del self._states[-1] - self._r.ending(last) - return - last = curr + self._r.ending(curr) + return self.state + + # Choose the most preferred unpinned criterion to try. + name, criterion = min( + unsatisfied_criterion_items, + key=self._get_criterion_item_preference, + ) + failure_causes = self._attempt_to_pin_criterion(name, criterion) + + # Backtrack if pinning fails. + if failure_causes: + result = self._backtrack() + if not result: + causes = [ + i for crit in failure_causes for i in crit.information + ] + raise ResolutionImpossible(causes) self._r.ending_round(round_index, curr) raise ResolutionTooDeep(max_rounds) -class Resolver(object): +def _has_route_to_root(criteria, key, all_keys, connected): + if key in connected: + return True + if key not in criteria: + return False + for p in criteria[key].iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey in connected: + connected.add(key) + return True + if _has_route_to_root(criteria, pkey, all_keys, connected): + connected.add(key) + return True + return False + + +Result = collections.namedtuple("Result", "mapping graph criteria") + + +def _build_result(state): + mapping = state.mapping + all_keys = {id(v): k for k, v in mapping.items()} + all_keys[id(None)] = None + + graph = DirectedGraph() + graph.add(None) # Sentinel as root dependencies' parent. + + connected = {None} + for key, criterion in state.criteria.items(): + if not _has_route_to_root(state.criteria, key, all_keys, connected): + continue + if key not in graph: + graph.add(key) + for p in criterion.iter_parent(): + try: + pkey = all_keys[id(p)] + except KeyError: + continue + if pkey not in graph: + graph.add(pkey) + graph.connect(pkey, key) + + return Result( + mapping={k: v for k, v in mapping.items() if k in connected}, + graph=graph, + criteria=state.criteria, + ) + + +class Resolver(AbstractResolver): """The thing that performs the actual resolution work. """ - def __init__(self, provider, reporter): - self.provider = provider - self.reporter = reporter - def resolve(self, requirements, max_rounds=20): + base_exception = ResolverException + + def resolve(self, requirements, max_rounds=100): """Take a collection of constraints, spit out the resolution result. The return value is a representation to the final resolution result. It - is a tuple subclass with two public members: + is a tuple subclass with three public members: * `mapping`: A dict of resolved candidates. Each key is an identifier of a requirement (as returned by the provider's `identify` method), @@ -271,17 +394,21 @@ class Resolver(object): The vertices are keys of `mapping`, and each edge represents *why* a particular package is included. A special vertex `None` is included to represent parents of user-supplied requirements. + * `criteria`: A dict of "criteria" that hold detailed information on + how edges in the graph are derived. Each key is an identifier of a + requirement, and the value is a `Criterion` instance. The following exceptions may be raised if a resolution cannot be found: - * `NoVersionsAvailable`: A requirement has no available candidates. * `ResolutionImpossible`: A resolution cannot be found for the given - combination of requirements. + combination of requirements. The `causes` attribute of the + exception is a list of (requirement, parent), giving the + requirements that could not be satisfied. * `ResolutionTooDeep`: The dependency tree is too deeply nested and the resolver gave up. This is usually caused by a circular dependency, but you can try to resolve this by increasing the `max_rounds` argument. """ resolution = Resolution(self.provider, self.reporter) - resolution.resolve(requirements, max_rounds=max_rounds) - return resolution.state + state = resolution.resolve(requirements, max_rounds=max_rounds) + return _build_result(state) diff --git a/pipenv/vendor/resolvelib/structs.py b/pipenv/vendor/resolvelib/structs.py index 97bd0095..1eee08b3 100644 --- a/pipenv/vendor/resolvelib/structs.py +++ b/pipenv/vendor/resolvelib/structs.py @@ -1,10 +1,11 @@ class DirectedGraph(object): """A graph structure with directed edges. """ + def __init__(self): self._vertices = set() - self._forwards = {} # <key> -> Set[<key>] - self._backwards = {} # <key> -> Set[<key>] + self._forwards = {} # <key> -> Set[<key>] + self._backwards = {} # <key> -> Set[<key>] def __iter__(self): return iter(self._vertices) @@ -28,7 +29,7 @@ class DirectedGraph(object): """Add a new vertex to the graph. """ if key in self._vertices: - raise ValueError('vertex exists') + raise ValueError("vertex exists") self._vertices.add(key) self._forwards[key] = set() self._backwards[key] = set() diff --git a/pipenv/vendor/scandir.py b/pipenv/vendor/scandir.py index 8bbae2c5..44f949fd 100644 --- a/pipenv/vendor/scandir.py +++ b/pipenv/vendor/scandir.py @@ -37,7 +37,7 @@ if _scandir is None and ctypes is None: warnings.warn("scandir can't find the compiled _scandir C module " "or ctypes, using slow generic fallback") -__version__ = '1.9.0' +__version__ = '1.10.0' __all__ = ['scandir', 'walk'] # Windows FILE_ATTRIBUTE constants for interpreting the @@ -583,7 +583,7 @@ elif sys.platform.startswith(('linux', 'darwin', 'sunos5')) or 'bsd' in sys.plat if _scandir is not None: scandir = scandir_c DirEntry = DirEntry_c - elif ctypes is not None: + elif ctypes is not None and have_dirent_d_type: scandir = scandir_python DirEntry = PosixDirEntry else: diff --git a/pipenv/vendor/semver.py b/pipenv/vendor/semver.py index 5f5be2c2..6bc9fcab 100644 --- a/pipenv/vendor/semver.py +++ b/pipenv/vendor/semver.py @@ -1,12 +1,16 @@ """ Python helper for Semantic Versioning (http://semver.org/) """ +from __future__ import print_function +import argparse import collections +from functools import wraps import re +import sys -__version__ = '2.8.1' +__version__ = '2.9.0' __author__ = 'Kostiantyn Rybnikov' __author_email__ = 'k-bx@k-bx.com' __maintainer__ = 'Sebastien Celles' @@ -33,6 +37,10 @@ _REGEX = re.compile( _LAST_NUMBER = re.compile(r'(?:[^\d]*(\d+)[^\d]*)+') +#: Contains the implemented semver.org version of the spec +SEMVER_SPEC_VERSION = "2.0.0" + + if not hasattr(__builtins__, 'cmp'): def cmp(a, b): return (a > b) - (a < b) @@ -47,7 +55,6 @@ def parse(version): if not provided :rtype: dict - >>> import semver >>> ver = semver.parse('3.4.5-pre.2+build.4') >>> ver['major'] 3 @@ -73,6 +80,18 @@ def parse(version): return version_parts +def comparator(operator): + """ Wrap a VersionInfo binary op method in a type-check """ + @wraps(operator) + def wrapper(self, other): + comparable_types = (VersionInfo, dict, tuple) + if not isinstance(other, comparable_types): + raise TypeError("other type %r must be in %r" + % (type(other), comparable_types)) + return operator(self, other) + return wrapper + + class VersionInfo(object): """ :param int major: version when you make incompatible API changes. @@ -84,33 +103,58 @@ class VersionInfo(object): """ __slots__ = ('_major', '_minor', '_patch', '_prerelease', '_build') - def __init__(self, major, minor, patch, prerelease=None, build=None): - self._major = major - self._minor = minor - self._patch = patch - self._prerelease = prerelease - self._build = build + def __init__(self, major, minor=0, patch=0, prerelease=None, build=None): + self._major = int(major) + self._minor = int(minor) + self._patch = int(patch) + self._prerelease = None if prerelease is None else str(prerelease) + self._build = None if build is None else str(build) @property def major(self): + """The major part of a version""" return self._major + @major.setter + def major(self, value): + raise AttributeError("attribute 'major' is readonly") + @property def minor(self): + """The minor part of a version""" return self._minor + @minor.setter + def minor(self, value): + raise AttributeError("attribute 'minor' is readonly") + @property def patch(self): + """The patch part of a version""" return self._patch + @patch.setter + def patch(self, value): + raise AttributeError("attribute 'patch' is readonly") + @property def prerelease(self): + """The prerelease part of a version""" return self._prerelease + @prerelease.setter + def prerelease(self, value): + raise AttributeError("attribute 'prerelease' is readonly") + @property def build(self): + """The build part of a version""" return self._build + @build.setter + def build(self, value): + raise AttributeError("attribute 'build' is readonly") + def _astuple(self): return (self.major, self.minor, self.patch, self.prerelease, self.build) @@ -124,40 +168,109 @@ class VersionInfo(object): ("build", self.build) )) + def __iter__(self): + """Implement iter(self).""" + # As long as we support Py2.7, we can't use the "yield from" syntax + for v in self._astuple(): + yield v + + def bump_major(self): + """Raise the major part of the version, return a new object + but leave self untouched + + :return: new object with the raised major part + :rtype: VersionInfo + + >>> ver = semver.parse_version_info("3.4.5") + >>> ver.bump_major() + VersionInfo(major=4, minor=0, patch=0, prerelease=None, build=None) + """ + return parse_version_info(bump_major(str(self))) + + def bump_minor(self): + """Raise the minor part of the version, return a new object + but leave self untouched + + :return: new object with the raised minor part + :rtype: VersionInfo + + >>> ver = semver.parse_version_info("3.4.5") + >>> ver.bump_minor() + VersionInfo(major=3, minor=5, patch=0, prerelease=None, build=None) + """ + return parse_version_info(bump_minor(str(self))) + + def bump_patch(self): + """Raise the patch part of the version, return a new object + but leave self untouched + + :return: new object with the raised patch part + :rtype: VersionInfo + + >>> ver = semver.parse_version_info("3.4.5") + >>> ver.bump_patch() + VersionInfo(major=3, minor=4, patch=6, prerelease=None, build=None) + """ + return parse_version_info(bump_patch(str(self))) + + def bump_prerelease(self, token='rc'): + """Raise the prerelease part of the version, return a new object + but leave self untouched + + :param token: defaults to 'rc' + :return: new object with the raised prerelease part + :rtype: str + + >>> ver = semver.parse_version_info("3.4.5-rc.1") + >>> ver.bump_prerelease() + VersionInfo(major=3, minor=4, patch=5, prerelease='rc.2', \ +build=None) + """ + return parse_version_info(bump_prerelease(str(self), token)) + + def bump_build(self, token='build'): + """Raise the build part of the version, return a new object + but leave self untouched + + :param token: defaults to 'build' + :return: new object with the raised build part + :rtype: str + + >>> ver = semver.parse_version_info("3.4.5-rc.1+build.9") + >>> ver.bump_build() + VersionInfo(major=3, minor=4, patch=5, prerelease='rc.1', \ +build='build.10') + """ + return parse_version_info(bump_build(str(self), token)) + + @comparator def __eq__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) == 0 + @comparator def __ne__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) != 0 + @comparator def __lt__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) < 0 + @comparator def __le__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) <= 0 + @comparator def __gt__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) > 0 + @comparator def __ge__(self, other): - if not isinstance(other, (VersionInfo, dict)): - return NotImplemented return _compare_by_keys(self._asdict(), _to_dict(other)) >= 0 def __repr__(self): s = ", ".join("%s=%r" % (key, val) for key, val in self._asdict().items()) - return "VersionInfo(%s)" % s + return "%s(%s)" % (type(self).__name__, s) def __str__(self): return format_version(*(self._astuple())) @@ -169,21 +282,44 @@ class VersionInfo(object): def parse(version): """Parse version string to a VersionInfo instance. - >>> from semver import VersionInfo - >>> VersionInfo.parse('3.4.5-pre.2+build.4') + :param version: version string + :return: a :class:`semver.VersionInfo` instance + :rtype: :class:`semver.VersionInfo` + + >>> semver.VersionInfo.parse('3.4.5-pre.2+build.4') VersionInfo(major=3, minor=4, patch=5, \ prerelease='pre.2', build='build.4') - - :param version: version string - :return: a :class:`VersionInfo` instance - :rtype: :class:`VersionInfo` """ return parse_version_info(version) + def replace(self, **parts): + """Replace one or more parts of a version and return a new + :class:`semver.VersionInfo` object, but leave self untouched + + :param dict parts: the parts to be updated. Valid keys are: + ``major``, ``minor``, ``patch``, ``prerelease``, or ``build`` + :return: the new :class:`semver.VersionInfo` object with the changed + parts + :raises: TypeError, if ``parts`` contains invalid keys + """ + version = self._asdict() + version.update(parts) + try: + return VersionInfo(**version) + except TypeError: + unknownkeys = set(parts) - set(self._asdict()) + error = ("replace() got %d unexpected keyword " + "argument(s): %s" % (len(unknownkeys), + ", ".join(unknownkeys)) + ) + raise TypeError(error) + def _to_dict(obj): if isinstance(obj, VersionInfo): return obj._asdict() + elif isinstance(obj, tuple): + return VersionInfo(*obj)._asdict() return obj @@ -194,7 +330,6 @@ def parse_version_info(version): :return: a :class:`VersionInfo` instance :rtype: :class:`VersionInfo` - >>> import semver >>> version_info = semver.parse_version_info("3.4.5-pre.2+build.4") >>> version_info.major 3 @@ -270,7 +405,6 @@ def compare(ver1, ver2): zero if ver1 == ver2 and strictly positive if ver1 > ver2 :rtype: int - >>> import semver >>> semver.compare("1.0.0", "2.0.0") -1 >>> semver.compare("2.0.0", "1.0.0") @@ -298,7 +432,6 @@ def match(version, match_expr): :return: True if the expression matches the version, otherwise False :rtype: bool - >>> import semver >>> semver.match("2.0.0", ">=1.0.0") True >>> semver.match("1.0.0", ">1.0.0") @@ -339,7 +472,6 @@ def max_ver(ver1, ver2): :return: the greater version of the two :rtype: :class:`VersionInfo` - >>> import semver >>> semver.max_ver("1.0.0", "2.0.0") '2.0.0' """ @@ -358,7 +490,6 @@ def min_ver(ver1, ver2): :return: the smaller version of the two :rtype: :class:`VersionInfo` - >>> import semver >>> semver.min_ver("1.0.0", "2.0.0") '1.0.0' """ @@ -372,15 +503,14 @@ def min_ver(ver1, ver2): def format_version(major, minor, patch, prerelease=None, build=None): """Format a version according to the Semantic Versioning specification - :param str major: the required major part of a version - :param str minor: the required minor part of a version - :param str patch: the required patch part of a version + :param int major: the required major part of a version + :param int minor: the required minor part of a version + :param int patch: the required patch part of a version :param str prerelease: the optional prerelease part of a version :param str build: the optional build part of a version :return: the formatted string :rtype: str - >>> import semver >>> semver.format_version(3, 4, 5, 'pre.2', 'build.4') '3.4.5-pre.2+build.4' """ @@ -414,7 +544,6 @@ def bump_major(version): :return: the raised version string :rtype: str - >>> import semver >>> semver.bump_major("3.4.5") '4.0.0' """ @@ -429,7 +558,6 @@ def bump_minor(version): :return: the raised version string :rtype: str - >>> import semver >>> semver.bump_minor("3.4.5") '3.5.0' """ @@ -444,7 +572,6 @@ def bump_patch(version): :return: the raised version string :rtype: str - >>> import semver >>> semver.bump_patch("3.4.5") '3.4.6' """ @@ -461,7 +588,7 @@ def bump_prerelease(version, token='rc'): :return: the raised version string :rtype: str - >>> bump_prerelease('3.4.5', 'dev') + >>> semver.bump_prerelease('3.4.5', 'dev') '3.4.5-dev.1' """ verinfo = parse(version) @@ -480,7 +607,7 @@ def bump_build(version, token='build'): :return: the raised version string :rtype: str - >>> bump_build('3.4.5-rc.1+build.9') + >>> semver.bump_build('3.4.5-rc.1+build.9') '3.4.5-rc.1+build.10' """ verinfo = parse(version) @@ -498,13 +625,125 @@ def finalize_version(version): :return: the finalized version string :rtype: str - >>> finalize_version('1.2.3-rc.5') + >>> semver.finalize_version('1.2.3-rc.5') '1.2.3' """ verinfo = parse(version) return format_version(verinfo['major'], verinfo['minor'], verinfo['patch']) +def createparser(): + """Create an :class:`argparse.ArgumentParser` instance + + :return: parser instance + :rtype: :class:`argparse.ArgumentParser` + """ + parser = argparse.ArgumentParser(prog=__package__, + description=__doc__) + s = parser.add_subparsers() + + # create compare subcommand + parser_compare = s.add_parser("compare", + help="Compare two versions" + ) + parser_compare.set_defaults(which="compare") + parser_compare.add_argument("version1", + help="First version" + ) + parser_compare.add_argument("version2", + help="Second version" + ) + + # create bump subcommand + parser_bump = s.add_parser("bump", + help="Bumps a version" + ) + parser_bump.set_defaults(which="bump") + sb = parser_bump.add_subparsers(title="Bump commands", + dest="bump") + + # Create subparsers for the bump subparser: + for p in (sb.add_parser("major", + help="Bump the major part of the version"), + sb.add_parser("minor", + help="Bump the minor part of the version"), + sb.add_parser("patch", + help="Bump the patch part of the version"), + sb.add_parser("prerelease", + help="Bump the prerelease part of the version"), + sb.add_parser("build", + help="Bump the build part of the version")): + p.add_argument("version", + help="Version to raise" + ) + + return parser + + +def process(args): + """Process the input from the CLI + + :param args: The parsed arguments + :type args: :class:`argparse.Namespace` + :param parser: the parser instance + :type parser: :class:`argparse.ArgumentParser` + :return: result of the selected action + :rtype: str + """ + if args.which == "bump": + maptable = {'major': 'bump_major', + 'minor': 'bump_minor', + 'patch': 'bump_patch', + 'prerelease': 'bump_prerelease', + 'build': 'bump_build', + } + ver = parse_version_info(args.version) + # get the respective method and call it + func = getattr(ver, maptable[args.bump]) + return str(func()) + + elif args.which == "compare": + return str(compare(args.version1, args.version2)) + + +def main(cliargs=None): + """Entry point for the application script + + :param list cliargs: Arguments to parse or None (=use :class:`sys.argv`) + :return: error code + :rtype: int + """ + try: + parser = createparser() + args = parser.parse_args(args=cliargs) + # args.parser = parser + result = process(args) + print(result) + return 0 + + except (ValueError, TypeError) as err: + print("ERROR", err, file=sys.stderr) + return 2 + + +def replace(version, **parts): + """Replace one or more parts of a version and return the new string + + :param str version: the version string to replace + :param dict parts: the parts to be updated. Valid keys are: + ``major``, ``minor``, ``patch``, ``prerelease``, or ``build`` + :return: the replaced version string + :raises: TypeError, if ``parts`` contains invalid keys + :rtype: str + + >>> import semver + >>> semver.replace("1.2.3", major=2, patch=10) + '2.2.10' + """ + version = parse_version_info(version) + return str(version.replace(**parts)) + + if __name__ == "__main__": import doctest doctest.testmod() diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 576c4224..2e7c0b79 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.7' +__version__ = '1.3.2' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/_core.py b/pipenv/vendor/shellingham/_core.py index fb988eb3..da103a94 100644 --- a/pipenv/vendor/shellingham/_core.py +++ b/pipenv/vendor/shellingham/_core.py @@ -1,5 +1,5 @@ SHELL_NAMES = { - 'sh', 'bash', 'dash', # Bourne. + 'sh', 'bash', 'dash', 'ash', # Bourne. 'csh', 'tcsh', # C. 'ksh', 'zsh', 'fish', # Common alternatives. 'cmd', 'powershell', 'pwsh', # Microsoft. diff --git a/pipenv/vendor/shellingham/nt.py b/pipenv/vendor/shellingham/nt.py index 7b3cc6b4..d66bc33f 100644 --- a/pipenv/vendor/shellingham/nt.py +++ b/pipenv/vendor/shellingham/nt.py @@ -75,7 +75,18 @@ def _iter_process(): # looking for. We can fix this when it actually matters. (#8) continue raise WinError() - info = {'executable': str(pe.szExeFile.decode('utf-8'))} + + # The executable name would be encoded with the current code page if + # we're in ANSI mode (usually). Try to decode it into str/unicode, + # replacing invalid characters to be safe (not thoeratically necessary, + # I think). Note that we need to use 'mbcs' instead of encoding + # settings from sys because this is from the Windows API, not Python + # internals (which those settings reflect). (pypa/pipenv#3382) + executable = pe.szExeFile + if isinstance(executable, bytes): + executable = executable.decode('mbcs', 'replace') + + info = {'executable': executable} if pe.th32ParentProcessID: info['parent_pid'] = pe.th32ParentProcessID yield pe.th32ProcessID, info diff --git a/pipenv/vendor/shellingham/posix/__init__.py b/pipenv/vendor/shellingham/posix/__init__.py index 923032b6..164bbc1d 100644 --- a/pipenv/vendor/shellingham/posix/__init__.py +++ b/pipenv/vendor/shellingham/posix/__init__.py @@ -1,4 +1,5 @@ import os +import re from .._core import SHELL_NAMES, ShellDetectionFailure from . import proc, ps @@ -19,20 +20,16 @@ def _get_process_mapping(): raise ShellDetectionFailure('compatible proc fs or ps utility is required') -def _iter_process_command(mapping, pid, max_depth): - """Iterator to traverse up the tree, yielding `argv[0]` of each process. +def _iter_process_args(mapping, pid, max_depth): + """Iterator to traverse up the tree, yielding each process's argument list. """ for _ in range(max_depth): try: proc = mapping[pid] except KeyError: # We've reached the root process. Give up. break - try: - cmd = proc.args[0] - except IndexError: # Process has no name? Whatever, ignore it. - pass - else: - yield cmd + if proc.args: # Persumably the process should always have a name? + yield proc.args pid = proc.ppid # Go up one level. @@ -47,15 +44,50 @@ def _get_login_shell(proc_cmd): return (os.path.basename(proc_cmd).lower(), proc_cmd) +_INTERPRETER_SHELL_NAMES = [ + (re.compile(r'^python(\d+(\.\d+)?)?$'), {'xonsh'}), +] + + +def _get_interpreter_shell(proc_name, proc_args): + """Get shell invoked via an interpreter. + + Some shells are implemented on, and invoked with an interpreter, e.g. xonsh + is commonly executed with an executable Python script. This detects what + script the interpreter is actually running, and check whether that looks + like a shell. + + See sarugaku/shellingham#26 for rational. + """ + for pattern, shell_names in _INTERPRETER_SHELL_NAMES: + if not pattern.match(proc_name): + continue + for arg in proc_args: + name = os.path.basename(arg).lower() + if os.path.isfile(arg) and name in shell_names: + return (name, arg) + return None + + +def _get_shell(cmd, *args): + if cmd.startswith('-'): # Login shell! Let's use this. + return _get_login_shell(cmd) + name = os.path.basename(cmd).lower() + if name in SHELL_NAMES: # Command looks like a shell. + return (name, cmd) + shell = _get_interpreter_shell(name, args) + if shell: + return shell + return None + + def get_shell(pid=None, max_depth=6): """Get the shell that the supplied pid or os.getpid() is running in. """ pid = str(pid or os.getpid()) mapping = _get_process_mapping() - for proc_cmd in _iter_process_command(mapping, pid, max_depth): - if proc_cmd.startswith('-'): # Login shell! Let's use this. - return _get_login_shell(proc_cmd) - name = os.path.basename(proc_cmd).lower() - if name in SHELL_NAMES: # The inner-most (non-login) shell. - return (name, proc_cmd) + for proc_args in _iter_process_args(mapping, pid, max_depth): + shell = _get_shell(*proc_args) + if shell: + return shell return None diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index 4a155ed5..cf7e56f2 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -1,5 +1,4 @@ import errno -import shlex import subprocess import sys @@ -34,9 +33,13 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) + # XXX: This is not right, but we are really out of options. + # ps does not offer a sane way to decode the argument display, + # and this is "Good Enough" for obtaining shell names. Hopefully + # people don't name their shell with a space, or have something + # like "/usr/bin/xonsh is uber". (sarugaku/shellingham#14) + args = tuple(a.strip() for a in args.split(' ')) except ValueError: continue + processes[pid] = Process(args=args, pid=pid, ppid=ppid) return processes diff --git a/pipenv/vendor/shutilwhich/LICENSE b/pipenv/vendor/shutilwhich/LICENSE deleted file mode 100644 index 41ec01db..00000000 --- a/pipenv/vendor/shutilwhich/LICENSE +++ /dev/null @@ -1,292 +0,0 @@ -shutilwhich contains code from Python 3.3 and uses the same license: - -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see http://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see http://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations (now Zope -Corporation, see http://www.zope.com). In 2001, the Python Software -Foundation (PSF, see http://www.python.org/psf/) was formed, a -non-profit organization created specifically to own Python-related -Intellectual Property. Zope Corporation is a sponsoring member of -the PSF. - -All Python releases are Open Source (see http://www.opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.2 2.1.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2.1 2.2 2002 PSF yes - 2.2.2 2.2.1 2002 PSF yes - 2.2.3 2.2.2 2003 PSF yes - 2.3 2.2.2 2002-2003 PSF yes - 2.3.1 2.3 2002-2003 PSF yes - 2.3.2 2.3.1 2002-2003 PSF yes - 2.3.3 2.3.2 2002-2003 PSF yes - 2.3.4 2.3.3 2004 PSF yes - 2.3.5 2.3.4 2005 PSF yes - 2.4 2.3 2004 PSF yes - 2.4.1 2.4 2005 PSF yes - 2.4.2 2.4.1 2005 PSF yes - 2.4.3 2.4.2 2006 PSF yes - 2.4.4 2.4.3 2006 PSF yes - 2.5 2.4 2006 PSF yes - 2.5.1 2.5 2007 PSF yes - 2.5.2 2.5.1 2008 PSF yes - 2.5.3 2.5.2 2008 PSF yes - 2.6 2.5 2008 PSF yes - 2.6.1 2.6 2008 PSF yes - 2.6.2 2.6.1 2009 PSF yes - 2.6.3 2.6.2 2009 PSF yes - 2.6.4 2.6.3 2009 PSF yes - 2.6.5 2.6.4 2010 PSF yes - 3.0 2.6 2008 PSF yes - 3.0.1 3.0 2009 PSF yes - 3.1 3.0.1 2009 PSF yes - 3.1.1 3.1 2009 PSF yes - 3.1.2 3.1.1 2010 PSF yes - 3.1.3 3.1.2 2010 PSF yes - 3.1.4 3.1.3 2011 PSF yes - 3.2 3.1 2011 PSF yes - 3.2.1 3.2 2011 PSF yes - 3.2.2 3.2.1 2011 PSF yes - 3.2.3 3.2.2 2012 PSF yes - 3.3.0 3.2 2012 PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012 Python Software Foundation; All Rights Reserved" are retained in Python -alone or in any derivative version prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the Internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the Internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/pipenv/vendor/shutilwhich/__init__.py b/pipenv/vendor/shutilwhich/__init__.py deleted file mode 100644 index 71528812..00000000 --- a/pipenv/vendor/shutilwhich/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- - -__version__ = '1.1.0' - -import shutil - -if not hasattr(shutil, 'which'): - from .lib import which - shutil.which = which -else: - from shutil import which diff --git a/pipenv/vendor/shutilwhich/lib.py b/pipenv/vendor/shutilwhich/lib.py deleted file mode 100644 index 6451083a..00000000 --- a/pipenv/vendor/shutilwhich/lib.py +++ /dev/null @@ -1,58 +0,0 @@ -import os -import sys - - -# Everything below this point has been copied verbatim from the Python-3.3 -# sources. -def which(cmd, mode=os.F_OK | os.X_OK, path=None): - """Given a command, mode, and a PATH string, return the path which - conforms to the given mode on the PATH, or None if there is no such - file. - - `mode` defaults to os.F_OK | os.X_OK. `path` defaults to the result - of os.environ.get("PATH"), or can be overridden with a custom search - path. - - """ - # Check that a given file can be accessed with the correct mode. - # Additionally check that `file` is not a directory, as on Windows - # directories pass the os.access check. - def _access_check(fn, mode): - return (os.path.exists(fn) and os.access(fn, mode) - and not os.path.isdir(fn)) - - # Short circuit. If we're given a full path which matches the mode - # and it exists, we're done here. - if _access_check(cmd, mode): - return cmd - - path = (path or os.environ.get("PATH", os.defpath)).split(os.pathsep) - - if sys.platform == "win32": - # The current directory takes precedence on Windows. - if not os.curdir in path: - path.insert(0, os.curdir) - - # PATHEXT is necessary to check on Windows. - pathext = os.environ.get("PATHEXT", "").split(os.pathsep) - # See if the given file matches any of the expected path extensions. - # This will allow us to short circuit when given "python.exe". - matches = [cmd for ext in pathext if cmd.lower().endswith(ext.lower())] - # If it does match, only test that one, otherwise we have to try - # others. - files = [cmd] if matches else [cmd + ext.lower() for ext in pathext] - else: - # On other platforms you don't have things like PATHEXT to tell you - # what file suffixes are executable, so just pass on cmd as-is. - files = [cmd] - - seen = set() - for dir in path: - dir = os.path.normcase(dir) - if not dir in seen: - seen.add(dir) - for thefile in files: - name = os.path.join(dir, thefile) - if _access_check(name, mode): - return name - return None diff --git a/pipenv/vendor/six.LICENSE b/pipenv/vendor/six.LICENSE index f3068bfd..de663311 100644 --- a/pipenv/vendor/six.LICENSE +++ b/pipenv/vendor/six.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Benjamin Peterson +Copyright (c) 2010-2020 Benjamin Peterson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/pipenv/vendor/six.py b/pipenv/vendor/six.py index 6bf4fd38..5fe9f8e1 100644 --- a/pipenv/vendor/six.py +++ b/pipenv/vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2020 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson <benjamin@python.org>" -__version__ = "1.11.0" +__version__ = "1.14.0" # Useful for very coarse version differentiation. @@ -255,9 +255,11 @@ _moved_attributes = [ MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), + MovedModule("collections_abc", "collections", "collections.abc" if sys.version_info >= (3, 3) else "collections"), MovedModule("copyreg", "copy_reg"), MovedModule("dbm_gnu", "gdbm", "dbm.gnu"), - MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread"), + MovedModule("dbm_ndbm", "dbm", "dbm.ndbm"), + MovedModule("_dummy_thread", "dummy_thread", "_dummy_thread" if sys.version_info < (3, 9) else "_thread"), MovedModule("http_cookiejar", "cookielib", "http.cookiejar"), MovedModule("http_cookies", "Cookie", "http.cookies"), MovedModule("html_entities", "htmlentitydefs", "html.entities"), @@ -637,13 +639,16 @@ if PY3: import io StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" else: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" + _assertNotRegex = "assertNotRegex" else: def b(s): return s @@ -665,6 +670,7 @@ else: _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" _assertRegex = "assertRegexpMatches" + _assertNotRegex = "assertNotRegexpMatches" _add_doc(b, """Byte literal""") _add_doc(u, """Text literal""") @@ -681,6 +687,10 @@ def assertRegex(self, *args, **kwargs): return getattr(self, _assertRegex)(*args, **kwargs) +def assertNotRegex(self, *args, **kwargs): + return getattr(self, _assertNotRegex)(*args, **kwargs) + + if PY3: exec_ = getattr(moves.builtins, "exec") @@ -716,16 +726,7 @@ else: """) -if sys.version_info[:2] == (3, 2): - exec_("""def raise_from(value, from_value): - try: - if from_value is None: - raise value - raise value from from_value - finally: - value = None -""") -elif sys.version_info[:2] > (3, 2): +if sys.version_info[:2] > (3,): exec_("""def raise_from(value, from_value): try: raise value from from_value @@ -805,13 +806,33 @@ if sys.version_info[:2] < (3, 3): _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): + # This does exactly the same what the :func:`py3:functools.update_wrapper` + # function does on Python versions after 3.2. It sets the ``__wrapped__`` + # attribute on ``wrapper`` object and it doesn't raise an error if any of + # the attributes mentioned in ``assigned`` and ``updated`` are missing on + # ``wrapped`` object. + def _update_wrapper(wrapper, wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES): + for attr in assigned: + try: + value = getattr(wrapped, attr) + except AttributeError: + continue + else: + setattr(wrapper, attr, value) + for attr in updated: + getattr(wrapper, attr).update(getattr(wrapped, attr, {})) + wrapper.__wrapped__ = wrapped + return wrapper + _update_wrapper.__doc__ = functools.update_wrapper.__doc__ + def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, updated=functools.WRAPPER_UPDATES): - def wrapper(f): - f = functools.wraps(wrapped, assigned, updated)(f) - f.__wrapped__ = wrapped - return f - return wrapper + return functools.partial(_update_wrapper, wrapped=wrapped, + assigned=assigned, updated=updated) + wraps.__doc__ = functools.wraps.__doc__ + else: wraps = functools.wraps @@ -824,7 +845,15 @@ def with_metaclass(meta, *bases): class metaclass(type): def __new__(cls, name, this_bases, d): - return meta(name, bases, d) + if sys.version_info[:2] >= (3, 7): + # This version introduced PEP 560 that requires a bit + # of extra care (we mimic what is done by __build_class__). + resolved_bases = types.resolve_bases(bases) + if resolved_bases is not bases: + d['__orig_bases__'] = bases + else: + resolved_bases = bases + return meta(name, resolved_bases, d) @classmethod def __prepare__(cls, name, this_bases): @@ -844,13 +873,73 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ - A decorator that defines __unicode__ and __str__ methods under Python 2. + A class decorator that defines __unicode__ and __str__ methods under Python 2. Under Python 3 it does nothing. To support Python 2 and 3 with a single code base, define a __str__ method diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 92bfa27c..ab126006 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.5.2" +__version__ = "0.5.11" diff --git a/pipenv/vendor/tomlkit/_compat.py b/pipenv/vendor/tomlkit/_compat.py index b7407af6..768ce338 100644 --- a/pipenv/vendor/tomlkit/_compat.py +++ b/pipenv/vendor/tomlkit/_compat.py @@ -67,12 +67,12 @@ except ImportError: if self._name is None: return "%s.%s(%r)" % ( self.__class__.__module__, - self.__class__.__qualname__, + self.__class__.__name__, self._offset, ) return "%s.%s(%r, %r)" % ( self.__class__.__module__, - self.__class__.__qualname__, + self.__class__.__name__, self._offset, self._name, ) @@ -137,6 +137,7 @@ except ImportError: PY2 = sys.version_info[0] == 2 PY36 = sys.version_info >= (3, 6) +PY38 = sys.version_info >= (3, 8) if PY2: unicode = unicode diff --git a/pipenv/vendor/tomlkit/api.py b/pipenv/vendor/tomlkit/api.py index 0ac26752..c9621830 100644 --- a/pipenv/vendor/tomlkit/api.py +++ b/pipenv/vendor/tomlkit/api.py @@ -109,7 +109,7 @@ def table(): # type: () -> Table def inline_table(): # type: () -> InlineTable - return InlineTable(Container(), Trivia()) + return InlineTable(Container(), Trivia(), new=True) def aot(): # type: () -> AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 9b5db5cb..96415901 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import copy + from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -14,6 +16,9 @@ from .items import Whitespace from .items import item as _item +_NOT_SET = object() + + class Container(dict): """ A container for items within a TOMLDocument. @@ -243,6 +248,9 @@ class Container(dict): item = _item(item) idx = self._map[key] + # Insert after the max index if there are many. + if isinstance(idx, tuple): + idx = max(idx) current_item = self._body[idx][1] if "\n" not in current_item.trivia.trail: current_item.trivia.trail += "\n" @@ -485,6 +493,36 @@ class Container(dict): for k, v in other.items(): self[k] = v + def get(self, key, default=None): # type: (Any, Optional[Any]) -> Any + if not isinstance(key, Key): + key = Key(key) + + if key not in self: + return default + + return self[key] + + def pop(self, key, default=_NOT_SET): + try: + value = self[key] + except KeyError: + if default is _NOT_SET: + raise + + return default + + del self[key] + + return value + + def setdefault( + self, key, default=None + ): # type: (Union[Key, str], Any) -> Union[Item, Container] + if key not in self: + self[key] = default + + return self[key] + def __contains__(self, key): # type: (Union[Key, str]) -> bool if not isinstance(key, Key): key = Key(key) @@ -500,22 +538,16 @@ class Container(dict): raise NonExistentKey(key) if isinstance(idx, tuple): - container = Container(True) - - for i in idx: - item = self._body[i][1] - - if isinstance(item, Table): - for k, v in item.value.body: - container.append(k, v) - else: - container.append(key, item) - - return container + # The item we are getting is an out of order table + # so we need a proxy to retrieve the proper objects + # from the parent container + return OutOfOrderTableProxy(self, idx) item = self._body[idx][1] + if item.is_boolean(): + return item.value - return item.value + return item def __setitem__(self, key, value): # type: (Union[Key, str], Any) -> None if key is not None and key in self: @@ -544,6 +576,9 @@ class Container(dict): def _replace_at( self, idx, new_key, value ): # type: (Union[int, Tuple[int]], Union[Key, str], Item) -> None + if not isinstance(new_key, Key): + new_key = Key(new_key) + if isinstance(idx, tuple): for i in idx[1:]: self._body[i] = (None, Null()) @@ -553,6 +588,8 @@ class Container(dict): k, v = self._body[idx] self._map[new_key] = self._map.pop(k) + if new_key != k: + super(Container, self).__delitem__(k) if isinstance(self._map[new_key], tuple): self._map[new_key] = self._map[new_key][0] @@ -600,3 +637,112 @@ class Container(dict): self._map = state[0] self._body = state[1] self._parsed = state[2] + + def copy(self): # type: () -> Container + return copy.copy(self) + + def __copy__(self): # type: () -> Container + c = self.__class__(self._parsed) + for k, v in super(Container, self).copy().items(): + super(Container, c).__setitem__(k, v) + + c._body += self.body + c._map.update(self._map) + + return c + + +class OutOfOrderTableProxy(dict): + def __init__(self, container, indices): # type: (Container, Tuple) -> None + self._container = container + self._internal_container = Container(self._container.parsing) + self._tables = [] + self._tables_map = {} + self._map = {} + + for i in indices: + key, item = self._container._body[i] + + if isinstance(item, Table): + self._tables.append(item) + table_idx = len(self._tables) - 1 + for k, v in item.value.body: + self._internal_container.append(k, v) + self._tables_map[k] = table_idx + else: + self._internal_container.append(key, item) + self._map[key] = i + + def __getitem__(self, key): # type: (Union[Key, str]) -> Any + if key not in self._internal_container: + raise NonExistentKey(key) + + return self._internal_container[key] + + def __setitem__(self, key, item): # type: (Union[Key, str], Any) -> None + if key in self._map: + idx = self._map[key] + self._container._replace_at(idx, key, item) + elif key in self._tables_map: + table = self._tables[self._tables_map[key]] + table[key] = item + elif self._tables: + table = self._tables[0] + table[key] = item + else: + self._container[key] = item + + def __delitem__(self, key): # type: (Union[Key, str]) -> None + if key in self._map: + idx = self._map[key] + del self._container[key] + del self._map[key] + elif key in self._tables_map: + table = self._tables[self._tables_map[key]] + del table[key] + del self._tables_map[key] + else: + raise NonExistentKey(key) + + del self._internal_container[key] + + def keys(self): + return self._internal_container.keys() + + def values(self): + return self._internal_container.values() + + def items(self): # type: () -> Generator[Item] + return self._internal_container.items() + + def update(self, other): # type: (Dict) -> None + self._internal_container.update(other) + + def get(self, key, default=None): # type: (Any, Optional[Any]) -> Any + return self._internal_container.get(key, default=default) + + def pop(self, key, default=_NOT_SET): + return self._internal_container.pop(key, default=default) + + def setdefault( + self, key, default=None + ): # type: (Union[Key, str], Any) -> Union[Item, Container] + return self._internal_container.setdefault(key, default=default) + + def __contains__(self, key): + return key in self._internal_container + + def __str__(self): + return str(self._internal_container) + + def __repr__(self): + return repr(self._internal_container) + + def __eq__(self, other): # type: (Dict) -> bool + if not isinstance(other, dict): + return NotImplemented + + return self._internal_container == other + + def __getattr__(self, attribute): + return getattr(self._internal_container, attribute) diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index cccfd4a1..309fe1d8 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -8,6 +8,7 @@ from datetime import datetime from datetime import time from ._compat import PY2 +from ._compat import PY38 from ._compat import decode from ._compat import long from ._compat import unicode @@ -35,6 +36,7 @@ def item(value, _parent=None): elif isinstance(value, float): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): + val = Table(Container(), Trivia(), False) if isinstance(value, InlineTableDict): val = InlineTable(Container(), Trivia()) else: @@ -70,13 +72,32 @@ def item(value, _parent=None): elif isinstance(value, (str, unicode)): escaped = escape_string(value) - return String(StringType.SLB, value, escaped, Trivia()) + return String(StringType.SLB, decode(value), escaped, Trivia()) elif isinstance(value, datetime): - return DateTime(value, Trivia(), value.isoformat().replace("+00:00", "Z")) + return DateTime( + value.year, + value.month, + value.day, + value.hour, + value.minute, + value.second, + value.microsecond, + value.tzinfo, + Trivia(), + value.isoformat().replace("+00:00", "Z"), + ) elif isinstance(value, date): - return Date(value, Trivia(), value.isoformat()) + return Date(value.year, value.month, value.day, Trivia(), value.isoformat()) elif isinstance(value, time): - return Time(value, Trivia(), value.isoformat()) + return Time( + value.hour, + value.minute, + value.second, + value.microsecond, + value.tzinfo, + Trivia(), + value.isoformat(), + ) raise ValueError("Invalid type {}".format(type(value))) @@ -260,6 +281,15 @@ class Item(object): return self + def is_boolean(self): # type: () -> bool + return isinstance(self, Bool) + + def is_table(self): # type: () -> bool + return isinstance(self, Table) + + def is_inline_table(self): # type: () -> bool + return isinstance(self, InlineTable) + def _getstate(self, protocol=3): return (self._trivia,) @@ -465,7 +495,7 @@ class Bool(Item): A boolean literal. """ - def __init__(self, t, trivia): # type: (float, Trivia) -> None + def __init__(self, t, trivia): # type: (int, Trivia) -> None super(Bool, self).__init__(trivia) self._value = bool(t) @@ -484,26 +514,53 @@ class Bool(Item): def _getstate(self, protocol=3): return self._value, self._trivia + def __bool__(self): + return self._value + + __nonzero__ = __bool__ + + def __eq__(self, other): + if not isinstance(other, bool): + return NotImplemented + + return other == self._value + class DateTime(Item, datetime): """ A datetime literal. """ - def __new__(cls, value, *_): # type: (..., datetime, ...) -> datetime + def __new__( + cls, + year, + month, + day, + hour, + minute, + second, + microsecond, + tzinfo, + trivia, + raw, + **kwargs + ): # type: (int, int, int, int, int, int, int, ..., Trivia, ...) -> datetime return datetime.__new__( cls, - value.year, - value.month, - value.day, - value.hour, - value.minute, - value.second, - value.microsecond, - tzinfo=value.tzinfo, + year, + month, + day, + hour, + minute, + second, + microsecond, + tzinfo=tzinfo, + **kwargs ) - def __init__(self, _, trivia, raw): # type: (datetime, Trivia, str) -> None + def __init__( + self, year, month, day, hour, minute, second, microsecond, tzinfo, trivia, raw + ): # type: (int, int, int, int, int, int, int, ..., Trivia) -> None super(DateTime, self).__init__(trivia) self._raw = raw @@ -520,23 +577,8 @@ class DateTime(Item, datetime): return self._raw def __add__(self, other): - result = super(DateTime, self).__add__(other) - - return self._new(result) - - def __sub__(self, other): - result = super(DateTime, self).__sub__(other) - - return self._new(result) - - def _new(self, result): - raw = result.isoformat() - - return DateTime(result, self._trivia, raw) - - def _getstate(self, protocol=3): - return ( - datetime( + if PY38: + result = datetime( self.year, self.month, self.day, @@ -545,7 +587,58 @@ class DateTime(Item, datetime): self.second, self.microsecond, self.tzinfo, - ), + ).__add__(other) + else: + result = super(DateTime, self).__add__(other) + + return self._new(result) + + def __sub__(self, other): + if PY38: + result = datetime( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, + ).__sub__(other) + else: + result = super(DateTime, self).__sub__(other) + + if isinstance(result, datetime): + result = self._new(result) + + return result + + def _new(self, result): + raw = result.isoformat() + + return DateTime( + result.year, + result.month, + result.day, + result.hour, + result.minute, + result.second, + result.microsecond, + result.tzinfo, + self._trivia, + raw, + ) + + def _getstate(self, protocol=3): + return ( + self.year, + self.month, + self.day, + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, self._trivia, self._raw, ) @@ -556,10 +649,12 @@ class Date(Item, date): A date literal. """ - def __new__(cls, value, *_): # type: (..., date, ...) -> date - return date.__new__(cls, value.year, value.month, value.day) + def __new__(cls, year, month, day, *_): # type: (int, int, int, ...) -> date + return date.__new__(cls, year, month, day) - def __init__(self, _, trivia, raw): # type: (date, Trivia, str) -> None + def __init__( + self, year, month, day, trivia, raw + ): # type: (int, int, int, Trivia, str) -> None super(Date, self).__init__(trivia) self._raw = raw @@ -576,22 +671,31 @@ class Date(Item, date): return self._raw def __add__(self, other): - result = super(Date, self).__add__(other) + if PY38: + result = date(self.year, self.month, self.day).__add__(other) + else: + result = super(Date, self).__add__(other) return self._new(result) def __sub__(self, other): - result = super(Date, self).__sub__(other) + if PY38: + result = date(self.year, self.month, self.day).__sub__(other) + else: + result = super(Date, self).__sub__(other) - return self._new(result) + if isinstance(result, date): + result = self._new(result) + + return result def _new(self, result): raw = result.isoformat() - return Date(result, self._trivia, raw) + return Date(result.year, result.month, result.day, self._trivia, raw) def _getstate(self, protocol=3): - return (datetime(self.year, self.month, self.day), self._trivia, self._raw) + return (self.year, self.month, self.day, self._trivia, self._raw) class Time(Item, time): @@ -599,12 +703,14 @@ class Time(Item, time): A time literal. """ - def __new__(cls, value, *_): # type: (time, ...) -> time - return time.__new__( - cls, value.hour, value.minute, value.second, value.microsecond - ) + def __new__( + cls, hour, minute, second, microsecond, tzinfo, *_ + ): # type: (int, int, int, int, ...) -> time + return time.__new__(cls, hour, minute, second, microsecond, tzinfo) - def __init__(self, _, trivia, raw): # type: (time, Trivia, str) -> None + def __init__( + self, hour, minute, second, microsecond, tzinfo, trivia, raw + ): # type: (int, int, int, int, Trivia, str) -> None super(Time, self).__init__(trivia) self._raw = raw @@ -622,7 +728,11 @@ class Time(Item, time): def _getstate(self, protocol=3): return ( - time(self.hour, self.minute, self.second, self.microsecond, self.tzinfo), + self.hour, + self.minute, + self.second, + self.microsecond, + self.tzinfo, self._trivia, self._raw, ) @@ -633,7 +743,7 @@ class Array(Item, list): An array literal """ - def __init__(self, value, trivia): # type: (list, Trivia) -> None + def __init__(self, value, trivia, multiline=False): # type: (list, Trivia) -> None super(Array, self).__init__(trivia) list.__init__( @@ -641,6 +751,7 @@ class Array(Item, list): ) self._value = value + self._multiline = multiline @property def discriminant(self): # type: () -> int @@ -662,8 +773,23 @@ class Array(Item, list): return len(set(discriminants)) == 1 + def multiline(self, multiline): # type: (bool) -> self + self._multiline = multiline + + return self + def as_string(self): # type: () -> str - return "[{}]".format("".join(v.as_string() for v in self._value)) + if not self._multiline: + return "[{}]".format("".join(v.as_string() for v in self._value)) + + s = "[\n" + self.trivia.indent + " " * 4 + s += (",\n" + self.trivia.indent + " " * 4).join( + v.as_string() for v in self._value if not isinstance(v, Whitespace) + ) + s += ",\n" + s += "]" + + return s def append(self, _item): # type: () -> None if self._value: @@ -804,6 +930,20 @@ class Table(Item, dict): return self + def raw_append(self, key, _item): # type: (Union[Key, str], Any) -> Table + if not isinstance(_item, Item): + _item = item(_item) + + self._value.append(key, _item) + + if isinstance(key, Key): + key = key.key + + if key is not None: + super(Table, self).__setitem__(key, _item) + + return self + def remove(self, key): # type: (Union[Key, str]) -> Table self._value.remove(key) @@ -857,6 +997,9 @@ class Table(Item, dict): for k, v in other.items(): self[k] = v + def get(self, key, default=None): # type: (Any, Optional[Any]) -> Any + return self._value.get(key, default) + def __contains__(self, key): # type: (Union[Key, str]) -> bool return key in self._value @@ -891,6 +1034,9 @@ class Table(Item, dict): def __repr__(self): return super(Table, self).__repr__() + def __str__(self): + return str(self.value) + def _getstate(self, protocol=3): return ( self._value, @@ -908,11 +1054,12 @@ class InlineTable(Item, dict): """ def __init__( - self, value, trivia - ): # type: (tomlkit.container.Container, Trivia) -> None + self, value, trivia, new=False + ): # type: (tomlkit.container.Container, Trivia, bool) -> None super(InlineTable, self).__init__(trivia) self._value = value + self._new = new for k, v in self._value.body: if k is not None: @@ -934,7 +1081,7 @@ class InlineTable(Item, dict): _item = item(_item) if not isinstance(_item, (Whitespace, Comment)): - if not _item.trivia.indent and len(self._value) > 0: + if not _item.trivia.indent and len(self._value) > 0 and not self._new: _item.trivia.indent = " " if _item.trivia.comment: _item.trivia.comment = "" @@ -965,7 +1112,10 @@ class InlineTable(Item, dict): for i, (k, v) in enumerate(self._value.body): if k is None: if i == len(self._value.body) - 1: - buf = buf.rstrip(",") + if self._new: + buf = buf.rstrip(", ") + else: + buf = buf.rstrip(",") buf += v.as_string() @@ -982,6 +1132,8 @@ class InlineTable(Item, dict): if i != len(self._value.body) - 1: buf += "," + if self._new: + buf += " " buf += "}" @@ -1003,6 +1155,9 @@ class InlineTable(Item, dict): for k, v in other.items(): self[k] = v + def get(self, key, default=None): # type: (Any, Optional[Any]) -> Any + return self._value.get(key, default) + def __contains__(self, key): # type: (Union[Key, str]) -> bool return key in self._value diff --git a/pipenv/vendor/tomlkit/parser.py b/pipenv/vendor/tomlkit/parser.py index 3f507bb4..13fd9f98 100644 --- a/pipenv/vendor/tomlkit/parser.py +++ b/pipenv/vendor/tomlkit/parser.py @@ -194,7 +194,8 @@ class Parser: in_name = False current = "" t = KeyType.Bare - for c in name: + parts = 0 + for c in name.strip(): c = TOMLChar(c) if c == ".": @@ -205,14 +206,20 @@ class Parser: if not current: raise self.parse_error() - yield Key(current, t=t, sep="") + yield Key(current.strip(), t=t, sep="") + parts += 1 current = "" t = KeyType.Bare continue elif c in {"'", '"'}: if in_name: - if t == KeyType.Literal and c == '"': + if ( + t == KeyType.Literal + and c == '"' + or t == KeyType.Basic + and c == "'" + ): current += c continue @@ -221,17 +228,35 @@ class Parser: in_name = False else: + if current and TOMLChar(current[-1]).is_spaces() and not parts: + raise self.parse_error() + in_name = True t = KeyType.Literal if c == "'" else KeyType.Basic continue elif in_name or c.is_bare_key_char(): + if ( + not in_name + and current + and TOMLChar(current[-1]).is_spaces() + and not parts + ): + raise self.parse_error() + current += c + elif c.is_spaces(): + # A space is only valid at this point + # if it's in between parts. + # We store it for now and will check + # later if it's valid + current += c + continue else: raise self.parse_error() - if current: - yield Key(current, t=t, sep="") + if current.strip(): + yield Key(current.strip(), t=t, sep="") def _parse_item(self): # type: () -> Optional[Tuple[Optional[Key], Item]] """ @@ -434,15 +459,21 @@ class Parser: def _handle_dotted_key( self, container, key, value - ): # type: (Container, Key, Any) -> None + ): # type: (Union[Container, Table], Key, Any) -> None names = tuple(self._split_table_name(key.key)) name = names[0] name._dotted = True if name in container: - table = container.item(name) + if isinstance(container, Table): + table = container.value.item(name) + else: + table = container.item(name) else: table = Table(Container(True), Trivia(), False, is_super_table=True) - container.append(name, table) + if isinstance(container, Table): + container.raw_append(name, table) + else: + container.append(name, table) for i, _name in enumerate(names[1:]): if i == len(names) - 2: @@ -517,19 +548,41 @@ class Parser: if m.group(1) and m.group(5): # datetime try: - return DateTime(parse_rfc3339(raw), trivia, raw) + dt = parse_rfc3339(raw) + return DateTime( + dt.year, + dt.month, + dt.day, + dt.hour, + dt.minute, + dt.second, + dt.microsecond, + dt.tzinfo, + trivia, + raw, + ) except ValueError: raise self.parse_error(InvalidDateTimeError) if m.group(1): try: - return Date(parse_rfc3339(raw), trivia, raw) + dt = parse_rfc3339(raw) + return Date(dt.year, dt.month, dt.day, trivia, raw) except ValueError: raise self.parse_error(InvalidDateError) if m.group(5): try: - return Time(parse_rfc3339(raw), trivia, raw) + t = parse_rfc3339(raw) + return Time( + t.hour, + t.minute, + t.second, + t.microsecond, + t.tzinfo, + trivia, + raw, + ) except ValueError: raise self.parse_error(InvalidTimeError) @@ -876,15 +929,46 @@ class Parser: is_aot = True - # Key + # Consume any whitespace self.mark() - while self._current != "]" and self.inc(): - if self.end(): - raise self.parse_error(UnexpectedEofError) - + while self._current.is_spaces() and self.inc(): pass - name = self.extract() + ws_prefix = self.extract() + + # Key + if self._current in [StringType.SLL.value, StringType.SLB.value]: + delimiter = ( + StringType.SLL + if self._current == StringType.SLL.value + else StringType.SLB + ) + name = self._parse_string(delimiter) + name = "{delimiter}{name}{delimiter}".format( + delimiter=delimiter.value, name=name + ) + + self.mark() + while self._current != "]" and self.inc(): + if self.end(): + raise self.parse_error(UnexpectedEofError) + + pass + + ws_suffix = self.extract() + name += ws_suffix + else: + self.mark() + while self._current != "]" and self.inc(): + if self.end(): + raise self.parse_error(UnexpectedEofError) + + pass + + name = self.extract() + + name = ws_prefix + name + if not name.strip(): raise self.parse_error(EmptyTableNameError) @@ -911,6 +995,13 @@ class Parser: cws, comment, trail = self._parse_comment_trail() result = Null() + table = Table( + values, + Trivia(indent, cws, comment, trail), + is_aot, + name=name, + display_name=name, + ) if len(name_parts) > 1: if missing_table: @@ -960,9 +1051,9 @@ class Parser: _key, item = item if not self._merge_ws(item, values): if _key is not None and _key.is_dotted(): - self._handle_dotted_key(values, _key, item) + self._handle_dotted_key(table, _key, item) else: - values.append(_key, item) + table.raw_append(_key, item) else: if self._current == "[": is_aot_next, name_next = self._peek_table() @@ -970,7 +1061,7 @@ class Parser: if self._is_child(name, name_next): key_next, table_next = self._parse_table(name) - values.append(key_next, table_next) + table.raw_append(key_next, table_next) # Picking up any sibling while not self.end(): @@ -981,7 +1072,7 @@ class Parser: key_next, table_next = self._parse_table(name) - values.append(key_next, table_next) + table.raw_append(key_next, table_next) break else: @@ -991,13 +1082,7 @@ class Parser: ) if isinstance(result, Null): - result = Table( - values, - Trivia(indent, cws, comment, trail), - is_aot, - name=name, - display_name=name, - ) + result = table if is_aot and (not self._aot_stack or name != self._aot_stack[-1]): result = self._parse_aot(result, name) diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py index dcfdafd0..ddb580e4 100644 --- a/pipenv/vendor/tomlkit/source.py +++ b/pipenv/vendor/tomlkit/source.py @@ -45,10 +45,6 @@ class _State: if self._save_marker: self._source._marker = self._marker - # Restore exceptions are silently consumed, other exceptions need to - # propagate - return exception_type is None - class _StateHandler: """ diff --git a/pipenv/vendor/urllib3/LICENSE.txt b/pipenv/vendor/urllib3/LICENSE.txt index 1c3283ee..c89cf27b 100644 --- a/pipenv/vendor/urllib3/LICENSE.txt +++ b/pipenv/vendor/urllib3/LICENSE.txt @@ -1,19 +1,21 @@ -This is the MIT license: http://www.opensource.org/licenses/mit-license.php +MIT License -Copyright 2008-2016 Andrey Petrov and contributors (see CONTRIBUTORS.txt) +Copyright (c) 2008-2019 Andrey Petrov and contributors (see CONTRIBUTORS.txt) -Permission is hereby granted, free of charge, to any person obtaining a copy of this -software and associated documentation files (the "Software"), to deal in the Software -without restriction, including without limitation the rights to use, copy, modify, merge, -publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons -to whom the Software is furnished to do so, subject to the following conditions: +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -The above copyright notice and this permission notice shall be included in all copies or -substantial portions of the Software. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, -INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR -PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE -FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR -OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER -DEALINGS IN THE SOFTWARE. +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py index 75725167..667e9bce 100644 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -1,15 +1,10 @@ """ urllib3 - Thread-safe connection pooling and re-using. """ - from __future__ import absolute_import import warnings -from .connectionpool import ( - HTTPConnectionPool, - HTTPSConnectionPool, - connection_from_url -) +from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool, connection_from_url from . import exceptions from .filepost import encode_multipart_formdata @@ -25,25 +20,25 @@ from .util.retry import Retry import logging from logging import NullHandler -__author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' -__license__ = 'MIT' -__version__ = '1.24' +__author__ = "Andrey Petrov (andrey.petrov@shazow.net)" +__license__ = "MIT" +__version__ = "1.25.9" __all__ = ( - 'HTTPConnectionPool', - 'HTTPSConnectionPool', - 'PoolManager', - 'ProxyManager', - 'HTTPResponse', - 'Retry', - 'Timeout', - 'add_stderr_logger', - 'connection_from_url', - 'disable_warnings', - 'encode_multipart_formdata', - 'get_host', - 'make_headers', - 'proxy_from_url', + "HTTPConnectionPool", + "HTTPSConnectionPool", + "PoolManager", + "ProxyManager", + "HTTPResponse", + "Retry", + "Timeout", + "add_stderr_logger", + "connection_from_url", + "disable_warnings", + "encode_multipart_formdata", + "get_host", + "make_headers", + "proxy_from_url", ) logging.getLogger(__name__).addHandler(NullHandler()) @@ -60,10 +55,10 @@ def add_stderr_logger(level=logging.DEBUG): # even if urllib3 is vendored within another package. logger = logging.getLogger(__name__) handler = logging.StreamHandler() - handler.setFormatter(logging.Formatter('%(asctime)s %(levelname)s %(message)s')) + handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) logger.addHandler(handler) logger.setLevel(level) - logger.debug('Added a stderr logging handler to logger: %s', __name__) + logger.debug("Added a stderr logging handler to logger: %s", __name__) return handler @@ -75,18 +70,17 @@ del NullHandler # shouldn't be: otherwise, it's very hard for users to use most Python # mechanisms to silence them. # SecurityWarning's always go off by default. -warnings.simplefilter('always', exceptions.SecurityWarning, append=True) +warnings.simplefilter("always", exceptions.SecurityWarning, append=True) # SubjectAltNameWarning's should go off once per host -warnings.simplefilter('default', exceptions.SubjectAltNameWarning, append=True) +warnings.simplefilter("default", exceptions.SubjectAltNameWarning, append=True) # InsecurePlatformWarning's don't vary between requests, so we keep it default. -warnings.simplefilter('default', exceptions.InsecurePlatformWarning, - append=True) +warnings.simplefilter("default", exceptions.InsecurePlatformWarning, append=True) # SNIMissingWarnings should go off only once. -warnings.simplefilter('default', exceptions.SNIMissingWarning, append=True) +warnings.simplefilter("default", exceptions.SNIMissingWarning, append=True) def disable_warnings(category=exceptions.HTTPWarning): """ Helper for quickly disabling all urllib3 warnings. """ - warnings.simplefilter('ignore', category) + warnings.simplefilter("ignore", category) diff --git a/pipenv/vendor/urllib3/_collections.py b/pipenv/vendor/urllib3/_collections.py index 34f23811..019d1511 100644 --- a/pipenv/vendor/urllib3/_collections.py +++ b/pipenv/vendor/urllib3/_collections.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + try: from collections.abc import Mapping, MutableMapping except ImportError: @@ -6,6 +7,7 @@ except ImportError: try: from threading import RLock except ImportError: # Platform-specific: No threads available + class RLock: def __enter__(self): pass @@ -19,7 +21,7 @@ from .exceptions import InvalidHeader from .packages.six import iterkeys, itervalues, PY3 -__all__ = ['RecentlyUsedContainer', 'HTTPHeaderDict'] +__all__ = ["RecentlyUsedContainer", "HTTPHeaderDict"] _Null = object() @@ -82,7 +84,9 @@ class RecentlyUsedContainer(MutableMapping): return len(self._container) def __iter__(self): - raise NotImplementedError('Iteration over this class is unlikely to be threadsafe.') + raise NotImplementedError( + "Iteration over this class is unlikely to be threadsafe." + ) def clear(self): with self.lock: @@ -150,7 +154,7 @@ class HTTPHeaderDict(MutableMapping): def __getitem__(self, key): val = self._container[key.lower()] - return ', '.join(val[1:]) + return ", ".join(val[1:]) def __delitem__(self, key): del self._container[key.lower()] @@ -159,12 +163,13 @@ class HTTPHeaderDict(MutableMapping): return key.lower() in self._container def __eq__(self, other): - if not isinstance(other, Mapping) and not hasattr(other, 'keys'): + if not isinstance(other, Mapping) and not hasattr(other, "keys"): return False if not isinstance(other, type(self)): other = type(self)(other) - return (dict((k.lower(), v) for k, v in self.itermerged()) == - dict((k.lower(), v) for k, v in other.itermerged())) + return dict((k.lower(), v) for k, v in self.itermerged()) == dict( + (k.lower(), v) for k, v in other.itermerged() + ) def __ne__(self, other): return not self.__eq__(other) @@ -184,9 +189,9 @@ class HTTPHeaderDict(MutableMapping): yield vals[0] def pop(self, key, default=__marker): - '''D.pop(k[,d]) -> v, remove specified key and return the corresponding value. + """D.pop(k[,d]) -> v, remove specified key and return the corresponding value. If key is not found, d is returned if given, otherwise KeyError is raised. - ''' + """ # Using the MutableMapping function directly fails due to the private marker. # Using ordinary dict.pop would expose the internal structures. # So let's reinvent the wheel. @@ -228,8 +233,10 @@ class HTTPHeaderDict(MutableMapping): with self.add instead of self.__setitem__ """ if len(args) > 1: - raise TypeError("extend() takes at most 1 positional " - "arguments ({0} given)".format(len(args))) + raise TypeError( + "extend() takes at most 1 positional " + "arguments ({0} given)".format(len(args)) + ) other = args[0] if len(args) >= 1 else () if isinstance(other, HTTPHeaderDict): @@ -295,7 +302,7 @@ class HTTPHeaderDict(MutableMapping): """Iterate over all headers, merging duplicate ones together.""" for key in self: val = self._container[key.lower()] - yield val[0], ', '.join(val[1:]) + yield val[0], ", ".join(val[1:]) def items(self): return list(self.iteritems()) @@ -306,7 +313,7 @@ class HTTPHeaderDict(MutableMapping): # python2.7 does not expose a proper API for exporting multiheaders # efficiently. This function re-reads raw lines from the message # object and extracts the multiheaders properly. - obs_fold_continued_leaders = (' ', '\t') + obs_fold_continued_leaders = (" ", "\t") headers = [] for line in message.headers: @@ -316,14 +323,14 @@ class HTTPHeaderDict(MutableMapping): # in RFC-7230 S3.2.4. This indicates a multiline header, but # there exists no previous header to which we can attach it. raise InvalidHeader( - 'Header continuation with no previous header: %s' % line + "Header continuation with no previous header: %s" % line ) else: key, value = headers[-1] - headers[-1] = (key, value + ' ' + line.strip()) + headers[-1] = (key, value + " " + line.strip()) continue - key, value = line.split(':', 1) + key, value = line.split(":", 1) headers.append((key, value.strip())) return cls(headers) diff --git a/pipenv/vendor/urllib3/connection.py b/pipenv/vendor/urllib3/connection.py index 02b36654..6da1cf4b 100644 --- a/pipenv/vendor/urllib3/connection.py +++ b/pipenv/vendor/urllib3/connection.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import re import datetime import logging import os @@ -11,6 +12,7 @@ from .packages.six.moves.http_client import HTTPException # noqa: F401 try: # Compiled with SSL? import ssl + BaseSSLError = ssl.SSLError except (ImportError, AttributeError): # Platform-specific: No SSL. ssl = None @@ -19,10 +21,11 @@ except (ImportError, AttributeError): # Platform-specific: No SSL. pass -try: # Python 3: - # Not a no-op, we're adding this to the namespace so it can be imported. +try: + # Python 3: not a no-op, we're adding this to the namespace so it can be imported. ConnectionError = ConnectionError -except NameError: # Python 2: +except NameError: + # Python 2 class ConnectionError(Exception): pass @@ -40,7 +43,7 @@ from .util.ssl_ import ( resolve_ssl_version, assert_fingerprint, create_urllib3_context, - ssl_wrap_socket + ssl_wrap_socket, ) @@ -50,20 +53,18 @@ from ._collections import HTTPHeaderDict log = logging.getLogger(__name__) -port_by_scheme = { - 'http': 80, - 'https': 443, -} +port_by_scheme = {"http": 80, "https": 443} -# When updating RECENT_DATE, move it to within two years of the current date, -# and not less than 6 months ago. -# Example: if Today is 2018-01-01, then RECENT_DATE should be any date on or -# after 2016-01-01 (today - 2 years) AND before 2017-07-01 (today - 6 months) -RECENT_DATE = datetime.date(2017, 6, 30) +# When it comes time to update this value as a part of regular maintenance +# (ie test_recent_date is failing) update it to ~6 months before the current date. +RECENT_DATE = datetime.date(2019, 1, 1) + +_CONTAINS_CONTROL_CHAR_RE = re.compile(r"[^-!#$%&'*+.^_`|~0-9a-zA-Z]") class DummyConnection(object): """Used to detect a failed ConnectionCls import.""" + pass @@ -91,7 +92,7 @@ class HTTPConnection(_HTTPConnection, object): Or you may want to disable the defaults by passing an empty list (e.g., ``[]``). """ - default_port = port_by_scheme['http'] + default_port = port_by_scheme["http"] #: Disable Nagle's algorithm by default. #: ``[(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)]`` @@ -101,15 +102,15 @@ class HTTPConnection(_HTTPConnection, object): is_verified = False def __init__(self, *args, **kw): - if six.PY3: # Python 3 - kw.pop('strict', None) + if not six.PY2: + kw.pop("strict", None) # Pre-set source_address. - self.source_address = kw.get('source_address') + self.source_address = kw.get("source_address") #: The socket options provided by the user. If no options are #: provided, we use the default options. - self.socket_options = kw.pop('socket_options', self.default_socket_options) + self.socket_options = kw.pop("socket_options", self.default_socket_options) _HTTPConnection.__init__(self, *args, **kw) @@ -130,7 +131,7 @@ class HTTPConnection(_HTTPConnection, object): those cases where it's appropriate (i.e., when doing DNS lookup to establish the actual TCP connection across which we're going to send HTTP requests). """ - return self._dns_host.rstrip('.') + return self._dns_host.rstrip(".") @host.setter def host(self, value): @@ -149,29 +150,34 @@ class HTTPConnection(_HTTPConnection, object): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = connection.create_connection( - (self._dns_host, self.port), self.timeout, **extra_kw) + (self._dns_host, self.port), self.timeout, **extra_kw + ) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except SocketError as e: raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn def _prepare_conn(self, conn): self.sock = conn - if self._tunnel_host: + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, "_tunnel_host", None): # TODO: Fix tunnel so it doesn't depend on self.sock state. self._tunnel() # Mark this connection as not reusable @@ -181,24 +187,32 @@ class HTTPConnection(_HTTPConnection, object): conn = self._new_conn() self._prepare_conn(conn) + def putrequest(self, method, url, *args, **kwargs): + """Send a request to the server""" + match = _CONTAINS_CONTROL_CHAR_RE.search(method) + if match: + raise ValueError( + "Method cannot contain non-token characters %r (found at least %r)" + % (method, match.group()) + ) + + return _HTTPConnection.putrequest(self, method, url, *args, **kwargs) + def request_chunked(self, method, url, body=None, headers=None): """ Alternative to the common request method, which sends the body with chunked encoding and not as one block """ headers = HTTPHeaderDict(headers if headers is not None else {}) - skip_accept_encoding = 'accept-encoding' in headers - skip_host = 'host' in headers + skip_accept_encoding = "accept-encoding" in headers + skip_host = "host" in headers self.putrequest( - method, - url, - skip_accept_encoding=skip_accept_encoding, - skip_host=skip_host + method, url, skip_accept_encoding=skip_accept_encoding, skip_host=skip_host ) for header, value in headers.items(): self.putheader(header, value) - if 'transfer-encoding' not in headers: - self.putheader('Transfer-Encoding', 'chunked') + if "transfer-encoding" not in headers: + self.putheader("Transfer-Encoding", "chunked") self.endheaders() if body is not None: @@ -209,99 +223,93 @@ class HTTPConnection(_HTTPConnection, object): if not chunk: continue if not isinstance(chunk, bytes): - chunk = chunk.encode('utf8') + chunk = chunk.encode("utf8") len_str = hex(len(chunk))[2:] - self.send(len_str.encode('utf-8')) - self.send(b'\r\n') + self.send(len_str.encode("utf-8")) + self.send(b"\r\n") self.send(chunk) - self.send(b'\r\n') + self.send(b"\r\n") # After the if clause, to always have a closed body - self.send(b'0\r\n\r\n') + self.send(b"0\r\n\r\n") class HTTPSConnection(HTTPConnection): - default_port = port_by_scheme['https'] + default_port = port_by_scheme["https"] + cert_reqs = None + ca_certs = None + ca_cert_dir = None + ca_cert_data = None ssl_version = None + assert_fingerprint = None - def __init__(self, host, port=None, key_file=None, cert_file=None, - strict=None, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - ssl_context=None, server_hostname=None, **kw): + def __init__( + self, + host, + port=None, + key_file=None, + cert_file=None, + key_password=None, + strict=None, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + ssl_context=None, + server_hostname=None, + **kw + ): - HTTPConnection.__init__(self, host, port, strict=strict, - timeout=timeout, **kw) + HTTPConnection.__init__(self, host, port, strict=strict, timeout=timeout, **kw) self.key_file = key_file self.cert_file = cert_file + self.key_password = key_password self.ssl_context = ssl_context self.server_hostname = server_hostname # Required property for Google AppEngine 1.9.0 which otherwise causes # HTTPS requests to go out as HTTP. (See Issue #356) - self._protocol = 'https' + self._protocol = "https" - def connect(self): - conn = self._new_conn() - self._prepare_conn(conn) - - if self.ssl_context is None: - self.ssl_context = create_urllib3_context( - ssl_version=resolve_ssl_version(None), - cert_reqs=resolve_cert_reqs(None), - ) - - self.sock = ssl_wrap_socket( - sock=conn, - keyfile=self.key_file, - certfile=self.cert_file, - ssl_context=self.ssl_context, - server_hostname=self.server_hostname - ) - - -class VerifiedHTTPSConnection(HTTPSConnection): - """ - Based on httplib.HTTPSConnection but wraps the socket with - SSL certification. - """ - cert_reqs = None - ca_certs = None - ca_cert_dir = None - ssl_version = None - assert_fingerprint = None - - def set_cert(self, key_file=None, cert_file=None, - cert_reqs=None, ca_certs=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None): + def set_cert( + self, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + ca_cert_data=None, + ): """ This method should only be called once, before the connection is used. """ - # If cert_reqs is not provided, we can try to guess. If the user gave - # us a cert database, we assume they want to use it: otherwise, if - # they gave us an SSL Context object we should use whatever is set for - # it. + # If cert_reqs is not provided we'll assume CERT_REQUIRED unless we also + # have an SSLContext object in which case we'll use its verify_mode. if cert_reqs is None: - if ca_certs or ca_cert_dir: - cert_reqs = 'CERT_REQUIRED' - elif self.ssl_context is not None: + if self.ssl_context is not None: cert_reqs = self.ssl_context.verify_mode + else: + cert_reqs = resolve_cert_reqs(None) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.assert_hostname = assert_hostname self.assert_fingerprint = assert_fingerprint self.ca_certs = ca_certs and os.path.expanduser(ca_certs) self.ca_cert_dir = ca_cert_dir and os.path.expanduser(ca_cert_dir) + self.ca_cert_data = ca_cert_data def connect(self): # Add certificate verification conn = self._new_conn() hostname = self.host - if self._tunnel_host: + # Google App Engine's httplib does not define _tunnel_host + if getattr(self, "_tunnel_host", None): self.sock = conn # Calls self._set_hostport(), so self.host is # self._tunnel_host below. @@ -318,15 +326,19 @@ class VerifiedHTTPSConnection(HTTPSConnection): is_time_off = datetime.date.today() < RECENT_DATE if is_time_off: - warnings.warn(( - 'System time is way off (before {0}). This will probably ' - 'lead to SSL verification errors').format(RECENT_DATE), - SystemTimeWarning + warnings.warn( + ( + "System time is way off (before {0}). This will probably " + "lead to SSL verification errors" + ).format(RECENT_DATE), + SystemTimeWarning, ) # Wrap socket using verification with the root certs in # trusted_root_certs + default_ssl_context = False if self.ssl_context is None: + default_ssl_context = True self.ssl_context = create_urllib3_context( ssl_version=resolve_ssl_version(self.ssl_version), cert_reqs=resolve_cert_reqs(self.cert_reqs), @@ -334,38 +346,58 @@ class VerifiedHTTPSConnection(HTTPSConnection): context = self.ssl_context context.verify_mode = resolve_cert_reqs(self.cert_reqs) + + # Try to load OS default certs if none are given. + # Works well on Windows (requires Python3.4+) + if ( + not self.ca_certs + and not self.ca_cert_dir + and not self.ca_cert_data + and default_ssl_context + and hasattr(context, "load_default_certs") + ): + context.load_default_certs() + self.sock = ssl_wrap_socket( sock=conn, keyfile=self.key_file, certfile=self.cert_file, + key_password=self.key_password, ca_certs=self.ca_certs, ca_cert_dir=self.ca_cert_dir, + ca_cert_data=self.ca_cert_data, server_hostname=server_hostname, - ssl_context=context) + ssl_context=context, + ) if self.assert_fingerprint: - assert_fingerprint(self.sock.getpeercert(binary_form=True), - self.assert_fingerprint) - elif context.verify_mode != ssl.CERT_NONE \ - and not getattr(context, 'check_hostname', False) \ - and self.assert_hostname is not False: + assert_fingerprint( + self.sock.getpeercert(binary_form=True), self.assert_fingerprint + ) + elif ( + context.verify_mode != ssl.CERT_NONE + and not getattr(context, "check_hostname", False) + and self.assert_hostname is not False + ): # While urllib3 attempts to always turn off hostname matching from # the TLS library, this cannot always be done. So we check whether # the TLS Library still thinks it's matching hostnames. cert = self.sock.getpeercert() - if not cert.get('subjectAltName', ()): - warnings.warn(( - 'Certificate for {0} has no `subjectAltName`, falling back to check for a ' - '`commonName` for now. This feature is being removed by major browsers and ' - 'deprecated by RFC 2818. (See https://github.com/shazow/urllib3/issues/497 ' - 'for details.)'.format(hostname)), - SubjectAltNameWarning + if not cert.get("subjectAltName", ()): + warnings.warn( + ( + "Certificate for {0} has no `subjectAltName`, falling back to check for a " + "`commonName` for now. This feature is being removed by major browsers and " + "deprecated by RFC 2818. (See https://github.com/urllib3/urllib3/issues/497 " + "for details.)".format(hostname) + ), + SubjectAltNameWarning, ) _match_hostname(cert, self.assert_hostname or server_hostname) self.is_verified = ( - context.verify_mode == ssl.CERT_REQUIRED or - self.assert_fingerprint is not None + context.verify_mode == ssl.CERT_REQUIRED + or self.assert_fingerprint is not None ) @@ -373,9 +405,10 @@ def _match_hostname(cert, asserted_hostname): try: match_hostname(cert, asserted_hostname) except CertificateError as e: - log.error( - 'Certificate did not match expected hostname: %s. ' - 'Certificate: %s', asserted_hostname, cert + log.warning( + "Certificate did not match expected hostname: %s. Certificate: %s", + asserted_hostname, + cert, ) # Add cert to exception and reraise so client code can inspect # the cert when catching the exception, if they want to @@ -383,9 +416,8 @@ def _match_hostname(cert, asserted_hostname): raise -if ssl: - # Make a copy for testing. - UnverifiedHTTPSConnection = HTTPSConnection - HTTPSConnection = VerifiedHTTPSConnection -else: - HTTPSConnection = DummyConnection +if not ssl: + HTTPSConnection = DummyConnection # noqa: F811 + + +VerifiedHTTPSConnection = HTTPSConnection diff --git a/pipenv/vendor/urllib3/connectionpool.py b/pipenv/vendor/urllib3/connectionpool.py index f7a8f193..5f044dbd 100644 --- a/pipenv/vendor/urllib3/connectionpool.py +++ b/pipenv/vendor/urllib3/connectionpool.py @@ -29,8 +29,11 @@ from .packages.six.moves import queue from .connection import ( port_by_scheme, DummyConnection, - HTTPConnection, HTTPSConnection, VerifiedHTTPSConnection, - HTTPException, BaseSSLError, + HTTPConnection, + HTTPSConnection, + VerifiedHTTPSConnection, + HTTPException, + BaseSSLError, ) from .request import RequestMethods from .response import HTTPResponse @@ -40,7 +43,13 @@ from .util.request import set_file_position from .util.response import assert_header_parsing from .util.retry import Retry from .util.timeout import Timeout -from .util.url import get_host, Url, NORMALIZABLE_SCHEMES +from .util.url import ( + get_host, + parse_url, + Url, + _normalize_host as normalize_host, + _encode_target, +) from .util.queue import LifoQueue @@ -56,6 +65,11 @@ class ConnectionPool(object): """ Base class for all connection pools, such as :class:`.HTTPConnectionPool` and :class:`.HTTPSConnectionPool`. + + .. note:: + ConnectionPool.urlopen() does not normalize or percent-encode target URIs + which is useful if your target server doesn't support percent-encoded + target URIs. """ scheme = None @@ -65,13 +79,12 @@ class ConnectionPool(object): if not host: raise LocationValueError("No host specified.") - self.host = _ipv6_host(host, self.scheme) + self.host = _normalize_host(host, scheme=self.scheme) self._proxy_host = host.lower() self.port = port def __str__(self): - return '%s(host=%r, port=%r)' % (type(self).__name__, - self.host, self.port) + return "%s(host=%r, port=%r)" % (type(self).__name__, self.host, self.port) def __enter__(self): return self @@ -152,15 +165,24 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): :class:`urllib3.connection.HTTPSConnection` instances. """ - scheme = 'http' + scheme = "http" ConnectionCls = HTTPConnection ResponseCls = HTTPResponse - def __init__(self, host, port=None, strict=False, - timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, block=False, - headers=None, retries=None, - _proxy=None, _proxy_headers=None, - **conn_kw): + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + **conn_kw + ): ConnectionPool.__init__(self, host, port) RequestMethods.__init__(self, headers) @@ -194,19 +216,27 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Enable Nagle's algorithm for proxies, to avoid packet fragmentation. # We cannot know if the user has added default socket options, so we cannot replace the # list. - self.conn_kw.setdefault('socket_options', []) + self.conn_kw.setdefault("socket_options", []) def _new_conn(self): """ Return a fresh :class:`HTTPConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTP connection (%d): %s:%s", - self.num_connections, self.host, self.port or "80") + log.debug( + "Starting new HTTP connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "80", + ) - conn = self.ConnectionCls(host=self.host, port=self.port, - timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + conn = self.ConnectionCls( + host=self.host, + port=self.port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + **self.conn_kw + ) return conn def _get_conn(self, timeout=None): @@ -230,16 +260,17 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): except queue.Empty: if self.block: - raise EmptyPoolError(self, - "Pool reached maximum size and no more " - "connections are allowed.") + raise EmptyPoolError( + self, + "Pool reached maximum size and no more connections are allowed.", + ) pass # Oh well, we'll create a new connection then # If this is a persistent connection, check if it got disconnected if conn and is_connection_dropped(conn): log.debug("Resetting dropped connection: %s", self.host) conn.close() - if getattr(conn, 'auto_open', 1) == 0: + if getattr(conn, "auto_open", 1) == 0: # This is a proxied connection that has been mutated by # httplib._tunnel() and cannot be reused (since it would # attempt to bypass the proxy) @@ -269,9 +300,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): pass except queue.Full: # This should never happen if self.block == True - log.warning( - "Connection pool is full, discarding connection: %s", - self.host) + log.warning("Connection pool is full, discarding connection: %s", self.host) # Connection never got put back into the pool, close it. if conn: @@ -303,21 +332,30 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): """Is the error actually a timeout? Will raise a ReadTimeout or pass""" if isinstance(err, SocketTimeout): - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # See the above comment about EAGAIN in Python 3. In Python 2 we have # to specifically catch it and throw the timeout error - if hasattr(err, 'errno') and err.errno in _blocking_errnos: - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + if hasattr(err, "errno") and err.errno in _blocking_errnos: + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) # Catch possible read timeouts thrown as SSL errors. If not the # case, rethrow the original. We need to do this because of: # http://bugs.python.org/issue10272 - if 'timed out' in str(err) or 'did not complete (read)' in str(err): # Python < 2.7.4 - raise ReadTimeoutError(self, url, "Read timed out. (read timeout=%s)" % timeout_value) + if "timed out" in str(err) or "did not complete (read)" in str( + err + ): # Python < 2.7.4 + raise ReadTimeoutError( + self, url, "Read timed out. (read timeout=%s)" % timeout_value + ) - def _make_request(self, conn, method, url, timeout=_Default, chunked=False, - **httplib_request_kw): + def _make_request( + self, conn, method, url, timeout=_Default, chunked=False, **httplib_request_kw + ): """ Perform a request on a given urllib connection object taken from our pool. @@ -357,7 +395,7 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): read_timeout = timeout_obj.read_timeout # App Engine doesn't have a sock attr - if getattr(conn, 'sock', None): + if getattr(conn, "sock", None): # In Python 3 socket.py will catch EAGAIN and return None when you # try and read into the file pointer created by http.client, which # instead raises a BadStatusLine exception. Instead of catching @@ -365,7 +403,8 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # timeouts, check for a zero timeout before making the request. if read_timeout == 0: raise ReadTimeoutError( - self, url, "Read timed out. (read timeout=%s)" % read_timeout) + self, url, "Read timed out. (read timeout=%s)" % read_timeout + ) if read_timeout is Timeout.DEFAULT_TIMEOUT: conn.sock.settimeout(socket.getdefaulttimeout()) else: # None or a value @@ -373,31 +412,45 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Receive the response from the server try: - try: # Python 2.7, use buffering of HTTP responses + try: + # Python 2.7, use buffering of HTTP responses httplib_response = conn.getresponse(buffering=True) - except TypeError: # Python 3 + except TypeError: + # Python 3 try: httplib_response = conn.getresponse() - except Exception as e: - # Remove the TypeError from the exception chain in Python 3; - # otherwise it looks like a programming error was the cause. + except BaseException as e: + # Remove the TypeError from the exception chain in + # Python 3 (including for exceptions like SystemExit). + # Otherwise it looks like a bug in the code. six.raise_from(e, None) except (SocketTimeout, BaseSSLError, SocketError) as e: self._raise_timeout(err=e, url=url, timeout_value=read_timeout) raise # AppEngine doesn't have a version attr. - http_version = getattr(conn, '_http_vsn_str', 'HTTP/?') - log.debug("%s://%s:%s \"%s %s %s\" %s %s", self.scheme, self.host, self.port, - method, url, http_version, httplib_response.status, - httplib_response.length) + http_version = getattr(conn, "_http_vsn_str", "HTTP/?") + log.debug( + '%s://%s:%s "%s %s %s" %s %s', + self.scheme, + self.host, + self.port, + method, + url, + http_version, + httplib_response.status, + httplib_response.length, + ) try: assert_header_parsing(httplib_response.msg) except (HeaderParsingError, TypeError) as hpe: # Platform-specific: Python 3 log.warning( - 'Failed to parse headers (url=%s): %s', - self._absolute_url(url), hpe, exc_info=True) + "Failed to parse headers (url=%s): %s", + self._absolute_url(url), + hpe, + exc_info=True, + ) return httplib_response @@ -427,13 +480,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): Check if the given ``url`` is a member of the same host as this connection pool. """ - if url.startswith('/'): + if url.startswith("/"): return True # TODO: Add optional support for socket.gethostbyname checking. scheme, host, port = get_host(url) - - host = _ipv6_host(host, self.scheme) + if host is not None: + host = _normalize_host(host, scheme=scheme) # Use explicit default port for comparison when none is given if self.port and not port: @@ -443,10 +496,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): return (scheme, host, port) == (self.scheme, self.host, self.port) - def urlopen(self, method, url, body=None, headers=None, retries=None, - redirect=True, assert_same_host=True, timeout=_Default, - pool_timeout=None, release_conn=None, chunked=False, - body_pos=None, **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + assert_same_host=True, + timeout=_Default, + pool_timeout=None, + release_conn=None, + chunked=False, + body_pos=None, + **response_kw + ): """ Get a connection from the pool and perform an HTTP request. This is the lowest level call for making a request, so you'll need to specify all @@ -544,12 +609,18 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if release_conn is None: - release_conn = response_kw.get('preload_content', True) + release_conn = response_kw.get("preload_content", True) # Check host if assert_same_host and not self.is_same_host(url): raise HostChangedError(self, url, retries) + # Ensure that the URL we're connecting to is properly encoded + if url.startswith("/"): + url = six.ensure_str(_encode_target(url)) + else: + url = six.ensure_str(parse_url(url).url) + conn = None # Track whether `conn` needs to be released before @@ -560,13 +631,13 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # # See issue #651 [1] for details. # - # [1] <https://github.com/shazow/urllib3/issues/651> + # [1] <https://github.com/urllib3/urllib3/issues/651> release_this_conn = release_conn # Merge the proxy headers. Only do this in HTTP. We have to copy the # headers dict so we can safely change it without those changes being # reflected in anyone else's copy. - if self.scheme == 'http': + if self.scheme == "http": headers = headers.copy() headers.update(self.proxy_headers) @@ -589,15 +660,22 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): conn.timeout = timeout_obj.connect_timeout - is_new_proxy_conn = self.proxy is not None and not getattr(conn, 'sock', None) + is_new_proxy_conn = self.proxy is not None and not getattr( + conn, "sock", None + ) if is_new_proxy_conn: self._prepare_proxy(conn) # Make the request on the httplib connection object. - httplib_response = self._make_request(conn, method, url, - timeout=timeout_obj, - body=body, headers=headers, - chunked=chunked) + httplib_response = self._make_request( + conn, + method, + url, + timeout=timeout_obj, + body=body, + headers=headers, + chunked=chunked, + ) # If we're going to release the connection in ``finally:``, then # the response doesn't need to know about the connection. Otherwise @@ -606,14 +684,16 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): response_conn = conn if not release_conn else None # Pass method to Response for length checking - response_kw['request_method'] = method + response_kw["request_method"] = method # Import httplib's response into our own wrapper object - response = self.ResponseCls.from_httplib(httplib_response, - pool=self, - connection=response_conn, - retries=retries, - **response_kw) + response = self.ResponseCls.from_httplib( + httplib_response, + pool=self, + connection=response_conn, + retries=retries, + **response_kw + ) # Everything went great! clean_exit = True @@ -622,20 +702,28 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): # Timed out by queue. raise EmptyPoolError(self, "No pool connections are available.") - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError, CertificateError) as e: + except ( + TimeoutError, + HTTPException, + SocketError, + ProtocolError, + BaseSSLError, + SSLError, + CertificateError, + ) as e: # Discard the connection for these exceptions. It will be # replaced during the next _get_conn() call. clean_exit = False if isinstance(e, (BaseSSLError, CertificateError)): e = SSLError(e) elif isinstance(e, (SocketError, NewConnectionError)) and self.proxy: - e = ProxyError('Cannot connect to proxy.', e) + e = ProxyError("Cannot connect to proxy.", e) elif isinstance(e, (SocketError, HTTPException)): - e = ProtocolError('Connection aborted.', e) + e = ProtocolError("Connection aborted.", e) - retries = retries.increment(method, url, error=e, _pool=self, - _stacktrace=sys.exc_info()[2]) + retries = retries.increment( + method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2] + ) retries.sleep() # Keep track of the error for the retry warning. @@ -658,77 +746,87 @@ class HTTPConnectionPool(ConnectionPool, RequestMethods): if not conn: # Try again - log.warning("Retrying (%r) after connection " - "broken by '%r': %s", retries, err, url) - return self.urlopen(method, url, body, headers, retries, - redirect, assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) - - def drain_and_release_conn(response): - try: - # discard any remaining response body, the connection will be - # released back to the pool once the entire response is read - response.read() - except (TimeoutError, HTTPException, SocketError, ProtocolError, - BaseSSLError, SSLError) as e: - pass + log.warning( + "Retrying (%r) after connection broken by '%r': %s", retries, err, url + ) + return self.urlopen( + method, + url, + body, + headers, + retries, + redirect, + assert_same_host, + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + chunked=chunked, + body_pos=body_pos, + **response_kw + ) # Handle redirect? redirect_location = redirect and response.get_redirect_location() if redirect_location: if response.status == 303: - method = 'GET' + method = "GET" try: retries = retries.increment(method, url, response=response, _pool=self) except MaxRetryError: if retries.raise_on_redirect: - # Drain and release the connection for this response, since - # we're not returning it to be released manually. - drain_and_release_conn(response) + response.drain_conn() raise return response - # drain and return the connection to the pool before recursing - drain_and_release_conn(response) - + response.drain_conn() retries.sleep_for_retry(response) log.debug("Redirecting %s -> %s", url, redirect_location) return self.urlopen( - method, redirect_location, body, headers, - retries=retries, redirect=redirect, + method, + redirect_location, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, - release_conn=release_conn, body_pos=body_pos, - **response_kw) + timeout=timeout, + pool_timeout=pool_timeout, + release_conn=release_conn, + chunked=chunked, + body_pos=body_pos, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(response.getheader('Retry-After')) + has_retry_after = bool(response.getheader("Retry-After")) if retries.is_retry(method, response.status, has_retry_after): try: retries = retries.increment(method, url, response=response, _pool=self) except MaxRetryError: if retries.raise_on_status: - # Drain and release the connection for this response, since - # we're not returning it to be released manually. - drain_and_release_conn(response) + response.drain_conn() raise return response - # drain and return the connection to the pool before recursing - drain_and_release_conn(response) - + response.drain_conn() retries.sleep(response) log.debug("Retry: %s", url) return self.urlopen( - method, url, body, headers, - retries=retries, redirect=redirect, + method, + url, + body, + headers, + retries=retries, + redirect=redirect, assert_same_host=assert_same_host, - timeout=timeout, pool_timeout=pool_timeout, + timeout=timeout, + pool_timeout=pool_timeout, release_conn=release_conn, - body_pos=body_pos, **response_kw) + chunked=chunked, + body_pos=body_pos, + **response_kw + ) return response @@ -746,33 +844,57 @@ class HTTPSConnectionPool(HTTPConnectionPool): If ``assert_hostname`` is False, no verification is done. The ``key_file``, ``cert_file``, ``cert_reqs``, ``ca_certs``, - ``ca_cert_dir``, and ``ssl_version`` are only used if :mod:`ssl` is - available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade + ``ca_cert_dir``, ``ssl_version``, ``key_password`` are only used if :mod:`ssl` + is available and are fed into :meth:`urllib3.util.ssl_wrap_socket` to upgrade the connection socket into an SSL socket. """ - scheme = 'https' + scheme = "https" ConnectionCls = HTTPSConnection - def __init__(self, host, port=None, - strict=False, timeout=Timeout.DEFAULT_TIMEOUT, maxsize=1, - block=False, headers=None, retries=None, - _proxy=None, _proxy_headers=None, - key_file=None, cert_file=None, cert_reqs=None, - ca_certs=None, ssl_version=None, - assert_hostname=None, assert_fingerprint=None, - ca_cert_dir=None, **conn_kw): + def __init__( + self, + host, + port=None, + strict=False, + timeout=Timeout.DEFAULT_TIMEOUT, + maxsize=1, + block=False, + headers=None, + retries=None, + _proxy=None, + _proxy_headers=None, + key_file=None, + cert_file=None, + cert_reqs=None, + key_password=None, + ca_certs=None, + ssl_version=None, + assert_hostname=None, + assert_fingerprint=None, + ca_cert_dir=None, + **conn_kw + ): - HTTPConnectionPool.__init__(self, host, port, strict, timeout, maxsize, - block, headers, retries, _proxy, _proxy_headers, - **conn_kw) - - if ca_certs and cert_reqs is None: - cert_reqs = 'CERT_REQUIRED' + HTTPConnectionPool.__init__( + self, + host, + port, + strict, + timeout, + maxsize, + block, + headers, + retries, + _proxy, + _proxy_headers, + **conn_kw + ) self.key_file = key_file self.cert_file = cert_file self.cert_reqs = cert_reqs + self.key_password = key_password self.ca_certs = ca_certs self.ca_cert_dir = ca_cert_dir self.ssl_version = ssl_version @@ -786,13 +908,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): """ if isinstance(conn, VerifiedHTTPSConnection): - conn.set_cert(key_file=self.key_file, - cert_file=self.cert_file, - cert_reqs=self.cert_reqs, - ca_certs=self.ca_certs, - ca_cert_dir=self.ca_cert_dir, - assert_hostname=self.assert_hostname, - assert_fingerprint=self.assert_fingerprint) + conn.set_cert( + key_file=self.key_file, + key_password=self.key_password, + cert_file=self.cert_file, + cert_reqs=self.cert_reqs, + ca_certs=self.ca_certs, + ca_cert_dir=self.ca_cert_dir, + assert_hostname=self.assert_hostname, + assert_fingerprint=self.assert_fingerprint, + ) conn.ssl_version = self.ssl_version return conn @@ -809,12 +934,17 @@ class HTTPSConnectionPool(HTTPConnectionPool): Return a fresh :class:`httplib.HTTPSConnection`. """ self.num_connections += 1 - log.debug("Starting new HTTPS connection (%d): %s:%s", - self.num_connections, self.host, self.port or "443") + log.debug( + "Starting new HTTPS connection (%d): %s:%s", + self.num_connections, + self.host, + self.port or "443", + ) if not self.ConnectionCls or self.ConnectionCls is DummyConnection: - raise SSLError("Can't connect to HTTPS URL because the SSL " - "module is not available.") + raise SSLError( + "Can't connect to HTTPS URL because the SSL module is not available." + ) actual_host = self.host actual_port = self.port @@ -822,9 +952,16 @@ class HTTPSConnectionPool(HTTPConnectionPool): actual_host = self.proxy.host actual_port = self.proxy.port - conn = self.ConnectionCls(host=actual_host, port=actual_port, - timeout=self.timeout.connect_timeout, - strict=self.strict, **self.conn_kw) + conn = self.ConnectionCls( + host=actual_host, + port=actual_port, + timeout=self.timeout.connect_timeout, + strict=self.strict, + cert_file=self.cert_file, + key_file=self.key_file, + key_password=self.key_password, + **self.conn_kw + ) return self._prepare_conn(conn) @@ -835,16 +972,19 @@ class HTTPSConnectionPool(HTTPConnectionPool): super(HTTPSConnectionPool, self)._validate_conn(conn) # Force connect early to allow us to validate the connection. - if not getattr(conn, 'sock', None): # AppEngine might not have `.sock` + if not getattr(conn, "sock", None): # AppEngine might not have `.sock` conn.connect() if not conn.is_verified: - warnings.warn(( - 'Unverified HTTPS request is being made. ' - 'Adding certificate verification is strongly advised. See: ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings'), - InsecureRequestWarning) + warnings.warn( + ( + "Unverified HTTPS request is being made to host '%s'. " + "Adding certificate verification is strongly advised. See: " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings" % conn.host + ), + InsecureRequestWarning, + ) def connection_from_url(url, **kw): @@ -869,28 +1009,25 @@ def connection_from_url(url, **kw): """ scheme, host, port = get_host(url) port = port or port_by_scheme.get(scheme, 80) - if scheme == 'https': + if scheme == "https": return HTTPSConnectionPool(host, port=port, **kw) else: return HTTPConnectionPool(host, port=port, **kw) -def _ipv6_host(host, scheme): +def _normalize_host(host, scheme): """ - Process IPv6 address literals + Normalize hosts for comparisons and use with sockets. """ + host = normalize_host(host, scheme) + # httplib doesn't like it when we include brackets in IPv6 addresses # Specifically, if we include brackets but also pass the port then # httplib crazily doubles up the square brackets on the Host header. # Instead, we need to make sure we never pass ``None`` as the port. # However, for backward compatibility reasons we can't actually # *assert* that. See http://bugs.python.org/issue28539 - # - # Also if an IPv6 address literal has a zone identifier, the - # percent sign might be URIencoded, convert it back into ASCII - if host.startswith('[') and host.endswith(']'): - host = host.replace('%25', '%').strip('[]') - if scheme in NORMALIZABLE_SCHEMES: - host = host.lower() + if host.startswith("[") and host.endswith("]"): + host = host[1:-1] return host diff --git a/pipenv/vendor/urllib3/contrib/_appengine_environ.py b/pipenv/vendor/urllib3/contrib/_appengine_environ.py index f3e00942..8765b907 100644 --- a/pipenv/vendor/urllib3/contrib/_appengine_environ.py +++ b/pipenv/vendor/urllib3/contrib/_appengine_environ.py @@ -6,25 +6,31 @@ import os def is_appengine(): - return (is_local_appengine() or - is_prod_appengine() or - is_prod_appengine_mvms()) + return is_local_appengine() or is_prod_appengine() def is_appengine_sandbox(): - return is_appengine() and not is_prod_appengine_mvms() + """Reports if the app is running in the first generation sandbox. + + The second generation runtimes are technically still in a sandbox, but it + is much less restrictive, so generally you shouldn't need to check for it. + see https://cloud.google.com/appengine/docs/standard/runtimes + """ + return is_appengine() and os.environ["APPENGINE_RUNTIME"] == "python27" def is_local_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Development/' in os.environ['SERVER_SOFTWARE']) + return "APPENGINE_RUNTIME" in os.environ and os.environ.get( + "SERVER_SOFTWARE", "" + ).startswith("Development/") def is_prod_appengine(): - return ('APPENGINE_RUNTIME' in os.environ and - 'Google App Engine/' in os.environ['SERVER_SOFTWARE'] and - not is_prod_appengine_mvms()) + return "APPENGINE_RUNTIME" in os.environ and os.environ.get( + "SERVER_SOFTWARE", "" + ).startswith("Google App Engine/") def is_prod_appengine_mvms(): - return os.environ.get('GAE_VM', False) == 'true' + """Deprecated.""" + return False diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py index bcf41c02..d9b67333 100644 --- a/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py +++ b/pipenv/vendor/urllib3/contrib/_securetransport/bindings.py @@ -34,29 +34,35 @@ from __future__ import absolute_import import platform from ctypes.util import find_library from ctypes import ( - c_void_p, c_int32, c_char_p, c_size_t, c_byte, c_uint32, c_ulong, c_long, - c_bool + c_void_p, + c_int32, + c_char_p, + c_size_t, + c_byte, + c_uint32, + c_ulong, + c_long, + c_bool, ) from ctypes import CDLL, POINTER, CFUNCTYPE -security_path = find_library('Security') +security_path = find_library("Security") if not security_path: - raise ImportError('The library Security could not be found') + raise ImportError("The library Security could not be found") -core_foundation_path = find_library('CoreFoundation') +core_foundation_path = find_library("CoreFoundation") if not core_foundation_path: - raise ImportError('The library CoreFoundation could not be found') + raise ImportError("The library CoreFoundation could not be found") version = platform.mac_ver()[0] -version_info = tuple(map(int, version.split('.'))) +version_info = tuple(map(int, version.split("."))) if version_info < (10, 8): raise OSError( - 'Only OS X 10.8 and newer are supported, not %s.%s' % ( - version_info[0], version_info[1] - ) + "Only OS X 10.8 and newer are supported, not %s.%s" + % (version_info[0], version_info[1]) ) Security = CDLL(security_path, use_errno=True) @@ -129,27 +135,19 @@ try: Security.SecKeyGetTypeID.argtypes = [] Security.SecKeyGetTypeID.restype = CFTypeID - Security.SecCertificateCreateWithData.argtypes = [ - CFAllocatorRef, - CFDataRef - ] + Security.SecCertificateCreateWithData.argtypes = [CFAllocatorRef, CFDataRef] Security.SecCertificateCreateWithData.restype = SecCertificateRef - Security.SecCertificateCopyData.argtypes = [ - SecCertificateRef - ] + Security.SecCertificateCopyData.argtypes = [SecCertificateRef] Security.SecCertificateCopyData.restype = CFDataRef - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SecIdentityCreateWithCertificate.argtypes = [ CFTypeRef, SecCertificateRef, - POINTER(SecIdentityRef) + POINTER(SecIdentityRef), ] Security.SecIdentityCreateWithCertificate.restype = OSStatus @@ -159,201 +157,126 @@ try: c_void_p, Boolean, c_void_p, - POINTER(SecKeychainRef) + POINTER(SecKeychainRef), ] Security.SecKeychainCreate.restype = OSStatus - Security.SecKeychainDelete.argtypes = [ - SecKeychainRef - ] + Security.SecKeychainDelete.argtypes = [SecKeychainRef] Security.SecKeychainDelete.restype = OSStatus Security.SecPKCS12Import.argtypes = [ CFDataRef, CFDictionaryRef, - POINTER(CFArrayRef) + POINTER(CFArrayRef), ] Security.SecPKCS12Import.restype = OSStatus SSLReadFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, c_void_p, POINTER(c_size_t)) - SSLWriteFunc = CFUNCTYPE(OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t)) + SSLWriteFunc = CFUNCTYPE( + OSStatus, SSLConnectionRef, POINTER(c_byte), POINTER(c_size_t) + ) - Security.SSLSetIOFuncs.argtypes = [ - SSLContextRef, - SSLReadFunc, - SSLWriteFunc - ] + Security.SSLSetIOFuncs.argtypes = [SSLContextRef, SSLReadFunc, SSLWriteFunc] Security.SSLSetIOFuncs.restype = OSStatus - Security.SSLSetPeerID.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerID.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerID.restype = OSStatus - Security.SSLSetCertificate.argtypes = [ - SSLContextRef, - CFArrayRef - ] + Security.SSLSetCertificate.argtypes = [SSLContextRef, CFArrayRef] Security.SSLSetCertificate.restype = OSStatus - Security.SSLSetCertificateAuthorities.argtypes = [ - SSLContextRef, - CFTypeRef, - Boolean - ] + Security.SSLSetCertificateAuthorities.argtypes = [SSLContextRef, CFTypeRef, Boolean] Security.SSLSetCertificateAuthorities.restype = OSStatus - Security.SSLSetConnection.argtypes = [ - SSLContextRef, - SSLConnectionRef - ] + Security.SSLSetConnection.argtypes = [SSLContextRef, SSLConnectionRef] Security.SSLSetConnection.restype = OSStatus - Security.SSLSetPeerDomainName.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t - ] + Security.SSLSetPeerDomainName.argtypes = [SSLContextRef, c_char_p, c_size_t] Security.SSLSetPeerDomainName.restype = OSStatus - Security.SSLHandshake.argtypes = [ - SSLContextRef - ] + Security.SSLHandshake.argtypes = [SSLContextRef] Security.SSLHandshake.restype = OSStatus - Security.SSLRead.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLRead.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLRead.restype = OSStatus - Security.SSLWrite.argtypes = [ - SSLContextRef, - c_char_p, - c_size_t, - POINTER(c_size_t) - ] + Security.SSLWrite.argtypes = [SSLContextRef, c_char_p, c_size_t, POINTER(c_size_t)] Security.SSLWrite.restype = OSStatus - Security.SSLClose.argtypes = [ - SSLContextRef - ] + Security.SSLClose.argtypes = [SSLContextRef] Security.SSLClose.restype = OSStatus - Security.SSLGetNumberSupportedCiphers.argtypes = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberSupportedCiphers.argtypes = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberSupportedCiphers.restype = OSStatus Security.SSLGetSupportedCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetSupportedCiphers.restype = OSStatus Security.SSLSetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - c_size_t + c_size_t, ] Security.SSLSetEnabledCiphers.restype = OSStatus - Security.SSLGetNumberEnabledCiphers.argtype = [ - SSLContextRef, - POINTER(c_size_t) - ] + Security.SSLGetNumberEnabledCiphers.argtype = [SSLContextRef, POINTER(c_size_t)] Security.SSLGetNumberEnabledCiphers.restype = OSStatus Security.SSLGetEnabledCiphers.argtypes = [ SSLContextRef, POINTER(SSLCipherSuite), - POINTER(c_size_t) + POINTER(c_size_t), ] Security.SSLGetEnabledCiphers.restype = OSStatus - Security.SSLGetNegotiatedCipher.argtypes = [ - SSLContextRef, - POINTER(SSLCipherSuite) - ] + Security.SSLGetNegotiatedCipher.argtypes = [SSLContextRef, POINTER(SSLCipherSuite)] Security.SSLGetNegotiatedCipher.restype = OSStatus Security.SSLGetNegotiatedProtocolVersion.argtypes = [ SSLContextRef, - POINTER(SSLProtocol) + POINTER(SSLProtocol), ] Security.SSLGetNegotiatedProtocolVersion.restype = OSStatus - Security.SSLCopyPeerTrust.argtypes = [ - SSLContextRef, - POINTER(SecTrustRef) - ] + Security.SSLCopyPeerTrust.argtypes = [SSLContextRef, POINTER(SecTrustRef)] Security.SSLCopyPeerTrust.restype = OSStatus - Security.SecTrustSetAnchorCertificates.argtypes = [ - SecTrustRef, - CFArrayRef - ] + Security.SecTrustSetAnchorCertificates.argtypes = [SecTrustRef, CFArrayRef] Security.SecTrustSetAnchorCertificates.restype = OSStatus - Security.SecTrustSetAnchorCertificatesOnly.argstypes = [ - SecTrustRef, - Boolean - ] + Security.SecTrustSetAnchorCertificatesOnly.argstypes = [SecTrustRef, Boolean] Security.SecTrustSetAnchorCertificatesOnly.restype = OSStatus - Security.SecTrustEvaluate.argtypes = [ - SecTrustRef, - POINTER(SecTrustResultType) - ] + Security.SecTrustEvaluate.argtypes = [SecTrustRef, POINTER(SecTrustResultType)] Security.SecTrustEvaluate.restype = OSStatus - Security.SecTrustGetCertificateCount.argtypes = [ - SecTrustRef - ] + Security.SecTrustGetCertificateCount.argtypes = [SecTrustRef] Security.SecTrustGetCertificateCount.restype = CFIndex - Security.SecTrustGetCertificateAtIndex.argtypes = [ - SecTrustRef, - CFIndex - ] + Security.SecTrustGetCertificateAtIndex.argtypes = [SecTrustRef, CFIndex] Security.SecTrustGetCertificateAtIndex.restype = SecCertificateRef Security.SSLCreateContext.argtypes = [ CFAllocatorRef, SSLProtocolSide, - SSLConnectionType + SSLConnectionType, ] Security.SSLCreateContext.restype = SSLContextRef - Security.SSLSetSessionOption.argtypes = [ - SSLContextRef, - SSLSessionOption, - Boolean - ] + Security.SSLSetSessionOption.argtypes = [SSLContextRef, SSLSessionOption, Boolean] Security.SSLSetSessionOption.restype = OSStatus - Security.SSLSetProtocolVersionMin.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMin.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMin.restype = OSStatus - Security.SSLSetProtocolVersionMax.argtypes = [ - SSLContextRef, - SSLProtocol - ] + Security.SSLSetProtocolVersionMax.argtypes = [SSLContextRef, SSLProtocol] Security.SSLSetProtocolVersionMax.restype = OSStatus - Security.SecCopyErrorMessageString.argtypes = [ - OSStatus, - c_void_p - ] + Security.SecCopyErrorMessageString.argtypes = [OSStatus, c_void_p] Security.SecCopyErrorMessageString.restype = CFStringRef Security.SSLReadFunc = SSLReadFunc @@ -369,64 +292,47 @@ try: Security.OSStatus = OSStatus Security.kSecImportExportPassphrase = CFStringRef.in_dll( - Security, 'kSecImportExportPassphrase' + Security, "kSecImportExportPassphrase" ) Security.kSecImportItemIdentity = CFStringRef.in_dll( - Security, 'kSecImportItemIdentity' + Security, "kSecImportItemIdentity" ) # CoreFoundation time! - CoreFoundation.CFRetain.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRetain.argtypes = [CFTypeRef] CoreFoundation.CFRetain.restype = CFTypeRef - CoreFoundation.CFRelease.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFRelease.argtypes = [CFTypeRef] CoreFoundation.CFRelease.restype = None - CoreFoundation.CFGetTypeID.argtypes = [ - CFTypeRef - ] + CoreFoundation.CFGetTypeID.argtypes = [CFTypeRef] CoreFoundation.CFGetTypeID.restype = CFTypeID CoreFoundation.CFStringCreateWithCString.argtypes = [ CFAllocatorRef, c_char_p, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringCreateWithCString.restype = CFStringRef - CoreFoundation.CFStringGetCStringPtr.argtypes = [ - CFStringRef, - CFStringEncoding - ] + CoreFoundation.CFStringGetCStringPtr.argtypes = [CFStringRef, CFStringEncoding] CoreFoundation.CFStringGetCStringPtr.restype = c_char_p CoreFoundation.CFStringGetCString.argtypes = [ CFStringRef, c_char_p, CFIndex, - CFStringEncoding + CFStringEncoding, ] CoreFoundation.CFStringGetCString.restype = c_bool - CoreFoundation.CFDataCreate.argtypes = [ - CFAllocatorRef, - c_char_p, - CFIndex - ] + CoreFoundation.CFDataCreate.argtypes = [CFAllocatorRef, c_char_p, CFIndex] CoreFoundation.CFDataCreate.restype = CFDataRef - CoreFoundation.CFDataGetLength.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetLength.argtypes = [CFDataRef] CoreFoundation.CFDataGetLength.restype = CFIndex - CoreFoundation.CFDataGetBytePtr.argtypes = [ - CFDataRef - ] + CoreFoundation.CFDataGetBytePtr.argtypes = [CFDataRef] CoreFoundation.CFDataGetBytePtr.restype = c_void_p CoreFoundation.CFDictionaryCreate.argtypes = [ @@ -435,14 +341,11 @@ try: POINTER(CFTypeRef), CFIndex, CFDictionaryKeyCallBacks, - CFDictionaryValueCallBacks + CFDictionaryValueCallBacks, ] CoreFoundation.CFDictionaryCreate.restype = CFDictionaryRef - CoreFoundation.CFDictionaryGetValue.argtypes = [ - CFDictionaryRef, - CFTypeRef - ] + CoreFoundation.CFDictionaryGetValue.argtypes = [CFDictionaryRef, CFTypeRef] CoreFoundation.CFDictionaryGetValue.restype = CFTypeRef CoreFoundation.CFArrayCreate.argtypes = [ @@ -456,36 +359,30 @@ try: CoreFoundation.CFArrayCreateMutable.argtypes = [ CFAllocatorRef, CFIndex, - CFArrayCallBacks + CFArrayCallBacks, ] CoreFoundation.CFArrayCreateMutable.restype = CFMutableArrayRef - CoreFoundation.CFArrayAppendValue.argtypes = [ - CFMutableArrayRef, - c_void_p - ] + CoreFoundation.CFArrayAppendValue.argtypes = [CFMutableArrayRef, c_void_p] CoreFoundation.CFArrayAppendValue.restype = None - CoreFoundation.CFArrayGetCount.argtypes = [ - CFArrayRef - ] + CoreFoundation.CFArrayGetCount.argtypes = [CFArrayRef] CoreFoundation.CFArrayGetCount.restype = CFIndex - CoreFoundation.CFArrayGetValueAtIndex.argtypes = [ - CFArrayRef, - CFIndex - ] + CoreFoundation.CFArrayGetValueAtIndex.argtypes = [CFArrayRef, CFIndex] CoreFoundation.CFArrayGetValueAtIndex.restype = c_void_p CoreFoundation.kCFAllocatorDefault = CFAllocatorRef.in_dll( - CoreFoundation, 'kCFAllocatorDefault' + CoreFoundation, "kCFAllocatorDefault" + ) + CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll( + CoreFoundation, "kCFTypeArrayCallBacks" ) - CoreFoundation.kCFTypeArrayCallBacks = c_void_p.in_dll(CoreFoundation, 'kCFTypeArrayCallBacks') CoreFoundation.kCFTypeDictionaryKeyCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryKeyCallBacks' + CoreFoundation, "kCFTypeDictionaryKeyCallBacks" ) CoreFoundation.kCFTypeDictionaryValueCallBacks = c_void_p.in_dll( - CoreFoundation, 'kCFTypeDictionaryValueCallBacks' + CoreFoundation, "kCFTypeDictionaryValueCallBacks" ) CoreFoundation.CFTypeRef = CFTypeRef @@ -494,7 +391,7 @@ try: CoreFoundation.CFDictionaryRef = CFDictionaryRef except (AttributeError): - raise ImportError('Error initializing ctypes') + raise ImportError("Error initializing ctypes") class CFConst(object): @@ -502,6 +399,7 @@ class CFConst(object): A class object that acts as essentially a namespace for CoreFoundation constants. """ + kCFStringEncodingUTF8 = CFStringEncoding(0x08000100) @@ -509,6 +407,7 @@ class SecurityConst(object): """ A class object that acts as essentially a namespace for Security constants. """ + kSSLSessionOptionBreakOnServerAuth = 0 kSSLProtocol2 = 1 @@ -516,6 +415,9 @@ class SecurityConst(object): kTLSProtocol1 = 4 kTLSProtocol11 = 7 kTLSProtocol12 = 8 + # SecureTransport does not support TLS 1.3 even if there's a constant for it + kTLSProtocol13 = 10 + kTLSProtocolMaxSupported = 999 kSSLClientSide = 1 kSSLStreamType = 0 @@ -558,30 +460,27 @@ class SecurityConst(object): errSecInvalidTrustSettings = -25262 # Cipher suites. We only pick the ones our default cipher string allows. + # Source: https://developer.apple.com/documentation/security/1550981-ssl_cipher_suite_values TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA9 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCCA8 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D @@ -590,4 +489,5 @@ class SecurityConst(object): TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F TLS_AES_128_GCM_SHA256 = 0x1301 TLS_AES_256_GCM_SHA384 = 0x1302 - TLS_CHACHA20_POLY1305_SHA256 = 0x1303 + TLS_AES_128_CCM_8_SHA256 = 0x1305 + TLS_AES_128_CCM_SHA256 = 0x1304 diff --git a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py index b13cd9e7..e60168ca 100644 --- a/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py +++ b/pipenv/vendor/urllib3/contrib/_securetransport/low_level.py @@ -66,22 +66,18 @@ def _cf_string_to_unicode(value): value_as_void_p = ctypes.cast(value, ctypes.POINTER(ctypes.c_void_p)) string = CoreFoundation.CFStringGetCStringPtr( - value_as_void_p, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, CFConst.kCFStringEncodingUTF8 ) if string is None: buffer = ctypes.create_string_buffer(1024) result = CoreFoundation.CFStringGetCString( - value_as_void_p, - buffer, - 1024, - CFConst.kCFStringEncodingUTF8 + value_as_void_p, buffer, 1024, CFConst.kCFStringEncodingUTF8 ) if not result: - raise OSError('Error copying C string from CFStringRef') + raise OSError("Error copying C string from CFStringRef") string = buffer.value if string is not None: - string = string.decode('utf-8') + string = string.decode("utf-8") return string @@ -97,8 +93,8 @@ def _assert_no_error(error, exception_class=None): output = _cf_string_to_unicode(cf_error_string) CoreFoundation.CFRelease(cf_error_string) - if output is None or output == u'': - output = u'OSStatus %s' % error + if output is None or output == u"": + output = u"OSStatus %s" % error if exception_class is None: exception_class = ssl.SSLError @@ -115,8 +111,7 @@ def _cert_array_from_pem(pem_bundle): pem_bundle = pem_bundle.replace(b"\r\n", b"\n") der_certs = [ - base64.b64decode(match.group(1)) - for match in _PEM_CERTS_RE.finditer(pem_bundle) + base64.b64decode(match.group(1)) for match in _PEM_CERTS_RE.finditer(pem_bundle) ] if not der_certs: raise ssl.SSLError("No root certificates specified") @@ -124,7 +119,7 @@ def _cert_array_from_pem(pem_bundle): cert_array = CoreFoundation.CFArrayCreateMutable( CoreFoundation.kCFAllocatorDefault, 0, - ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks) + ctypes.byref(CoreFoundation.kCFTypeArrayCallBacks), ) if not cert_array: raise ssl.SSLError("Unable to allocate memory!") @@ -186,21 +181,16 @@ def _temporary_keychain(): # some random bytes to password-protect the keychain we're creating, so we # ask for 40 random bytes. random_bytes = os.urandom(40) - filename = base64.b16encode(random_bytes[:8]).decode('utf-8') + filename = base64.b16encode(random_bytes[:8]).decode("utf-8") password = base64.b16encode(random_bytes[8:]) # Must be valid UTF-8 tempdirectory = tempfile.mkdtemp() - keychain_path = os.path.join(tempdirectory, filename).encode('utf-8') + keychain_path = os.path.join(tempdirectory, filename).encode("utf-8") # We now want to create the keychain itself. keychain = Security.SecKeychainRef() status = Security.SecKeychainCreate( - keychain_path, - len(password), - password, - False, - None, - ctypes.byref(keychain) + keychain_path, len(password), password, False, None, ctypes.byref(keychain) ) _assert_no_error(status) @@ -219,14 +209,12 @@ def _load_items_from_file(keychain, path): identities = [] result_array = None - with open(path, 'rb') as f: + with open(path, "rb") as f: raw_filedata = f.read() try: filedata = CoreFoundation.CFDataCreate( - CoreFoundation.kCFAllocatorDefault, - raw_filedata, - len(raw_filedata) + CoreFoundation.kCFAllocatorDefault, raw_filedata, len(raw_filedata) ) result_array = CoreFoundation.CFArrayRef() result = Security.SecItemImport( @@ -237,7 +225,7 @@ def _load_items_from_file(keychain, path): 0, # import flags None, # key params, can include passphrase in the future keychain, # The keychain to insert into - ctypes.byref(result_array) # Results + ctypes.byref(result_array), # Results ) _assert_no_error(result) @@ -247,9 +235,7 @@ def _load_items_from_file(keychain, path): # keychain already has them! result_count = CoreFoundation.CFArrayGetCount(result_array) for index in range(result_count): - item = CoreFoundation.CFArrayGetValueAtIndex( - result_array, index - ) + item = CoreFoundation.CFArrayGetValueAtIndex(result_array, index) item = ctypes.cast(item, CoreFoundation.CFTypeRef) if _is_cert(item): @@ -307,9 +293,7 @@ def _load_client_cert_chain(keychain, *paths): try: for file_path in paths: - new_identities, new_certs = _load_items_from_file( - keychain, file_path - ) + new_identities, new_certs = _load_items_from_file(keychain, file_path) identities.extend(new_identities) certificates.extend(new_certs) @@ -318,9 +302,7 @@ def _load_client_cert_chain(keychain, *paths): if not identities: new_identity = Security.SecIdentityRef() status = Security.SecIdentityCreateWithCertificate( - keychain, - certificates[0], - ctypes.byref(new_identity) + keychain, certificates[0], ctypes.byref(new_identity) ) _assert_no_error(status) identities.append(new_identity) diff --git a/pipenv/vendor/urllib3/contrib/appengine.py b/pipenv/vendor/urllib3/contrib/appengine.py index 2952f114..9b7044ff 100644 --- a/pipenv/vendor/urllib3/contrib/appengine.py +++ b/pipenv/vendor/urllib3/contrib/appengine.py @@ -50,7 +50,7 @@ from ..exceptions import ( MaxRetryError, ProtocolError, TimeoutError, - SSLError + SSLError, ) from ..request import RequestMethods @@ -96,23 +96,24 @@ class AppEngineManager(RequestMethods): Beyond those cases, it will raise normal urllib3 errors. """ - def __init__(self, headers=None, retries=None, validate_certificate=True, - urlfetch_retries=True): + def __init__( + self, + headers=None, + retries=None, + validate_certificate=True, + urlfetch_retries=True, + ): if not urlfetch: raise AppEnginePlatformError( - "URLFetch is not available in this environment.") - - if is_prod_appengine_mvms(): - raise AppEnginePlatformError( - "Use normal urllib3.PoolManager instead of AppEngineManager" - "on Managed VMs, as using URLFetch is not necessary in " - "this environment.") + "URLFetch is not available in this environment." + ) warnings.warn( "urllib3 is using URLFetch on Google App Engine sandbox instead " "of sockets. To use sockets directly instead of URLFetch see " "https://urllib3.readthedocs.io/en/latest/reference/urllib3.contrib.html.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) RequestMethods.__init__(self, headers) self.validate_certificate = validate_certificate @@ -127,17 +128,22 @@ class AppEngineManager(RequestMethods): # Return False to re-raise any potential exceptions return False - def urlopen(self, method, url, body=None, headers=None, - retries=None, redirect=True, timeout=Timeout.DEFAULT_TIMEOUT, - **response_kw): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=None, + redirect=True, + timeout=Timeout.DEFAULT_TIMEOUT, + **response_kw + ): retries = self._get_retries(retries, redirect) try: - follow_redirects = ( - redirect and - retries.redirect != 0 and - retries.total) + follow_redirects = redirect and retries.redirect != 0 and retries.total response = urlfetch.fetch( url, payload=body, @@ -152,44 +158,52 @@ class AppEngineManager(RequestMethods): raise TimeoutError(self, e) except urlfetch.InvalidURLError as e: - if 'too large' in str(e): + if "too large" in str(e): raise AppEnginePlatformError( "URLFetch request too large, URLFetch only " - "supports requests up to 10mb in size.", e) + "supports requests up to 10mb in size.", + e, + ) raise ProtocolError(e) except urlfetch.DownloadError as e: - if 'Too many redirects' in str(e): + if "Too many redirects" in str(e): raise MaxRetryError(self, url, reason=e) raise ProtocolError(e) except urlfetch.ResponseTooLargeError as e: raise AppEnginePlatformError( "URLFetch response too large, URLFetch only supports" - "responses up to 32mb in size.", e) + "responses up to 32mb in size.", + e, + ) except urlfetch.SSLCertificateError as e: raise SSLError(e) except urlfetch.InvalidMethodError as e: raise AppEnginePlatformError( - "URLFetch does not support method: %s" % method, e) + "URLFetch does not support method: %s" % method, e + ) http_response = self._urlfetch_response_to_http_response( - response, retries=retries, **response_kw) + response, retries=retries, **response_kw + ) # Handle redirect? redirect_location = redirect and http_response.get_redirect_location() if redirect_location: # Check for redirect response - if (self.urlfetch_retries and retries.raise_on_redirect): + if self.urlfetch_retries and retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") else: if http_response.status == 303: - method = 'GET' + method = "GET" try: - retries = retries.increment(method, url, response=http_response, _pool=self) + retries = retries.increment( + method, url, response=http_response, _pool=self + ) except MaxRetryError: if retries.raise_on_redirect: raise MaxRetryError(self, url, "too many redirects") @@ -199,22 +213,32 @@ class AppEngineManager(RequestMethods): log.debug("Redirecting %s -> %s", url, redirect_location) redirect_url = urljoin(url, redirect_location) return self.urlopen( - method, redirect_url, body, headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + redirect_url, + body, + headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) # Check if we should retry the HTTP response. - has_retry_after = bool(http_response.getheader('Retry-After')) + has_retry_after = bool(http_response.getheader("Retry-After")) if retries.is_retry(method, http_response.status, has_retry_after): - retries = retries.increment( - method, url, response=http_response, _pool=self) + retries = retries.increment(method, url, response=http_response, _pool=self) log.debug("Retry: %s", url) retries.sleep(http_response) return self.urlopen( - method, url, - body=body, headers=headers, - retries=retries, redirect=redirect, - timeout=timeout, **response_kw) + method, + url, + body=body, + headers=headers, + retries=retries, + redirect=redirect, + timeout=timeout, + **response_kw + ) return http_response @@ -223,18 +247,18 @@ class AppEngineManager(RequestMethods): if is_prod_appengine(): # Production GAE handles deflate encoding automatically, but does # not remove the encoding header. - content_encoding = urlfetch_resp.headers.get('content-encoding') + content_encoding = urlfetch_resp.headers.get("content-encoding") - if content_encoding == 'deflate': - del urlfetch_resp.headers['content-encoding'] + if content_encoding == "deflate": + del urlfetch_resp.headers["content-encoding"] - transfer_encoding = urlfetch_resp.headers.get('transfer-encoding') + transfer_encoding = urlfetch_resp.headers.get("transfer-encoding") # We have a full response's content, # so let's make sure we don't report ourselves as chunked data. - if transfer_encoding == 'chunked': + if transfer_encoding == "chunked": encodings = transfer_encoding.split(",") - encodings.remove('chunked') - urlfetch_resp.headers['transfer-encoding'] = ','.join(encodings) + encodings.remove("chunked") + urlfetch_resp.headers["transfer-encoding"] = ",".join(encodings) original_response = HTTPResponse( # In order for decoding to work, we must present the content as @@ -262,20 +286,21 @@ class AppEngineManager(RequestMethods): warnings.warn( "URLFetch does not support granular timeout settings, " "reverting to total or default URLFetch timeout.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return timeout.total return timeout def _get_retries(self, retries, redirect): if not isinstance(retries, Retry): - retries = Retry.from_int( - retries, redirect=redirect, default=self.retries) + retries = Retry.from_int(retries, redirect=redirect, default=self.retries) if retries.connect or retries.read or retries.redirect: warnings.warn( "URLFetch only supports total retries and does not " "recognize connect, read, or redirect retry parameters.", - AppEnginePlatformWarning) + AppEnginePlatformWarning, + ) return retries diff --git a/pipenv/vendor/urllib3/contrib/ntlmpool.py b/pipenv/vendor/urllib3/contrib/ntlmpool.py index 8ea127c5..1fd242a6 100644 --- a/pipenv/vendor/urllib3/contrib/ntlmpool.py +++ b/pipenv/vendor/urllib3/contrib/ntlmpool.py @@ -20,7 +20,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): Implements an NTLM authentication version of an urllib3 connection pool """ - scheme = 'https' + scheme = "https" def __init__(self, user, pw, authurl, *args, **kwargs): """ @@ -31,7 +31,7 @@ class NTLMConnectionPool(HTTPSConnectionPool): super(NTLMConnectionPool, self).__init__(*args, **kwargs) self.authurl = authurl self.rawuser = user - user_parts = user.split('\\', 1) + user_parts = user.split("\\", 1) self.domain = user_parts[0].upper() self.user = user_parts[1] self.pw = pw @@ -40,72 +40,82 @@ class NTLMConnectionPool(HTTPSConnectionPool): # Performs the NTLM handshake that secures the connection. The socket # must be kept open while requests are performed. self.num_connections += 1 - log.debug('Starting NTLM HTTPS connection no. %d: https://%s%s', - self.num_connections, self.host, self.authurl) + log.debug( + "Starting NTLM HTTPS connection no. %d: https://%s%s", + self.num_connections, + self.host, + self.authurl, + ) - headers = {'Connection': 'Keep-Alive'} - req_header = 'Authorization' - resp_header = 'www-authenticate' + headers = {"Connection": "Keep-Alive"} + req_header = "Authorization" + resp_header = "www-authenticate" conn = HTTPSConnection(host=self.host, port=self.port) # Send negotiation message - headers[req_header] = ( - 'NTLM %s' % ntlm.create_NTLM_NEGOTIATE_MESSAGE(self.rawuser)) - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + headers[req_header] = "NTLM %s" % ntlm.create_NTLM_NEGOTIATE_MESSAGE( + self.rawuser + ) + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() reshdr = dict(res.getheaders()) - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', reshdr) - log.debug('Response data: %s [...]', res.read(100)) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", reshdr) + log.debug("Response data: %s [...]", res.read(100)) # Remove the reference to the socket, so that it can not be closed by # the response object (we want to keep the socket open) res.fp = None # Server should respond with a challenge message - auth_header_values = reshdr[resp_header].split(', ') + auth_header_values = reshdr[resp_header].split(", ") auth_header_value = None for s in auth_header_values: - if s[:5] == 'NTLM ': + if s[:5] == "NTLM ": auth_header_value = s[5:] if auth_header_value is None: - raise Exception('Unexpected %s response header: %s' % - (resp_header, reshdr[resp_header])) + raise Exception( + "Unexpected %s response header: %s" % (resp_header, reshdr[resp_header]) + ) # Send authentication message - ServerChallenge, NegotiateFlags = \ - ntlm.parse_NTLM_CHALLENGE_MESSAGE(auth_header_value) - auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE(ServerChallenge, - self.user, - self.domain, - self.pw, - NegotiateFlags) - headers[req_header] = 'NTLM %s' % auth_msg - log.debug('Request headers: %s', headers) - conn.request('GET', self.authurl, None, headers) + ServerChallenge, NegotiateFlags = ntlm.parse_NTLM_CHALLENGE_MESSAGE( + auth_header_value + ) + auth_msg = ntlm.create_NTLM_AUTHENTICATE_MESSAGE( + ServerChallenge, self.user, self.domain, self.pw, NegotiateFlags + ) + headers[req_header] = "NTLM %s" % auth_msg + log.debug("Request headers: %s", headers) + conn.request("GET", self.authurl, None, headers) res = conn.getresponse() - log.debug('Response status: %s %s', res.status, res.reason) - log.debug('Response headers: %s', dict(res.getheaders())) - log.debug('Response data: %s [...]', res.read()[:100]) + log.debug("Response status: %s %s", res.status, res.reason) + log.debug("Response headers: %s", dict(res.getheaders())) + log.debug("Response data: %s [...]", res.read()[:100]) if res.status != 200: if res.status == 401: - raise Exception('Server rejected request: wrong ' - 'username or password') - raise Exception('Wrong server response: %s %s' % - (res.status, res.reason)) + raise Exception("Server rejected request: wrong username or password") + raise Exception("Wrong server response: %s %s" % (res.status, res.reason)) res.fp = None - log.debug('Connection established') + log.debug("Connection established") return conn - def urlopen(self, method, url, body=None, headers=None, retries=3, - redirect=True, assert_same_host=True): + def urlopen( + self, + method, + url, + body=None, + headers=None, + retries=3, + redirect=True, + assert_same_host=True, + ): if headers is None: headers = {} - headers['Connection'] = 'Keep-Alive' - return super(NTLMConnectionPool, self).urlopen(method, url, body, - headers, retries, - redirect, - assert_same_host) + headers["Connection"] = "Keep-Alive" + return super(NTLMConnectionPool, self).urlopen( + method, url, body, headers, retries, redirect, assert_same_host + ) diff --git a/pipenv/vendor/urllib3/contrib/pyopenssl.py b/pipenv/vendor/urllib3/contrib/pyopenssl.py index 7c0e9465..81a80651 100644 --- a/pipenv/vendor/urllib3/contrib/pyopenssl.py +++ b/pipenv/vendor/urllib3/contrib/pyopenssl.py @@ -47,6 +47,7 @@ import OpenSSL.SSL from cryptography import x509 from cryptography.hazmat.backends.openssl import backend as openssl_backend from cryptography.hazmat.backends.openssl.x509 import _Certificate + try: from cryptography.x509 import UnsupportedExtension except ImportError: @@ -54,6 +55,7 @@ except ImportError: class UnsupportedExtension(Exception): pass + from socket import timeout, error as SocketError from io import BytesIO @@ -70,37 +72,35 @@ import sys from .. import util -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] + +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works. HAS_SNI = True # Map from urllib3 to PyOpenSSL compatible parameter-values. _openssl_versions = { - ssl.PROTOCOL_SSLv23: OpenSSL.SSL.SSLv23_METHOD, + util.PROTOCOL_TLS: OpenSSL.SSL.SSLv23_METHOD, ssl.PROTOCOL_TLSv1: OpenSSL.SSL.TLSv1_METHOD, } -if hasattr(ssl, 'PROTOCOL_TLSv1_1') and hasattr(OpenSSL.SSL, 'TLSv1_1_METHOD'): +if hasattr(ssl, "PROTOCOL_SSLv3") and hasattr(OpenSSL.SSL, "SSLv3_METHOD"): + _openssl_versions[ssl.PROTOCOL_SSLv3] = OpenSSL.SSL.SSLv3_METHOD + +if hasattr(ssl, "PROTOCOL_TLSv1_1") and hasattr(OpenSSL.SSL, "TLSv1_1_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_1] = OpenSSL.SSL.TLSv1_1_METHOD -if hasattr(ssl, 'PROTOCOL_TLSv1_2') and hasattr(OpenSSL.SSL, 'TLSv1_2_METHOD'): +if hasattr(ssl, "PROTOCOL_TLSv1_2") and hasattr(OpenSSL.SSL, "TLSv1_2_METHOD"): _openssl_versions[ssl.PROTOCOL_TLSv1_2] = OpenSSL.SSL.TLSv1_2_METHOD -try: - _openssl_versions.update({ssl.PROTOCOL_SSLv3: OpenSSL.SSL.SSLv3_METHOD}) -except AttributeError: - pass _stdlib_to_openssl_verify = { ssl.CERT_NONE: OpenSSL.SSL.VERIFY_NONE, ssl.CERT_OPTIONAL: OpenSSL.SSL.VERIFY_PEER, - ssl.CERT_REQUIRED: - OpenSSL.SSL.VERIFY_PEER + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ssl.CERT_REQUIRED: OpenSSL.SSL.VERIFY_PEER + + OpenSSL.SSL.VERIFY_FAIL_IF_NO_PEER_CERT, } -_openssl_to_stdlib_verify = dict( - (v, k) for k, v in _stdlib_to_openssl_verify.items() -) +_openssl_to_stdlib_verify = dict((v, k) for k, v in _stdlib_to_openssl_verify.items()) # OpenSSL will only write 16K at a time SSL_WRITE_BLOCKSIZE = 16384 @@ -113,10 +113,11 @@ log = logging.getLogger(__name__) def inject_into_urllib3(): - 'Monkey-patch urllib3 with PyOpenSSL-backed SSL-support.' + "Monkey-patch urllib3 with PyOpenSSL-backed SSL-support." _validate_dependencies_met() + util.SSLContext = PyOpenSSLContext util.ssl_.SSLContext = PyOpenSSLContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -125,8 +126,9 @@ def inject_into_urllib3(): def extract_from_urllib3(): - 'Undo monkey-patching by :func:`inject_into_urllib3`.' + "Undo monkey-patching by :func:`inject_into_urllib3`." + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -141,17 +143,23 @@ def _validate_dependencies_met(): """ # Method added in `cryptography==1.1`; not available in older versions from cryptography.x509.extensions import Extensions + if getattr(Extensions, "get_extension_for_class", None) is None: - raise ImportError("'cryptography' module missing required functionality. " - "Try upgrading to v1.3.4 or newer.") + raise ImportError( + "'cryptography' module missing required functionality. " + "Try upgrading to v1.3.4 or newer." + ) # pyOpenSSL 0.14 and above use cryptography for OpenSSL bindings. The _x509 # attribute is only present on those versions. from OpenSSL.crypto import X509 + x509 = X509() if getattr(x509, "_x509", None) is None: - raise ImportError("'pyOpenSSL' module missing required functionality. " - "Try upgrading to v0.14 or newer.") + raise ImportError( + "'pyOpenSSL' module missing required functionality. " + "Try upgrading to v0.14 or newer." + ) def _dnsname_to_stdlib(name): @@ -167,6 +175,7 @@ def _dnsname_to_stdlib(name): If the name cannot be idna-encoded then we return None signalling that the name given should be skipped. """ + def idna_encode(name): """ Borrowed wholesale from the Python Cryptography Project. It turns out @@ -176,19 +185,23 @@ def _dnsname_to_stdlib(name): import idna try: - for prefix in [u'*.', u'.']: + for prefix in [u"*.", u"."]: if name.startswith(prefix): - name = name[len(prefix):] - return prefix.encode('ascii') + idna.encode(name) + name = name[len(prefix) :] + return prefix.encode("ascii") + idna.encode(name) return idna.encode(name) except idna.core.IDNAError: return None + # Don't send IPv6 addresses through the IDNA encoder. + if ":" in name: + return name + name = idna_encode(name) if name is None: return None elif sys.version_info >= (3, 0): - name = name.decode('utf-8') + name = name.decode("utf-8") return name @@ -207,14 +220,16 @@ def get_subj_alt_name(peer_cert): # We want to find the SAN extension. Ask Cryptography to locate it (it's # faster than looping in Python) try: - ext = cert.extensions.get_extension_for_class( - x509.SubjectAlternativeName - ).value + ext = cert.extensions.get_extension_for_class(x509.SubjectAlternativeName).value except x509.ExtensionNotFound: # No such extension, return the empty list. return [] - except (x509.DuplicateExtension, UnsupportedExtension, - x509.UnsupportedGeneralNameType, UnicodeError) as e: + except ( + x509.DuplicateExtension, + UnsupportedExtension, + x509.UnsupportedGeneralNameType, + UnicodeError, + ) as e: # A problem has been found with the quality of the certificate. Assume # no SAN field is present. log.warning( @@ -233,23 +248,23 @@ def get_subj_alt_name(peer_cert): # does with certificates, and so we need to attempt to do the same. # We also want to skip over names which cannot be idna encoded. names = [ - ('DNS', name) for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) + ("DNS", name) + for name in map(_dnsname_to_stdlib, ext.get_values_for_type(x509.DNSName)) if name is not None ] names.extend( - ('IP Address', str(name)) - for name in ext.get_values_for_type(x509.IPAddress) + ("IP Address", str(name)) for name in ext.get_values_for_type(x509.IPAddress) ) return names class WrappedSocket(object): - '''API-compatibility wrapper for Python OpenSSL's Connection-class. + """API-compatibility wrapper for Python OpenSSL's Connection-class. Note: _makefile_refs, _drop() and _reuse() are needed for the garbage collector of pypy. - ''' + """ def __init__(self, connection, socket, suppress_ragged_eofs=True): self.connection = connection @@ -272,20 +287,24 @@ class WrappedSocket(object): try: data = self.connection.recv(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): - return b'' + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): + return b"" else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: - return b'' + return b"" else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv(*args, **kwargs) + + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) else: return data @@ -293,21 +312,25 @@ class WrappedSocket(object): try: return self.connection.recv_into(*args, **kwargs) except OpenSSL.SSL.SysCallError as e: - if self.suppress_ragged_eofs and e.args == (-1, 'Unexpected EOF'): + if self.suppress_ragged_eofs and e.args == (-1, "Unexpected EOF"): return 0 else: raise SocketError(str(e)) - except OpenSSL.SSL.ZeroReturnError as e: + except OpenSSL.SSL.ZeroReturnError: if self.connection.get_shutdown() == OpenSSL.SSL.RECEIVED_SHUTDOWN: return 0 else: raise except OpenSSL.SSL.WantReadError: if not util.wait_for_read(self.socket, self.socket.gettimeout()): - raise timeout('The read operation timed out') + raise timeout("The read operation timed out") else: return self.recv_into(*args, **kwargs) + # TLS 1.3 post-handshake authentication + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("read error: %r" % e) + def settimeout(self, timeout): return self.socket.settimeout(timeout) @@ -325,7 +348,9 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self._send_until_done(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self._send_until_done( + data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE] + ) total_sent += sent def shutdown(self): @@ -349,17 +374,16 @@ class WrappedSocket(object): return x509 if binary_form: - return OpenSSL.crypto.dump_certificate( - OpenSSL.crypto.FILETYPE_ASN1, - x509) + return OpenSSL.crypto.dump_certificate(OpenSSL.crypto.FILETYPE_ASN1, x509) return { - 'subject': ( - (('commonName', x509.get_subject().CN),), - ), - 'subjectAltName': get_subj_alt_name(x509) + "subject": ((("commonName", x509.get_subject().CN),),), + "subjectAltName": get_subj_alt_name(x509), } + def version(self): + return self.connection.get_protocol_version_name() + def _reuse(self): self._makefile_refs += 1 @@ -371,9 +395,12 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 makefile = backport_makefile @@ -386,6 +413,7 @@ class PyOpenSSLContext(object): for translating the interface of the standard library ``SSLContext`` object to calls into PyOpenSSL. """ + def __init__(self, protocol): self.protocol = _openssl_versions[protocol] self._ctx = OpenSSL.SSL.Context(self.protocol) @@ -407,41 +435,48 @@ class PyOpenSSLContext(object): @verify_mode.setter def verify_mode(self, value): - self._ctx.set_verify( - _stdlib_to_openssl_verify[value], - _verify_callback - ) + self._ctx.set_verify(_stdlib_to_openssl_verify[value], _verify_callback) def set_default_verify_paths(self): self._ctx.set_default_verify_paths() def set_ciphers(self, ciphers): if isinstance(ciphers, six.text_type): - ciphers = ciphers.encode('utf-8') + ciphers = ciphers.encode("utf-8") self._ctx.set_cipher_list(ciphers) def load_verify_locations(self, cafile=None, capath=None, cadata=None): if cafile is not None: - cafile = cafile.encode('utf-8') + cafile = cafile.encode("utf-8") if capath is not None: - capath = capath.encode('utf-8') - self._ctx.load_verify_locations(cafile, capath) - if cadata is not None: - self._ctx.load_verify_locations(BytesIO(cadata)) + capath = capath.encode("utf-8") + try: + self._ctx.load_verify_locations(cafile, capath) + if cadata is not None: + self._ctx.load_verify_locations(BytesIO(cadata)) + except OpenSSL.SSL.Error as e: + raise ssl.SSLError("unable to load trusted certificates: %r" % e) def load_cert_chain(self, certfile, keyfile=None, password=None): self._ctx.use_certificate_chain_file(certfile) if password is not None: - self._ctx.set_passwd_cb(lambda max_length, prompt_twice, userdata: password) + if not isinstance(password, six.binary_type): + password = password.encode("utf-8") + self._ctx.set_passwd_cb(lambda *_: password) self._ctx.use_privatekey_file(keyfile or certfile) - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): cnx = OpenSSL.SSL.Connection(self._ctx, sock) if isinstance(server_hostname, six.text_type): # Platform-specific: Python 3 - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") if server_hostname is not None: cnx.set_tlsext_host_name(server_hostname) @@ -453,10 +488,10 @@ class PyOpenSSLContext(object): cnx.do_handshake() except OpenSSL.SSL.WantReadError: if not util.wait_for_read(sock, sock.gettimeout()): - raise timeout('select timed out') + raise timeout("select timed out") continue except OpenSSL.SSL.Error as e: - raise ssl.SSLError('bad handshake: %r' % e) + raise ssl.SSLError("bad handshake: %r" % e) break return WrappedSocket(cnx, sock) diff --git a/pipenv/vendor/urllib3/contrib/securetransport.py b/pipenv/vendor/urllib3/contrib/securetransport.py index 77cb59ed..a6b7e94a 100644 --- a/pipenv/vendor/urllib3/contrib/securetransport.py +++ b/pipenv/vendor/urllib3/contrib/securetransport.py @@ -23,6 +23,31 @@ To use this module, simply import and inject it:: urllib3.contrib.securetransport.inject_into_urllib3() Happy TLSing! + +This code is a bastardised version of the code found in Will Bond's oscrypto +library. An enormous debt is owed to him for blazing this trail for us. For +that reason, this code should be considered to be covered both by urllib3's +license and by oscrypto's: + + Copyright (c) 2015-2016 Will Bond <will@wbond.net> + + Permission is hereby granted, free of charge, to any person obtaining a + copy of this software and associated documentation files (the "Software"), + to deal in the Software without restriction, including without limitation + the rights to use, copy, modify, merge, publish, distribute, sublicense, + and/or sell copies of the Software, and to permit persons to whom the + Software is furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + DEALINGS IN THE SOFTWARE. """ from __future__ import absolute_import @@ -37,12 +62,12 @@ import threading import weakref from .. import util -from ._securetransport.bindings import ( - Security, SecurityConst, CoreFoundation -) +from ._securetransport.bindings import Security, SecurityConst, CoreFoundation from ._securetransport.low_level import ( - _assert_no_error, _cert_array_from_pem, _temporary_keychain, - _load_client_cert_chain + _assert_no_error, + _cert_array_from_pem, + _temporary_keychain, + _load_client_cert_chain, ) try: # Platform-specific: Python 2 @@ -51,7 +76,7 @@ except ImportError: # Platform-specific: Python 3 _fileobject = None from ..packages.backports.makefile import backport_makefile -__all__ = ['inject_into_urllib3', 'extract_from_urllib3'] +__all__ = ["inject_into_urllib3", "extract_from_urllib3"] # SNI always works HAS_SNI = True @@ -86,35 +111,32 @@ SSL_WRITE_BLOCKSIZE = 16384 # individual cipher suites. We need to do this because this is how # SecureTransport wants them. CIPHER_SUITES = [ - SecurityConst.TLS_AES_256_GCM_SHA384, - SecurityConst.TLS_CHACHA20_POLY1305_SHA256, - SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_GCM_SHA384, + SecurityConst.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + SecurityConst.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_256_GCM_SHA384, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_GCM_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA256, - SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_256_CBC_SHA, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA256, + SecurityConst.TLS_DHE_RSA_WITH_AES_256_CBC_SHA, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA256, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_DHE_RSA_WITH_AES_128_CBC_SHA, - SecurityConst.TLS_DHE_DSS_WITH_AES_128_CBC_SHA, + SecurityConst.TLS_AES_256_GCM_SHA384, + SecurityConst.TLS_AES_128_GCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_GCM_SHA384, SecurityConst.TLS_RSA_WITH_AES_128_GCM_SHA256, + SecurityConst.TLS_AES_128_CCM_8_SHA256, + SecurityConst.TLS_AES_128_CCM_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_128_CBC_SHA256, SecurityConst.TLS_RSA_WITH_AES_256_CBC_SHA, @@ -123,38 +145,43 @@ CIPHER_SUITES = [ # Basically this is simple: for PROTOCOL_SSLv23 we turn it into a low of # TLSv1 and a high of TLSv1.2. For everything else, we pin to that version. +# TLSv1 to 1.2 are supported on macOS 10.8+ _protocol_to_min_max = { - ssl.PROTOCOL_SSLv23: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12), + util.PROTOCOL_TLS: (SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol12) } if hasattr(ssl, "PROTOCOL_SSLv2"): _protocol_to_min_max[ssl.PROTOCOL_SSLv2] = ( - SecurityConst.kSSLProtocol2, SecurityConst.kSSLProtocol2 + SecurityConst.kSSLProtocol2, + SecurityConst.kSSLProtocol2, ) if hasattr(ssl, "PROTOCOL_SSLv3"): _protocol_to_min_max[ssl.PROTOCOL_SSLv3] = ( - SecurityConst.kSSLProtocol3, SecurityConst.kSSLProtocol3 + SecurityConst.kSSLProtocol3, + SecurityConst.kSSLProtocol3, ) if hasattr(ssl, "PROTOCOL_TLSv1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1] = ( - SecurityConst.kTLSProtocol1, SecurityConst.kTLSProtocol1 + SecurityConst.kTLSProtocol1, + SecurityConst.kTLSProtocol1, ) if hasattr(ssl, "PROTOCOL_TLSv1_1"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_1] = ( - SecurityConst.kTLSProtocol11, SecurityConst.kTLSProtocol11 + SecurityConst.kTLSProtocol11, + SecurityConst.kTLSProtocol11, ) if hasattr(ssl, "PROTOCOL_TLSv1_2"): _protocol_to_min_max[ssl.PROTOCOL_TLSv1_2] = ( - SecurityConst.kTLSProtocol12, SecurityConst.kTLSProtocol12 + SecurityConst.kTLSProtocol12, + SecurityConst.kTLSProtocol12, ) -if hasattr(ssl, "PROTOCOL_TLS"): - _protocol_to_min_max[ssl.PROTOCOL_TLS] = _protocol_to_min_max[ssl.PROTOCOL_SSLv23] def inject_into_urllib3(): """ Monkey-patch urllib3 with SecureTransport-backed SSL-support. """ + util.SSLContext = SecureTransportContext util.ssl_.SSLContext = SecureTransportContext util.HAS_SNI = HAS_SNI util.ssl_.HAS_SNI = HAS_SNI @@ -166,6 +193,7 @@ def extract_from_urllib3(): """ Undo monkey-patching by :func:`inject_into_urllib3`. """ + util.SSLContext = orig_util_SSLContext util.ssl_.SSLContext = orig_util_SSLContext util.HAS_SNI = orig_util_HAS_SNI util.ssl_.HAS_SNI = orig_util_HAS_SNI @@ -195,7 +223,7 @@ def _read_callback(connection_id, data_buffer, data_length_pointer): while read_count < requested_length: if timeout is None or timeout >= 0: if not util.wait_for_read(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") remaining = requested_length - read_count buffer = (ctypes.c_char * remaining).from_address( @@ -251,7 +279,7 @@ def _write_callback(connection_id, data_buffer, data_length_pointer): while sent < bytes_to_write: if timeout is None or timeout >= 0: if not util.wait_for_write(base_socket, timeout): - raise socket.error(errno.EAGAIN, 'timed out') + raise socket.error(errno.EAGAIN, "timed out") chunk_sent = base_socket.send(data) sent += chunk_sent @@ -293,6 +321,7 @@ class WrappedSocket(object): Note: _makefile_refs, _drop(), and _reuse() are needed for the garbage collector of PyPy. """ + def __init__(self, socket): self.socket = socket self.context = None @@ -357,7 +386,7 @@ class WrappedSocket(object): # We want data in memory, so load it up. if os.path.isfile(trust_bundle): - with open(trust_bundle, 'rb') as f: + with open(trust_bundle, "rb") as f: trust_bundle = f.read() cert_array = None @@ -371,9 +400,7 @@ class WrappedSocket(object): # created for this connection, shove our CAs into it, tell ST to # ignore everything else it knows, and then ask if it can build a # chain. This is a buuuunch of code. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: raise ssl.SSLError("Failed to copy trust reference") @@ -385,9 +412,7 @@ class WrappedSocket(object): _assert_no_error(result) trust_result = Security.SecTrustResultType() - result = Security.SecTrustEvaluate( - trust, ctypes.byref(trust_result) - ) + result = Security.SecTrustEvaluate(trust, ctypes.byref(trust_result)) _assert_no_error(result) finally: if trust: @@ -399,23 +424,24 @@ class WrappedSocket(object): # Ok, now we can look at what the result was. successes = ( SecurityConst.kSecTrustResultUnspecified, - SecurityConst.kSecTrustResultProceed + SecurityConst.kSecTrustResultProceed, ) if trust_result.value not in successes: raise ssl.SSLError( - "certificate verify failed, error code: %d" % - trust_result.value + "certificate verify failed, error code: %d" % trust_result.value ) - def handshake(self, - server_hostname, - verify, - trust_bundle, - min_version, - max_version, - client_cert, - client_key, - client_key_passphrase): + def handshake( + self, + server_hostname, + verify, + trust_bundle, + min_version, + max_version, + client_cert, + client_key, + client_key_passphrase, + ): """ Actually performs the TLS handshake. This is run automatically by wrapped socket, and shouldn't be needed in user code. @@ -445,7 +471,7 @@ class WrappedSocket(object): # If we have a server hostname, we should set that too. if server_hostname: if not isinstance(server_hostname, bytes): - server_hostname = server_hostname.encode('utf-8') + server_hostname = server_hostname.encode("utf-8") result = Security.SSLSetPeerDomainName( self.context, server_hostname, len(server_hostname) @@ -458,6 +484,7 @@ class WrappedSocket(object): # Set the minimum and maximum TLS versions. result = Security.SSLSetProtocolVersionMin(self.context, min_version) _assert_no_error(result) + result = Security.SSLSetProtocolVersionMax(self.context, max_version) _assert_no_error(result) @@ -467,9 +494,7 @@ class WrappedSocket(object): # authing in that case. if not verify or trust_bundle is not None: result = Security.SSLSetSessionOption( - self.context, - SecurityConst.kSSLSessionOptionBreakOnServerAuth, - True + self.context, SecurityConst.kSSLSessionOptionBreakOnServerAuth, True ) _assert_no_error(result) @@ -479,9 +504,7 @@ class WrappedSocket(object): self._client_cert_chain = _load_client_cert_chain( self._keychain, client_cert, client_key ) - result = Security.SSLSetCertificate( - self.context, self._client_cert_chain - ) + result = Security.SSLSetCertificate(self.context, self._client_cert_chain) _assert_no_error(result) while True: @@ -532,7 +555,7 @@ class WrappedSocket(object): # There are some result codes that we want to treat as "not always # errors". Specifically, those are errSSLWouldBlock, # errSSLClosedGraceful, and errSSLClosedNoNotify. - if (result == SecurityConst.errSSLWouldBlock): + if result == SecurityConst.errSSLWouldBlock: # If we didn't process any bytes, then this was just a time out. # However, we can get errSSLWouldBlock in situations when we *did* # read some data, and in those cases we should just read "short" @@ -540,7 +563,10 @@ class WrappedSocket(object): if processed_bytes.value == 0: # Timed out, no data read. raise socket.timeout("recv timed out") - elif result in (SecurityConst.errSSLClosedGraceful, SecurityConst.errSSLClosedNoNotify): + elif result in ( + SecurityConst.errSSLClosedGraceful, + SecurityConst.errSSLClosedNoNotify, + ): # The remote peer has closed this connection. We should do so as # well. Note that we don't actually return here because in # principle this could actually be fired along with return data. @@ -579,7 +605,7 @@ class WrappedSocket(object): def sendall(self, data): total_sent = 0 while total_sent < len(data): - sent = self.send(data[total_sent:total_sent + SSL_WRITE_BLOCKSIZE]) + sent = self.send(data[total_sent : total_sent + SSL_WRITE_BLOCKSIZE]) total_sent += sent def shutdown(self): @@ -626,18 +652,14 @@ class WrappedSocket(object): # instead to just flag to urllib3 that it shouldn't do its own hostname # validation when using SecureTransport. if not binary_form: - raise ValueError( - "SecureTransport only supports dumping binary certs" - ) + raise ValueError("SecureTransport only supports dumping binary certs") trust = Security.SecTrustRef() certdata = None der_bytes = None try: # Grab the trust store. - result = Security.SSLCopyPeerTrust( - self.context, ctypes.byref(trust) - ) + result = Security.SSLCopyPeerTrust(self.context, ctypes.byref(trust)) _assert_no_error(result) if not trust: # Probably we haven't done the handshake yet. No biggie. @@ -667,6 +689,27 @@ class WrappedSocket(object): return der_bytes + def version(self): + protocol = Security.SSLProtocol() + result = Security.SSLGetNegotiatedProtocolVersion( + self.context, ctypes.byref(protocol) + ) + _assert_no_error(result) + if protocol.value == SecurityConst.kTLSProtocol13: + raise ssl.SSLError("SecureTransport does not support TLS 1.3") + elif protocol.value == SecurityConst.kTLSProtocol12: + return "TLSv1.2" + elif protocol.value == SecurityConst.kTLSProtocol11: + return "TLSv1.1" + elif protocol.value == SecurityConst.kTLSProtocol1: + return "TLSv1" + elif protocol.value == SecurityConst.kSSLProtocol3: + return "SSLv3" + elif protocol.value == SecurityConst.kSSLProtocol2: + return "SSLv2" + else: + raise ssl.SSLError("Unknown TLS version: %r" % protocol) + def _reuse(self): self._makefile_refs += 1 @@ -678,16 +721,21 @@ class WrappedSocket(object): if _fileobject: # Platform-specific: Python 2 + def makefile(self, mode, bufsize=-1): self._makefile_refs += 1 return _fileobject(self, mode, bufsize, close=True) + + else: # Platform-specific: Python 3 + def makefile(self, mode="r", buffering=None, *args, **kwargs): # We disable buffering with SecureTransport because it conflicts with # the buffering that ST does internally (see issue #1153 for more). buffering = 0 return backport_makefile(self, mode, buffering, *args, **kwargs) + WrappedSocket.makefile = makefile @@ -697,6 +745,7 @@ class SecureTransportContext(object): interface of the standard library ``SSLContext`` object to calls into SecureTransport. """ + def __init__(self, protocol): self._min_version, self._max_version = _protocol_to_min_max[protocol] self._options = 0 @@ -763,16 +812,17 @@ class SecureTransportContext(object): def set_ciphers(self, ciphers): # For now, we just require the default cipher string. if ciphers != util.ssl_.DEFAULT_CIPHERS: - raise ValueError( - "SecureTransport doesn't support custom cipher strings" - ) + raise ValueError("SecureTransport doesn't support custom cipher strings") def load_verify_locations(self, cafile=None, capath=None, cadata=None): # OK, we only really support cadata and cafile. if capath is not None: - raise ValueError( - "SecureTransport does not support cert directories" - ) + raise ValueError("SecureTransport does not support cert directories") + + # Raise if cafile does not exist. + if cafile is not None: + with open(cafile): + pass self._trust_bundle = cafile or cadata @@ -781,9 +831,14 @@ class SecureTransportContext(object): self._client_key = keyfile self._client_cert_passphrase = password - def wrap_socket(self, sock, server_side=False, - do_handshake_on_connect=True, suppress_ragged_eofs=True, - server_hostname=None): + def wrap_socket( + self, + sock, + server_side=False, + do_handshake_on_connect=True, + suppress_ragged_eofs=True, + server_hostname=None, + ): # So, what do we do here? Firstly, we assert some properties. This is a # stripped down shim, so there is some functionality we don't support. # See PEP 543 for the real deal. @@ -797,8 +852,13 @@ class SecureTransportContext(object): # Now we can handshake wrapped_socket.handshake( - server_hostname, self._verify, self._trust_bundle, - self._min_version, self._max_version, self._client_cert, - self._client_key, self._client_key_passphrase + server_hostname, + self._verify, + self._trust_bundle, + self._min_version, + self._max_version, + self._client_cert, + self._client_key, + self._client_key_passphrase, ) return wrapped_socket diff --git a/pipenv/vendor/urllib3/contrib/socks.py b/pipenv/vendor/urllib3/contrib/socks.py index 811e312e..9e97f7aa 100644 --- a/pipenv/vendor/urllib3/contrib/socks.py +++ b/pipenv/vendor/urllib3/contrib/socks.py @@ -1,25 +1,38 @@ # -*- coding: utf-8 -*- """ This module contains provisional support for SOCKS proxies from within -urllib3. This module supports SOCKS4 (specifically the SOCKS4A variant) and +urllib3. This module supports SOCKS4, SOCKS4A (an extension of SOCKS4), and SOCKS5. To enable its functionality, either install PySocks or install this module with the ``socks`` extra. The SOCKS implementation supports the full range of urllib3 features. It also supports the following SOCKS features: -- SOCKS4 -- SOCKS4a -- SOCKS5 +- SOCKS4A (``proxy_url='socks4a://...``) +- SOCKS4 (``proxy_url='socks4://...``) +- SOCKS5 with remote DNS (``proxy_url='socks5h://...``) +- SOCKS5 with local DNS (``proxy_url='socks5://...``) - Usernames and passwords for the SOCKS proxy -Known Limitations: + .. note:: + It is recommended to use ``socks5h://`` or ``socks4a://`` schemes in + your ``proxy_url`` to ensure that DNS resolution is done from the remote + server instead of client-side when connecting to a domain name. + +SOCKS4 supports IPv4 and domain names with the SOCKS4A extension. SOCKS5 +supports IPv4, IPv6, and domain names. + +When connecting to a SOCKS4 proxy the ``username`` portion of the ``proxy_url`` +will be sent as the ``userid`` section of the SOCKS request:: + + proxy_url="socks4a://<userid>@proxy-host" + +When connecting to a SOCKS5 proxy the ``username`` and ``password`` portion +of the ``proxy_url`` will be sent as the username/password to authenticate +with the proxy:: + + proxy_url="socks5h://<username>:<password>@proxy-host" -- Currently PySocks does not support contacting remote websites via literal - IPv6 addresses. Any such connection attempt will fail. You must use a domain - name. -- Currently PySocks does not support IPv6 connections to the SOCKS proxy. Any - such connection attempt will fail. """ from __future__ import absolute_import @@ -29,23 +42,20 @@ except ImportError: import warnings from ..exceptions import DependencyWarning - warnings.warn(( - 'SOCKS support in urllib3 requires the installation of optional ' - 'dependencies: specifically, PySocks. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies' + warnings.warn( + ( + "SOCKS support in urllib3 requires the installation of optional " + "dependencies: specifically, PySocks. For more information, see " + "https://urllib3.readthedocs.io/en/latest/contrib.html#socks-proxies" ), - DependencyWarning + DependencyWarning, ) raise from socket import error as SocketError, timeout as SocketTimeout -from ..connection import ( - HTTPConnection, HTTPSConnection -) -from ..connectionpool import ( - HTTPConnectionPool, HTTPSConnectionPool -) +from ..connection import HTTPConnection, HTTPSConnection +from ..connectionpool import HTTPConnectionPool, HTTPSConnectionPool from ..exceptions import ConnectTimeoutError, NewConnectionError from ..poolmanager import PoolManager from ..util.url import parse_url @@ -60,8 +70,9 @@ class SOCKSConnection(HTTPConnection): """ A plain-text HTTP connection that connects via a SOCKS proxy. """ + def __init__(self, *args, **kwargs): - self._socks_options = kwargs.pop('_socks_options') + self._socks_options = kwargs.pop("_socks_options") super(SOCKSConnection, self).__init__(*args, **kwargs) def _new_conn(self): @@ -70,28 +81,30 @@ class SOCKSConnection(HTTPConnection): """ extra_kw = {} if self.source_address: - extra_kw['source_address'] = self.source_address + extra_kw["source_address"] = self.source_address if self.socket_options: - extra_kw['socket_options'] = self.socket_options + extra_kw["socket_options"] = self.socket_options try: conn = socks.create_connection( (self.host, self.port), - proxy_type=self._socks_options['socks_version'], - proxy_addr=self._socks_options['proxy_host'], - proxy_port=self._socks_options['proxy_port'], - proxy_username=self._socks_options['username'], - proxy_password=self._socks_options['password'], - proxy_rdns=self._socks_options['rdns'], + proxy_type=self._socks_options["socks_version"], + proxy_addr=self._socks_options["proxy_host"], + proxy_port=self._socks_options["proxy_port"], + proxy_username=self._socks_options["username"], + proxy_password=self._socks_options["password"], + proxy_rdns=self._socks_options["rdns"], timeout=self.timeout, **extra_kw ) - except SocketTimeout as e: + except SocketTimeout: raise ConnectTimeoutError( - self, "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout)) + self, + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), + ) except socks.ProxyError as e: # This is fragile as hell, but it seems to be the only way to raise @@ -101,23 +114,22 @@ class SOCKSConnection(HTTPConnection): if isinstance(error, SocketTimeout): raise ConnectTimeoutError( self, - "Connection to %s timed out. (connect timeout=%s)" % - (self.host, self.timeout) + "Connection to %s timed out. (connect timeout=%s)" + % (self.host, self.timeout), ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % error + self, "Failed to establish a new connection: %s" % error ) else: raise NewConnectionError( - self, - "Failed to establish a new connection: %s" % e + self, "Failed to establish a new connection: %s" % e ) except SocketError as e: # Defensive: PySocks should catch all these. raise NewConnectionError( - self, "Failed to establish a new connection: %s" % e) + self, "Failed to establish a new connection: %s" % e + ) return conn @@ -143,47 +155,53 @@ class SOCKSProxyManager(PoolManager): A version of the urllib3 ProxyManager that routes connections via the defined SOCKS proxy. """ + pool_classes_by_scheme = { - 'http': SOCKSHTTPConnectionPool, - 'https': SOCKSHTTPSConnectionPool, + "http": SOCKSHTTPConnectionPool, + "https": SOCKSHTTPSConnectionPool, } - def __init__(self, proxy_url, username=None, password=None, - num_pools=10, headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + username=None, + password=None, + num_pools=10, + headers=None, + **connection_pool_kw + ): parsed = parse_url(proxy_url) if username is None and password is None and parsed.auth is not None: - split = parsed.auth.split(':') + split = parsed.auth.split(":") if len(split) == 2: username, password = split - if parsed.scheme == 'socks5': + if parsed.scheme == "socks5": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = False - elif parsed.scheme == 'socks5h': + elif parsed.scheme == "socks5h": socks_version = socks.PROXY_TYPE_SOCKS5 rdns = True - elif parsed.scheme == 'socks4': + elif parsed.scheme == "socks4": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = False - elif parsed.scheme == 'socks4a': + elif parsed.scheme == "socks4a": socks_version = socks.PROXY_TYPE_SOCKS4 rdns = True else: - raise ValueError( - "Unable to determine SOCKS version from %s" % proxy_url - ) + raise ValueError("Unable to determine SOCKS version from %s" % proxy_url) self.proxy_url = proxy_url socks_options = { - 'socks_version': socks_version, - 'proxy_host': parsed.host, - 'proxy_port': parsed.port, - 'username': username, - 'password': password, - 'rdns': rdns + "socks_version": socks_version, + "proxy_host": parsed.host, + "proxy_port": parsed.port, + "username": username, + "password": password, + "rdns": rdns, } - connection_pool_kw['_socks_options'] = socks_options + connection_pool_kw["_socks_options"] = socks_options super(SOCKSProxyManager, self).__init__( num_pools, headers, **connection_pool_kw diff --git a/pipenv/vendor/urllib3/exceptions.py b/pipenv/vendor/urllib3/exceptions.py index 7bbaa987..5cc4d8a4 100644 --- a/pipenv/vendor/urllib3/exceptions.py +++ b/pipenv/vendor/urllib3/exceptions.py @@ -1,7 +1,6 @@ from __future__ import absolute_import -from .packages.six.moves.http_client import ( - IncompleteRead as httplib_IncompleteRead -) +from .packages.six.moves.http_client import IncompleteRead as httplib_IncompleteRead + # Base Exceptions @@ -17,6 +16,7 @@ class HTTPWarning(Warning): class PoolError(HTTPError): "Base exception for errors caused within a pool." + def __init__(self, pool, message): self.pool = pool HTTPError.__init__(self, "%s: %s" % (pool, message)) @@ -28,6 +28,7 @@ class PoolError(HTTPError): class RequestError(PoolError): "Base exception for PoolErrors that have associated URLs." + def __init__(self, pool, url, message): self.url = url PoolError.__init__(self, pool, message) @@ -44,7 +45,10 @@ class SSLError(HTTPError): class ProxyError(HTTPError): "Raised when the connection to a proxy fails." - pass + + def __init__(self, message, error, *args): + super(ProxyError, self).__init__(message, error, *args) + self.original_error = error class DecodeError(HTTPError): @@ -63,6 +67,7 @@ ConnectionError = ProtocolError # Leaf Exceptions + class MaxRetryError(RequestError): """Raised when the maximum number of retries is exceeded. @@ -76,8 +81,7 @@ class MaxRetryError(RequestError): def __init__(self, pool, url, reason=None): self.reason = reason - message = "Max retries exceeded with url: %s (Caused by %r)" % ( - url, reason) + message = "Max retries exceeded with url: %s (Caused by %r)" % (url, reason) RequestError.__init__(self, pool, url, message) @@ -93,6 +97,7 @@ class HostChangedError(RequestError): class TimeoutStateError(HTTPError): """ Raised when passing an invalid state to a timeout """ + pass @@ -102,6 +107,7 @@ class TimeoutError(HTTPError): Catching this error will catch both :exc:`ReadTimeoutErrors <ReadTimeoutError>` and :exc:`ConnectTimeoutErrors <ConnectTimeoutError>`. """ + pass @@ -149,8 +155,8 @@ class LocationParseError(LocationValueError): class ResponseError(HTTPError): "Used as a container for an error reason supplied in a MaxRetryError." - GENERIC_ERROR = 'too many error responses' - SPECIFIC_ERROR = 'too many {status_code} error responses' + GENERIC_ERROR = "too many error responses" + SPECIFIC_ERROR = "too many {status_code} error responses" class SecurityWarning(HTTPWarning): @@ -188,6 +194,21 @@ class DependencyWarning(HTTPWarning): Warned when an attempt is made to import a module with missing optional dependencies. """ + + pass + + +class InvalidProxyConfigurationWarning(HTTPWarning): + """ + Warned when using an HTTPS proxy and an HTTPS URL. Currently + urllib3 doesn't support HTTPS proxies and the proxy will be + contacted via HTTP instead. This warning can be fixed by + changing your HTTPS proxy URL into an HTTP proxy URL. + + If you encounter this warning read this: + https://github.com/urllib3/urllib3/issues/1850 + """ + pass @@ -201,6 +222,7 @@ class BodyNotHttplibCompatible(HTTPError): Body should be httplib.HTTPResponse like (have an fp attribute which returns raw chunks) for read_chunked(). """ + pass @@ -212,12 +234,15 @@ class IncompleteRead(HTTPError, httplib_IncompleteRead): for `partial` to avoid creating large objects on streamed reads. """ + def __init__(self, partial, expected): super(IncompleteRead, self).__init__(partial, expected) def __repr__(self): - return ('IncompleteRead(%i bytes read, ' - '%i more expected)' % (self.partial, self.expected)) + return "IncompleteRead(%i bytes read, %i more expected)" % ( + self.partial, + self.expected, + ) class InvalidHeader(HTTPError): @@ -236,8 +261,9 @@ class ProxySchemeUnknown(AssertionError, ValueError): class HeaderParsingError(HTTPError): "Raised by assert_header_parsing, but we convert it to a log.warning statement." + def __init__(self, defects, unparsed_data): - message = '%s, unparsed data: %r' % (defects or 'Unknown', unparsed_data) + message = "%s, unparsed data: %r" % (defects or "Unknown", unparsed_data) super(HeaderParsingError, self).__init__(message) diff --git a/pipenv/vendor/urllib3/fields.py b/pipenv/vendor/urllib3/fields.py index 37fe64a3..8715b220 100644 --- a/pipenv/vendor/urllib3/fields.py +++ b/pipenv/vendor/urllib3/fields.py @@ -1,11 +1,12 @@ from __future__ import absolute_import import email.utils import mimetypes +import re from .packages import six -def guess_content_type(filename, default='application/octet-stream'): +def guess_content_type(filename, default="application/octet-stream"): """ Guess the "Content-Type" of a file. @@ -19,57 +20,143 @@ def guess_content_type(filename, default='application/octet-stream'): return default -def format_header_param(name, value): +def format_header_param_rfc2231(name, value): """ - Helper function to format and quote a single header parameter. + Helper function to format and quote a single header parameter using the + strategy defined in RFC 2231. Particularly useful for header parameters which might contain - non-ASCII values, like file names. This follows RFC 2231, as - suggested by RFC 2388 Section 4.4. + non-ASCII values, like file names. This follows RFC 2388 Section 4.4. :param name: The name of the parameter, a string expected to be ASCII only. :param value: - The value of the parameter, provided as a unicode string. + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + An RFC-2231-formatted unicode string. """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + if not any(ch in value for ch in '"\\\r\n'): - result = '%s="%s"' % (name, value) + result = u'%s="%s"' % (name, value) try: - result.encode('ascii') + result.encode("ascii") except (UnicodeEncodeError, UnicodeDecodeError): pass else: return result - if not six.PY3 and isinstance(value, six.text_type): # Python 2: - value = value.encode('utf-8') - value = email.utils.encode_rfc2231(value, 'utf-8') - value = '%s*=%s' % (name, value) + + if six.PY2: # Python 2: + value = value.encode("utf-8") + + # encode_rfc2231 accepts an encoded string and returns an ascii-encoded + # string in Python 2 but accepts and returns unicode strings in Python 3 + value = email.utils.encode_rfc2231(value, "utf-8") + value = "%s*=%s" % (name, value) + + if six.PY2: # Python 2: + value = value.decode("utf-8") + return value +_HTML5_REPLACEMENTS = { + u"\u0022": u"%22", + # Replace "\" with "\\". + u"\u005C": u"\u005C\u005C", + u"\u005C": u"\u005C\u005C", +} + +# All control characters from 0x00 to 0x1F *except* 0x1B. +_HTML5_REPLACEMENTS.update( + { + six.unichr(cc): u"%{:02X}".format(cc) + for cc in range(0x00, 0x1F + 1) + if cc not in (0x1B,) + } +) + + +def _replace_multiple(value, needles_and_replacements): + def replacer(match): + return needles_and_replacements[match.group(0)] + + pattern = re.compile( + r"|".join([re.escape(needle) for needle in needles_and_replacements.keys()]) + ) + + result = pattern.sub(replacer, value) + + return result + + +def format_header_param_html5(name, value): + """ + Helper function to format and quote a single header parameter using the + HTML5 strategy. + + Particularly useful for header parameters which might contain + non-ASCII values, like file names. This follows the `HTML5 Working Draft + Section 4.10.22.7`_ and matches the behavior of curl and modern browsers. + + .. _HTML5 Working Draft Section 4.10.22.7: + https://w3c.github.io/html/sec-forms.html#multipart-form-data + + :param name: + The name of the parameter, a string expected to be ASCII only. + :param value: + The value of the parameter, provided as ``bytes`` or `str``. + :ret: + A unicode string, stripped of troublesome characters. + """ + if isinstance(value, six.binary_type): + value = value.decode("utf-8") + + value = _replace_multiple(value, _HTML5_REPLACEMENTS) + + return u'%s="%s"' % (name, value) + + +# For backwards-compatibility. +format_header_param = format_header_param_html5 + + class RequestField(object): """ A data container for request body parameters. :param name: - The name of this request field. + The name of this request field. Must be unicode. :param data: The data/value body. :param filename: - An optional filename of the request field. + An optional filename of the request field. Must be unicode. :param headers: An optional dict-like object of headers to initially use for the field. + :param header_formatter: + An optional callable that is used to encode and format the headers. By + default, this is :func:`format_header_param_html5`. """ - def __init__(self, name, data, filename=None, headers=None): + + def __init__( + self, + name, + data, + filename=None, + headers=None, + header_formatter=format_header_param_html5, + ): self._name = name self._filename = filename self.data = data self.headers = {} if headers: self.headers = dict(headers) + self.header_formatter = header_formatter @classmethod - def from_tuples(cls, fieldname, value): + def from_tuples(cls, fieldname, value, header_formatter=format_header_param_html5): """ A :class:`~urllib3.fields.RequestField` factory from old-style tuple parameters. @@ -97,21 +184,25 @@ class RequestField(object): content_type = None data = value - request_param = cls(fieldname, data, filename=filename) + request_param = cls( + fieldname, data, filename=filename, header_formatter=header_formatter + ) request_param.make_multipart(content_type=content_type) return request_param def _render_part(self, name, value): """ - Overridable helper function to format a single header parameter. + Overridable helper function to format a single header parameter. By + default, this calls ``self.header_formatter``. :param name: The name of the parameter, a string expected to be ASCII only. :param value: The value of the parameter, provided as a unicode string. """ - return format_header_param(name, value) + + return self.header_formatter(name, value) def _render_parts(self, header_parts): """ @@ -133,7 +224,7 @@ class RequestField(object): if value is not None: parts.append(self._render_part(name, value)) - return '; '.join(parts) + return u"; ".join(parts) def render_headers(self): """ @@ -141,21 +232,22 @@ class RequestField(object): """ lines = [] - sort_keys = ['Content-Disposition', 'Content-Type', 'Content-Location'] + sort_keys = ["Content-Disposition", "Content-Type", "Content-Location"] for sort_key in sort_keys: if self.headers.get(sort_key, False): - lines.append('%s: %s' % (sort_key, self.headers[sort_key])) + lines.append(u"%s: %s" % (sort_key, self.headers[sort_key])) for header_name, header_value in self.headers.items(): if header_name not in sort_keys: if header_value: - lines.append('%s: %s' % (header_name, header_value)) + lines.append(u"%s: %s" % (header_name, header_value)) - lines.append('\r\n') - return '\r\n'.join(lines) + lines.append(u"\r\n") + return u"\r\n".join(lines) - def make_multipart(self, content_disposition=None, content_type=None, - content_location=None): + def make_multipart( + self, content_disposition=None, content_type=None, content_location=None + ): """ Makes this request field into a multipart request field. @@ -168,11 +260,14 @@ class RequestField(object): The 'Content-Location' of the request body. """ - self.headers['Content-Disposition'] = content_disposition or 'form-data' - self.headers['Content-Disposition'] += '; '.join([ - '', self._render_parts( - (('name', self._name), ('filename', self._filename)) - ) - ]) - self.headers['Content-Type'] = content_type - self.headers['Content-Location'] = content_location + self.headers["Content-Disposition"] = content_disposition or u"form-data" + self.headers["Content-Disposition"] += u"; ".join( + [ + u"", + self._render_parts( + ((u"name", self._name), (u"filename", self._filename)) + ), + ] + ) + self.headers["Content-Type"] = content_type + self.headers["Content-Location"] = content_location diff --git a/pipenv/vendor/urllib3/filepost.py b/pipenv/vendor/urllib3/filepost.py index 78f1e19b..b7b00992 100644 --- a/pipenv/vendor/urllib3/filepost.py +++ b/pipenv/vendor/urllib3/filepost.py @@ -9,7 +9,7 @@ from .packages import six from .packages.six import b from .fields import RequestField -writer = codecs.lookup('utf-8')[3] +writer = codecs.lookup("utf-8")[3] def choose_boundary(): @@ -17,8 +17,8 @@ def choose_boundary(): Our embarrassingly-simple replacement for mimetools.choose_boundary. """ boundary = binascii.hexlify(os.urandom(16)) - if six.PY3: - boundary = boundary.decode('ascii') + if not six.PY2: + boundary = boundary.decode("ascii") return boundary @@ -76,7 +76,7 @@ def encode_multipart_formdata(fields, boundary=None): boundary = choose_boundary() for field in iter_field_objects(fields): - body.write(b('--%s\r\n' % (boundary))) + body.write(b("--%s\r\n" % (boundary))) writer(body).write(field.render_headers()) data = field.data @@ -89,10 +89,10 @@ def encode_multipart_formdata(fields, boundary=None): else: body.write(data) - body.write(b'\r\n') + body.write(b"\r\n") - body.write(b('--%s--\r\n' % (boundary))) + body.write(b("--%s--\r\n" % (boundary))) - content_type = str('multipart/form-data; boundary=%s' % boundary) + content_type = str("multipart/form-data; boundary=%s" % boundary) return body.getvalue(), content_type diff --git a/pipenv/vendor/urllib3/packages/__init__.py b/pipenv/vendor/urllib3/packages/__init__.py index 170e974c..fce4caa6 100644 --- a/pipenv/vendor/urllib3/packages/__init__.py +++ b/pipenv/vendor/urllib3/packages/__init__.py @@ -2,4 +2,4 @@ from __future__ import absolute_import from . import ssl_match_hostname -__all__ = ('ssl_match_hostname', ) +__all__ = ("ssl_match_hostname",) diff --git a/pipenv/vendor/urllib3/packages/backports/makefile.py b/pipenv/vendor/urllib3/packages/backports/makefile.py index 740db377..a3156a69 100644 --- a/pipenv/vendor/urllib3/packages/backports/makefile.py +++ b/pipenv/vendor/urllib3/packages/backports/makefile.py @@ -11,15 +11,14 @@ import io from socket import SocketIO -def backport_makefile(self, mode="r", buffering=None, encoding=None, - errors=None, newline=None): +def backport_makefile( + self, mode="r", buffering=None, encoding=None, errors=None, newline=None +): """ Backport of ``socket.makefile`` from Python 3.5. """ if not set(mode) <= {"r", "w", "b"}: - raise ValueError( - "invalid mode %r (only r, w, b allowed)" % (mode,) - ) + raise ValueError("invalid mode %r (only r, w, b allowed)" % (mode,)) writing = "w" in mode reading = "r" in mode or not writing assert reading or writing diff --git a/pipenv/vendor/urllib3/packages/six.py b/pipenv/vendor/urllib3/packages/six.py index 190c0239..31442409 100644 --- a/pipenv/vendor/urllib3/packages/six.py +++ b/pipenv/vendor/urllib3/packages/six.py @@ -1,6 +1,4 @@ -"""Utilities for writing code that runs on Python 2 and 3""" - -# Copyright (c) 2010-2015 Benjamin Peterson +# Copyright (c) 2010-2019 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -20,6 +18,8 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. +"""Utilities for writing code that runs on Python 2 and 3""" + from __future__ import absolute_import import functools @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson <benjamin@python.org>" -__version__ = "1.10.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -38,15 +38,15 @@ PY3 = sys.version_info[0] == 3 PY34 = sys.version_info[0:2] >= (3, 4) if PY3: - string_types = str, - integer_types = int, - class_types = type, + string_types = (str,) + integer_types = (int,) + class_types = (type,) text_type = str binary_type = bytes MAXSIZE = sys.maxsize else: - string_types = basestring, + string_types = (basestring,) integer_types = (int, long) class_types = (type, types.ClassType) text_type = unicode @@ -58,9 +58,9 @@ else: else: # It's possible to have sizeof(long) != sizeof(Py_ssize_t). class X(object): - def __len__(self): return 1 << 31 + try: len(X()) except OverflowError: @@ -84,7 +84,6 @@ def _import_module(name): class _LazyDescr(object): - def __init__(self, name): self.name = name @@ -101,7 +100,6 @@ class _LazyDescr(object): class MovedModule(_LazyDescr): - def __init__(self, name, old, new=None): super(MovedModule, self).__init__(name) if PY3: @@ -122,7 +120,6 @@ class MovedModule(_LazyDescr): class _LazyModule(types.ModuleType): - def __init__(self, name): super(_LazyModule, self).__init__(name) self.__doc__ = self.__class__.__doc__ @@ -137,7 +134,6 @@ class _LazyModule(types.ModuleType): class MovedAttribute(_LazyDescr): - def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None): super(MovedAttribute, self).__init__(name) if PY3: @@ -221,28 +217,36 @@ class _SixMetaPathImporter(object): Required, if is_package is implemented""" self.__get_module(fullname) # eventually raises ImportError return None + get_source = get_code # same as get_code + _importer = _SixMetaPathImporter(__name__) class _MovedItems(_LazyModule): """Lazy loading of moved objects""" + __path__ = [] # mark as package _moved_attributes = [ MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"), MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"), - MovedAttribute("filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse"), + MovedAttribute( + "filterfalse", "itertools", "itertools", "ifilterfalse", "filterfalse" + ), MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"), MovedAttribute("intern", "__builtin__", "sys"), MovedAttribute("map", "itertools", "builtins", "imap", "map"), MovedAttribute("getcwd", "os", "os", "getcwdu", "getcwd"), MovedAttribute("getcwdb", "os", "os", "getcwd", "getcwdb"), + MovedAttribute("getoutput", "commands", "subprocess"), MovedAttribute("range", "__builtin__", "builtins", "xrange", "range"), - MovedAttribute("reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload"), + MovedAttribute( + "reload_module", "__builtin__", "importlib" if PY34 else "imp", "reload" + ), MovedAttribute("reduce", "__builtin__", "functools"), MovedAttribute("shlex_quote", "pipes", "shlex", "quote"), MovedAttribute("StringIO", "StringIO", "io"), @@ -251,7 +255,9 @@ _moved_attributes = [ MovedAttribute("UserString", "UserString", "collections"), MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"), MovedAttribute("zip", "itertools", "builtins", "izip", "zip"), - MovedAttribute("zip_longest", "itertools", "itertools", "izip_longest", "zip_longest"), + MovedAttribute( + "zip_longest", "itertools", "itertools", "izip_longest", "zip_longest" + ), MovedModule("builtins", "__builtin__"), MovedModule("configparser", "ConfigParser"), MovedModule("copyreg", "copy_reg"), @@ -262,10 +268,13 @@ _moved_attributes = [ MovedModule("html_entities", "htmlentitydefs", "html.entities"), MovedModule("html_parser", "HTMLParser", "html.parser"), MovedModule("http_client", "httplib", "http.client"), - MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), - MovedModule("email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart"), - MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"), + MovedModule("email_mime_image", "email.MIMEImage", "email.mime.image"), + MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"), + MovedModule( + "email_mime_nonmultipart", "email.MIMENonMultipart", "email.mime.nonmultipart" + ), + MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"), MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"), MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"), MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"), @@ -283,15 +292,12 @@ _moved_attributes = [ MovedModule("tkinter_ttk", "ttk", "tkinter.ttk"), MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"), MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"), - MovedModule("tkinter_colorchooser", "tkColorChooser", - "tkinter.colorchooser"), - MovedModule("tkinter_commondialog", "tkCommonDialog", - "tkinter.commondialog"), + MovedModule("tkinter_colorchooser", "tkColorChooser", "tkinter.colorchooser"), + MovedModule("tkinter_commondialog", "tkCommonDialog", "tkinter.commondialog"), MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"), MovedModule("tkinter_font", "tkFont", "tkinter.font"), MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"), - MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", - "tkinter.simpledialog"), + MovedModule("tkinter_tksimpledialog", "tkSimpleDialog", "tkinter.simpledialog"), MovedModule("urllib_parse", __name__ + ".moves.urllib_parse", "urllib.parse"), MovedModule("urllib_error", __name__ + ".moves.urllib_error", "urllib.error"), MovedModule("urllib", __name__ + ".moves.urllib", __name__ + ".moves.urllib"), @@ -301,9 +307,7 @@ _moved_attributes = [ ] # Add windows specific modules. if sys.platform == "win32": - _moved_attributes += [ - MovedModule("winreg", "_winreg"), - ] + _moved_attributes += [MovedModule("winreg", "_winreg")] for attr in _moved_attributes: setattr(_MovedItems, attr.name, attr) @@ -337,10 +341,14 @@ _urllib_parse_moved_attributes = [ MovedAttribute("quote_plus", "urllib", "urllib.parse"), MovedAttribute("unquote", "urllib", "urllib.parse"), MovedAttribute("unquote_plus", "urllib", "urllib.parse"), + MovedAttribute( + "unquote_to_bytes", "urllib", "urllib.parse", "unquote", "unquote_to_bytes" + ), MovedAttribute("urlencode", "urllib", "urllib.parse"), MovedAttribute("splitquery", "urllib", "urllib.parse"), MovedAttribute("splittag", "urllib", "urllib.parse"), MovedAttribute("splituser", "urllib", "urllib.parse"), + MovedAttribute("splitvalue", "urllib", "urllib.parse"), MovedAttribute("uses_fragment", "urlparse", "urllib.parse"), MovedAttribute("uses_netloc", "urlparse", "urllib.parse"), MovedAttribute("uses_params", "urlparse", "urllib.parse"), @@ -353,8 +361,11 @@ del attr Module_six_moves_urllib_parse._moved_attributes = _urllib_parse_moved_attributes -_importer._add_module(Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), - "moves.urllib_parse", "moves.urllib.parse") +_importer._add_module( + Module_six_moves_urllib_parse(__name__ + ".moves.urllib_parse"), + "moves.urllib_parse", + "moves.urllib.parse", +) class Module_six_moves_urllib_error(_LazyModule): @@ -373,8 +384,11 @@ del attr Module_six_moves_urllib_error._moved_attributes = _urllib_error_moved_attributes -_importer._add_module(Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), - "moves.urllib_error", "moves.urllib.error") +_importer._add_module( + Module_six_moves_urllib_error(__name__ + ".moves.urllib.error"), + "moves.urllib_error", + "moves.urllib.error", +) class Module_six_moves_urllib_request(_LazyModule): @@ -416,6 +430,8 @@ _urllib_request_moved_attributes = [ MovedAttribute("URLopener", "urllib", "urllib.request"), MovedAttribute("FancyURLopener", "urllib", "urllib.request"), MovedAttribute("proxy_bypass", "urllib", "urllib.request"), + MovedAttribute("parse_http_list", "urllib2", "urllib.request"), + MovedAttribute("parse_keqv_list", "urllib2", "urllib.request"), ] for attr in _urllib_request_moved_attributes: setattr(Module_six_moves_urllib_request, attr.name, attr) @@ -423,8 +439,11 @@ del attr Module_six_moves_urllib_request._moved_attributes = _urllib_request_moved_attributes -_importer._add_module(Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), - "moves.urllib_request", "moves.urllib.request") +_importer._add_module( + Module_six_moves_urllib_request(__name__ + ".moves.urllib.request"), + "moves.urllib_request", + "moves.urllib.request", +) class Module_six_moves_urllib_response(_LazyModule): @@ -444,8 +463,11 @@ del attr Module_six_moves_urllib_response._moved_attributes = _urllib_response_moved_attributes -_importer._add_module(Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), - "moves.urllib_response", "moves.urllib.response") +_importer._add_module( + Module_six_moves_urllib_response(__name__ + ".moves.urllib.response"), + "moves.urllib_response", + "moves.urllib.response", +) class Module_six_moves_urllib_robotparser(_LazyModule): @@ -454,21 +476,27 @@ class Module_six_moves_urllib_robotparser(_LazyModule): _urllib_robotparser_moved_attributes = [ - MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser"), + MovedAttribute("RobotFileParser", "robotparser", "urllib.robotparser") ] for attr in _urllib_robotparser_moved_attributes: setattr(Module_six_moves_urllib_robotparser, attr.name, attr) del attr -Module_six_moves_urllib_robotparser._moved_attributes = _urllib_robotparser_moved_attributes +Module_six_moves_urllib_robotparser._moved_attributes = ( + _urllib_robotparser_moved_attributes +) -_importer._add_module(Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), - "moves.urllib_robotparser", "moves.urllib.robotparser") +_importer._add_module( + Module_six_moves_urllib_robotparser(__name__ + ".moves.urllib.robotparser"), + "moves.urllib_robotparser", + "moves.urllib.robotparser", +) class Module_six_moves_urllib(types.ModuleType): """Create a six.moves.urllib namespace that resembles the Python 3 namespace""" + __path__ = [] # mark as package parse = _importer._get_module("moves.urllib_parse") error = _importer._get_module("moves.urllib_error") @@ -477,10 +505,12 @@ class Module_six_moves_urllib(types.ModuleType): robotparser = _importer._get_module("moves.urllib_robotparser") def __dir__(self): - return ['parse', 'error', 'request', 'response', 'robotparser'] + return ["parse", "error", "request", "response", "robotparser"] -_importer._add_module(Module_six_moves_urllib(__name__ + ".moves.urllib"), - "moves.urllib") + +_importer._add_module( + Module_six_moves_urllib(__name__ + ".moves.urllib"), "moves.urllib" +) def add_move(move): @@ -520,19 +550,24 @@ else: try: advance_iterator = next except NameError: + def advance_iterator(it): return it.next() + + next = advance_iterator try: callable = callable except NameError: + def callable(obj): return any("__call__" in klass.__dict__ for klass in type(obj).__mro__) if PY3: + def get_unbound_function(unbound): return unbound @@ -543,6 +578,7 @@ if PY3: Iterator = object else: + def get_unbound_function(unbound): return unbound.im_func @@ -553,13 +589,13 @@ else: return types.MethodType(func, None, cls) class Iterator(object): - def next(self): return type(self).__next__(self) callable = callable -_add_doc(get_unbound_function, - """Get the function out of a possibly unbound function""") +_add_doc( + get_unbound_function, """Get the function out of a possibly unbound function""" +) get_method_function = operator.attrgetter(_meth_func) @@ -571,6 +607,7 @@ get_function_globals = operator.attrgetter(_func_globals) if PY3: + def iterkeys(d, **kw): return iter(d.keys(**kw)) @@ -589,6 +626,7 @@ if PY3: viewitems = operator.methodcaller("items") else: + def iterkeys(d, **kw): return d.iterkeys(**kw) @@ -609,28 +647,33 @@ else: _add_doc(iterkeys, "Return an iterator over the keys of a dictionary.") _add_doc(itervalues, "Return an iterator over the values of a dictionary.") -_add_doc(iteritems, - "Return an iterator over the (key, value) pairs of a dictionary.") -_add_doc(iterlists, - "Return an iterator over the (key, [values]) pairs of a dictionary.") +_add_doc(iteritems, "Return an iterator over the (key, value) pairs of a dictionary.") +_add_doc( + iterlists, "Return an iterator over the (key, [values]) pairs of a dictionary." +) if PY3: + def b(s): return s.encode("latin-1") def u(s): return s + unichr = chr import struct + int2byte = struct.Struct(">B").pack del struct byte2int = operator.itemgetter(0) indexbytes = operator.getitem iterbytes = iter import io + StringIO = io.StringIO BytesIO = io.BytesIO + del io _assertCountEqual = "assertCountEqual" if sys.version_info[1] <= 1: _assertRaisesRegex = "assertRaisesRegexp" @@ -639,12 +682,15 @@ if PY3: _assertRaisesRegex = "assertRaisesRegex" _assertRegex = "assertRegex" else: + def b(s): return s + # Workaround for standalone backslash def u(s): - return unicode(s.replace(r'\\', r'\\\\'), "unicode_escape") + return unicode(s.replace(r"\\", r"\\\\"), "unicode_escape") + unichr = unichr int2byte = chr @@ -653,8 +699,10 @@ else: def indexbytes(buf, i): return ord(buf[i]) + iterbytes = functools.partial(itertools.imap, ord) import StringIO + StringIO = BytesIO = StringIO.StringIO _assertCountEqual = "assertItemsEqual" _assertRaisesRegex = "assertRaisesRegexp" @@ -679,13 +727,19 @@ if PY3: exec_ = getattr(moves.builtins, "exec") def reraise(tp, value, tb=None): - if value is None: - value = tp() - if value.__traceback__ is not tb: - raise value.with_traceback(tb) - raise value + try: + if value is None: + value = tp() + if value.__traceback__ is not tb: + raise value.with_traceback(tb) + raise value + finally: + value = None + tb = None + else: + def exec_(_code_, _globs_=None, _locs_=None): """Execute code in a namespace.""" if _globs_ is None: @@ -698,28 +752,45 @@ else: _locs_ = _globs_ exec("""exec _code_ in _globs_, _locs_""") - exec_("""def reraise(tp, value, tb=None): - raise tp, value, tb -""") + exec_( + """def reraise(tp, value, tb=None): + try: + raise tp, value, tb + finally: + tb = None +""" + ) if sys.version_info[:2] == (3, 2): - exec_("""def raise_from(value, from_value): - if from_value is None: - raise value - raise value from from_value -""") + exec_( + """def raise_from(value, from_value): + try: + if from_value is None: + raise value + raise value from from_value + finally: + value = None +""" + ) elif sys.version_info[:2] > (3, 2): - exec_("""def raise_from(value, from_value): - raise value from from_value -""") + exec_( + """def raise_from(value, from_value): + try: + raise value from from_value + finally: + value = None +""" + ) else: + def raise_from(value, from_value): raise value print_ = getattr(moves.builtins, "print", None) if print_ is None: + def print_(*args, **kwargs): """The new-style print function for Python 2.4 and 2.5.""" fp = kwargs.pop("file", sys.stdout) @@ -730,14 +801,17 @@ if print_ is None: if not isinstance(data, basestring): data = str(data) # If the file has an encoding, encode unicode with it. - if (isinstance(fp, file) and - isinstance(data, unicode) and - fp.encoding is not None): + if ( + isinstance(fp, file) + and isinstance(data, unicode) + and fp.encoding is not None + ): errors = getattr(fp, "errors", None) if errors is None: errors = "strict" data = data.encode(fp.encoding, errors) fp.write(data) + want_unicode = False sep = kwargs.pop("sep", None) if sep is not None: @@ -773,6 +847,8 @@ if print_ is None: write(sep) write(arg) write(end) + + if sys.version_info[:2] < (3, 3): _print = print_ @@ -783,16 +859,24 @@ if sys.version_info[:2] < (3, 3): if flush and fp is not None: fp.flush() + _add_doc(reraise, """Reraise an exception.""") if sys.version_info[0:2] < (3, 4): - def wraps(wrapped, assigned=functools.WRAPPER_ASSIGNMENTS, - updated=functools.WRAPPER_UPDATES): + + def wraps( + wrapped, + assigned=functools.WRAPPER_ASSIGNMENTS, + updated=functools.WRAPPER_UPDATES, + ): def wrapper(f): f = functools.wraps(wrapped, assigned, updated)(f) f.__wrapped__ = wrapped return f + return wrapper + + else: wraps = functools.wraps @@ -802,29 +886,95 @@ def with_metaclass(meta, *bases): # This requires a bit of explanation: the basic idea is to make a dummy # metaclass for one level of class instantiation that replaces itself with # the actual metaclass. - class metaclass(meta): - + class metaclass(type): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + @classmethod + def __prepare__(cls, name, this_bases): + return meta.__prepare__(name, bases) + + return type.__new__(metaclass, "temporary_class", (), {}) def add_metaclass(metaclass): """Class decorator for creating a class with a metaclass.""" + def wrapper(cls): orig_vars = cls.__dict__.copy() - slots = orig_vars.get('__slots__') + slots = orig_vars.get("__slots__") if slots is not None: if isinstance(slots, str): slots = [slots] for slots_var in slots: orig_vars.pop(slots_var) - orig_vars.pop('__dict__', None) - orig_vars.pop('__weakref__', None) + orig_vars.pop("__dict__", None) + orig_vars.pop("__weakref__", None) + if hasattr(cls, "__qualname__"): + orig_vars["__qualname__"] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) + return wrapper +def ensure_binary(s, encoding="utf-8", errors="strict"): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding="utf-8", errors="strict"): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding="utf-8", errors="strict"): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. @@ -834,12 +984,13 @@ def python_2_unicode_compatible(klass): returning text and apply this decorator to the class. """ if PY2: - if '__str__' not in klass.__dict__: - raise ValueError("@python_2_unicode_compatible cannot be applied " - "to %s because it doesn't define __str__()." % - klass.__name__) + if "__str__" not in klass.__dict__: + raise ValueError( + "@python_2_unicode_compatible cannot be applied " + "to %s because it doesn't define __str__()." % klass.__name__ + ) klass.__unicode__ = klass.__str__ - klass.__str__ = lambda self: self.__unicode__().encode('utf-8') + klass.__str__ = lambda self: self.__unicode__().encode("utf-8") return klass @@ -859,8 +1010,10 @@ if sys.meta_path: # be floating around. Therefore, we can't use isinstance() to check for # the six meta path importer, since the other six instance will have # inserted an importer with different class. - if (type(importer).__name__ == "_SixMetaPathImporter" and - importer.name == __name__): + if ( + type(importer).__name__ == "_SixMetaPathImporter" + and importer.name == __name__ + ): del sys.meta_path[i] break del i, importer diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py index d6594eb2..75b6bb1c 100644 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/__init__.py @@ -16,4 +16,4 @@ except ImportError: from ._implementation import CertificateError, match_hostname # Not needed, but documenting what we provide. -__all__ = ('CertificateError', 'match_hostname') +__all__ = ("CertificateError", "match_hostname") diff --git a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py index d6e66c01..689208d3 100644 --- a/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py +++ b/pipenv/vendor/urllib3/packages/ssl_match_hostname/_implementation.py @@ -15,7 +15,7 @@ try: except ImportError: ipaddress = None -__version__ = '3.5.0.1' +__version__ = "3.5.0.1" class CertificateError(ValueError): @@ -33,18 +33,19 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # Ported from python3-syntax: # leftmost, *remainder = dn.split(r'.') - parts = dn.split(r'.') + parts = dn.split(r".") leftmost = parts[0] remainder = parts[1:] - wildcards = leftmost.count('*') + wildcards = leftmost.count("*") if wildcards > max_wildcards: # Issue #17980: avoid denials of service by refusing more # than one wildcard per fragment. A survey of established # policy among SSL implementations showed it to be a # reasonable choice. raise CertificateError( - "too many wildcards in certificate DNS name: " + repr(dn)) + "too many wildcards in certificate DNS name: " + repr(dn) + ) # speed up common case w/o wildcards if not wildcards: @@ -53,11 +54,11 @@ def _dnsname_match(dn, hostname, max_wildcards=1): # RFC 6125, section 6.4.3, subitem 1. # The client SHOULD NOT attempt to match a presented identifier in which # the wildcard character comprises a label other than the left-most label. - if leftmost == '*': + if leftmost == "*": # When '*' is a fragment by itself, it matches a non-empty dotless # fragment. - pats.append('[^.]+') - elif leftmost.startswith('xn--') or hostname.startswith('xn--'): + pats.append("[^.]+") + elif leftmost.startswith("xn--") or hostname.startswith("xn--"): # RFC 6125, section 6.4.3, subitem 3. # The client SHOULD NOT attempt to match a presented identifier # where the wildcard character is embedded within an A-label or @@ -65,21 +66,22 @@ def _dnsname_match(dn, hostname, max_wildcards=1): pats.append(re.escape(leftmost)) else: # Otherwise, '*' matches any dotless string, e.g. www* - pats.append(re.escape(leftmost).replace(r'\*', '[^.]*')) + pats.append(re.escape(leftmost).replace(r"\*", "[^.]*")) # add the remaining fragments, ignore any wildcards for frag in remainder: pats.append(re.escape(frag)) - pat = re.compile(r'\A' + r'\.'.join(pats) + r'\Z', re.IGNORECASE) + pat = re.compile(r"\A" + r"\.".join(pats) + r"\Z", re.IGNORECASE) return pat.match(hostname) def _to_unicode(obj): if isinstance(obj, str) and sys.version_info < (3,): - obj = unicode(obj, encoding='ascii', errors='strict') + obj = unicode(obj, encoding="ascii", errors="strict") return obj + def _ipaddress_match(ipname, host_ip): """Exact matching of IP addresses. @@ -101,9 +103,11 @@ def match_hostname(cert, hostname): returns nothing. """ if not cert: - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") + raise ValueError( + "empty or no certificate, match_hostname needs a " + "SSL socket or SSL context with either " + "CERT_OPTIONAL or CERT_REQUIRED" + ) try: # Divergence from upstream: ipaddress can't handle byte str host_ip = ipaddress.ip_address(_to_unicode(hostname)) @@ -122,35 +126,35 @@ def match_hostname(cert, hostname): else: raise dnsnames = [] - san = cert.get('subjectAltName', ()) + san = cert.get("subjectAltName", ()) for key, value in san: - if key == 'DNS': + if key == "DNS": if host_ip is None and _dnsname_match(value, hostname): return dnsnames.append(value) - elif key == 'IP Address': + elif key == "IP Address": if host_ip is not None and _ipaddress_match(value, host_ip): return dnsnames.append(value) if not dnsnames: # The subject is only checked when there is no dNSName entry # in subjectAltName - for sub in cert.get('subject', ()): + for sub in cert.get("subject", ()): for key, value in sub: # XXX according to RFC 2818, the most specific Common Name # must be used. - if key == 'commonName': + if key == "commonName": if _dnsname_match(value, hostname): return dnsnames.append(value) if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) + raise CertificateError( + "hostname %r " + "doesn't match either of %s" % (hostname, ", ".join(map(repr, dnsnames))) + ) elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) + raise CertificateError("hostname %r doesn't match %r" % (hostname, dnsnames[0])) else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") + raise CertificateError( + "no appropriate commonName or subjectAltName fields were found" + ) diff --git a/pipenv/vendor/urllib3/poolmanager.py b/pipenv/vendor/urllib3/poolmanager.py index fe5491cf..e2bd3bd8 100644 --- a/pipenv/vendor/urllib3/poolmanager.py +++ b/pipenv/vendor/urllib3/poolmanager.py @@ -2,57 +2,73 @@ from __future__ import absolute_import import collections import functools import logging +import warnings from ._collections import RecentlyUsedContainer from .connectionpool import HTTPConnectionPool, HTTPSConnectionPool from .connectionpool import port_by_scheme -from .exceptions import LocationValueError, MaxRetryError, ProxySchemeUnknown +from .exceptions import ( + LocationValueError, + MaxRetryError, + ProxySchemeUnknown, + InvalidProxyConfigurationWarning, +) +from .packages import six from .packages.six.moves.urllib.parse import urljoin from .request import RequestMethods from .util.url import parse_url from .util.retry import Retry -__all__ = ['PoolManager', 'ProxyManager', 'proxy_from_url'] +__all__ = ["PoolManager", "ProxyManager", "proxy_from_url"] log = logging.getLogger(__name__) -SSL_KEYWORDS = ('key_file', 'cert_file', 'cert_reqs', 'ca_certs', - 'ssl_version', 'ca_cert_dir', 'ssl_context') +SSL_KEYWORDS = ( + "key_file", + "cert_file", + "cert_reqs", + "ca_certs", + "ssl_version", + "ca_cert_dir", + "ssl_context", + "key_password", +) # All known keyword arguments that could be provided to the pool manager, its # pools, or the underlying connections. This is used to construct a pool key. _key_fields = ( - 'key_scheme', # str - 'key_host', # str - 'key_port', # int - 'key_timeout', # int or float or Timeout - 'key_retries', # int or Retry - 'key_strict', # bool - 'key_block', # bool - 'key_source_address', # str - 'key_key_file', # str - 'key_cert_file', # str - 'key_cert_reqs', # str - 'key_ca_certs', # str - 'key_ssl_version', # str - 'key_ca_cert_dir', # str - 'key_ssl_context', # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext - 'key_maxsize', # int - 'key_headers', # dict - 'key__proxy', # parsed proxy url - 'key__proxy_headers', # dict - 'key_socket_options', # list of (level (int), optname (int), value (int or str)) tuples - 'key__socks_options', # dict - 'key_assert_hostname', # bool or string - 'key_assert_fingerprint', # str - 'key_server_hostname', #str + "key_scheme", # str + "key_host", # str + "key_port", # int + "key_timeout", # int or float or Timeout + "key_retries", # int or Retry + "key_strict", # bool + "key_block", # bool + "key_source_address", # str + "key_key_file", # str + "key_key_password", # str + "key_cert_file", # str + "key_cert_reqs", # str + "key_ca_certs", # str + "key_ssl_version", # str + "key_ca_cert_dir", # str + "key_ssl_context", # instance of ssl.SSLContext or urllib3.util.ssl_.SSLContext + "key_maxsize", # int + "key_headers", # dict + "key__proxy", # parsed proxy url + "key__proxy_headers", # dict + "key_socket_options", # list of (level (int), optname (int), value (int or str)) tuples + "key__socks_options", # dict + "key_assert_hostname", # bool or string + "key_assert_fingerprint", # str + "key_server_hostname", # str ) #: The namedtuple class used to construct keys for the connection pool. #: All custom key schemes should include the fields in this key at a minimum. -PoolKey = collections.namedtuple('PoolKey', _key_fields) +PoolKey = collections.namedtuple("PoolKey", _key_fields) def _default_key_normalizer(key_class, request_context): @@ -77,24 +93,24 @@ def _default_key_normalizer(key_class, request_context): """ # Since we mutate the dictionary, make a copy first context = request_context.copy() - context['scheme'] = context['scheme'].lower() - context['host'] = context['host'].lower() + context["scheme"] = context["scheme"].lower() + context["host"] = context["host"].lower() # These are both dictionaries and need to be transformed into frozensets - for key in ('headers', '_proxy_headers', '_socks_options'): + for key in ("headers", "_proxy_headers", "_socks_options"): if key in context and context[key] is not None: context[key] = frozenset(context[key].items()) # The socket_options key may be a list and needs to be transformed into a # tuple. - socket_opts = context.get('socket_options') + socket_opts = context.get("socket_options") if socket_opts is not None: - context['socket_options'] = tuple(socket_opts) + context["socket_options"] = tuple(socket_opts) # Map the kwargs to the names in the namedtuple - this is necessary since # namedtuples can't have fields starting with '_'. for key in list(context.keys()): - context['key_' + key] = context.pop(key) + context["key_" + key] = context.pop(key) # Default to ``None`` for keys missing from the context for field in key_class._fields: @@ -109,14 +125,11 @@ def _default_key_normalizer(key_class, request_context): #: Each PoolManager makes a copy of this dictionary so they can be configured #: globally here, or individually on the instance. key_fn_by_scheme = { - 'http': functools.partial(_default_key_normalizer, PoolKey), - 'https': functools.partial(_default_key_normalizer, PoolKey), + "http": functools.partial(_default_key_normalizer, PoolKey), + "https": functools.partial(_default_key_normalizer, PoolKey), } -pool_classes_by_scheme = { - 'http': HTTPConnectionPool, - 'https': HTTPSConnectionPool, -} +pool_classes_by_scheme = {"http": HTTPConnectionPool, "https": HTTPSConnectionPool} class PoolManager(RequestMethods): @@ -152,8 +165,7 @@ class PoolManager(RequestMethods): def __init__(self, num_pools=10, headers=None, **connection_pool_kw): RequestMethods.__init__(self, headers) self.connection_pool_kw = connection_pool_kw - self.pools = RecentlyUsedContainer(num_pools, - dispose_func=lambda p: p.close()) + self.pools = RecentlyUsedContainer(num_pools, dispose_func=lambda p: p.close()) # Locally set the pool classes and keys so other PoolManagers can # override them. @@ -186,10 +198,10 @@ class PoolManager(RequestMethods): # this function has historically only used the scheme, host, and port # in the positional args. When an API change is acceptable these can # be removed. - for key in ('scheme', 'host', 'port'): + for key in ("scheme", "host", "port"): request_context.pop(key, None) - if scheme == 'http': + if scheme == "http": for kw in SSL_KEYWORDS: request_context.pop(kw, None) @@ -204,7 +216,7 @@ class PoolManager(RequestMethods): """ self.pools.clear() - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): """ Get a :class:`ConnectionPool` based on the host, port, and scheme. @@ -219,11 +231,11 @@ class PoolManager(RequestMethods): raise LocationValueError("No host specified.") request_context = self._merge_pool_kwargs(pool_kwargs) - request_context['scheme'] = scheme or 'http' + request_context["scheme"] = scheme or "http" if not port: - port = port_by_scheme.get(request_context['scheme'].lower(), 80) - request_context['port'] = port - request_context['host'] = host + port = port_by_scheme.get(request_context["scheme"].lower(), 80) + request_context["port"] = port + request_context["host"] = host return self.connection_from_context(request_context) @@ -234,7 +246,7 @@ class PoolManager(RequestMethods): ``request_context`` must at least contain the ``scheme`` key and its value must be a key in ``key_fn_by_scheme`` instance variable. """ - scheme = request_context['scheme'].lower() + scheme = request_context["scheme"].lower() pool_key_constructor = self.key_fn_by_scheme[scheme] pool_key = pool_key_constructor(request_context) @@ -256,9 +268,9 @@ class PoolManager(RequestMethods): return pool # Make a fresh ConnectionPool of the desired type - scheme = request_context['scheme'] - host = request_context['host'] - port = request_context['port'] + scheme = request_context["scheme"] + host = request_context["host"] + port = request_context["port"] pool = self._new_pool(scheme, host, port, request_context=request_context) self.pools[pool_key] = pool @@ -276,8 +288,9 @@ class PoolManager(RequestMethods): not used. """ u = parse_url(url) - return self.connection_from_host(u.host, port=u.port, scheme=u.scheme, - pool_kwargs=pool_kwargs) + return self.connection_from_host( + u.host, port=u.port, scheme=u.scheme, pool_kwargs=pool_kwargs + ) def _merge_pool_kwargs(self, override): """ @@ -311,11 +324,11 @@ class PoolManager(RequestMethods): u = parse_url(url) conn = self.connection_from_host(u.host, port=u.port, scheme=u.scheme) - kw['assert_same_host'] = False - kw['redirect'] = False + kw["assert_same_host"] = False + kw["redirect"] = False - if 'headers' not in kw: - kw['headers'] = self.headers.copy() + if "headers" not in kw: + kw["headers"] = self.headers.copy() if self.proxy is not None and u.scheme == "http": response = conn.urlopen(method, url, **kw) @@ -331,31 +344,37 @@ class PoolManager(RequestMethods): # RFC 7231, Section 6.4.4 if response.status == 303: - method = 'GET' + method = "GET" - retries = kw.get('retries') + retries = kw.get("retries") if not isinstance(retries, Retry): retries = Retry.from_int(retries, redirect=redirect) # Strip headers marked as unsafe to forward to the redirected location. # Check remove_headers_on_redirect to avoid a potential network call within # conn.is_same_host() which may use socket.gethostbyname() in the future. - if (retries.remove_headers_on_redirect - and not conn.is_same_host(redirect_location)): - for header in retries.remove_headers_on_redirect: - kw['headers'].pop(header, None) + if retries.remove_headers_on_redirect and not conn.is_same_host( + redirect_location + ): + headers = list(six.iterkeys(kw["headers"])) + for header in headers: + if header.lower() in retries.remove_headers_on_redirect: + kw["headers"].pop(header, None) try: retries = retries.increment(method, url, response=response, _pool=conn) except MaxRetryError: if retries.raise_on_redirect: + response.drain_conn() raise return response - kw['retries'] = retries - kw['redirect'] = redirect + kw["retries"] = retries + kw["redirect"] = redirect log.info("Redirecting %s -> %s", url, redirect_location) + + response.drain_conn() return self.urlopen(method, redirect_location, **kw) @@ -386,12 +405,21 @@ class ProxyManager(PoolManager): """ - def __init__(self, proxy_url, num_pools=10, headers=None, - proxy_headers=None, **connection_pool_kw): + def __init__( + self, + proxy_url, + num_pools=10, + headers=None, + proxy_headers=None, + **connection_pool_kw + ): if isinstance(proxy_url, HTTPConnectionPool): - proxy_url = '%s://%s:%i' % (proxy_url.scheme, proxy_url.host, - proxy_url.port) + proxy_url = "%s://%s:%i" % ( + proxy_url.scheme, + proxy_url.host, + proxy_url.port, + ) proxy = parse_url(proxy_url) if not proxy.port: port = port_by_scheme.get(proxy.scheme, 80) @@ -403,45 +431,59 @@ class ProxyManager(PoolManager): self.proxy = proxy self.proxy_headers = proxy_headers or {} - connection_pool_kw['_proxy'] = self.proxy - connection_pool_kw['_proxy_headers'] = self.proxy_headers + connection_pool_kw["_proxy"] = self.proxy + connection_pool_kw["_proxy_headers"] = self.proxy_headers - super(ProxyManager, self).__init__( - num_pools, headers, **connection_pool_kw) + super(ProxyManager, self).__init__(num_pools, headers, **connection_pool_kw) - def connection_from_host(self, host, port=None, scheme='http', pool_kwargs=None): + def connection_from_host(self, host, port=None, scheme="http", pool_kwargs=None): if scheme == "https": return super(ProxyManager, self).connection_from_host( - host, port, scheme, pool_kwargs=pool_kwargs) + host, port, scheme, pool_kwargs=pool_kwargs + ) return super(ProxyManager, self).connection_from_host( - self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs) + self.proxy.host, self.proxy.port, self.proxy.scheme, pool_kwargs=pool_kwargs + ) def _set_proxy_headers(self, url, headers=None): """ Sets headers needed by proxies: specifically, the Accept and Host headers. Only sets headers not provided by the user. """ - headers_ = {'Accept': '*/*'} + headers_ = {"Accept": "*/*"} netloc = parse_url(url).netloc if netloc: - headers_['Host'] = netloc + headers_["Host"] = netloc if headers: headers_.update(headers) return headers_ + def _validate_proxy_scheme_url_selection(self, url_scheme): + if url_scheme == "https" and self.proxy.scheme == "https": + warnings.warn( + "Your proxy configuration specified an HTTPS scheme for the proxy. " + "Are you sure you want to use HTTPS to contact the proxy? " + "This most likely indicates an error in your configuration. " + "Read this issue for more info: " + "https://github.com/urllib3/urllib3/issues/1850", + InvalidProxyConfigurationWarning, + stacklevel=3, + ) + def urlopen(self, method, url, redirect=True, **kw): "Same as HTTP(S)ConnectionPool.urlopen, ``url`` must be absolute." u = parse_url(url) + self._validate_proxy_scheme_url_selection(u.scheme) if u.scheme == "http": # For proxied HTTPS requests, httplib sets the necessary headers # on the CONNECT to the proxy. For HTTP, we'll definitely # need to set 'Host' at the very least. - headers = kw.get('headers', self.headers) - kw['headers'] = self._set_proxy_headers(url, headers) + headers = kw.get("headers", self.headers) + kw["headers"] = self._set_proxy_headers(url, headers) return super(ProxyManager, self).urlopen(method, url, redirect=redirect, **kw) diff --git a/pipenv/vendor/urllib3/request.py b/pipenv/vendor/urllib3/request.py index 8f2f44bb..55f160bb 100644 --- a/pipenv/vendor/urllib3/request.py +++ b/pipenv/vendor/urllib3/request.py @@ -4,7 +4,7 @@ from .filepost import encode_multipart_formdata from .packages.six.moves.urllib.parse import urlencode -__all__ = ['RequestMethods'] +__all__ = ["RequestMethods"] class RequestMethods(object): @@ -36,16 +36,25 @@ class RequestMethods(object): explicitly. """ - _encode_url_methods = {'DELETE', 'GET', 'HEAD', 'OPTIONS'} + _encode_url_methods = {"DELETE", "GET", "HEAD", "OPTIONS"} def __init__(self, headers=None): self.headers = headers or {} - def urlopen(self, method, url, body=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **kw): # Abstract - raise NotImplementedError("Classes extending RequestMethods must implement " - "their own ``urlopen`` method.") + def urlopen( + self, + method, + url, + body=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **kw + ): # Abstract + raise NotImplementedError( + "Classes extending RequestMethods must implement " + "their own ``urlopen`` method." + ) def request(self, method, url, fields=None, headers=None, **urlopen_kw): """ @@ -60,19 +69,18 @@ class RequestMethods(object): """ method = method.upper() - urlopen_kw['request_url'] = url + urlopen_kw["request_url"] = url if method in self._encode_url_methods: - return self.request_encode_url(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_url( + method, url, fields=fields, headers=headers, **urlopen_kw + ) else: - return self.request_encode_body(method, url, fields=fields, - headers=headers, - **urlopen_kw) + return self.request_encode_body( + method, url, fields=fields, headers=headers, **urlopen_kw + ) - def request_encode_url(self, method, url, fields=None, headers=None, - **urlopen_kw): + def request_encode_url(self, method, url, fields=None, headers=None, **urlopen_kw): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the url. This is useful for request methods like GET, HEAD, DELETE, etc. @@ -80,17 +88,24 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': headers} + extra_kw = {"headers": headers} extra_kw.update(urlopen_kw) if fields: - url += '?' + urlencode(fields) + url += "?" + urlencode(fields) return self.urlopen(method, url, **extra_kw) - def request_encode_body(self, method, url, fields=None, headers=None, - encode_multipart=True, multipart_boundary=None, - **urlopen_kw): + def request_encode_body( + self, + method, + url, + fields=None, + headers=None, + encode_multipart=True, + multipart_boundary=None, + **urlopen_kw + ): """ Make a request using :meth:`urlopen` with the ``fields`` encoded in the body. This is useful for request methods like POST, PUT, PATCH, etc. @@ -129,22 +144,28 @@ class RequestMethods(object): if headers is None: headers = self.headers - extra_kw = {'headers': {}} + extra_kw = {"headers": {}} if fields: - if 'body' in urlopen_kw: + if "body" in urlopen_kw: raise TypeError( - "request got values for both 'fields' and 'body', can only specify one.") + "request got values for both 'fields' and 'body', can only specify one." + ) if encode_multipart: - body, content_type = encode_multipart_formdata(fields, boundary=multipart_boundary) + body, content_type = encode_multipart_formdata( + fields, boundary=multipart_boundary + ) else: - body, content_type = urlencode(fields), 'application/x-www-form-urlencoded' + body, content_type = ( + urlencode(fields), + "application/x-www-form-urlencoded", + ) - extra_kw['body'] = body - extra_kw['headers'] = {'Content-Type': content_type} + extra_kw["body"] = body + extra_kw["headers"] = {"Content-Type": content_type} - extra_kw['headers'].update(headers) + extra_kw["headers"].update(headers) extra_kw.update(urlopen_kw) return self.urlopen(method, url, **extra_kw) diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py index f0cfbb54..7dc9b93c 100644 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -6,10 +6,21 @@ import logging from socket import timeout as SocketTimeout from socket import error as SocketError +try: + import brotli +except ImportError: + brotli = None + from ._collections import HTTPHeaderDict from .exceptions import ( - BodyNotHttplibCompatible, ProtocolError, DecodeError, ReadTimeoutError, - ResponseNotChunked, IncompleteRead, InvalidHeader + BodyNotHttplibCompatible, + ProtocolError, + DecodeError, + ReadTimeoutError, + ResponseNotChunked, + IncompleteRead, + InvalidHeader, + HTTPError, ) from .packages.six import string_types as basestring, PY3 from .packages.six.moves import http_client as httplib @@ -20,10 +31,9 @@ log = logging.getLogger(__name__) class DeflateDecoder(object): - def __init__(self): self._first_try = True - self._data = b'' + self._data = b"" self._obj = zlib.decompressobj() def __getattr__(self, name): @@ -60,7 +70,6 @@ class GzipDecoderState(object): class GzipDecoder(object): - def __init__(self): self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) self._state = GzipDecoderState.FIRST_MEMBER @@ -69,9 +78,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = b'' + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,15 +90,35 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) +if brotli is not None: + + class BrotliDecoder(object): + # Supports both 'brotlipy' and 'Brotli' packages + # since they share an import name. The top branches + # are for 'brotlipy' and bottom branches for 'Brotli' + def __init__(self): + self._obj = brotli.Decompressor() + + def decompress(self, data): + if hasattr(self._obj, "decompress"): + return self._obj.decompress(data) + return self._obj.process(data) + + def flush(self): + if hasattr(self._obj, "flush"): + return self._obj.flush() + return b"" + + class MultiDecoder(object): """ From RFC7231: @@ -100,7 +129,7 @@ class MultiDecoder(object): """ def __init__(self, modes): - self._decoders = [_get_decoder(m.strip()) for m in modes.split(',')] + self._decoders = [_get_decoder(m.strip()) for m in modes.split(",")] def flush(self): return self._decoders[0].flush() @@ -112,12 +141,15 @@ class MultiDecoder(object): def _get_decoder(mode): - if ',' in mode: + if "," in mode: return MultiDecoder(mode) - if mode == 'gzip': + if mode == "gzip": return GzipDecoder() + if brotli is not None and mode == "br": + return BrotliDecoder() + return DeflateDecoder() @@ -154,14 +186,31 @@ class HTTPResponse(io.IOBase): value of Content-Length header, if present. Otherwise, raise error. """ - CONTENT_DECODERS = ['gzip', 'deflate'] + CONTENT_DECODERS = ["gzip", "deflate"] + if brotli is not None: + CONTENT_DECODERS += ["br"] REDIRECT_STATUSES = [301, 302, 303, 307, 308] - def __init__(self, body='', headers=None, status=0, version=0, reason=None, - strict=0, preload_content=True, decode_content=True, - original_response=None, pool=None, connection=None, msg=None, - retries=None, enforce_content_length=False, - request_method=None, request_url=None): + def __init__( + self, + body="", + headers=None, + status=0, + version=0, + reason=None, + strict=0, + preload_content=True, + decode_content=True, + original_response=None, + pool=None, + connection=None, + msg=None, + retries=None, + enforce_content_length=False, + request_method=None, + request_url=None, + auto_close=True, + ): if isinstance(headers, HTTPHeaderDict): self.headers = headers @@ -174,6 +223,7 @@ class HTTPResponse(io.IOBase): self.decode_content = decode_content self.retries = retries self.enforce_content_length = enforce_content_length + self.auto_close = auto_close self._decoder = None self._body = None @@ -189,13 +239,13 @@ class HTTPResponse(io.IOBase): self._pool = pool self._connection = connection - if hasattr(body, 'read'): + if hasattr(body, "read"): self._fp = body # Are we using the chunked-style of transfer encoding? self.chunked = False self.chunk_left = None - tr_enc = self.headers.get('transfer-encoding', '').lower() + tr_enc = self.headers.get("transfer-encoding", "").lower() # Don't incur the penalty of creating a list and then discarding it encodings = (enc.strip() for enc in tr_enc.split(",")) if "chunked" in encodings: @@ -217,7 +267,7 @@ class HTTPResponse(io.IOBase): location. ``False`` if not a redirect status code. """ if self.status in self.REDIRECT_STATUSES: - return self.headers.get('location') + return self.headers.get("location") return False @@ -228,6 +278,17 @@ class HTTPResponse(io.IOBase): self._pool._put_conn(self._connection) self._connection = None + def drain_conn(self): + """ + Read and discard any remaining HTTP response data in the response connection. + + Unread data in the HTTPResponse connection blocks the connection from being released back to the pool. + """ + try: + self.read() + except (HTTPError, SocketError, BaseSSLError, HTTPException): + pass + @property def data(self): # For backwords-compat with earlier urllib3 0.4 and earlier. @@ -256,18 +317,20 @@ class HTTPResponse(io.IOBase): """ Set initial length value for Response content if available. """ - length = self.headers.get('content-length') + length = self.headers.get("content-length") if length is not None: if self.chunked: # This Response will fail with an IncompleteRead if it can't be # received as chunked. This method falls back to attempt reading # the response before raising an exception. - log.warning("Received response with both Content-Length and " - "Transfer-Encoding set. This is expressly forbidden " - "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " - "attempting to process response as Transfer-Encoding: " - "chunked.") + log.warning( + "Received response with both Content-Length and " + "Transfer-Encoding set. This is expressly forbidden " + "by RFC 7230 sec 3.3.2. Ignoring Content-Length and " + "attempting to process response as Transfer-Encoding: " + "chunked." + ) return None try: @@ -276,10 +339,12 @@ class HTTPResponse(io.IOBase): # (e.g. Content-Length: 42, 42). This line ensures the values # are all valid ints and that as long as the `set` length is 1, # all values are the same. Otherwise, the header is invalid. - lengths = set([int(val) for val in length.split(',')]) + lengths = set([int(val) for val in length.split(",")]) if len(lengths) > 1: - raise InvalidHeader("Content-Length contained multiple " - "unmatching values (%s)" % length) + raise InvalidHeader( + "Content-Length contained multiple " + "unmatching values (%s)" % length + ) length = lengths.pop() except ValueError: length = None @@ -295,7 +360,7 @@ class HTTPResponse(io.IOBase): status = 0 # Check for responses that shouldn't include a body - if status in (204, 304) or 100 <= status < 200 or request_method == 'HEAD': + if status in (204, 304) or 100 <= status < 200 or request_method == "HEAD": length = 0 return length @@ -306,29 +371,41 @@ class HTTPResponse(io.IOBase): """ # Note: content-encoding value should be case-insensitive, per RFC 7230 # Section 3.2 - content_encoding = self.headers.get('content-encoding', '').lower() + content_encoding = self.headers.get("content-encoding", "").lower() if self._decoder is None: if content_encoding in self.CONTENT_DECODERS: self._decoder = _get_decoder(content_encoding) - elif ',' in content_encoding: - encodings = [e.strip() for e in content_encoding.split(',') if e.strip() in self.CONTENT_DECODERS] + elif "," in content_encoding: + encodings = [ + e.strip() + for e in content_encoding.split(",") + if e.strip() in self.CONTENT_DECODERS + ] if len(encodings): self._decoder = _get_decoder(content_encoding) + DECODER_ERROR_CLASSES = (IOError, zlib.error) + if brotli is not None: + DECODER_ERROR_CLASSES += (brotli.error,) + def _decode(self, data, decode_content, flush_decoder): """ Decode the data passed in and potentially flush the decoder. """ + if not decode_content: + return data + try: - if decode_content and self._decoder: + if self._decoder: data = self._decoder.decompress(data) - except (IOError, zlib.error) as e: - content_encoding = self.headers.get('content-encoding', '').lower() + except self.DECODER_ERROR_CLASSES as e: + content_encoding = self.headers.get("content-encoding", "").lower() raise DecodeError( "Received response with content-encoding: %s, but " - "failed to decode it." % content_encoding, e) - - if flush_decoder and decode_content: + "failed to decode it." % content_encoding, + e, + ) + if flush_decoder: data += self._flush_decoder() return data @@ -339,10 +416,10 @@ class HTTPResponse(io.IOBase): being used. """ if self._decoder: - buf = self._decoder.decompress(b'') + buf = self._decoder.decompress(b"") return buf + self._decoder.flush() - return b'' + return b"" @contextmanager def _error_catcher(self): @@ -362,20 +439,20 @@ class HTTPResponse(io.IOBase): except SocketTimeout: # FIXME: Ideally we'd like to include the url in the ReadTimeoutError but # there is yet no clean way to get at it from this context. - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except BaseSSLError as e: # FIXME: Is there a better way to differentiate between SSLErrors? - if 'read operation timed out' not in str(e): # Defensive: + if "read operation timed out" not in str(e): # Defensive: # This shouldn't happen but just in case we're missing an edge # case, let's avoid swallowing SSL errors. raise - raise ReadTimeoutError(self._pool, None, 'Read timed out.') + raise ReadTimeoutError(self._pool, None, "Read timed out.") except (HTTPException, SocketError) as e: # This includes IncompleteRead. - raise ProtocolError('Connection broken: %r' % e, e) + raise ProtocolError("Connection broken: %r" % e, e) # If no exception is thrown, we should avoid cleaning up # unnecessarily. @@ -430,17 +507,19 @@ class HTTPResponse(io.IOBase): return flush_decoder = False - data = None + fp_closed = getattr(self._fp, "closed", False) with self._error_catcher(): if amt is None: # cStringIO doesn't like amt=None - data = self._fp.read() + data = self._fp.read() if not fp_closed else b"" flush_decoder = True else: cache_content = False - data = self._fp.read(amt) - if amt != 0 and not data: # Platform-specific: Buggy versions of Python. + data = self._fp.read(amt) if not fp_closed else b"" + if ( + amt != 0 and not data + ): # Platform-specific: Buggy versions of Python. # Close the connection when no data is returned # # This is redundant to what httplib/http.client _should_ @@ -450,7 +529,10 @@ class HTTPResponse(io.IOBase): # no harm in redundantly calling close. self._fp.close() flush_decoder = True - if self.enforce_content_length and self.length_remaining not in (0, None): + if self.enforce_content_length and self.length_remaining not in ( + 0, + None, + ): # This is an edge case that httplib failed to cover due # to concerns of backward compatibility. We're # addressing it here to make sure IncompleteRead is @@ -470,7 +552,7 @@ class HTTPResponse(io.IOBase): return data - def stream(self, amt=2**16, decode_content=None): + def stream(self, amt=2 ** 16, decode_content=None): """ A generator wrapper for the read() method. A call will block until ``amt`` bytes have been read from the connection or until the @@ -508,21 +590,24 @@ class HTTPResponse(io.IOBase): headers = r.msg if not isinstance(headers, HTTPHeaderDict): - if PY3: # Python 3 + if PY3: headers = HTTPHeaderDict(headers.items()) - else: # Python 2 + else: + # Python 2.7 headers = HTTPHeaderDict.from_httplib(headers) # HTTPResponse objects in Python 3 don't have a .strict attribute - strict = getattr(r, 'strict', 0) - resp = ResponseCls(body=r, - headers=headers, - status=r.status, - version=r.version, - reason=r.reason, - strict=strict, - original_response=r, - **response_kw) + strict = getattr(r, "strict", 0) + resp = ResponseCls( + body=r, + headers=headers, + status=r.status, + version=r.version, + reason=r.reason, + strict=strict, + original_response=r, + **response_kw + ) return resp # Backwards-compatibility methods for httplib.HTTPResponse @@ -544,13 +629,18 @@ class HTTPResponse(io.IOBase): if self._connection: self._connection.close() + if not self.auto_close: + io.IOBase.close(self) + @property def closed(self): - if self._fp is None: + if not self.auto_close: + return io.IOBase.closed.__get__(self) + elif self._fp is None: return True - elif hasattr(self._fp, 'isclosed'): + elif hasattr(self._fp, "isclosed"): return self._fp.isclosed() - elif hasattr(self._fp, 'closed'): + elif hasattr(self._fp, "closed"): return self._fp.closed else: return True @@ -561,11 +651,17 @@ class HTTPResponse(io.IOBase): elif hasattr(self._fp, "fileno"): return self._fp.fileno() else: - raise IOError("The file-like object this HTTPResponse is wrapped " - "around has no file descriptor") + raise IOError( + "The file-like object this HTTPResponse is wrapped " + "around has no file descriptor" + ) def flush(self): - if self._fp is not None and hasattr(self._fp, 'flush'): + if ( + self._fp is not None + and hasattr(self._fp, "flush") + and not getattr(self._fp, "closed", False) + ): return self._fp.flush() def readable(self): @@ -578,7 +674,7 @@ class HTTPResponse(io.IOBase): if len(temp) == 0: return 0 else: - b[:len(temp)] = temp + b[: len(temp)] = temp return len(temp) def supports_chunked_reads(self): @@ -588,7 +684,7 @@ class HTTPResponse(io.IOBase): attribute. If it is present we assume it returns raw chunks as processed by read_chunked(). """ - return hasattr(self._fp, 'fp') + return hasattr(self._fp, "fp") def _update_chunk_length(self): # First, we'll figure out length of a chunk and then @@ -596,7 +692,7 @@ class HTTPResponse(io.IOBase): if self.chunk_left is not None: return line = self._fp.fp.readline() - line = line.split(b';', 1)[0] + line = line.split(b";", 1)[0] try: self.chunk_left = int(line, 16) except ValueError: @@ -645,11 +741,13 @@ class HTTPResponse(io.IOBase): if not self.chunked: raise ResponseNotChunked( "Response is not chunked. " - "Header 'transfer-encoding: chunked' is missing.") + "Header 'transfer-encoding: chunked' is missing." + ) if not self.supports_chunked_reads(): raise BodyNotHttplibCompatible( "Body should be httplib.HTTPResponse like. " - "It should have have an fp attribute which returns raw chunks.") + "It should have have an fp attribute which returns raw chunks." + ) with self._error_catcher(): # Don't bother reading the body of a HEAD request. @@ -667,8 +765,9 @@ class HTTPResponse(io.IOBase): if self.chunk_left == 0: break chunk = self._handle_chunk(amt) - decoded = self._decode(chunk, decode_content=decode_content, - flush_decoder=False) + decoded = self._decode( + chunk, decode_content=decode_content, flush_decoder=False + ) if decoded: yield decoded @@ -686,7 +785,7 @@ class HTTPResponse(io.IOBase): if not line: # Some sites may not end with '\r\n'. break - if line == b'\r\n': + if line == b"\r\n": break # We read everything; close the "file". @@ -703,3 +802,20 @@ class HTTPResponse(io.IOBase): return self.retries.history[-1].redirect_location else: return self._request_url + + def __iter__(self): + buffer = [] + for chunk in self.stream(decode_content=True): + if b"\n" in chunk: + chunk = chunk.split(b"\n") + yield b"".join(buffer) + chunk[0] + b"\n" + for x in chunk[1:-1]: + yield x + b"\n" + if chunk[-1]: + buffer = [chunk[-1]] + else: + buffer = [] + else: + buffer.append(chunk) + if buffer: + yield b"".join(buffer) diff --git a/pipenv/vendor/urllib3/util/__init__.py b/pipenv/vendor/urllib3/util/__init__.py index 2f2770b6..a96c73a9 100644 --- a/pipenv/vendor/urllib3/util/__init__.py +++ b/pipenv/vendor/urllib3/util/__init__.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # For backwards compatibility, provide imports that used to be here. from .connection import is_connection_dropped from .request import make_headers @@ -12,43 +13,34 @@ from .ssl_ import ( resolve_cert_reqs, resolve_ssl_version, ssl_wrap_socket, + PROTOCOL_TLS, ) -from .timeout import ( - current_time, - Timeout, -) +from .timeout import current_time, Timeout from .retry import Retry -from .url import ( - get_host, - parse_url, - split_first, - Url, -) -from .wait import ( - wait_for_read, - wait_for_write -) +from .url import get_host, parse_url, split_first, Url +from .wait import wait_for_read, wait_for_write __all__ = ( - 'HAS_SNI', - 'IS_PYOPENSSL', - 'IS_SECURETRANSPORT', - 'SSLContext', - 'Retry', - 'Timeout', - 'Url', - 'assert_fingerprint', - 'current_time', - 'is_connection_dropped', - 'is_fp_closed', - 'get_host', - 'parse_url', - 'make_headers', - 'resolve_cert_reqs', - 'resolve_ssl_version', - 'split_first', - 'ssl_wrap_socket', - 'wait_for_read', - 'wait_for_write' + "HAS_SNI", + "IS_PYOPENSSL", + "IS_SECURETRANSPORT", + "SSLContext", + "PROTOCOL_TLS", + "Retry", + "Timeout", + "Url", + "assert_fingerprint", + "current_time", + "is_connection_dropped", + "is_fp_closed", + "get_host", + "parse_url", + "make_headers", + "resolve_cert_reqs", + "resolve_ssl_version", + "split_first", + "ssl_wrap_socket", + "wait_for_read", + "wait_for_write", ) diff --git a/pipenv/vendor/urllib3/util/connection.py b/pipenv/vendor/urllib3/util/connection.py index 5ad70b2f..86f0a3b0 100644 --- a/pipenv/vendor/urllib3/util/connection.py +++ b/pipenv/vendor/urllib3/util/connection.py @@ -14,7 +14,7 @@ def is_connection_dropped(conn): # Platform-specific Note: For platforms like AppEngine, this will always return ``False`` to let the platform handle connection recycling transparently for us. """ - sock = getattr(conn, 'sock', False) + sock = getattr(conn, "sock", False) if sock is False: # Platform-specific: AppEngine return False if sock is None: # Connection already closed (such as by httplib). @@ -30,8 +30,12 @@ def is_connection_dropped(conn): # Platform-specific # library test suite. Added to its signature is only `socket_options`. # One additional modification is that we avoid binding to IPv6 servers # discovered in DNS if the system doesn't have IPv6 functionality. -def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, - source_address=None, socket_options=None): +def create_connection( + address, + timeout=socket._GLOBAL_DEFAULT_TIMEOUT, + source_address=None, + socket_options=None, +): """Connect to *address* and return the socket object. Convenience function. Connect to *address* (a 2-tuple ``(host, @@ -45,8 +49,8 @@ def create_connection(address, timeout=socket._GLOBAL_DEFAULT_TIMEOUT, """ host, port = address - if host.startswith('['): - host = host.strip('[]') + if host.startswith("["): + host = host.strip("[]") err = None # Using the value from allowed_gai_family() in the context of getaddrinfo lets @@ -117,7 +121,7 @@ def _has_ipv6(host): # has_ipv6 returns true if cPython was compiled with IPv6 support. # It does not tell us if the system has IPv6 support enabled. To # determine that we must bind to an IPv6 address. - # https://github.com/shazow/urllib3/pull/611 + # https://github.com/urllib3/urllib3/pull/611 # https://bugs.python.org/issue658327 try: sock = socket.socket(socket.AF_INET6) @@ -131,4 +135,4 @@ def _has_ipv6(host): return has_ipv6 -HAS_IPV6 = _has_ipv6('::1') +HAS_IPV6 = _has_ipv6("::1") diff --git a/pipenv/vendor/urllib3/util/request.py b/pipenv/vendor/urllib3/util/request.py index 3ddfcd55..3b7bb54d 100644 --- a/pipenv/vendor/urllib3/util/request.py +++ b/pipenv/vendor/urllib3/util/request.py @@ -4,12 +4,25 @@ from base64 import b64encode from ..packages.six import b, integer_types from ..exceptions import UnrewindableBodyError -ACCEPT_ENCODING = 'gzip,deflate' +ACCEPT_ENCODING = "gzip,deflate" +try: + import brotli as _unused_module_brotli # noqa: F401 +except ImportError: + pass +else: + ACCEPT_ENCODING += ",br" + _FAILEDTELL = object() -def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, - basic_auth=None, proxy_basic_auth=None, disable_cache=None): +def make_headers( + keep_alive=None, + accept_encoding=None, + user_agent=None, + basic_auth=None, + proxy_basic_auth=None, + disable_cache=None, +): """ Shortcuts for generating request headers. @@ -49,27 +62,27 @@ def make_headers(keep_alive=None, accept_encoding=None, user_agent=None, if isinstance(accept_encoding, str): pass elif isinstance(accept_encoding, list): - accept_encoding = ','.join(accept_encoding) + accept_encoding = ",".join(accept_encoding) else: accept_encoding = ACCEPT_ENCODING - headers['accept-encoding'] = accept_encoding + headers["accept-encoding"] = accept_encoding if user_agent: - headers['user-agent'] = user_agent + headers["user-agent"] = user_agent if keep_alive: - headers['connection'] = 'keep-alive' + headers["connection"] = "keep-alive" if basic_auth: - headers['authorization'] = 'Basic ' + \ - b64encode(b(basic_auth)).decode('utf-8') + headers["authorization"] = "Basic " + b64encode(b(basic_auth)).decode("utf-8") if proxy_basic_auth: - headers['proxy-authorization'] = 'Basic ' + \ - b64encode(b(proxy_basic_auth)).decode('utf-8') + headers["proxy-authorization"] = "Basic " + b64encode( + b(proxy_basic_auth) + ).decode("utf-8") if disable_cache: - headers['cache-control'] = 'no-cache' + headers["cache-control"] = "no-cache" return headers @@ -81,7 +94,7 @@ def set_file_position(body, pos): """ if pos is not None: rewind_body(body, pos) - elif getattr(body, 'tell', None) is not None: + elif getattr(body, "tell", None) is not None: try: pos = body.tell() except (IOError, OSError): @@ -103,16 +116,20 @@ def rewind_body(body, body_pos): :param int pos: Position to seek to in file. """ - body_seek = getattr(body, 'seek', None) + body_seek = getattr(body, "seek", None) if body_seek is not None and isinstance(body_pos, integer_types): try: body_seek(body_pos) except (IOError, OSError): - raise UnrewindableBodyError("An error occurred when rewinding request " - "body for redirect/retry.") + raise UnrewindableBodyError( + "An error occurred when rewinding request body for redirect/retry." + ) elif body_pos is _FAILEDTELL: - raise UnrewindableBodyError("Unable to record file position for rewinding " - "request body during a redirect/retry.") + raise UnrewindableBodyError( + "Unable to record file position for rewinding " + "request body during a redirect/retry." + ) else: - raise ValueError("body_pos must be of type integer, " - "instead it was %s." % type(body_pos)) + raise ValueError( + "body_pos must be of type integer, instead it was %s." % type(body_pos) + ) diff --git a/pipenv/vendor/urllib3/util/response.py b/pipenv/vendor/urllib3/util/response.py index 3d548648..715868dd 100644 --- a/pipenv/vendor/urllib3/util/response.py +++ b/pipenv/vendor/urllib3/util/response.py @@ -52,11 +52,10 @@ def assert_header_parsing(headers): # This will fail silently if we pass in the wrong kind of parameter. # To make debugging easier add an explicit check. if not isinstance(headers, httplib.HTTPMessage): - raise TypeError('expected httplib.Message, got {0}.'.format( - type(headers))) + raise TypeError("expected httplib.Message, got {0}.".format(type(headers))) - defects = getattr(headers, 'defects', None) - get_payload = getattr(headers, 'get_payload', None) + defects = getattr(headers, "defects", None) + get_payload = getattr(headers, "get_payload", None) unparsed_data = None if get_payload: @@ -84,4 +83,4 @@ def is_response_to_head(response): method = response._method if isinstance(method, int): # Platform-specific: Appengine return method == 3 - return method.upper() == 'HEAD' + return method.upper() == "HEAD" diff --git a/pipenv/vendor/urllib3/util/retry.py b/pipenv/vendor/urllib3/util/retry.py index e7d0abd6..ee30c91b 100644 --- a/pipenv/vendor/urllib3/util/retry.py +++ b/pipenv/vendor/urllib3/util/retry.py @@ -13,6 +13,7 @@ from ..exceptions import ( ReadTimeoutError, ResponseError, InvalidHeader, + ProxyError, ) from ..packages import six @@ -21,8 +22,9 @@ log = logging.getLogger(__name__) # Data structure for representing the metadata of requests that result in a retry. -RequestHistory = namedtuple('RequestHistory', ["method", "url", "error", - "status", "redirect_location"]) +RequestHistory = namedtuple( + "RequestHistory", ["method", "url", "error", "status", "redirect_location"] +) class Retry(object): @@ -146,21 +148,33 @@ class Retry(object): request. """ - DEFAULT_METHOD_WHITELIST = frozenset([ - 'HEAD', 'GET', 'PUT', 'DELETE', 'OPTIONS', 'TRACE']) + DEFAULT_METHOD_WHITELIST = frozenset( + ["HEAD", "GET", "PUT", "DELETE", "OPTIONS", "TRACE"] + ) RETRY_AFTER_STATUS_CODES = frozenset([413, 429, 503]) - DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(['Authorization']) + DEFAULT_REDIRECT_HEADERS_BLACKLIST = frozenset(["Authorization"]) #: Maximum backoff time. BACKOFF_MAX = 120 - def __init__(self, total=10, connect=None, read=None, redirect=None, status=None, - method_whitelist=DEFAULT_METHOD_WHITELIST, status_forcelist=None, - backoff_factor=0, raise_on_redirect=True, raise_on_status=True, - history=None, respect_retry_after_header=True, - remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST): + def __init__( + self, + total=10, + connect=None, + read=None, + redirect=None, + status=None, + method_whitelist=DEFAULT_METHOD_WHITELIST, + status_forcelist=None, + backoff_factor=0, + raise_on_redirect=True, + raise_on_status=True, + history=None, + respect_retry_after_header=True, + remove_headers_on_redirect=DEFAULT_REDIRECT_HEADERS_BLACKLIST, + ): self.total = total self.connect = connect @@ -179,19 +193,25 @@ class Retry(object): self.raise_on_status = raise_on_status self.history = history or tuple() self.respect_retry_after_header = respect_retry_after_header - self.remove_headers_on_redirect = remove_headers_on_redirect + self.remove_headers_on_redirect = frozenset( + [h.lower() for h in remove_headers_on_redirect] + ) def new(self, **kw): params = dict( total=self.total, - connect=self.connect, read=self.read, redirect=self.redirect, status=self.status, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, method_whitelist=self.method_whitelist, status_forcelist=self.status_forcelist, backoff_factor=self.backoff_factor, raise_on_redirect=self.raise_on_redirect, raise_on_status=self.raise_on_status, history=self.history, - remove_headers_on_redirect=self.remove_headers_on_redirect + remove_headers_on_redirect=self.remove_headers_on_redirect, + respect_retry_after_header=self.respect_retry_after_header, ) params.update(kw) return type(self)(**params) @@ -216,8 +236,11 @@ class Retry(object): :rtype: float """ # We want to consider only the last consecutive errors sequence (Ignore redirects). - consecutive_errors_len = len(list(takewhile(lambda x: x.redirect_location is None, - reversed(self.history)))) + consecutive_errors_len = len( + list( + takewhile(lambda x: x.redirect_location is None, reversed(self.history)) + ) + ) if consecutive_errors_len <= 1: return 0 @@ -273,7 +296,7 @@ class Retry(object): this method will return immediately. """ - if response: + if self.respect_retry_after_header and response: slept = self.sleep_for_retry(response) if slept: return @@ -284,6 +307,8 @@ class Retry(object): """ Errors when we're fairly sure that the server did not receive the request, so it should be safe to retry. """ + if isinstance(err, ProxyError): + err = err.original_error return isinstance(err, ConnectTimeoutError) def _is_read_error(self, err): @@ -314,8 +339,12 @@ class Retry(object): if self.status_forcelist and status_code in self.status_forcelist: return True - return (self.total and self.respect_retry_after_header and - has_retry_after and (status_code in self.RETRY_AFTER_STATUS_CODES)) + return ( + self.total + and self.respect_retry_after_header + and has_retry_after + and (status_code in self.RETRY_AFTER_STATUS_CODES) + ) def is_exhausted(self): """ Are we out of retries? """ @@ -326,8 +355,15 @@ class Retry(object): return min(retry_counts) < 0 - def increment(self, method=None, url=None, response=None, error=None, - _pool=None, _stacktrace=None): + def increment( + self, + method=None, + url=None, + response=None, + error=None, + _pool=None, + _stacktrace=None, + ): """ Return a new Retry object with incremented retry counters. :param response: A response object, or None, if the server did not @@ -350,7 +386,7 @@ class Retry(object): read = self.read redirect = self.redirect status_count = self.status - cause = 'unknown' + cause = "unknown" status = None redirect_location = None @@ -372,7 +408,7 @@ class Retry(object): # Redirect retry? if redirect is not None: redirect -= 1 - cause = 'too many redirects' + cause = "too many redirects" redirect_location = response.get_redirect_location() status = response.status @@ -383,16 +419,21 @@ class Retry(object): if response and response.status: if status_count is not None: status_count -= 1 - cause = ResponseError.SPECIFIC_ERROR.format( - status_code=response.status) + cause = ResponseError.SPECIFIC_ERROR.format(status_code=response.status) status = response.status - history = self.history + (RequestHistory(method, url, error, status, redirect_location),) + history = self.history + ( + RequestHistory(method, url, error, status, redirect_location), + ) new_retry = self.new( total=total, - connect=connect, read=read, redirect=redirect, status=status_count, - history=history) + connect=connect, + read=read, + redirect=redirect, + status=status_count, + history=history, + ) if new_retry.is_exhausted(): raise MaxRetryError(_pool, url, error or ResponseError(cause)) @@ -402,9 +443,10 @@ class Retry(object): return new_retry def __repr__(self): - return ('{cls.__name__}(total={self.total}, connect={self.connect}, ' - 'read={self.read}, redirect={self.redirect}, status={self.status})').format( - cls=type(self), self=self) + return ( + "{cls.__name__}(total={self.total}, connect={self.connect}, " + "read={self.read}, redirect={self.redirect}, status={self.status})" + ).format(cls=type(self), self=self) # For backwards compatibility (equivalent to pre-v1.9): diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py index 24ee26d6..f7e2b705 100644 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -2,11 +2,12 @@ from __future__ import absolute_import import errno import warnings import hmac -import socket +import sys from binascii import hexlify, unhexlify from hashlib import md5, sha1, sha256 +from .url import IPV4_RE, BRACELESS_IPV6_ADDRZ_RE from ..exceptions import SSLError, InsecurePlatformWarning, SNIMissingWarning from ..packages import six @@ -17,11 +18,7 @@ IS_PYOPENSSL = False IS_SECURETRANSPORT = False # Maps the length of a digest to a possible hash function producing this digest -HASHFUNC_MAP = { - 32: md5, - 40: sha1, - 64: sha256, -} +HASHFUNC_MAP = {32: md5, 40: sha1, 64: sha256} def _const_compare_digest_backport(a, b): @@ -37,17 +34,27 @@ def _const_compare_digest_backport(a, b): return result == 0 -_const_compare_digest = getattr(hmac, 'compare_digest', - _const_compare_digest_backport) - +_const_compare_digest = getattr(hmac, "compare_digest", _const_compare_digest_backport) try: # Test for SSL features import ssl - from ssl import wrap_socket, CERT_NONE, PROTOCOL_SSLv23 + from ssl import wrap_socket, CERT_REQUIRED from ssl import HAS_SNI # Has SNI? except ImportError: pass +try: # Platform-specific: Python 3.6 + from ssl import PROTOCOL_TLS + + PROTOCOL_SSLv23 = PROTOCOL_TLS +except ImportError: + try: + from ssl import PROTOCOL_SSLv23 as PROTOCOL_TLS + + PROTOCOL_SSLv23 = PROTOCOL_TLS + except ImportError: + PROTOCOL_SSLv23 = PROTOCOL_TLS = 2 + try: from ssl import OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION @@ -56,25 +63,6 @@ except ImportError: OP_NO_COMPRESSION = 0x20000 -# Python 2.7 doesn't have inet_pton on non-Linux so we fallback on inet_aton in -# those cases. This means that we can only detect IPv4 addresses in this case. -if hasattr(socket, 'inet_pton'): - inet_pton = socket.inet_pton -else: - # Maybe we can use ipaddress if the user has urllib3[secure]? - try: - import ipaddress - - def inet_pton(_, host): - if isinstance(host, bytes): - host = host.decode('ascii') - return ipaddress.ip_address(host) - - except ImportError: # Platform-specific: Non-Linux - def inet_pton(_, host): - return socket.inet_aton(host) - - # A secure default. # Sources for more information on TLS ciphers: # @@ -83,36 +71,37 @@ else: # - https://hynek.me/articles/hardening-your-web-servers-ssl-ciphers/ # # The general intent is: -# - Prefer TLS 1.3 cipher suites # - prefer cipher suites that offer perfect forward secrecy (DHE/ECDHE), # - prefer ECDHE over DHE for better performance, # - prefer any AES-GCM and ChaCha20 over any AES-CBC for better performance and # security, # - prefer AES-GCM over ChaCha20 because hardware-accelerated AES is common, -# - disable NULL authentication, MD5 MACs and DSS for security reasons. -DEFAULT_CIPHERS = ':'.join([ - 'TLS13-AES-256-GCM-SHA384', - 'TLS13-CHACHA20-POLY1305-SHA256', - 'TLS13-AES-128-GCM-SHA256', - 'ECDH+AESGCM', - 'ECDH+CHACHA20', - 'DH+AESGCM', - 'DH+CHACHA20', - 'ECDH+AES256', - 'DH+AES256', - 'ECDH+AES128', - 'DH+AES', - 'RSA+AESGCM', - 'RSA+AES', - '!aNULL', - '!eNULL', - '!MD5', -]) +# - disable NULL authentication, MD5 MACs, DSS, and other +# insecure ciphers for security reasons. +# - NOTE: TLS 1.3 cipher suites are managed through a different interface +# not exposed by CPython (yet!) and are enabled by default if they're available. +DEFAULT_CIPHERS = ":".join( + [ + "ECDHE+AESGCM", + "ECDHE+CHACHA20", + "DHE+AESGCM", + "DHE+CHACHA20", + "ECDH+AESGCM", + "DH+AESGCM", + "ECDH+AES", + "DH+AES", + "RSA+AESGCM", + "RSA+AES", + "!aNULL", + "!eNULL", + "!MD5", + "!DSS", + ] +) try: from ssl import SSLContext # Modern SSL? except ImportError: - import sys class SSLContext(object): # Platform-specific: Python 2 def __init__(self, protocol_version): @@ -130,32 +119,35 @@ except ImportError: self.certfile = certfile self.keyfile = keyfile - def load_verify_locations(self, cafile=None, capath=None): + def load_verify_locations(self, cafile=None, capath=None, cadata=None): self.ca_certs = cafile if capath is not None: raise SSLError("CA directories not supported in older Pythons") + if cadata is not None: + raise SSLError("CA data not supported in older Pythons") + def set_ciphers(self, cipher_suite): self.ciphers = cipher_suite def wrap_socket(self, socket, server_hostname=None, server_side=False): warnings.warn( - 'A true SSLContext object is not available. This prevents ' - 'urllib3 from configuring SSL appropriately and may cause ' - 'certain SSL connections to fail. You can upgrade to a newer ' - 'version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - InsecurePlatformWarning + "A true SSLContext object is not available. This prevents " + "urllib3 from configuring SSL appropriately and may cause " + "certain SSL connections to fail. You can upgrade to a newer " + "version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + InsecurePlatformWarning, ) kwargs = { - 'keyfile': self.keyfile, - 'certfile': self.certfile, - 'ca_certs': self.ca_certs, - 'cert_reqs': self.verify_mode, - 'ssl_version': self.protocol, - 'server_side': server_side, + "keyfile": self.keyfile, + "certfile": self.certfile, + "ca_certs": self.ca_certs, + "cert_reqs": self.verify_mode, + "ssl_version": self.protocol, + "server_side": server_side, } return wrap_socket(socket, ciphers=self.ciphers, **kwargs) @@ -170,12 +162,11 @@ def assert_fingerprint(cert, fingerprint): Fingerprint as string of hexdigits, can be interspersed by colons. """ - fingerprint = fingerprint.replace(':', '').lower() + fingerprint = fingerprint.replace(":", "").lower() digest_length = len(fingerprint) hashfunc = HASHFUNC_MAP.get(digest_length) if not hashfunc: - raise SSLError( - 'Fingerprint of invalid length: {0}'.format(fingerprint)) + raise SSLError("Fingerprint of invalid length: {0}".format(fingerprint)) # We need encode() here for py32; works on py2 and p33. fingerprint_bytes = unhexlify(fingerprint.encode()) @@ -183,15 +174,18 @@ def assert_fingerprint(cert, fingerprint): cert_digest = hashfunc(cert).digest() if not _const_compare_digest(cert_digest, fingerprint_bytes): - raise SSLError('Fingerprints did not match. Expected "{0}", got "{1}".' - .format(fingerprint, hexlify(cert_digest))) + raise SSLError( + 'Fingerprints did not match. Expected "{0}", got "{1}".'.format( + fingerprint, hexlify(cert_digest) + ) + ) def resolve_cert_reqs(candidate): """ Resolves the argument to a numeric constant, which can be passed to the wrap_socket function/method from the ssl module. - Defaults to :data:`ssl.CERT_NONE`. + Defaults to :data:`ssl.CERT_REQUIRED`. If given a string it is assumed to be the name of the constant in the :mod:`ssl` module or its abbreviation. (So you can specify `REQUIRED` instead of `CERT_REQUIRED`. @@ -199,12 +193,12 @@ def resolve_cert_reqs(candidate): constant which can directly be passed to wrap_socket. """ if candidate is None: - return CERT_NONE + return CERT_REQUIRED if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'CERT_' + candidate) + res = getattr(ssl, "CERT_" + candidate) return res return candidate @@ -215,19 +209,20 @@ def resolve_ssl_version(candidate): like resolve_cert_reqs """ if candidate is None: - return PROTOCOL_SSLv23 + return PROTOCOL_TLS if isinstance(candidate, str): res = getattr(ssl, candidate, None) if res is None: - res = getattr(ssl, 'PROTOCOL_' + candidate) + res = getattr(ssl, "PROTOCOL_" + candidate) return res return candidate -def create_urllib3_context(ssl_version=None, cert_reqs=None, - options=None, ciphers=None): +def create_urllib3_context( + ssl_version=None, cert_reqs=None, options=None, ciphers=None +): """All arguments have the same meaning as ``ssl_wrap_socket``. By default, this function does a lot of the same work that @@ -261,7 +256,9 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, Constructed SSLContext object with specified options :rtype: SSLContext """ - context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context = SSLContext(ssl_version or PROTOCOL_TLS) + + context.set_ciphers(ciphers or DEFAULT_CIPHERS) # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs @@ -278,18 +275,41 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, context.options |= options + # Enable post-handshake authentication for TLS 1.3, see GH #1634. PHA is + # necessary for conditional client cert authentication with TLS 1.3. + # The attribute is None for OpenSSL <= 1.1.0 or does not exist in older + # versions of Python. We only enable on Python 3.7.4+ or if certificate + # verification is enabled to work around Python issue #37428 + # See: https://bugs.python.org/issue37428 + if (cert_reqs == ssl.CERT_REQUIRED or sys.version_info >= (3, 7, 4)) and getattr( + context, "post_handshake_auth", None + ) is not None: + context.post_handshake_auth = True + context.verify_mode = cert_reqs - if getattr(context, 'check_hostname', None) is not None: # Platform-specific: Python 3.2 + if ( + getattr(context, "check_hostname", None) is not None + ): # Platform-specific: Python 3.2 # We do our own verification, including fingerprints and alternative # hostnames. So disable it here context.check_hostname = False return context -def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, - ca_certs=None, server_hostname=None, - ssl_version=None, ciphers=None, ssl_context=None, - ca_cert_dir=None): +def ssl_wrap_socket( + sock, + keyfile=None, + certfile=None, + cert_reqs=None, + ca_certs=None, + server_hostname=None, + ssl_version=None, + ciphers=None, + ssl_context=None, + ca_cert_dir=None, + key_password=None, + ca_cert_data=None, +): """ All arguments except for server_hostname, ssl_context, and ca_cert_dir have the same meaning as they do when using :func:`ssl.wrap_socket`. @@ -305,18 +325,22 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, A directory containing CA certificates in multiple separate files, as supported by OpenSSL's -CApath flag or the capath argument to SSLContext.load_verify_locations(). + :param key_password: + Optional password if the keyfile is encrypted. + :param ca_cert_data: + Optional string containing CA certificates in PEM format suitable for + passing as the cadata parameter to SSLContext.load_verify_locations() """ context = ssl_context if context is None: # Note: This branch of code and all the variables in it are no longer # used by urllib3 itself. We should consider deprecating and removing # this code. - context = create_urllib3_context(ssl_version, cert_reqs, - ciphers=ciphers) + context = create_urllib3_context(ssl_version, cert_reqs, ciphers=ciphers) - if ca_certs or ca_cert_dir: + if ca_certs or ca_cert_dir or ca_cert_data: try: - context.load_verify_locations(ca_certs, ca_cert_dir) + context.load_verify_locations(ca_certs, ca_cert_dir, ca_cert_data) except IOError as e: # Platform-specific: Python 2.7 raise SSLError(e) # Py33 raises FileNotFoundError which subclasses OSError @@ -325,55 +349,66 @@ def ssl_wrap_socket(sock, keyfile=None, certfile=None, cert_reqs=None, if e.errno == errno.ENOENT: raise SSLError(e) raise - elif getattr(context, 'load_default_certs', None) is not None: + + elif ssl_context is None and hasattr(context, "load_default_certs"): # try to load OS default certs; works well on Windows (require Python3.4+) context.load_default_certs() + # Attempt to detect if we get the goofy behavior of the + # keyfile being encrypted and OpenSSL asking for the + # passphrase via the terminal and instead error out. + if keyfile and key_password is None and _is_key_file_encrypted(keyfile): + raise SSLError("Client private key is encrypted, password is required") + if certfile: - context.load_cert_chain(certfile, keyfile) + if key_password is None: + context.load_cert_chain(certfile, keyfile) + else: + context.load_cert_chain(certfile, keyfile, key_password) # If we detect server_hostname is an IP address then the SNI # extension should not be used according to RFC3546 Section 3.1 # We shouldn't warn the user if SNI isn't available but we would # not be using SNI anyways due to IP address for server_hostname. - if ((server_hostname is not None and not is_ipaddress(server_hostname)) - or IS_SECURETRANSPORT): + if ( + server_hostname is not None and not is_ipaddress(server_hostname) + ) or IS_SECURETRANSPORT: if HAS_SNI and server_hostname is not None: return context.wrap_socket(sock, server_hostname=server_hostname) warnings.warn( - 'An HTTPS request has been made, but the SNI (Server Name ' - 'Indication) extension to TLS is not available on this platform. ' - 'This may cause the server to present an incorrect TLS ' - 'certificate, which can cause validation failures. You can upgrade to ' - 'a newer version of Python to solve this. For more information, see ' - 'https://urllib3.readthedocs.io/en/latest/advanced-usage.html' - '#ssl-warnings', - SNIMissingWarning + "An HTTPS request has been made, but the SNI (Server Name " + "Indication) extension to TLS is not available on this platform. " + "This may cause the server to present an incorrect TLS " + "certificate, which can cause validation failures. You can upgrade to " + "a newer version of Python to solve this. For more information, see " + "https://urllib3.readthedocs.io/en/latest/advanced-usage.html" + "#ssl-warnings", + SNIMissingWarning, ) return context.wrap_socket(sock) def is_ipaddress(hostname): - """Detects whether the hostname given is an IP address. + """Detects whether the hostname given is an IPv4 or IPv6 address. + Also detects IPv6 addresses with Zone IDs. :param str hostname: Hostname to examine. :return: True if the hostname is an IP address, False otherwise. """ - if six.PY3 and isinstance(hostname, bytes): + if not six.PY2 and isinstance(hostname, bytes): # IDN A-label bytes are ASCII compatible. - hostname = hostname.decode('ascii') + hostname = hostname.decode("ascii") + return bool(IPV4_RE.match(hostname) or BRACELESS_IPV6_ADDRZ_RE.match(hostname)) - families = [socket.AF_INET] - if hasattr(socket, 'AF_INET6'): - families.append(socket.AF_INET6) - for af in families: - try: - inet_pton(af, hostname) - except (socket.error, ValueError, OSError): - pass - else: - return True +def _is_key_file_encrypted(key_file): + """Detects if a key file is encrypted or not.""" + with open(key_file, "r") as f: + for line in f: + # Look for Proc-Type: 4,ENCRYPTED + if "ENCRYPTED" in line: + return True + return False diff --git a/pipenv/vendor/urllib3/util/timeout.py b/pipenv/vendor/urllib3/util/timeout.py index cec817e6..b61fea75 100644 --- a/pipenv/vendor/urllib3/util/timeout.py +++ b/pipenv/vendor/urllib3/util/timeout.py @@ -1,4 +1,5 @@ from __future__ import absolute_import + # The default socket timeout, used by httplib to indicate that no timeout was # specified by the user from socket import _GLOBAL_DEFAULT_TIMEOUT @@ -45,19 +46,20 @@ class Timeout(object): :type total: integer, float, or None :param connect: - The maximum amount of time to wait for a connection attempt to a server - to succeed. Omitting the parameter will default the connect timeout to - the system default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait for a connection + attempt to a server to succeed. Omitting the parameter will default the + connect timeout to the system default, probably `the global default + timeout in socket.py <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. None will set an infinite timeout for connection attempts. :type connect: integer, float, or None :param read: - The maximum amount of time to wait between consecutive - read operations for a response from the server. Omitting - the parameter will default the read timeout to the system - default, probably `the global default timeout in socket.py + The maximum amount of time (in seconds) to wait between consecutive + read operations for a response from the server. Omitting the parameter + will default the read timeout to the system default, probably `the + global default timeout in socket.py <http://hg.python.org/cpython/file/603b4d593758/Lib/socket.py#l535>`_. None will set an infinite timeout. @@ -91,14 +93,21 @@ class Timeout(object): DEFAULT_TIMEOUT = _GLOBAL_DEFAULT_TIMEOUT def __init__(self, total=None, connect=_Default, read=_Default): - self._connect = self._validate_timeout(connect, 'connect') - self._read = self._validate_timeout(read, 'read') - self.total = self._validate_timeout(total, 'total') + self._connect = self._validate_timeout(connect, "connect") + self._read = self._validate_timeout(read, "read") + self.total = self._validate_timeout(total, "total") self._start_connect = None - def __str__(self): - return '%s(connect=%r, read=%r, total=%r)' % ( - type(self).__name__, self._connect, self._read, self.total) + def __repr__(self): + return "%s(connect=%r, read=%r, total=%r)" % ( + type(self).__name__, + self._connect, + self._read, + self.total, + ) + + # __str__ provided for backwards compatibility + __str__ = __repr__ @classmethod def _validate_timeout(cls, value, name): @@ -118,22 +127,31 @@ class Timeout(object): return value if isinstance(value, bool): - raise ValueError("Timeout cannot be a boolean value. It must " - "be an int, float or None.") + raise ValueError( + "Timeout cannot be a boolean value. It must " + "be an int, float or None." + ) try: float(value) except (TypeError, ValueError): - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) try: if value <= 0: - raise ValueError("Attempted to set %s timeout to %s, but the " - "timeout cannot be set to a value less " - "than or equal to 0." % (name, value)) - except TypeError: # Python 3 - raise ValueError("Timeout value %s was %s, but it must be an " - "int, float or None." % (name, value)) + raise ValueError( + "Attempted to set %s timeout to %s, but the " + "timeout cannot be set to a value less " + "than or equal to 0." % (name, value) + ) + except TypeError: + # Python 3 + raise ValueError( + "Timeout value %s was %s, but it must be an " + "int, float or None." % (name, value) + ) return value @@ -165,8 +183,7 @@ class Timeout(object): # We can't use copy.deepcopy because that will also create a new object # for _GLOBAL_DEFAULT_TIMEOUT, which socket.py uses as a sentinel to # detect the user default. - return Timeout(connect=self._connect, read=self._read, - total=self.total) + return Timeout(connect=self._connect, read=self._read, total=self.total) def start_connect(self): """ Start the timeout clock, used during a connect() attempt @@ -182,14 +199,15 @@ class Timeout(object): def get_connect_duration(self): """ Gets the time elapsed since the call to :meth:`start_connect`. - :return: Elapsed time. + :return: Elapsed time in seconds. :rtype: float :raises urllib3.exceptions.TimeoutStateError: if you attempt to get duration for a timer that hasn't been started. """ if self._start_connect is None: - raise TimeoutStateError("Can't get connect duration for timer " - "that has not started.") + raise TimeoutStateError( + "Can't get connect duration for timer that has not started." + ) return current_time() - self._start_connect @property @@ -227,15 +245,16 @@ class Timeout(object): :raises urllib3.exceptions.TimeoutStateError: If :meth:`start_connect` has not yet been called on this object. """ - if (self.total is not None and - self.total is not self.DEFAULT_TIMEOUT and - self._read is not None and - self._read is not self.DEFAULT_TIMEOUT): + if ( + self.total is not None + and self.total is not self.DEFAULT_TIMEOUT + and self._read is not None + and self._read is not self.DEFAULT_TIMEOUT + ): # In case the connect timeout has not yet been established. if self._start_connect is None: return self._read - return max(0, min(self.total - self.get_connect_duration(), - self._read)) + return max(0, min(self.total - self.get_connect_duration(), self._read)) elif self.total is not None and self.total is not self.DEFAULT_TIMEOUT: return max(0, self.total - self.get_connect_duration()) else: diff --git a/pipenv/vendor/urllib3/util/url.py b/pipenv/vendor/urllib3/util/url.py index 6b6f9968..793324e5 100644 --- a/pipenv/vendor/urllib3/util/url.py +++ b/pipenv/vendor/urllib3/util/url.py @@ -1,34 +1,110 @@ from __future__ import absolute_import +import re from collections import namedtuple from ..exceptions import LocationParseError +from ..packages import six -url_attrs = ['scheme', 'auth', 'host', 'port', 'path', 'query', 'fragment'] +url_attrs = ["scheme", "auth", "host", "port", "path", "query", "fragment"] # We only want to normalize urls with an HTTP(S) scheme. # urllib3 infers URLs without a scheme (None) to be http. -NORMALIZABLE_SCHEMES = ('http', 'https', None) +NORMALIZABLE_SCHEMES = ("http", "https", None) + +# Almost all of these patterns were derived from the +# 'rfc3986' module: https://github.com/python-hyper/rfc3986 +PERCENT_RE = re.compile(r"%[a-fA-F0-9]{2}") +SCHEME_RE = re.compile(r"^(?:[a-zA-Z][a-zA-Z0-9+-]*:|/)") +URI_RE = re.compile( + r"^(?:([a-zA-Z][a-zA-Z0-9+.-]*):)?" + r"(?://([^\\/?#]*))?" + r"([^?#]*)" + r"(?:\?([^#]*))?" + r"(?:#(.*))?$", + re.UNICODE | re.DOTALL, +) + +IPV4_PAT = r"(?:[0-9]{1,3}\.){3}[0-9]{1,3}" +HEX_PAT = "[0-9A-Fa-f]{1,4}" +LS32_PAT = "(?:{hex}:{hex}|{ipv4})".format(hex=HEX_PAT, ipv4=IPV4_PAT) +_subs = {"hex": HEX_PAT, "ls32": LS32_PAT} +_variations = [ + # 6( h16 ":" ) ls32 + "(?:%(hex)s:){6}%(ls32)s", + # "::" 5( h16 ":" ) ls32 + "::(?:%(hex)s:){5}%(ls32)s", + # [ h16 ] "::" 4( h16 ":" ) ls32 + "(?:%(hex)s)?::(?:%(hex)s:){4}%(ls32)s", + # [ *1( h16 ":" ) h16 ] "::" 3( h16 ":" ) ls32 + "(?:(?:%(hex)s:)?%(hex)s)?::(?:%(hex)s:){3}%(ls32)s", + # [ *2( h16 ":" ) h16 ] "::" 2( h16 ":" ) ls32 + "(?:(?:%(hex)s:){0,2}%(hex)s)?::(?:%(hex)s:){2}%(ls32)s", + # [ *3( h16 ":" ) h16 ] "::" h16 ":" ls32 + "(?:(?:%(hex)s:){0,3}%(hex)s)?::%(hex)s:%(ls32)s", + # [ *4( h16 ":" ) h16 ] "::" ls32 + "(?:(?:%(hex)s:){0,4}%(hex)s)?::%(ls32)s", + # [ *5( h16 ":" ) h16 ] "::" h16 + "(?:(?:%(hex)s:){0,5}%(hex)s)?::%(hex)s", + # [ *6( h16 ":" ) h16 ] "::" + "(?:(?:%(hex)s:){0,6}%(hex)s)?::", +] + +UNRESERVED_PAT = r"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._!\-~" +IPV6_PAT = "(?:" + "|".join([x % _subs for x in _variations]) + ")" +ZONE_ID_PAT = "(?:%25|%)(?:[" + UNRESERVED_PAT + "]|%[a-fA-F0-9]{2})+" +IPV6_ADDRZ_PAT = r"\[" + IPV6_PAT + r"(?:" + ZONE_ID_PAT + r")?\]" +REG_NAME_PAT = r"(?:[^\[\]%:/?#]|%[a-fA-F0-9]{2})*" +TARGET_RE = re.compile(r"^(/[^?#]*)(?:\?([^#]*))?(?:#.*)?$") + +IPV4_RE = re.compile("^" + IPV4_PAT + "$") +IPV6_RE = re.compile("^" + IPV6_PAT + "$") +IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT + "$") +BRACELESS_IPV6_ADDRZ_RE = re.compile("^" + IPV6_ADDRZ_PAT[2:-2] + "$") +ZONE_ID_RE = re.compile("(" + ZONE_ID_PAT + r")\]$") + +SUBAUTHORITY_PAT = (u"^(?:(.*)@)?(%s|%s|%s)(?::([0-9]{0,5}))?$") % ( + REG_NAME_PAT, + IPV4_PAT, + IPV6_ADDRZ_PAT, +) +SUBAUTHORITY_RE = re.compile(SUBAUTHORITY_PAT, re.UNICODE | re.DOTALL) + +UNRESERVED_CHARS = set( + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789._-~" +) +SUB_DELIM_CHARS = set("!$&'()*+,;=") +USERINFO_CHARS = UNRESERVED_CHARS | SUB_DELIM_CHARS | {":"} +PATH_CHARS = USERINFO_CHARS | {"@", "/"} +QUERY_CHARS = FRAGMENT_CHARS = PATH_CHARS | {"?"} -class Url(namedtuple('Url', url_attrs)): +class Url(namedtuple("Url", url_attrs)): """ - Datastructure for representing an HTTP URL. Used as a return value for + Data structure for representing an HTTP URL. Used as a return value for :func:`parse_url`. Both the scheme and host are normalized as they are both case-insensitive according to RFC 3986. """ + __slots__ = () - def __new__(cls, scheme=None, auth=None, host=None, port=None, path=None, - query=None, fragment=None): - if path and not path.startswith('/'): - path = '/' + path - if scheme: + def __new__( + cls, + scheme=None, + auth=None, + host=None, + port=None, + path=None, + query=None, + fragment=None, + ): + if path and not path.startswith("/"): + path = "/" + path + if scheme is not None: scheme = scheme.lower() - if host and scheme in NORMALIZABLE_SCHEMES: - host = host.lower() - return super(Url, cls).__new__(cls, scheme, auth, host, port, path, - query, fragment) + return super(Url, cls).__new__( + cls, scheme, auth, host, port, path, query, fragment + ) @property def hostname(self): @@ -38,10 +114,10 @@ class Url(namedtuple('Url', url_attrs)): @property def request_uri(self): """Absolute path including the query string.""" - uri = self.path or '/' + uri = self.path or "/" if self.query is not None: - uri += '?' + self.query + uri += "?" + self.query return uri @@ -49,7 +125,7 @@ class Url(namedtuple('Url', url_attrs)): def netloc(self): """Network location including host and port""" if self.port: - return '%s:%d' % (self.host, self.port) + return "%s:%d" % (self.host, self.port) return self.host @property @@ -72,23 +148,23 @@ class Url(namedtuple('Url', url_attrs)): 'http://username:password@host.com:80/path?query#fragment' """ scheme, auth, host, port, path, query, fragment = self - url = '' + url = u"" # We use "is not None" we want things to happen with empty strings (or 0 port) if scheme is not None: - url += scheme + '://' + url += scheme + u"://" if auth is not None: - url += auth + '@' + url += auth + u"@" if host is not None: url += host if port is not None: - url += ':' + str(port) + url += u":" + str(port) if path is not None: url += path if query is not None: - url += '?' + query + url += u"?" + query if fragment is not None: - url += '#' + fragment + url += u"#" + fragment return url @@ -98,6 +174,8 @@ class Url(namedtuple('Url', url_attrs)): def split_first(s, delims): """ + .. deprecated:: 1.25 + Given a string and an iterable of delimiters, split on the first found delimiter. Return two split parts and the matched delimiter. @@ -124,15 +202,141 @@ def split_first(s, delims): min_delim = d if min_idx is None or min_idx < 0: - return s, '', None + return s, "", None - return s[:min_idx], s[min_idx + 1:], min_delim + return s[:min_idx], s[min_idx + 1 :], min_delim + + +def _encode_invalid_chars(component, allowed_chars, encoding="utf-8"): + """Percent-encodes a URI component without reapplying + onto an already percent-encoded component. + """ + if component is None: + return component + + component = six.ensure_text(component) + + # Normalize existing percent-encoded bytes. + # Try to see if the component we're encoding is already percent-encoded + # so we can skip all '%' characters but still encode all others. + component, percent_encodings = PERCENT_RE.subn( + lambda match: match.group(0).upper(), component + ) + + uri_bytes = component.encode("utf-8", "surrogatepass") + is_percent_encoded = percent_encodings == uri_bytes.count(b"%") + encoded_component = bytearray() + + for i in range(0, len(uri_bytes)): + # Will return a single character bytestring on both Python 2 & 3 + byte = uri_bytes[i : i + 1] + byte_ord = ord(byte) + if (is_percent_encoded and byte == b"%") or ( + byte_ord < 128 and byte.decode() in allowed_chars + ): + encoded_component += byte + continue + encoded_component.extend(b"%" + (hex(byte_ord)[2:].encode().zfill(2).upper())) + + return encoded_component.decode(encoding) + + +def _remove_path_dot_segments(path): + # See http://tools.ietf.org/html/rfc3986#section-5.2.4 for pseudo-code + segments = path.split("/") # Turn the path into a list of segments + output = [] # Initialize the variable to use to store output + + for segment in segments: + # '.' is the current directory, so ignore it, it is superfluous + if segment == ".": + continue + # Anything other than '..', should be appended to the output + elif segment != "..": + output.append(segment) + # In this case segment == '..', if we can, we should pop the last + # element + elif output: + output.pop() + + # If the path starts with '/' and the output is empty or the first string + # is non-empty + if path.startswith("/") and (not output or output[0]): + output.insert(0, "") + + # If the path starts with '/.' or '/..' ensure we add one more empty + # string to add a trailing '/' + if path.endswith(("/.", "/..")): + output.append("") + + return "/".join(output) + + +def _normalize_host(host, scheme): + if host: + if isinstance(host, six.binary_type): + host = six.ensure_str(host) + + if scheme in NORMALIZABLE_SCHEMES: + is_ipv6 = IPV6_ADDRZ_RE.match(host) + if is_ipv6: + match = ZONE_ID_RE.search(host) + if match: + start, end = match.span(1) + zone_id = host[start:end] + + if zone_id.startswith("%25") and zone_id != "%25": + zone_id = zone_id[3:] + else: + zone_id = zone_id[1:] + zone_id = "%" + _encode_invalid_chars(zone_id, UNRESERVED_CHARS) + return host[:start].lower() + zone_id + host[end:] + else: + return host.lower() + elif not IPV4_RE.match(host): + return six.ensure_str( + b".".join([_idna_encode(label) for label in host.split(".")]) + ) + return host + + +def _idna_encode(name): + if name and any([ord(x) > 128 for x in name]): + try: + import idna + except ImportError: + six.raise_from( + LocationParseError("Unable to parse URL without the 'idna' module"), + None, + ) + try: + return idna.encode(name.lower(), strict=True, std3_rules=True) + except idna.IDNAError: + six.raise_from( + LocationParseError(u"Name '%s' is not a valid IDNA label" % name), None + ) + return name.lower().encode("ascii") + + +def _encode_target(target): + """Percent-encodes a request target so that there are no invalid characters""" + path, query = TARGET_RE.match(target).groups() + target = _encode_invalid_chars(path, PATH_CHARS) + query = _encode_invalid_chars(query, QUERY_CHARS) + if query is not None: + target += "?" + query + return target def parse_url(url): """ Given a url, return a parsed :class:`.Url` namedtuple. Best-effort is performed to parse incomplete urls. Fields not provided will be None. + This parser is RFC 3986 compliant. + + The parser logic and helper functions are based heavily on + work done in the ``rfc3986`` module. + + :param str url: URL to parse into a :class:`.Url` namedtuple. Partly backwards-compatible with :mod:`urlparse`. @@ -145,81 +349,77 @@ def parse_url(url): >>> parse_url('/foo?bar') Url(scheme=None, host=None, port=None, path='/foo', query='bar', ...) """ - - # While this code has overlap with stdlib's urlparse, it is much - # simplified for our needs and less annoying. - # Additionally, this implementations does silly things to be optimal - # on CPython. - if not url: # Empty return Url() - scheme = None - auth = None - host = None - port = None - path = None - fragment = None - query = None + source_url = url + if not SCHEME_RE.search(url): + url = "//" + url - # Scheme - if '://' in url: - scheme, url = url.split('://', 1) + try: + scheme, authority, path, query, fragment = URI_RE.match(url).groups() + normalize_uri = scheme is None or scheme.lower() in NORMALIZABLE_SCHEMES - # Find the earliest Authority Terminator - # (http://tools.ietf.org/html/rfc3986#section-3.2) - url, path_, delim = split_first(url, ['/', '?', '#']) + if scheme: + scheme = scheme.lower() - if delim: - # Reassemble the path - path = delim + path_ - - # Auth - if '@' in url: - # Last '@' denotes end of auth part - auth, url = url.rsplit('@', 1) - - # IPv6 - if url and url[0] == '[': - host, url = url.split(']', 1) - host += ']' - - # Port - if ':' in url: - _host, port = url.split(':', 1) - - if not host: - host = _host - - if port: - # If given, ports must be integers. No whitespace, no plus or - # minus prefixes, no non-integer digits such as ^2 (superscript). - if not port.isdigit(): - raise LocationParseError(url) - try: - port = int(port) - except ValueError: - raise LocationParseError(url) + if authority: + auth, host, port = SUBAUTHORITY_RE.match(authority).groups() + if auth and normalize_uri: + auth = _encode_invalid_chars(auth, USERINFO_CHARS) + if port == "": + port = None else: - # Blank ports are cool, too. (rfc3986#section-3.2.3) - port = None + auth, host, port = None, None, None - elif not host and url: - host = url + if port is not None: + port = int(port) + if not (0 <= port <= 65535): + raise LocationParseError(url) + host = _normalize_host(host, scheme) + + if normalize_uri and path: + path = _remove_path_dot_segments(path) + path = _encode_invalid_chars(path, PATH_CHARS) + if normalize_uri and query: + query = _encode_invalid_chars(query, QUERY_CHARS) + if normalize_uri and fragment: + fragment = _encode_invalid_chars(fragment, FRAGMENT_CHARS) + + except (ValueError, AttributeError): + return six.raise_from(LocationParseError(source_url), None) + + # For the sake of backwards compatibility we put empty + # string values for path if there are any defined values + # beyond the path in the URL. + # TODO: Remove this when we break backwards compatibility. if not path: - return Url(scheme, auth, host, port, path, query, fragment) + if query is not None or fragment is not None: + path = "" + else: + path = None - # Fragment - if '#' in path: - path, fragment = path.split('#', 1) + # Ensure that each part of the URL is a `str` for + # backwards compatibility. + if isinstance(url, six.text_type): + ensure_func = six.ensure_text + else: + ensure_func = six.ensure_str - # Query - if '?' in path: - path, query = path.split('?', 1) + def ensure_type(x): + return x if x is None else ensure_func(x) - return Url(scheme, auth, host, port, path, query, fragment) + return Url( + scheme=ensure_type(scheme), + auth=ensure_type(auth), + host=ensure_type(host), + port=port, + path=ensure_type(path), + query=ensure_type(query), + fragment=ensure_type(fragment), + ) def get_host(url): @@ -227,4 +427,4 @@ def get_host(url): Deprecated. Use :func:`parse_url` instead. """ p = parse_url(url) - return p.scheme or 'http', p.hostname, p.port + return p.scheme or "http", p.hostname, p.port diff --git a/pipenv/vendor/urllib3/util/wait.py b/pipenv/vendor/urllib3/util/wait.py index 4db71baf..d71d2fd7 100644 --- a/pipenv/vendor/urllib3/util/wait.py +++ b/pipenv/vendor/urllib3/util/wait.py @@ -2,6 +2,7 @@ import errno from functools import partial import select import sys + try: from time import monotonic except ImportError: @@ -40,6 +41,8 @@ if sys.version_info >= (3, 5): # Modern Python, that retries syscalls by default def _retry_on_intr(fn, timeout): return fn(timeout) + + else: # Old and broken Pythons. def _retry_on_intr(fn, timeout): diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 1a419eef..457c127c 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -1,52 +1,58 @@ appdirs==1.4.3 backports.shutil_get_terminal_size==1.0.0 backports.weakref==1.0.post1 -blindspin==2.0.1 -click==7.0 -click-completion==0.5.0 +click==7.1.1 +click-completion==0.5.2 click-didyoumean==0.0.3 -colorama==0.3.9 +colorama==0.4.3 delegator.py==0.1.1 - pexpect==4.6.0 + pexpect==4.8.0 ptyprocess==0.6.0 -python-dotenv==0.9.1 +python-dotenv==0.10.3 first==2.0.1 iso8601==0.1.12 -jinja2==2.10 -markupsafe==1.0 -parse==1.9.0 -pathlib2==2.3.2 - scandir==1.9 -pipdeptree==0.13.0 -pipreqs==0.4.9 +jinja2==2.11.1 +markupsafe==1.1.1 +parse==1.15.0 +pathlib2==2.3.5 + scandir==1.10 +pipdeptree==0.13.2 +pipreqs==0.4.10 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.10 -requests==2.20.1 +pythonfinder==1.2.2 +requests==2.23.0 chardet==3.0.4 - idna==2.7 - urllib3==1.24 - certifi==2018.10.15 -requirementslib==1.3.3 - attrs==18.2.0 - distlib==0.2.8 - packaging==18.0 - pyparsing==2.2.2 + idna==2.9 + urllib3==1.25.9 + certifi==2020.4.5.1 +requirementslib==1.5.7 + attrs==19.3.0 + distlib==0.3.0 + packaging==20.3 + pyparsing==2.4.7 git+https://github.com/sarugaku/plette.git@master#egg=plette - tomlkit==0.5.2 -shellingham==1.2.7 -six==1.11.0 -semver==2.8.1 -shutilwhich==1.1.0 + tomlkit==0.5.11 +shellingham==1.3.2 +six==1.14.0 +semver==2.9.0 toml==0.10.0 -cached-property==1.4.3 -vistir==0.2.5 -pip-shims==0.3.2 -ptyprocess==0.6.0 +cached-property==1.5.1 +vistir==0.5.0 +pip-shims==0.5.2 + contextlib2==0.6.0.post1 + funcsigs==1.0.2 enum34==1.1.6 -yaspin==0.14.0 -cerberus==1.2 -git+https://github.com/sarugaku/passa.git@master#egg=passa -cursor==1.2.0 -resolvelib==0.2.2 +# yaspin==0.15.0 +yaspin==0.14.3 +cerberus==1.3.2 +resolvelib==0.3.0 backports.functools_lru_cache==1.5 +pep517==0.8.2 + zipp==0.6.0 + importlib_metadata==1.6.0 + importlib-resources==1.4.0 + more-itertools==5.0.0 +git+https://github.com/sarugaku/passa.git@master#egg=passa +orderedmultidict==1.0.1 +dparse==0.5.0 diff --git a/pipenv/vendor/vendor_pip.txt b/pipenv/vendor/vendor_pip.txt index 9389dd94..aadd3526 100644 --- a/pipenv/vendor/vendor_pip.txt +++ b/pipenv/vendor/vendor_pip.txt @@ -1,23 +1,23 @@ appdirs==1.4.3 -distlib==0.2.7 -distro==1.3.0 -html5lib==1.0.1 -six==1.11.0 -colorama==0.3.9 CacheControl==0.12.5 -msgpack-python==0.5.6 -lockfile==0.12.2 -progress==1.4 +colorama==0.4.1 +contextlib2==0.6.0 +distlib==0.2.9.post0 +distro==1.4.0 +html5lib==1.0.1 ipaddress==1.0.22 # Only needed on 2.6 and 2.7 -packaging==18.0 -pep517==0.2 -pyparsing==2.2.1 -pytoml==0.1.19 -retrying==1.3.3 -requests==2.19.1 +msgpack==0.6.2 +packaging==19.2 +pep517==0.7.0 +progress==1.5 +pyparsing==2.4.2 +pytoml==0.1.21 +requests==2.22.0 + certifi==2019.9.11 chardet==3.0.4 - idna==2.7 - urllib3==1.23 - certifi==2018.8.24 -setuptools==40.4.3 + idna==2.8 + urllib3==1.25.6 +retrying==1.3.3 +setuptools==41.4.0 +six==1.12.0 webencodings==0.5.1 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 4c5c9061..fe78c8d5 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals from .compat import ( NamedTemporaryFile, + StringIO, TemporaryDirectory, partialmethod, to_native_string, @@ -11,27 +12,31 @@ from .contextmanagers import ( atomic_open_for_write, cd, open_file, + replaced_stream, + replaced_streams, + spinner, temp_environ, temp_path, - spinner, ) +from .cursor import hide_cursor, show_cursor from .misc import ( + StreamWrapper, + chunked, + decode_for_output, + divide, + get_wrapped_stream, load_path, partialclass, run, shell_escape, - decode_for_output, - to_text, - to_bytes, take, - chunked, - divide, + to_bytes, + to_text, ) -from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile -from .spin import VistirSpinner, create_spinner +from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree +from .spin import create_spinner - -__version__ = '0.2.5' +__version__ = "0.5.0" __all__ = [ @@ -50,7 +55,6 @@ __all__ = [ "NamedTemporaryFile", "partialmethod", "spinner", - "VistirSpinner", "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", @@ -61,4 +65,11 @@ __all__ = [ "take", "chunked", "divide", + "StringIO", + "get_wrapped_stream", + "StreamWrapper", + "replaced_stream", + "replaced_streams", + "show_cursor", + "hide_cursor", ] diff --git a/pipenv/vendor/vistir/_winconsole.py b/pipenv/vendor/vistir/_winconsole.py new file mode 100644 index 00000000..a8be4772 --- /dev/null +++ b/pipenv/vendor/vistir/_winconsole.py @@ -0,0 +1,523 @@ +# -*- coding: utf-8 -*- + +# This Module is taken in part from the click project and expanded +# see https://github.com/pallets/click/blob/6cafd32/click/_winconsole.py +# Copyright © 2014 by the Pallets team. + +# Some rights reserved. + +# Redistribution and use in source and binary forms of the software as well as +# documentation, with or without modification, are permitted provided that the +# following conditions are met: +# Redistributions of source code must retain the above copyright notice, +# this list of conditions and the following disclaimer. +# Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# Neither the name of the copyright holder nor the names of its contributors +# may be used to endorse or promote products derived from this +# software without specific prior written permission. + +# THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND +# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT +# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND +# DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +# This module is based on the excellent work by Adam Bartoš who +# provided a lot of what went into the implementation here in +# the discussion to issue1602 in the Python bug tracker. +# +# There are some general differences in regards to how this works +# compared to the original patches as we do not need to patch +# the entire interpreter but just work in our little world of +# echo and prmopt. + +import ctypes +import io +import os +import sys +import time +import zlib +from ctypes import ( + POINTER, + WINFUNCTYPE, + Structure, + byref, + c_char, + c_char_p, + c_int, + c_ssize_t, + c_ulong, + c_void_p, + create_unicode_buffer, + py_object, + windll, +) +from ctypes.wintypes import LPCWSTR, LPWSTR +from itertools import count + +import msvcrt +from six import PY2, text_type + +from .compat import IS_TYPE_CHECKING +from .misc import StreamWrapper, run, to_text + +try: + from ctypes import pythonapi + + PyObject_GetBuffer = pythonapi.PyObject_GetBuffer + PyBuffer_Release = pythonapi.PyBuffer_Release +except ImportError: + pythonapi = None + + +if IS_TYPE_CHECKING: + from typing import Text + + +c_ssize_p = POINTER(c_ssize_t) + +kernel32 = windll.kernel32 +GetStdHandle = kernel32.GetStdHandle +ReadConsoleW = kernel32.ReadConsoleW +WriteConsoleW = kernel32.WriteConsoleW +GetLastError = kernel32.GetLastError +GetConsoleCursorInfo = kernel32.GetConsoleCursorInfo +SetConsoleCursorInfo = kernel32.SetConsoleCursorInfo +GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) +CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( + ("CommandLineToArgvW", windll.shell32) +) + + +# XXX: Added for cursor hiding on windows +STDOUT_HANDLE_ID = ctypes.c_ulong(-11) +STDERR_HANDLE_ID = ctypes.c_ulong(-12) +STDIN_HANDLE = GetStdHandle(-10) +STDOUT_HANDLE = GetStdHandle(-11) +STDERR_HANDLE = GetStdHandle(-12) + +STREAM_MAP = {0: STDIN_HANDLE, 1: STDOUT_HANDLE, 2: STDERR_HANDLE} + + +PyBUF_SIMPLE = 0 +PyBUF_WRITABLE = 1 + +ERROR_SUCCESS = 0 +ERROR_NOT_ENOUGH_MEMORY = 8 +ERROR_OPERATION_ABORTED = 995 + +STDIN_FILENO = 0 +STDOUT_FILENO = 1 +STDERR_FILENO = 2 + +EOF = b"\x1a" +MAX_BYTES_WRITTEN = 32767 + + +class Py_buffer(Structure): + _fields_ = [ + ("buf", c_void_p), + ("obj", py_object), + ("len", c_ssize_t), + ("itemsize", c_ssize_t), + ("readonly", c_int), + ("ndim", c_int), + ("format", c_char_p), + ("shape", c_ssize_p), + ("strides", c_ssize_p), + ("suboffsets", c_ssize_p), + ("internal", c_void_p), + ] + + if PY2: + _fields_.insert(-1, ("smalltable", c_ssize_t * 2)) + + +# XXX: This was added for the use of cursors +class CONSOLE_CURSOR_INFO(Structure): + _fields_ = [("dwSize", ctypes.c_int), ("bVisible", ctypes.c_int)] + + +# On PyPy we cannot get buffers so our ability to operate here is +# serverly limited. +if pythonapi is None: + get_buffer = None +else: + + def get_buffer(obj, writable=False): + buf = Py_buffer() + flags = PyBUF_WRITABLE if writable else PyBUF_SIMPLE + PyObject_GetBuffer(py_object(obj), byref(buf), flags) + try: + buffer_type = c_char * buf.len + return buffer_type.from_address(buf.buf) + finally: + PyBuffer_Release(byref(buf)) + + +def get_long_path(short_path): + # type: (Text, str) -> Text + BUFFER_SIZE = 500 + buffer = create_unicode_buffer(BUFFER_SIZE) + get_long_path_name = windll.kernel32.GetLongPathNameW + get_long_path_name(to_text(short_path), buffer, BUFFER_SIZE) + return buffer.value + + +class _WindowsConsoleRawIOBase(io.RawIOBase): + def __init__(self, handle): + self.handle = handle + + def isatty(self): + io.RawIOBase.isatty(self) + return True + + +class _WindowsConsoleReader(_WindowsConsoleRawIOBase): + def readable(self): + return True + + def readinto(self, b): + bytes_to_be_read = len(b) + if not bytes_to_be_read: + return 0 + elif bytes_to_be_read % 2: + raise ValueError( + "cannot read odd number of bytes from " "UTF-16-LE encoded console" + ) + + buffer = get_buffer(b, writable=True) + code_units_to_be_read = bytes_to_be_read // 2 + code_units_read = c_ulong() + + rv = ReadConsoleW( + self.handle, buffer, code_units_to_be_read, byref(code_units_read), None + ) + if GetLastError() == ERROR_OPERATION_ABORTED: + # wait for KeyboardInterrupt + time.sleep(0.1) + if not rv: + raise OSError("Windows error: %s" % GetLastError()) + + if buffer[0] == EOF: + return 0 + return 2 * code_units_read.value + + +class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): + def writable(self): + return True + + @staticmethod + def _get_error_message(errno): + if errno == ERROR_SUCCESS: + return "ERROR_SUCCESS" + elif errno == ERROR_NOT_ENOUGH_MEMORY: + return "ERROR_NOT_ENOUGH_MEMORY" + return "Windows error %s" % errno + + def write(self, b): + bytes_to_be_written = len(b) + buf = get_buffer(b) + code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 + code_units_written = c_ulong() + + WriteConsoleW( + self.handle, buf, code_units_to_be_written, byref(code_units_written), None + ) + bytes_written = 2 * code_units_written.value + + if bytes_written == 0 and bytes_to_be_written > 0: + raise OSError(self._get_error_message(GetLastError())) + return bytes_written + + +class ConsoleStream(object): + def __init__(self, text_stream, byte_stream): + self._text_stream = text_stream + self.buffer = byte_stream + + @property + def name(self): + return self.buffer.name + + @property + def fileno(self): + return self.buffer.fileno + + def write(self, x): + if isinstance(x, text_type): + return self._text_stream.write(x) + try: + self.flush() + except Exception: + pass + return self.buffer.write(x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __getattr__(self, name): + try: + return getattr(self._text_stream, name) + except io.UnsupportedOperation: + return getattr(self.buffer, name) + + def isatty(self): + return self.buffer.isatty() + + def __repr__(self): + return "<ConsoleStream name=%r encoding=%r>" % (self.name, self.encoding) + + +class WindowsChunkedWriter(object): + """ + Wraps a stream (such as stdout), acting as a transparent proxy for all + attribute access apart from method 'write()' which we wrap to write in + limited chunks due to a Windows limitation on binary console streams. + """ + + def __init__(self, wrapped): + # double-underscore everything to prevent clashes with names of + # attributes on the wrapped stream object. + self.__wrapped = wrapped + + def __getattr__(self, name): + return getattr(self.__wrapped, name) + + def write(self, text): + total_to_write = len(text) + written = 0 + + while written < total_to_write: + to_write = min(total_to_write - written, MAX_BYTES_WRITTEN) + self.__wrapped.write(text[written : written + to_write]) + written += to_write + + +_wrapped_std_streams = set() + + +def _wrap_std_stream(name): + # Python 2 & Windows 7 and below + if PY2 and sys.getwindowsversion()[:2] <= (6, 1) and name not in _wrapped_std_streams: + setattr(sys, name, WindowsChunkedWriter(getattr(sys, name))) + _wrapped_std_streams.add(name) + + +def _get_text_stdin(buffer_stream): + text_stream = StreamWrapper( + io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +def _get_text_stdout(buffer_stream): + text_stream = StreamWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +def _get_text_stderr(buffer_stream): + text_stream = StreamWrapper( + io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), + "utf-16-le", + "strict", + line_buffering=True, + ) + return ConsoleStream(text_stream, buffer_stream) + + +if PY2: + + def _hash_py_argv(): + return zlib.crc32("\x00".join(sys.argv[1:])) + + _initial_argv_hash = _hash_py_argv() + + def _get_windows_argv(): + argc = c_int(0) + argv_unicode = CommandLineToArgvW(GetCommandLineW(), byref(argc)) + argv = [argv_unicode[i] for i in range(0, argc.value)] + + if not hasattr(sys, "frozen"): + argv = argv[1:] + while len(argv) > 0: + arg = argv[0] + if not arg.startswith("-") or arg == "-": + break + argv = argv[1:] + if arg.startswith(("-c", "-m")): + break + + return argv[1:] + + +_stream_factories = {0: _get_text_stdin, 1: _get_text_stdout, 2: _get_text_stderr} + + +def _get_windows_console_stream(f, encoding, errors): + if ( + get_buffer is not None + and encoding in ("utf-16-le", None) + and errors in ("strict", None) + and hasattr(f, "isatty") + and f.isatty() + ): + if isinstance(f, ConsoleStream): + return f + func = _stream_factories.get(f.fileno()) + if func is not None: + if not PY2: + f = getattr(f, "buffer", None) + if f is None: + return None + else: + # If we are on Python 2 we need to set the stream that we + # deal with to binary mode as otherwise the exercise if a + # bit moot. The same problems apply as for + # get_binary_stdin and friends from _compat. + msvcrt.setmode(f.fileno(), os.O_BINARY) + return func(f) + + +def hide_cursor(): + cursor_info = CONSOLE_CURSOR_INFO() + GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + cursor_info.visible = False + SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + + +def show_cursor(): + cursor_info = CONSOLE_CURSOR_INFO() + GetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + cursor_info.visible = True + SetConsoleCursorInfo(STDOUT_HANDLE, ctypes.byref(cursor_info)) + + +def get_stream_handle(stream): + return STREAM_MAP.get(stream.fileno()) + + +def _walk_for_powershell(directory): + for path, dirs, files in os.walk(directory): + powershell = next( + iter(fn for fn in files if fn.lower() == "powershell.exe"), None + ) + if powershell is not None: + return os.path.join(directory, powershell) + for subdir in dirs: + powershell = _walk_for_powershell(os.path.join(directory, subdir)) + if powershell: + return powershell + return None + + +def _get_powershell_path(): + paths = [ + os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir) + for subdir in ("SysWOW64", "system32") + ] + powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None) + if not powershell_path: + powershell_path, _ = run( + ["where", "powershell"], block=True, nospin=True, return_object=False + ) + if powershell_path: + return powershell_path.strip() + return None + + +def _get_sid_with_powershell(): + powershell_path = _get_powershell_path() + if not powershell_path: + return None + args = [ + powershell_path, + "-ExecutionPolicy", + "Bypass", + "-Command", + "Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'", + ] + sid, _ = run(args, nospin=True) + return sid.strip() + + +def _get_sid_from_registry(): + try: + import winreg + except ImportError: + import _winreg as winreg + var_names = ("%USERPROFILE%", "%HOME%") + current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None) + root, subkey = ( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList", + ) + subkey_names = [] + value = None + matching_key = None + try: + with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key: + for i in count(): + key_name = winreg.EnumKey(key, i) + subkey_names.append(key_name) + value = query_registry_value( + root, r"{0}\{1}".format(subkey, key_name), "ProfileImagePath" + ) + if value and value.lower() == current_user_home.lower(): + matching_key = key_name + break + except OSError: + pass + if matching_key is not None: + return matching_key + + +def get_value_from_tuple(value, value_type): + try: + import winreg + except ImportError: + import _winreg as winreg + if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ): + if "\0" in value: + return value[: value.index("\0")] + return value + return None + + +def query_registry_value(root, key_name, value): + try: + import winreg + except ImportError: + import _winreg as winreg + try: + with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key: + return get_value_from_tuple(*winreg.QueryValueEx(key, value)) + except OSError: + return None + + +def get_current_user(): + fns = (_get_sid_from_registry, _get_sid_with_powershell) + for fn in fns: + result = fn() + if result: + return result + return None diff --git a/pipenv/vendor/vistir/backports/__init__.py b/pipenv/vendor/vistir/backports/__init__.py index 0bdac1ea..b5ed6562 100644 --- a/pipenv/vendor/vistir/backports/__init__.py +++ b/pipenv/vendor/vistir/backports/__init__.py @@ -2,10 +2,7 @@ from __future__ import absolute_import, unicode_literals from .functools import partialmethod +from .surrogateescape import register_surrogateescape from .tempfile import NamedTemporaryFile - -__all__ = [ - "NamedTemporaryFile", - "partialmethod" -] +__all__ = ["NamedTemporaryFile", "partialmethod", "register_surrogateescape"] diff --git a/pipenv/vendor/vistir/backports/functools.py b/pipenv/vendor/vistir/backports/functools.py index 8060d183..5d98f41a 100644 --- a/pipenv/vendor/vistir/backports/functools.py +++ b/pipenv/vendor/vistir/backports/functools.py @@ -3,8 +3,7 @@ from __future__ import absolute_import, unicode_literals from functools import partial - -__all__ = ["partialmethod",] +__all__ = ["partialmethod"] class partialmethod(object): @@ -16,8 +15,7 @@ class partialmethod(object): def __init__(self, func, *args, **keywords): if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) + raise TypeError("{!r} is not callable or a descriptor".format(func)) # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) @@ -36,26 +34,28 @@ class partialmethod(object): def __repr__(self): args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) + keywords = ", ".join("{}={!r}".format(k, v) for k, v in self.keywords.items()) format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + return format_string.format( + module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords, + ) def _make_unbound_method(self): def _method(*args, **keywords): call_keywords = self.keywords.copy() call_keywords.update(keywords) if len(args) > 1: - cls_or_self, rest = args[0], tuple(args[1:],) + cls_or_self, rest = args[0], tuple(args[1:]) else: cls_or_self = args[0] rest = tuple() call_args = (cls_or_self,) + self.args + tuple(rest) return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ _method._partialmethod = self return _method diff --git a/pipenv/vendor/vistir/backports/surrogateescape.py b/pipenv/vendor/vistir/backports/surrogateescape.py new file mode 100644 index 00000000..0532be08 --- /dev/null +++ b/pipenv/vendor/vistir/backports/surrogateescape.py @@ -0,0 +1,196 @@ +""" +This is Victor Stinner's pure-Python implementation of PEP 383: the "surrogateescape" error +handler of Python 3. +Source: misc/python/surrogateescape.py in https://bitbucket.org/haypo/misc +""" + +# This code is released under the Python license and the BSD 2-clause license + +import codecs +import sys + +import six + +FS_ERRORS = "surrogateescape" + +# # -- Python 2/3 compatibility ------------------------------------- +# FS_ERRORS = 'my_surrogateescape' + + +def u(text): + if six.PY3: + return text + else: + return text.decode("unicode_escape") + + +def b(data): + if six.PY3: + return data.encode("latin1") + else: + return data + + +if six.PY3: + _unichr = chr + bytes_chr = lambda code: bytes((code,)) +else: + _unichr = unichr + bytes_chr = chr + + +def surrogateescape_handler(exc): + """ + Pure Python implementation of the PEP 383: the "surrogateescape" error + handler of Python 3. Undecodable bytes will be replaced by a Unicode + character U+DCxx on decoding, and these are translated into the + original bytes on encoding. + """ + mystring = exc.object[exc.start : exc.end] + + try: + if isinstance(exc, UnicodeDecodeError): + # mystring is a byte-string in this case + decoded = replace_surrogate_decode(mystring) + elif isinstance(exc, UnicodeEncodeError): + # In the case of u'\udcc3'.encode('ascii', + # 'this_surrogateescape_handler'), both Python 2.x and 3.x raise an + # exception anyway after this function is called, even though I think + # it's doing what it should. It seems that the strict encoder is called + # to encode the unicode string that this function returns ... + decoded = replace_surrogate_encode(mystring) + else: + raise exc + except NotASurrogateError: + raise exc + return (decoded, exc.end) + + +class NotASurrogateError(Exception): + pass + + +def replace_surrogate_encode(mystring): + """ + Returns a (unicode) string, not the more logical bytes, because the codecs + register_error functionality expects this. + """ + decoded = [] + for ch in mystring: + # if utils.PY3: + # code = ch + # else: + code = ord(ch) + + # The following magic comes from Py3.3's Python/codecs.c file: + if not 0xD800 <= code <= 0xDCFF: + # Not a surrogate. Fail with the original exception. + raise NotASurrogateError + # mybytes = [0xe0 | (code >> 12), + # 0x80 | ((code >> 6) & 0x3f), + # 0x80 | (code & 0x3f)] + # Is this a good idea? + if 0xDC00 <= code <= 0xDC7F: + decoded.append(_unichr(code - 0xDC00)) + elif code <= 0xDCFF: + decoded.append(_unichr(code - 0xDC00)) + else: + raise NotASurrogateError + return str().join(decoded) + + +def replace_surrogate_decode(mybytes): + """ + Returns a (unicode) string + """ + decoded = [] + for ch in mybytes: + # We may be parsing newbytes (in which case ch is an int) or a native + # str on Py2 + if isinstance(ch, int): + code = ch + else: + code = ord(ch) + if 0x80 <= code <= 0xFF: + decoded.append(_unichr(0xDC00 + code)) + elif code <= 0x7F: + decoded.append(_unichr(code)) + else: + # # It may be a bad byte + # # Try swallowing it. + # continue + # print("RAISE!") + raise NotASurrogateError + return str().join(decoded) + + +def encodefilename(fn): + if FS_ENCODING == "ascii": + # ASCII encoder of Python 2 expects that the error handler returns a + # Unicode string encodable to ASCII, whereas our surrogateescape error + # handler has to return bytes in 0x80-0xFF range. + encoded = [] + for index, ch in enumerate(fn): + code = ord(ch) + if code < 128: + ch = bytes_chr(code) + elif 0xDC80 <= code <= 0xDCFF: + ch = bytes_chr(code - 0xDC00) + else: + raise UnicodeEncodeError( + FS_ENCODING, fn, index, index + 1, "ordinal not in range(128)" + ) + encoded.append(ch) + return bytes().join(encoded) + elif FS_ENCODING == "utf-8": + # UTF-8 encoder of Python 2 encodes surrogates, so U+DC80-U+DCFF + # doesn't go through our error handler + encoded = [] + for index, ch in enumerate(fn): + code = ord(ch) + if 0xD800 <= code <= 0xDFFF: + if 0xDC80 <= code <= 0xDCFF: + ch = bytes_chr(code - 0xDC00) + encoded.append(ch) + else: + raise UnicodeEncodeError( + FS_ENCODING, fn, index, index + 1, "surrogates not allowed" + ) + else: + ch_utf8 = ch.encode("utf-8") + encoded.append(ch_utf8) + return bytes().join(encoded) + else: + return fn.encode(FS_ENCODING, FS_ERRORS) + + +def decodefilename(fn): + return fn.decode(FS_ENCODING, FS_ERRORS) + + +FS_ENCODING = "ascii" +fn = b("[abc\xff]") +encoded = u("[abc\udcff]") +# FS_ENCODING = 'cp932'; fn = b('[abc\x81\x00]'); encoded = u('[abc\udc81\x00]') +# FS_ENCODING = 'UTF-8'; fn = b('[abc\xff]'); encoded = u('[abc\udcff]') + + +# normalize the filesystem encoding name. +# For example, we expect "utf-8", not "UTF8". +FS_ENCODING = codecs.lookup(FS_ENCODING).name + + +def register_surrogateescape(): + """ + Registers the surrogateescape error handler on Python 2 (only) + """ + if six.PY3: + return + try: + codecs.lookup_error(FS_ERRORS) + except LookupError: + codecs.register_error(FS_ERRORS, surrogateescape_handler) + + +if __name__ == "__main__": + pass diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index fb044acf..a3d7f3df 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -5,7 +5,6 @@ import functools import io import os import sys - from tempfile import _bin_openflags, _mkstemp_inner, gettempdir import six @@ -16,6 +15,24 @@ except ImportError: from pipenv.vendor.backports.weakref import finalize +def fs_encode(path): + try: + return os.fsencode(path) + except AttributeError: + from ..compat import fs_encode + + return fs_encode(path) + + +def fs_decode(path): + try: + return os.fsdecode(path) + except AttributeError: + from ..compat import fs_decode + + return fs_decode(path) + + __all__ = ["finalize", "NamedTemporaryFile"] @@ -49,7 +66,7 @@ def _sanitize_params(prefix, suffix, dir): if output_type is str: dir = gettempdir() else: - dir = os.fsencode(gettempdir()) + dir = fs_encode(gettempdir()) return prefix, suffix, dir, output_type @@ -175,7 +192,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, - wrapper_class_override=None + wrapper_class_override=None, ): """Create and return a temporary file. Arguments: @@ -203,13 +220,11 @@ def NamedTemporaryFile( else: (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: - file = io.open( - fd, mode, buffering=buffering, newline=newline, encoding=encoding - ) + file = io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) if wrapper_class_override is not None: - return type( - str("_TempFileWrapper"), (wrapper_class_override, object), {} - )(file, name, delete) + return type(str("_TempFileWrapper"), (wrapper_class_override, object), {})( + file, name, delete + ) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/cmdparse.py b/pipenv/vendor/vistir/cmdparse.py index 07326c93..664ae7df 100644 --- a/pipenv/vendor/vistir/cmdparse.py +++ b/pipenv/vendor/vistir/cmdparse.py @@ -1,12 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import itertools import re import shlex import six - __all__ = ["ScriptEmptyError", "Script"] @@ -14,6 +14,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -72,7 +78,8 @@ class Script(object): See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts + itertools.chain( + [_quote_if_contains(self.command, r"[\s^()]")], + (_quote_if_contains(arg, r"[\s^]") for arg in self.args), + ) ) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 27d1e75c..ee96f761 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,15 +1,16 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import codecs import errno import os import sys import warnings - from tempfile import mkdtemp import six +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile __all__ = [ "Path", @@ -19,47 +20,57 @@ __all__ = [ "JSONDecodeError", "FileNotFoundError", "ResourceWarning", - "FileNotFoundError", "PermissionError", + "is_type_checking", + "IS_TYPE_CHECKING", "IsADirectoryError", "fs_str", "lru_cache", "TemporaryDirectory", "NamedTemporaryFile", "to_native_string", + "Iterable", + "Mapping", + "Sequence", + "Set", + "ItemsView", + "fs_encode", + "fs_decode", + "_fs_encode_errors", + "_fs_decode_errors", ] -if sys.version_info >= (3, 5): +if sys.version_info >= (3, 5): # pragma: no cover from pathlib import Path - from functools import lru_cache -else: - from pathlib2 import Path - from pipenv.vendor.backports.functools_lru_cache import lru_cache +else: # pragma: no cover + from pipenv.vendor.pathlib2 import Path -from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile -if sys.version_info < (3, 3): - from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size - NamedTemporaryFile = _NamedTemporaryFile -else: +if six.PY3: # pragma: no cover + # Only Python 3.4+ is supported + from functools import lru_cache, partialmethod from tempfile import NamedTemporaryFile from shutil import get_terminal_size - -try: from weakref import finalize -except ImportError: - from pipenv.vendor.backports.weakref import finalize - -try: - from functools import partialmethod -except Exception: - from .backports.functools import partialmethod +else: # pragma: no cover + # Only Python 2.7 is supported + from pipenv.vendor.backports.functools_lru_cache import lru_cache + from .backports.functools import partialmethod # type: ignore + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + from .backports.surrogateescape import register_surrogateescape + + register_surrogateescape() + NamedTemporaryFile = _NamedTemporaryFile + from pipenv.vendor.backports.weakref import finalize # type: ignore try: + # Introduced Python 3.5 from json import JSONDecodeError -except ImportError: # Old Pythons. - JSONDecodeError = ValueError +except ImportError: # pragma: no cover + JSONDecodeError = ValueError # type: ignore -if six.PY2: +if six.PY2: # pragma: no cover + + from io import BytesIO as StringIO class ResourceWarning(Warning): pass @@ -76,23 +87,75 @@ if six.PY2: self.errno = errno.EACCES super(PermissionError, self).__init__(*args, **kwargs) + class TimeoutError(OSError): + """Timeout expired.""" + + def __init__(self, *args, **kwargs): + self.errno = errno.ETIMEDOUT + super(TimeoutError, self).__init__(*args, **kwargs) + class IsADirectoryError(OSError): """The command does not work on directories""" - pass -else: - from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError + def __init__(self, *args, **kwargs): + self.errno = errno.EISDIR + super(IsADirectoryError, self).__init__(*args, **kwargs) -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable + class FileExistsError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EEXIST + super(FileExistsError, self).__init__(*args, **kwargs) + + +else: # pragma: no cover + from builtins import ( + ResourceWarning, + FileNotFoundError, + PermissionError, + IsADirectoryError, + FileExistsError, + TimeoutError, + ) + from io import StringIO + +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore + +# fmt: off +from six.moves import ItemsView, Iterable, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on if not sys.warnoptions: warnings.simplefilter("default", ResourceWarning) +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +IS_TYPE_CHECKING = os.environ.get("MYPY_RUNNING", is_type_checking()) + + class TemporaryDirectory(object): - """Create and return a temporary directory. This has the same + + """ + Create and return a temporary directory. This has the same behavior as mkdtemp but can be used as a context manager. For example: @@ -128,32 +191,7 @@ class TemporaryDirectory(object): def _rmtree(cls, name): from .path import rmtree - def onerror(func, path, exc_info): - if issubclass(exc_info[0], (PermissionError, OSError)): - try: - try: - if path != name: - os.chflags(os.path.dirname(path), 0) - os.chflags(path, 0) - except AttributeError: - pass - if path != name: - os.chmod(os.path.dirname(path), 0o70) - os.chmod(path, 0o700) - - try: - os.unlink(path) - # PermissionError is raised on FreeBSD for directories - except (IsADirectoryError, PermissionError, OSError): - cls._rmtree(path) - except FileNotFoundError: - pass - elif issubclass(exc_info[0], FileNotFoundError): - pass - else: - raise - - rmtree(name, onerror=onerror) + rmtree(name) @classmethod def _cleanup(cls, name, warn_message): @@ -174,22 +212,197 @@ class TemporaryDirectory(object): self._rmtree(self.name) +def is_bytes(string): + """Check if a string is a bytes instance + + :param Union[str, bytes] string: A string that may be string or bytes like + :return: Whether the provided string is a bytes type or not + :rtype: bool + """ + if six.PY3 and isinstance(string, (bytes, memoryview, bytearray)): # noqa + return True + elif six.PY2 and isinstance(string, (buffer, bytearray)): # noqa + return True + return False + + def fs_str(string): """Encodes a string into the proper filesystem encoding Borrowed from pip-tools """ + if isinstance(string, str): return string assert not isinstance(string, bytes) return string.encode(_fs_encoding) -_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +def _get_path(path): + """ + Fetch the string value from a path-like object + + Returns **None** if there is no string value. + """ + + if isinstance(path, (six.string_types, bytes)): + return path + path_type = type(path) + try: + path_repr = path_type.__fspath__(path) + except AttributeError: + return + if isinstance(path_repr, (six.string_types, bytes)): + return path_repr + return + + +# copied from the os backport which in turn copied this from +# the pyutf8 package -- +# URL: https://github.com/etrepum/pyutf8/blob/master/pyutf8/ref.py +# +def _invalid_utf8_indexes(bytes): + skips = [] + i = 0 + len_bytes = len(bytes) + while i < len_bytes: + c1 = bytes[i] + if c1 < 0x80: + # U+0000 - U+007F - 7 bits + i += 1 + continue + try: + c2 = bytes[i + 1] + if (c1 & 0xE0 == 0xC0) and (c2 & 0xC0 == 0x80): + # U+0080 - U+07FF - 11 bits + c = ((c1 & 0x1F) << 6) | (c2 & 0x3F) + if c < 0x80: # pragma: no cover + # Overlong encoding + skips.extend([i, i + 1]) # pragma: no cover + i += 2 + continue + c3 = bytes[i + 2] + if (c1 & 0xF0 == 0xE0) and (c2 & 0xC0 == 0x80) and (c3 & 0xC0 == 0x80): + # U+0800 - U+FFFF - 16 bits + c = ((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F) + if (c < 0x800) or (0xD800 <= c <= 0xDFFF): + # Overlong encoding or surrogate. + skips.extend([i, i + 1, i + 2]) + i += 3 + continue + c4 = bytes[i + 3] + if ( + (c1 & 0xF8 == 0xF0) + and (c2 & 0xC0 == 0x80) + and (c3 & 0xC0 == 0x80) + and (c4 & 0xC0 == 0x80) + ): + # U+10000 - U+10FFFF - 21 bits + c = ((((((c1 & 0x0F) << 6) | (c2 & 0x3F)) << 6) | (c3 & 0x3F)) << 6) | ( + c4 & 0x3F + ) + if (c < 0x10000) or (c > 0x10FFFF): # pragma: no cover + # Overlong encoding or invalid code point. + skips.extend([i, i + 1, i + 2, i + 3]) + i += 4 + continue + except IndexError: + pass + skips.append(i) + i += 1 + return skips + + +# XXX backport: Another helper to support the Python 2 UTF-8 decoding hack. +def _chunks(b, indexes): + i = 0 + for j in indexes: + yield b[i:j] + yield b[j : j + 1] + i = j + 1 + yield b[i:] + + +def fs_encode(path): + """ + Encode a filesystem path to the proper filesystem encoding + + :param Union[str, bytes] path: A string-like path + :returns: A bytes-encoded filesystem path representation + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to encode") + if isinstance(path, six.text_type): + if six.PY2: + return b"".join( + ( + _byte(ord(c) - 0xDC00) + if 0xDC00 <= ord(c) <= 0xDCFF + else c.encode(_fs_encoding, _fs_encode_errors) + ) + for c in path + ) + return path.encode(_fs_encoding, _fs_encode_errors) + return path + + +def fs_decode(path): + """ + Decode a filesystem path using the proper filesystem encoding + + :param path: The filesystem path to decode from bytes or string + :return: The filesystem path, decoded with the determined encoding + :rtype: Text + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to decode") + if isinstance(path, six.binary_type): + import array + + indexes = _invalid_utf8_indexes(array.array(str("B"), path)) + if six.PY2: + return "".join( + chunk.decode(_fs_encoding, _fs_decode_errors) + for chunk in _chunks(path, indexes) + ) + if indexes and os.name == "nt": + return path.decode(_fs_encoding, "surrogateescape") + return path.decode(_fs_encoding, _fs_decode_errors) + return path + + +if sys.version_info[0] < 3: # pragma: no cover + _fs_encode_errors = "surrogateescape" + _fs_decode_errors = "surrogateescape" + _fs_encoding = "utf-8" +else: # pragma: no cover + _fs_encoding = "utf-8" + if sys.platform.startswith("win"): + _fs_error_fn = None + if sys.version_info[:2] > (3, 4): + alt_strategy = "surrogatepass" + else: + alt_strategy = "surrogateescape" + else: + if sys.version_info >= (3, 3): + _fs_encoding = sys.getfilesystemencoding() + if not _fs_encoding: + _fs_encoding = sys.getdefaultencoding() + alt_strategy = "surrogateescape" + _fs_error_fn = getattr(sys, "getfilesystemencodeerrors", None) + _fs_encode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy + _fs_decode_errors = _fs_error_fn() if _fs_error_fn else alt_strategy + +_byte = chr if sys.version_info < (3,) else lambda i: bytes([i]) def to_native_string(string): from .misc import to_text, to_bytes + if six.PY2: return to_bytes(string) return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 77fbb9df..66fde577 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,21 +1,27 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import io import os import stat import sys - -from contextlib import contextmanager +from contextlib import closing, contextmanager import six from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path - __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" + "temp_environ", + "temp_path", + "cd", + "atomic_open_for_write", + "open_file", + "spinner", + "dummy_spinner", + "replaced_stream", + "replaced_streams", ] @@ -103,7 +109,13 @@ def dummy_spinner(spin_type, text, **kwargs): @contextmanager -def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, write_to_stdout=True): +def spinner( + spinner_name=None, + start_text=None, + handler_map=None, + nospin=False, + write_to_stdout=True, +): """Get a spinner object or a dummy spinner to wrap a context. :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) @@ -119,6 +131,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, """ from .spin import create_spinner + has_yaspin = None try: import yaspin @@ -145,7 +158,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, handler_map=handler_map, nospin=nospin, use_yaspin=use_yaspin, - write_to_stdout=write_to_stdout + write_to_stdout=write_to_stdout, ) as _spinner: yield _spinner @@ -266,23 +279,84 @@ def open_file(link, session=None, stream=True): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with io.open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} if not session: - from requests import Session - - session = Session() - with session.get(link, headers=headers, stream=stream) as resp: try: - raw = getattr(resp, "raw", None) - result = raw if raw else resp - yield result - finally: - if raw: - conn = getattr(raw, "_connection") - if conn is not None: - conn.close() - result.close() + from requests import Session + except ImportError: + session = None + else: + session = Session() + if session is None: + with closing(six.moves.urllib.request.urlopen(link)) as f: + yield f + else: + with session.get(link, headers=headers, stream=stream) as resp: + try: + raw = getattr(resp, "raw", None) + result = raw if raw else resp + yield result + finally: + if raw: + conn = getattr(raw, "_connection") + if conn is not None: + conn.close() + result.close() + + +@contextmanager +def replaced_stream(stream_name): + """ + Context manager to temporarily swap out *stream_name* with a stream wrapper. + + :param str stream_name: The name of a sys stream to wrap + :returns: A ``StreamWrapper`` replacement, temporarily + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + + >>> sys.stdout.write("hello") + 'hello' + """ + + orig_stream = getattr(sys, stream_name) + new_stream = six.StringIO() + try: + setattr(sys, stream_name, new_stream) + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stream) + + +@contextmanager +def replaced_streams(): + """ + Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using + ``replaced_stream`` + + returns: *(stdout, stderr)* + + >>> import sys + >>> with vistir.contextmanagers.replaced_streams() as streams: + >>> stdout, stderr = streams + >>> sys.stderr.write("test") + >>> sys.stdout.write("hello") + >>> assert stdout.getvalue() == "hello" + >>> assert stderr.getvalue() == "test" + + >>> stdout.getvalue() + 'hello' + + >>> stderr.getvalue() + 'test' + """ + + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) diff --git a/pipenv/vendor/vistir/cursor.py b/pipenv/vendor/vistir/cursor.py new file mode 100644 index 00000000..bdb281f6 --- /dev/null +++ b/pipenv/vendor/vistir/cursor.py @@ -0,0 +1,61 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import os +import sys + +__all__ = ["hide_cursor", "show_cursor", "get_stream_handle"] + + +def get_stream_handle(stream=sys.stdout): + """ + Get the OS appropriate handle for the corresponding output stream. + + :param str stream: The the stream to get the handle for + :return: A handle to the appropriate stream, either a ctypes buffer + or **sys.stdout** or **sys.stderr**. + """ + handle = stream + if os.name == "nt": + from ._winconsole import get_stream_handle as get_win_stream_handle + + return get_win_stream_handle(stream) + return handle + + +def hide_cursor(stream=sys.stdout): + """ + Hide the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ._winconsole import hide_cursor + + hide_cursor() + else: + handle.write("\033[?25l") + handle.flush() + + +def show_cursor(stream=sys.stdout): + """ + Show the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ._winconsole import show_cursor + + show_cursor() + else: + handle.write("\033[?25h") + handle.flush() diff --git a/pipenv/vendor/vistir/environment.py b/pipenv/vendor/vistir/environment.py new file mode 100644 index 00000000..b8490c00 --- /dev/null +++ b/pipenv/vendor/vistir/environment.py @@ -0,0 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function + +from .compat import IS_TYPE_CHECKING + +MYPY_RUNNING = IS_TYPE_CHECKING diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 11480e2f..54e0d2a0 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -1,24 +1,37 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import io import json -import logging import locale +import logging import os import subprocess import sys - from collections import OrderedDict from functools import partial from itertools import islice, tee +from weakref import WeakKeyDictionary import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string, Iterable +from .compat import ( + Iterable, + Path, + StringIO, + TimeoutError, + fs_str, + is_bytes, + partialmethod, + to_native_string, +) from .contextmanagers import spinner as spinner +from .environment import MYPY_RUNNING +from .termcolors import ANSI_REMOVAL_RE, colorize if os.name != "nt": + class WindowsError(OSError): pass @@ -38,10 +51,19 @@ __all__ = [ "divide", "getpreferredencoding", "decode_for_output", + "get_canonical_encoding_name", + "get_wrapped_stream", + "StreamWrapper", ] +if MYPY_RUNNING: + from typing import Any, Dict, List, Optional, Union + from .spin import VistirSpinner + + def _get_logger(name=None, level="ERROR"): + # type: (Optional[str], str) -> logging.Logger if not name: name = __name__ if isinstance(level, six.string_types): @@ -58,6 +80,7 @@ def _get_logger(name=None, level="ERROR"): def shell_escape(cmd): + # type: (Union[str, List[str]]) -> str """Escape strings for use in :func:`~subprocess.Popen` and :func:`run`. This is a passthrough method for instantiating a :class:`~vistir.cmdparse.Script` @@ -68,6 +91,7 @@ def shell_escape(cmd): def unnest(elem): + # type: (Iterable) -> Any """Flatten an arbitrarily nested iterable :param elem: An iterable to flatten @@ -82,22 +106,27 @@ def unnest(elem): elem, target = tee(elem, 2) else: target = elem - for el in target: - if isinstance(el, Iterable) and not isinstance(el, six.string_types): - el, el_copy = tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el + if not target or not _is_iterable(target): + yield target + else: + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el def _is_iterable(elem): - if getattr(elem, "__iter__", False): + # type: (Any) -> bool + if getattr(elem, "__iter__", False) or isinstance(elem, Iterable): return True return False def dedup(iterable): + # type: (Iterable) -> Iterable """Deduplicate an iterable object like iter(set(iterable)) but order-reserved. """ @@ -105,6 +134,7 @@ def dedup(iterable): def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=True): + # type: (Union[str, List[str]], Optional[Dict[str, str], bool, Optional[str], bool]) -> subprocess.Popen from distutils.spawn import find_executable if not env: @@ -132,7 +162,7 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru # a "command" that is non-executable. See pypa/pipenv#2727. try: return subprocess.Popen(cmd, **options) - except WindowsError as e: + except WindowsError as e: # pragma: no cover if getattr(e, "winerror", 9999) != 193: raise options["shell"] = True @@ -140,6 +170,85 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru return subprocess.Popen(script.cmdify(), **options) +def _read_streams(stream_dict): + results = {} + for outstream in stream_dict.keys(): + stream = stream_dict[outstream] + if not stream: + results[outstream] = None + continue + line = to_text(stream.readline()) + if not line: + results[outstream] = None + continue + line = to_text("{0}".format(line.rstrip())) + results[outstream] = line + return results + + +def get_stream_results(cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False): + stream_results = {"stdout": [], "stderr": []} + streams = {"stderr": cmd_instance.stderr, "stdout": cmd_instance.stdout} + while True: + stream_contents = _read_streams(streams) + stdout_line = stream_contents["stdout"] + stderr_line = stream_contents["stderr"] + if not (stdout_line or stderr_line): + break + last_changed = 0 + display_line = "" + for stream_name in stream_contents.keys(): + if stream_contents[stream_name] and stream_name in stream_results: + line = stream_contents[stream_name] + stream_results[stream_name].append(line) + display_line = ( + fs_str("{0}".format(line)) + if stream_name == "stderr" + else display_line + ) + if display_line and last_changed > 10: + last_changed = 0 + display_line = "" + elif display_line: + last_changed += 1 + if len(display_line) > maxlen: + display_line = "{0}...".format(display_line[:maxlen]) + if verbose: + use_stderr = not stdout_allowed or stream_name != "stdout" + if spinner: + target = spinner.stderr if use_stderr else spinner.stdout + spinner.hide_and_write(display_line, target=target) + else: + target = sys.stderr if use_stderr else sys.stdout + target.write(display_line) + target.flush() + if spinner: + spinner.text = to_native_string( + "{0} {1}".format(spinner.text, display_line) + ) + continue + return stream_results + + +def _handle_nonblocking_subprocess(c, spinner=None): + # type: (subprocess.Popen, VistirSpinner) -> subprocess.Popen + try: + c.wait() + finally: + if c.stdout: + c.stdout.close() + if c.stderr: + c.stderr.close() + if spinner: + if c.returncode > 0: + spinner.fail(to_native_string("Failed...cleaning up...")) + if not os.name == "nt": + spinner.ok(to_native_string("✔ Complete")) + else: + spinner.ok(to_native_string("Complete")) + return c + + def _create_subprocess( cmd, env=None, @@ -151,89 +260,49 @@ def _create_subprocess( combine_stderr=False, display_limit=200, start_text="", - write_to_stdout=True + write_to_stdout=True, ): if not env: env = os.environ.copy() try: - c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, - combine_stderr=combine_stderr) - except Exception as exc: - sys.stderr.write("Error %s while executing command %s", exc, " ".join(cmd._parts)) - raise + c = _spawn_subprocess( + cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr + ) + except Exception as exc: # pragma: no cover + import traceback + + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) + sys.stderr.write( + "Error while executing command %s:" % to_native_string(" ".join(cmd._parts)) + ) + sys.stderr.write(formatted_tb) + raise exc if not block: c.stdin.close() - output = [] - err = [] - spinner_orig_text = None - if spinner: - spinner_orig_text = getattr(spinner, "text", None) - if spinner_orig_text is None: - spinner_orig_text = start_text if start_text is not None else "" - streams = { - "stdout": c.stdout, - "stderr": c.stderr - } - while True: - stdout_line = None - stderr_line = None - for outstream in streams.keys(): - stream = streams[outstream] - if not stream: - continue - line = to_text(stream.readline()) - if not line: - continue - line = to_text("{0}".format(line.rstrip())) - if outstream == "stderr": - stderr_line = line - else: - stdout_line = line - if not (stdout_line or stderr_line): - break - if stderr_line is not None: - err.append(stderr_line) - err_line = fs_str("{0}".format(stderr_line)) - if verbose and err_line is not None: - if spinner: - spinner.hide_and_write(err_line, target=spinner.stderr) - else: - sys.stderr.write(err_line) - sys.stderr.flush() - if stdout_line is not None: - output.append(stdout_line) - display_line = fs_str("{0}".format(stdout_line)) - if len(stdout_line) > display_limit: - display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose and display_line is not None: - if spinner: - target = spinner.stdout if write_to_stdout else spinner.stderr - spinner.hide_and_write(display_line, target=target) - else: - target = sys.stdout if write_to_stdout else sys.stderr - target.write(display_line) - target.flush() - if spinner: - spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) - continue - try: - c.wait() - finally: - if c.stdout: - c.stdout.close() - if c.stderr: - c.stderr.close() - if spinner: - if c.returncode > 0: - spinner.fail(to_native_string("Failed...cleaning up...")) - if not os.name == "nt": - spinner.ok(to_native_string("✔ Complete")) - else: - spinner.ok(to_native_string("Complete")) + spinner_orig_text = "" + if spinner and getattr(spinner, "text", None) is not None: + spinner_orig_text = spinner.text + if not spinner_orig_text and start_text is not None: + spinner_orig_text = start_text + stream_results = get_stream_results( + c, + verbose=verbose, + maxlen=display_limit, + spinner=spinner, + stdout_allowed=write_to_stdout, + ) + _handle_nonblocking_subprocess(c, spinner) + output = stream_results["stdout"] + err = stream_results["stderr"] c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: - c.out, c.err = c.communicate() + try: + c.out, c.err = c.communicate() + except (SystemExit, KeyboardInterrupt, TimeoutError): # pragma: no cover + c.terminate() + c.out, c.err = c.communicate() + raise if not block: c.wait() c.out = to_text("{0}".format(c.out)) if c.out else fs_str("") @@ -254,7 +323,7 @@ def run( spinner_name=None, combine_stderr=True, display_limit=200, - write_to_stdout=True + write_to_stdout=True, ): """Use `subprocess.Popen` to get the output of a command and decode it. @@ -279,14 +348,11 @@ def run( _env = os.environ.copy() if env: _env.update(env) - env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) - _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} - for key, val in env.items(): - _env[fs_encode(key)] = fs_encode(val) + _env = {fs_encode(k): fs_encode(v) for k, v in _env.items()} else: - _env = {k: fs_str(v) for k, v in os.environ.items()} + _env = {k: fs_str(v) for k, v in _env.items()} if not spinner_name: spinner_name = "bouncingBar" if six.PY2: @@ -299,8 +365,12 @@ def run( if block or not return_object: combine_stderr = False start_text = "" - with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin, - write_to_stdout=write_to_stdout) as sp: + with spinner( + spinner_name=spinner_name, + start_text=start_text, + nospin=nospin, + write_to_stdout=write_to_stdout, + ) as sp: return _create_subprocess( cmd, env=_env, @@ -311,11 +381,10 @@ def run( spinner=sp, combine_stderr=combine_stderr, start_text=start_text, - write_to_stdout=True + write_to_stdout=write_to_stdout, ) - def load_path(python): """Load the :mod:`sys.path` from the given python executable's environment as json @@ -328,8 +397,9 @@ def load_path(python): """ python = Path(python).as_posix() - out, err = run([python, "-c", "import json, sys; print(json.dumps(sys.path))"], - nospin=True) + out, err = run( + [python, "-c", "import json, sys; print(json.dumps(sys.path))"], nospin=True + ) if out: return json.loads(out) else: @@ -367,14 +437,14 @@ def partialclass(cls, *args, **kwargs): # Swiped from attrs.make_class try: type_.__module__ = sys._getframe(1).f_globals.get("__name__", "__main__") - except (AttributeError, ValueError): - pass + except (AttributeError, ValueError): # pragma: no cover + pass # pragma: no cover return type_ # Borrowed from django -- force bytes and decode -- see link for details: # https://github.com/django/django/blob/fc6b90b/django/utils/encoding.py#L112 -def to_bytes(string, encoding="utf-8", errors="ignore"): +def to_bytes(string, encoding="utf-8", errors=None): """Force a value to bytes. :param string: Some input that can be converted to a bytes. @@ -385,19 +455,23 @@ def to_bytes(string, encoding="utf-8", errors="ignore"): :rtype: bytes """ + unicode_name = get_canonical_encoding_name("utf-8") if not errors: - if encoding.lower() == "utf-8": - errors = "surrogateescape" if six.PY3 else "ignore" + if get_canonical_encoding_name(encoding) == unicode_name: + if six.PY3 and os.name == "nt": + errors = "surrogatepass" + else: + errors = "surrogateescape" if six.PY3 else "ignore" else: errors = "strict" if isinstance(string, bytes): - if encoding.lower() == "utf-8": + if get_canonical_encoding_name(encoding) == unicode_name: return string else: - return string.decode("utf-8").encode(encoding, errors) + return string.decode(unicode_name).encode(encoding, errors) elif isinstance(string, memoryview): - return bytes(string) - elif not isinstance(string, six.string_types): + return string.tobytes() + elif not isinstance(string, six.string_types): # pragma: no cover try: if six.PY3: return six.text_type(string).encode(encoding, errors) @@ -422,9 +496,13 @@ def to_text(string, encoding="utf-8", errors=None): :rtype: str """ + unicode_name = get_canonical_encoding_name("utf-8") if not errors: - if encoding.lower() == "utf-8": - errors = "surrogateescape" if six.PY3 else "ignore" + if get_canonical_encoding_name(encoding) == unicode_name: + if six.PY3 and os.name == "nt": + errors = "surrogatepass" + else: + errors = "surrogateescape" if six.PY3 else "ignore" else: errors = "strict" if issubclass(type(string), six.text_type): @@ -436,13 +514,13 @@ def to_text(string, encoding="utf-8", errors=None): string = six.text_type(string, encoding, errors) else: string = six.text_type(string) - elif hasattr(string, "__unicode__"): + elif hasattr(string, "__unicode__"): # pragma: no cover string = six.text_type(string) else: string = six.text_type(bytes(string), encoding, errors) else: string = string.decode(encoding, errors) - except UnicodeDecodeError as e: + except UnicodeDecodeError: # pragma: no cover string = " ".join(to_text(arg, encoding, errors) for arg in string) return string @@ -494,7 +572,7 @@ def chunked(n, iterable): try: - locale_encoding = locale.getdefaultencoding()[1] or "ascii" + locale_encoding = locale.getdefaultlocale()[1] or "ascii" except Exception: locale_encoding = "ascii" @@ -515,19 +593,443 @@ def getpreferredencoding(): PREFERRED_ENCODING = getpreferredencoding() -def decode_for_output(output): +def get_output_encoding(source_encoding): + """ + Given a source encoding, determine the preferred output encoding. + + :param str source_encoding: The encoding of the source material. + :returns: The output encoding to decode to. + :rtype: str + """ + + if source_encoding is not None: + if get_canonical_encoding_name(source_encoding) == "ascii": + return "utf-8" + return get_canonical_encoding_name(source_encoding) + return get_canonical_encoding_name(PREFERRED_ENCODING) + + +def _encode(output, encoding=None, errors=None, translation_map=None): + if encoding is None: + encoding = PREFERRED_ENCODING + try: + output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( # noqa: F821 + to_text(output, encoding=encoding, errors=errors), translation_map + ) + else: + output = output.translate(translation_map) + else: + output = to_text(output, encoding=encoding, errors=errors) + except AttributeError: + pass + return output + + +def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal + :param target_stream: A stream to write to, we will encode to target this stream if possible. + :param dict translation_map: A mapping of unicode character ordinals to replacement strings. :return: A re-encoded string using the preferred encoding :rtype: str """ if not isinstance(output, six.string_types): return output + encoding = None + if target_stream is not None: + encoding = getattr(target_stream, "encoding", None) + encoding = get_output_encoding(encoding) try: - output = output.encode(PREFERRED_ENCODING) - except AttributeError: - pass - output = output.decode(PREFERRED_ENCODING) - return output + output = _encode(output, encoding=encoding, translation_map=translation_map) + except (UnicodeDecodeError, UnicodeEncodeError): + output = to_native_string(output) + output = _encode( + output, encoding=encoding, errors="replace", translation_map=translation_map + ) + return to_text(output, encoding=encoding, errors="replace") + + +def get_canonical_encoding_name(name): + # type: (str) -> str + """ + Given an encoding name, get the canonical name from a codec lookup. + + :param str name: The name of the codec to lookup + :return: The canonical version of the codec name + :rtype: str + """ + + import codecs + + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + +def _is_binary_buffer(stream): + try: + stream.write(b"") + except Exception: + try: + stream.write("") + except Exception: + pass + return False + return True + + +def _get_binary_buffer(stream): + if six.PY3 and not _is_binary_buffer(stream): + stream = getattr(stream, "buffer", None) + if stream is not None and _is_binary_buffer(stream): + return stream + return stream + + +def get_wrapped_stream(stream, encoding=None, errors="replace"): + """ + Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + + :param stream: A stream instance to wrap + :param str encoding: The encoding to use for the stream + :param str errors: The error handler to use, default "replace" + :returns: A new, wrapped stream + :rtype: :class:`StreamWrapper` + """ + + if stream is None: + raise TypeError("must provide a stream to wrap") + stream = _get_binary_buffer(stream) + if stream is not None and encoding is None: + encoding = "utf-8" + if not encoding: + encoding = get_output_encoding(stream) + else: + encoding = get_canonical_encoding_name(encoding) + return StreamWrapper(stream, encoding, errors, line_buffering=True) + + +class StreamWrapper(io.TextIOWrapper): + + """ + This wrapper class will wrap a provided stream and supply an interface + for compatibility. + """ + + def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): + self._stream = stream = _StreamProvider(stream) + io.TextIOWrapper.__init__( + self, stream, encoding, errors, line_buffering=line_buffering, **kwargs + ) + + # borrowed from click's implementation of stream wrappers, see + # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 + if six.PY2: + + def write(self, x): + if isinstance(x, (str, buffer, bytearray)): # noqa: F821 + try: + self.flush() + except Exception: + pass + # This is modified from the initial implementation to rely on + # our own decoding functionality to preserve unicode strings where + # possible + return self.buffer.write(str(x)) + return io.TextIOWrapper.write(self, x) + + else: + + def write(self, x): + # try to use backslash and surrogate escape strategies before failing + self._errors = ( + "backslashescape" if self.encoding != "mbcs" else "surrogateescape" + ) + try: + return io.TextIOWrapper.write(self, to_text(x, errors=self._errors)) + except UnicodeDecodeError: + if self._errors != "surrogateescape": + self._errors = "surrogateescape" + else: + self._errors = "replace" + return io.TextIOWrapper.write(self, to_text(x, errors=self._errors)) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + return self._stream.isatty() + + +# More things borrowed from click, this is because we are using `TextIOWrapper` instead of +# just a normal StringIO +class _StreamProvider(object): + def __init__(self, stream): + self._stream = stream + super(_StreamProvider, self).__init__() + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + fn = getattr(self._stream, "read1", None) + if fn is not None: + return fn(size) + if six.PY2: + return self._stream.readline(size) + return self._stream.read(size) + + def readable(self): + fn = getattr(self._stream, "readable", None) + if fn is not None: + return fn() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self): + fn = getattr(self._stream, "writable", None) + if fn is not None: + return fn() + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + fn = getattr(self._stream, "seekable", None) + if fn is not None: + return fn() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True + + +# XXX: The approach here is inspired somewhat by click with details taken from various +# XXX: other sources. Specifically we are using a stream cache and stream wrapping +# XXX: techniques from click (loosely inspired for the most part, with many details) +# XXX: heavily modified to suit our needs + + +def _isatty(stream): + try: + is_a_tty = stream.isatty() + except Exception: # pragma: no cover + is_a_tty = False + return is_a_tty + + +_wrap_for_color = None + +try: + import colorama +except ImportError: + colorama = None + +_color_stream_cache = WeakKeyDictionary() + +if os.name == "nt" or sys.platform.startswith("win"): + + if colorama is not None: + + def _is_wrapped_for_color(stream): + return isinstance( + stream, (colorama.AnsiToWin32, colorama.ansitowin32.StreamWrapper) + ) + + def _wrap_for_color(stream, color=None): + try: + cached = _color_stream_cache.get(stream) + except KeyError: + cached = None + if cached is not None: + return cached + strip = not _can_use_color(stream, color) + _color_wrapper = colorama.AnsiToWin32(stream, strip=strip) + result = _color_wrapper.stream + _write = result.write + + def _write_with_color(s): + try: + return _write(s) + except Exception: + _color_wrapper.reset_all() + raise + + result.write = _write_with_color + try: + _color_stream_cache[stream] = result + except Exception: + pass + return result + + +def _cached_stream_lookup(stream_lookup_func, stream_resolution_func): + stream_cache = WeakKeyDictionary() + + def lookup(): + stream = stream_lookup_func() + result = None + if stream in stream_cache: + result = stream_cache.get(stream, None) + if result is not None: + return result + result = stream_resolution_func() + try: + stream = stream_lookup_func() + stream_cache[stream] = result + except Exception: + pass + return result + + return lookup + + +def get_text_stream(stream="stdout", encoding=None): + """Retrieve a unicode stream wrapper around **sys.stdout** or **sys.stderr**. + + :param str stream: The name of the stream to wrap from the :mod:`sys` module. + :param str encoding: An optional encoding to use. + :return: A new :class:`~vistir.misc.StreamWrapper` instance around the stream + :rtype: `vistir.misc.StreamWrapper` + """ + + stream_map = {"stdin": sys.stdin, "stdout": sys.stdout, "stderr": sys.stderr} + if os.name == "nt" or sys.platform.startswith("win"): + from ._winconsole import _get_windows_console_stream, _wrap_std_stream + + else: + _get_windows_console_stream = lambda *args: None # noqa + _wrap_std_stream = lambda *args: None # noqa + + if six.PY2 and stream != "stdin": + _wrap_std_stream(stream) + sys_stream = stream_map[stream] + windows_console = _get_windows_console_stream(sys_stream, encoding, None) + if windows_console is not None: + if _can_use_color(windows_console): + return _wrap_for_color(windows_console) + return windows_console + return get_wrapped_stream(sys_stream, encoding) + + +def get_text_stdout(): + return get_text_stream("stdout") + + +def get_text_stderr(): + return get_text_stream("stderr") + + +def get_text_stdin(): + return get_text_stream("stdin") + + +_text_stdin = _cached_stream_lookup(lambda: sys.stdin, get_text_stdin) +_text_stdout = _cached_stream_lookup(lambda: sys.stdout, get_text_stdout) +_text_stderr = _cached_stream_lookup(lambda: sys.stderr, get_text_stderr) + + +TEXT_STREAMS = { + "stdin": get_text_stdin, + "stdout": get_text_stdout, + "stderr": get_text_stderr, +} + + +def replace_with_text_stream(stream_name): + """Given a stream name, replace the target stream with a text-converted equivalent + + :param str stream_name: The name of a target stream, such as **stdout** or **stderr** + :return: None + """ + new_stream = TEXT_STREAMS.get(stream_name) + if new_stream is not None: + new_stream = new_stream() + setattr(sys, stream_name, new_stream) + return None + + +def _can_use_color(stream=None, color=None): + from .termcolors import DISABLE_COLORS + + if DISABLE_COLORS: + return False + if not color: + if not stream: + stream = sys.stdin + return _isatty(stream) + return bool(color) + + +def echo(text, fg=None, bg=None, style=None, file=None, err=False, color=None): + """Write the given text to the provided stream or **sys.stdout** by default. + + Provides optional foreground and background colors from the ansi defaults: + **grey**, **red**, **green**, **yellow**, **blue**, **magenta**, **cyan** + or **white**. + + Available styles include **bold**, **dark**, **underline**, **blink**, **reverse**, + **concealed** + + :param str text: Text to write + :param str fg: Foreground color to use (default: None) + :param str bg: Foreground color to use (default: None) + :param str style: Style to use (default: None) + :param stream file: File to write to (default: None) + :param bool color: Whether to force color (i.e. ANSI codes are in the text) + """ + + if file and not hasattr(file, "write"): + raise TypeError("Expected a writable stream, received {0!r}".format(file)) + if not file: + if err: + file = _text_stderr() + else: + file = _text_stdout() + if text and not isinstance(text, (six.string_types, bytes, bytearray)): + text = six.text_type(text) + text = "" if not text else text + if isinstance(text, six.text_type): + text += "\n" + else: + text += b"\n" + if text and six.PY3 and is_bytes(text): + buffer = _get_binary_buffer(file) + if buffer is not None: + file.flush() + buffer.write(text) + buffer.flush() + return + if text and not is_bytes(text): + can_use_color = _can_use_color(file, color=color) + if any([fg, bg, style]): + text = colorize(text, fg=fg, bg=bg, attrs=style) + if not can_use_color or (os.name == "nt" and not _wrap_for_color): + text = ANSI_REMOVAL_RE.sub("", text) + elif os.name == "nt" and _wrap_for_color and not _is_wrapped_for_color(file): + file = _wrap_for_color(file, color=color) + if text: + file.write(text) + file.flush() diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 6e9a7f65..25d29eb9 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import atexit import errno @@ -8,23 +8,38 @@ import os import posixpath import shutil import stat +import time import warnings import six - from six.moves import urllib_parse from six.moves.urllib import request as urllib_request from .backports.tempfile import _TemporaryFileWrapper from .compat import ( - _NamedTemporaryFile, + IS_TYPE_CHECKING, + FileNotFoundError, Path, + PermissionError, ResourceWarning, TemporaryDirectory, _fs_encoding, + _NamedTemporaryFile, finalize, + fs_decode, + fs_encode, ) +# fmt: off +if six.PY3: + from urllib.parse import quote_from_bytes as quote +else: + from urllib import quote +# fmt: on + + +if IS_TYPE_CHECKING: + from typing import Optional, Callable, Text, ByteString, AnyStr __all__ = [ "check_for_unc_path", @@ -49,7 +64,11 @@ __all__ = [ if os.name == "nt": - warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message="The Windows bytes API has been deprecated.*", + ) def unicode_path(path): @@ -83,6 +102,7 @@ else: def normalize_path(path): + # type: (AnyStr) -> AnyStr """ Return a case-normalized absolute variable-expanded path. @@ -91,12 +111,17 @@ def normalize_path(path): :rtype: str """ - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + path = os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + if os.name == "nt" and os.path.exists(path): + from ._winconsole import get_long_path + + path = get_long_path(path) + + return os.path.normpath(os.path.normcase(path)) def is_in_path(path, parent): + # type: (AnyStr, AnyStr) -> bool """ Determine if the provided full path is in the given parent root. @@ -110,6 +135,7 @@ def is_in_path(path, parent): def normalize_drive(path): + # type: (str) -> Text """Normalize drive in path so they stay consistent. This currently only affects local drives on Windows, which can be @@ -130,6 +156,7 @@ def normalize_drive(path): def path_to_url(path): + # type: (str) -> Text """Convert the supplied local path to a file uri. :param str path: A string pointing to or representing a local path @@ -139,21 +166,32 @@ def path_to_url(path): >>> path_to_url("/home/user/code/myrepo/myfile.zip") 'file:///home/user/code/myrepo/myfile.zip' """ - from .misc import to_text, to_bytes + from .misc import to_bytes if not path: return path - path = to_bytes(path, encoding="utf-8") - normalized_path = to_text(normalize_drive(os.path.abspath(path)), encoding="utf-8") - return to_text(Path(normalized_path).as_uri(), encoding="utf-8") + normalized_path = Path(normalize_drive(os.path.abspath(path))).as_posix() + if os.name == "nt" and normalized_path[1] == ":": + drive, _, path = normalized_path.partition(":") + # XXX: This enables us to handle half-surrogates that were never + # XXX: actually part of a surrogate pair, but were just incidentally + # XXX: passed in as a piece of a filename + quoted_path = quote(fs_encode(path)) + return fs_decode("file:///{0}:{1}".format(drive, quoted_path)) + # XXX: This is also here to help deal with incidental dangling surrogates + # XXX: on linux, by making sure they are preserved during encoding so that + # XXX: we can urlencode the backslash correctly + bytes_path = to_bytes(normalized_path, errors="backslashreplace") + return fs_decode("file://{0}".format(quote(bytes_path))) def url_to_path(url): - """Convert a valid file url to a local filesystem path + # type: (str) -> ByteString + """ + Convert a valid file url to a local filesystem path Follows logic taken from pip's equivalent function """ - from .misc import to_bytes assert is_file_url(url), "Only file: urls can be converted to local paths" _, netloc, path, _, _ = urllib_parse.urlsplit(url) @@ -162,7 +200,7 @@ def url_to_path(url): netloc = "\\\\" + netloc path = urllib_request.url2pathname(netloc + path) - return to_bytes(path, encoding="utf-8") + return urllib_parse.unquote(path) def is_valid_url(url): @@ -195,9 +233,8 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - from .compat import to_native_string - fn = to_native_string(fn) + fn = fs_encode(fn) if os.path.exists(fn): file_stat = os.stat(fn).st_mode return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) @@ -212,20 +249,19 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_bytes, to_text - newdir = to_bytes(newdir, "utf-8") + newdir = fs_encode(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( "a file with the same name as the desired dir, '{0}', already exists.".format( - newdir + fs_decode(newdir) ) ) else: - head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) + head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - curdir = to_bytes(".", encoding="utf-8") + curdir = fs_encode(".") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -233,8 +269,8 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', already exists.".format( - to_text(newdir, encoding="utf-8") + "A file with the same name as the desired dir, '{0}', already exists.".format( + fs_decode(newdir) ) ) os.makedirs(os.path.join(head, tail), mode) @@ -289,34 +325,79 @@ def create_tracked_tempfile(*args, **kwargs): return _NamedTemporaryFile(*args, **kwargs) +def _find_icacls_exe(): + if os.name == "nt": + paths = [ + os.path.expandvars(r"%windir%\{0}").format(subdir) + for subdir in ("system32", "SysWOW64") + ] + for path in paths: + icacls_path = next( + iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None + ) + if icacls_path is not None: + icacls_path = os.path.join(path, icacls_path) + return icacls_path + return None + + def set_write_bit(fn): - """Set read-write permissions for the current user on the target path. Fail silently + # type: (str) -> None + """ + Set read-write permissions for the current user on the target path. Fail silently if the path doesn't exist. :param str fn: The target filename or path + :return: None """ - from .compat import to_native_string - - fn = to_native_string(fn) + fn = fs_encode(fn) if not os.path.exists(fn): return file_stat = os.stat(fn).st_mode os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if os.name == "nt": + from ._winconsole import get_current_user + + user_sid = get_current_user() + icacls_exe = _find_icacls_exe() or "icacls" + from .misc import run + + if user_sid: + c = run( + [ + icacls_exe, + "''{0}''".format(fn), + "/grant", + "{0}:WD".format(user_sid), + "/T", + "/C", + "/Q", + ], + nospin=True, + return_object=True, + ) + if not c.err and c.returncode == 0: + return + if not os.path.isdir(fn): - try: - os.chflags(fn, 0) - except AttributeError: - pass + for path in [fn, os.path.dirname(fn)]: + try: + os.chflags(path, 0) + except AttributeError: + pass + return None for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root,d) for d in dirs]: + for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) def rmtree(directory, ignore_errors=False, onerror=None): - """Stand-in for :func:`~shutil.rmtree` with additional error-handling. + # type: (str, bool, Optional[Callable]) -> None + """ + Stand-in for :func:`~shutil.rmtree` with additional error-handling. This version of `rmtree` handles read-only paths, especially in the case of index files written by certain source control systems. @@ -330,20 +411,48 @@ def rmtree(directory, ignore_errors=False, onerror=None): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - from .compat import to_native_string - - directory = to_native_string(directory) + directory = fs_encode(directory) if onerror is None: onerror = handle_remove_readonly try: - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=onerror - ) - except (IOError, OSError, FileNotFoundError) as exc: + shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # Ignore removal failures where the file doesn't exist - if exc.errno == errno.ENOENT: - pass - raise + if exc.errno != errno.ENOENT: + raise + + +def _wait_for_files(path): # pragma: no cover + """ + Retry with backoff up to 1 second to delete files from a directory. + + :param str path: The path to crawl to delete files from + :return: A list of remaining paths or None + :rtype: Optional[List[str]] + """ + timeout = 0.001 + remaining = [] + while timeout < 1.0: + remaining = [] + if os.path.isdir(path): + L = os.listdir(path) + for target in L: + _remaining = _wait_for_files(target) + if _remaining: + remaining.extend(_remaining) + continue + try: + os.unlink(path) + except FileNotFoundError as e: + if e.errno == errno.ENOENT: + return + except (OSError, IOError, PermissionError): + time.sleep(timeout) + timeout *= 2 + remaining.append(path) + else: + return + return remaining def handle_remove_readonly(func, path, exc): @@ -360,45 +469,45 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ( - ResourceWarning, FileNotFoundError, PermissionError, to_native_string - ) + from .compat import ResourceWarning, FileNotFoundError, PermissionError PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) - default_warning_message = ( - "Unable to remove file due to permissions restriction: {!r}" - ) + default_warning_message = "Unable to remove file due to permissions restriction: {!r}" # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) try: func(path) - except (OSError, IOError, FileNotFoundError, PermissionError) as e: - if e.errno == errno.ENOENT: - return - elif e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) + except ( + OSError, + IOError, + FileNotFoundError, + PermissionError, + ) as e: # pragma: no cover + if e.errno in PERM_ERRORS: + if e.errno == errno.ENOENT: + return + remaining = None + if os.path.isdir(path): + remaining = _wait_for_files(path) + if remaining: + warnings.warn(default_warning_message.format(path), ResourceWarning) + else: + func(path, ignore_errors=True) return if exc_exception.errno in PERM_ERRORS: set_write_bit(path) + remaining = _wait_for_files(path) try: func(path) except (OSError, IOError, FileNotFoundError, PermissionError) as e: if e.errno in PERM_ERRORS: - warnings.warn(default_warning_message.format(path), ResourceWarning) - pass - elif e.errno == errno.ENOENT: # File already gone - pass - else: - raise - else: + if e.errno != errno.ENOENT: # File still exists + warnings.warn(default_warning_message.format(path), ResourceWarning) return - elif exc_exception.errno == errno.ENOENT: - pass else: raise exc_exception @@ -494,8 +603,8 @@ def get_converted_relative_path(path, relative_to=None): raise ValueError("The path argument does not currently accept UNC paths") relpath_s = to_text(posixpath.normpath(path.as_posix())) - if not (relpath_s == u"." or relpath_s.startswith(u"./")): - relpath_s = posixpath.join(u".", relpath_s) + if not (relpath_s == "." or relpath_s.startswith("./")): + relpath_s = posixpath.join(".", relpath_s) return relpath_s diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index a2455b7d..64b615de 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import functools import os @@ -6,43 +7,73 @@ import signal import sys import threading import time +from io import StringIO import colorama -import cursor import six from .compat import to_native_string -from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS -from io import StringIO +from .cursor import hide_cursor, show_cursor +from .misc import decode_for_output, to_text +from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored try: import yaspin -except ImportError: +except ImportError: # pragma: no cover yaspin = None Spinners = None -else: - from yaspin.spinners import Spinners + SpinBase = None +else: # pragma: no cover + import yaspin.spinners + import yaspin.core + + Spinners = yaspin.spinners.Spinners + SpinBase = yaspin.core.Yaspin + +if os.name == "nt": # pragma: no cover + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.fail() + spinner.stop() + + +else: # pragma: no cover + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.red.fail("✘") + spinner.stop() -handler = None -if yaspin and os.name == "nt": - handler = yaspin.signal_handlers.default_handler -elif yaspin and os.name != "nt": - handler = yaspin.signal_handlers.fancy_handler CLEAR_LINE = chr(27) + "[K" +TRANSLATION_MAP = {10004: u"OK", 10008: u"x"} + + +decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) + class DummySpinner(object): def __init__(self, text="", **kwargs): - super(DummySpinner, self).__init__() if DISABLE_COLORS: colorama.init() - from .misc import decode_for_output - self.text = to_native_string(decode_for_output(text)) if text else "" + self.text = to_native_string(decode_output(text)) if text else "" self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() self.write_to_stdout = kwargs.get("write_to_stdout", False) + super(DummySpinner, self).__init__() def __enter__(self): if self.text and self.text != "None": @@ -50,15 +81,16 @@ class DummySpinner(object): self.write(self.text) return self - def __exit__(self, exc_type, exc_val, traceback): + def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback - from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) + self.write_err("".join(formatted_tb)) self._close_output_buffer() return False - def __getattr__(self, k): + def __getattr__(self, k): # pragma: no cover try: retval = super(DummySpinner, self).__getattribute__(k) except AttributeError: @@ -76,56 +108,63 @@ class DummySpinner(object): pass def fail(self, exitcode=1, text="FAIL"): - from .misc import decode_for_output - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.write(decode_for_output(text)) + self.write(text) else: - self.write_err(decode_for_output(text)) + self.write_err(text) self._close_output_buffer() def ok(self, text="OK"): - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.stdout.write(self.text) + self.write(text) else: - self.stderr.write(self.text) + self.write_err(text) self._close_output_buffer() return 0 def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output(u"\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output(u"{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text=None): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stdout.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stdout.write(line) - self.stdout.write(CLEAR_LINE) + if not self.stdout.closed: + stdout = self.stdout + else: + stdout = sys.stdout + stdout.write(decode_output(u"\r", target_stream=stdout)) + text = to_text(text) + line = decode_output(u"{0}\n".format(text), target_stream=stdout) + stdout.write(line) + stdout.write(CLEAR_LINE) def write_err(self, text=None): - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stderr.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stderr.write(line) - self.stderr.write(CLEAR_LINE) + text = to_text(text) + if not self.stderr.closed: + stderr = self.stderr + else: + if sys.stderr.closed: + print(text) + return + stderr = sys.stderr + stderr.write(decode_output(u"\r", target_stream=stderr)) + line = decode_output(u"{0}\n".format(text), target_stream=stderr) + stderr.write(line) + stderr.write(CLEAR_LINE) @staticmethod def _hide_cursor(target=None): @@ -136,10 +175,11 @@ class DummySpinner(object): pass -base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner +if SpinBase is None: + SpinBase = DummySpinner -class VistirSpinner(base_obj): +class VistirSpinner(SpinBase): "A spinner class for handling spinners on windows and posix." def __init__(self, *args, **kwargs): @@ -158,10 +198,7 @@ class VistirSpinner(base_obj): colorama.init() sigmap = {} if handler: - sigmap.update({ - signal.SIGINT: handler, - signal.SIGTERM: handler - }) + sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler}) handler_map = kwargs.pop("handler_map", {}) if os.name == "nt": sigmap[signal.SIGBREAK] = handler @@ -182,60 +219,64 @@ class VistirSpinner(base_obj): self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) super(VistirSpinner, self).__init__(*args, **kwargs) + if DISABLE_COLORS: + colorama.deinit() - def ok(self, text="OK", err=False): + def ok(self, text=u"OK", err=False): """Set Ok (success) finalizer to a spinner.""" # Do not display spin text for ok state self._text = None - _text = text if text else "OK" + _text = to_text(text) if text else u"OK" err = err or not self.write_to_stdout self._freeze(_text, err=err) - def fail(self, text="FAIL", err=False): + def fail(self, text=u"FAIL", err=False): """Set fail finalizer to a spinner.""" # Do not display spin text for fail state self._text = None - _text = text if text else "FAIL" + _text = text if text else u"FAIL" err = err or not self.write_to_stdout self._freeze(_text, err=err) def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output - if text is None or isinstance(text, six.string_types) and text == "None": + if text is None or isinstance(text, six.string_types) and text == u"None": pass - target.write(decode_for_output("\r")) + target.write(decode_output(u"\r")) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output(u"{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) - def write(self, text): + def write(self, text): # pragma: no cover if not self.write_to_stdout: return self.write_err(text) - from .misc import to_text - sys.stdout.write("\r") - self.stdout.write(CLEAR_LINE) + stdout = self.stdout + if self.stdout.closed: + stdout = sys.stdout + stdout.write(decode_output(u"\r", target_stream=stdout)) + stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) - self.stdout.write(text) - self.out_buff.write(to_text(text)) + text = decode_output(u"{0}\n".format(text), target_stream=stdout) + stdout.write(text) + self.out_buff.write(text) - def write_err(self, text): + def write_err(self, text): # pragma: no cover """Write error text in the terminal without breaking the spinner.""" - from .misc import to_text - - self.stderr.write("\r") - self.stderr.write(CLEAR_LINE) + stderr = self.stderr + if self.stderr.closed: + stderr = sys.stderr + stderr.write(decode_output(u"\r", target_stream=stderr)) + stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) + text = decode_output(u"{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(to_text(text)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -270,52 +311,47 @@ class VistirSpinner(base_obj): if target.isatty(): self._show_cursor(target=target) - if self.stderr and self.stderr != sys.stderr: - self.stderr.close() - if self.stdout and self.stdout != sys.stdout: - self.stdout.close() self.out_buff.close() def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" if not final_text: final_text = "" - text = to_native_string(final_text) - self._last_frame = self._compose_out(text, mode="last") + target = self.stderr if err else self.stdout + if target.closed: + target = sys.stderr if err else sys.stdout + text = to_text(final_text) + last_frame = self._compose_out(text, mode="last") + self._last_frame = decode_output(last_frame, target_stream=target) # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - if err or not self.write_to_stdout: - self.stderr.write(self._last_frame) - else: - self.stdout.write(self._last_frame) + target.write(self._last_frame) def _compose_color_func(self): fn = functools.partial( - colored, - color=self._color, - on_color=self._on_color, - attrs=list(self._attrs), + colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs) ) return fn def _compose_out(self, frame, mode=None): # Ensure Unicode input - frame = to_native_string(frame) + frame = to_text(frame) if self._text is None: - self._text = "" - text = to_native_string(self._text) + self._text = u"" + text = to_text(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode + frame = to_text(frame) if not mode: - out = to_native_string("\r{0} {1}".format(frame, text)) + out = u"\r{0} {1}".format(frame, text) else: - out = to_native_string("{0} {1}\n".format(frame, text)) + out = u"{0} {1}\n".format(frame, text) return out def _spin(self): @@ -331,6 +367,7 @@ class VistirSpinner(base_obj): # Compose output spin_phase = next(self._cycle) out = self._compose_out(spin_phase) + out = decode_output(out, target) # Write target.write(out) @@ -381,13 +418,13 @@ class VistirSpinner(base_obj): def _hide_cursor(target=None): if not target: target = sys.stdout - cursor.hide(stream=target) + hide_cursor(stream=target) @staticmethod def _show_cursor(target=None): if not target: target = sys.stdout - cursor.show(stream=target) + show_cursor(stream=target) @staticmethod def _clear_err(): diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index d72e8ebe..27b5ff44 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -1,62 +1,27 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import colorama + import os +import re + +import colorama +import six + from .compat import to_native_string - -DISABLE_COLORS = os.getenv("CI", False) or os.getenv("ANSI_COLORS_DISABLED", - os.getenv("VISTIR_DISABLE_COLORS", False) +DISABLE_COLORS = os.getenv("CI", False) or os.getenv( + "ANSI_COLORS_DISABLED", os.getenv("VISTIR_DISABLE_COLORS", False) ) -ATTRIBUTES = dict( - list(zip([ - 'bold', - 'dark', - '', - 'underline', - 'blink', - '', - 'reverse', - 'concealed' - ], - list(range(1, 9)) - )) - ) -del ATTRIBUTES[''] +ATTRIBUTE_NAMES = ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"] +ATTRIBUTES = dict(zip(ATTRIBUTE_NAMES, range(1, 9))) +del ATTRIBUTES[""] - -HIGHLIGHTS = dict( - list(zip([ - 'on_grey', - 'on_red', - 'on_green', - 'on_yellow', - 'on_blue', - 'on_magenta', - 'on_cyan', - 'on_white' - ], - list(range(40, 48)) - )) - ) - - -COLORS = dict( - list(zip([ - 'grey', - 'red', - 'green', - 'yellow', - 'blue', - 'magenta', - 'cyan', - 'white', - ], - list(range(30, 38)) - )) - ) +colors = ["grey", "red", "green", "yellow", "blue", "magenta", "cyan", "white"] +COLORS = dict(zip(colors, range(30, 38))) +HIGHLIGHTS = dict(zip(["on_{0}".format(c) for c in colors], range(40, 48))) +ANSI_REMOVAL_RE = re.compile(r"\033\[((?:\d|;)*)([a-zA-Z])") COLOR_MAP = { @@ -106,39 +71,49 @@ def colored(text, color=None, on_color=None, attrs=None): colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) colored('Hello, World!', 'green') """ - if os.getenv('ANSI_COLORS_DISABLED') is None: + return colorize(text, fg=color, bg=on_color, attrs=attrs) + + +def colorize(text, fg=None, bg=None, attrs=None): + if os.getenv("ANSI_COLORS_DISABLED") is None: style = "NORMAL" - if 'bold' in attrs: + if attrs is not None and not isinstance(attrs, list): + _attrs = [] + if isinstance(attrs, six.string_types): + _attrs.append(attrs) + else: + _attrs = list(attrs) + attrs = _attrs + if attrs and "bold" in attrs: style = "BRIGHT" - attrs.remove('bold') - if color is not None: - color = color.upper() + attrs.remove("bold") + if fg is not None: + fg = fg.upper() text = to_native_string("%s%s%s%s%s") % ( - to_native_string(getattr(colorama.Fore, color)), + to_native_string(getattr(colorama.Fore, fg)), to_native_string(getattr(colorama.Style, style)), to_native_string(text), to_native_string(colorama.Fore.RESET), to_native_string(colorama.Style.NORMAL), ) - if on_color is not None: - on_color = on_color.upper() + if bg is not None: + bg = bg.upper() text = to_native_string("%s%s%s%s") % ( - to_native_string(getattr(colorama.Back, on_color)), + to_native_string(getattr(colorama.Back, bg)), to_native_string(text), to_native_string(colorama.Back.RESET), to_native_string(colorama.Style.NORMAL), ) if attrs is not None: - fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( - chr(27), - chr(27) - ) + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % (chr(27), chr(27)) for attr in attrs: text = fmt_str % (ATTRIBUTES[attr], text) text += RESET + else: + text = ANSI_REMOVAL_RE.sub("", text) return text diff --git a/pipenv/vendor/yaspin/__version__.py b/pipenv/vendor/yaspin/__version__.py index 9e78220f..23f00709 100644 --- a/pipenv/vendor/yaspin/__version__.py +++ b/pipenv/vendor/yaspin/__version__.py @@ -1 +1 @@ -__version__ = "0.14.0" +__version__ = "0.14.3" diff --git a/pipenv/vendor/yaspin/api.py b/pipenv/vendor/yaspin/api.py index 156630db..f59ce002 100644 --- a/pipenv/vendor/yaspin/api.py +++ b/pipenv/vendor/yaspin/api.py @@ -84,5 +84,7 @@ def kbi_safe_yaspin(*args, **kwargs): return Yaspin(*args, **kwargs) -_kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") -kbi_safe_yaspin.__doc__ = _kbi_safe_doc +# Handle PYTHONOPTIMIZE=2 case, when docstrings are set to None. +if yaspin.__doc__: + _kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") + kbi_safe_yaspin.__doc__ = _kbi_safe_doc diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b621..12960b3b 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -17,7 +17,7 @@ import threading import time import colorama -import cursor +from pipenv.vendor.vistir import cursor from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -82,6 +82,7 @@ class Yaspin(object): self._hide_spin = None self._spin_thread = None self._last_frame = None + self._stdout_lock = threading.Lock() # Signals @@ -253,43 +254,47 @@ class Yaspin(object): thr_is_alive = self._spin_thread and self._spin_thread.is_alive() if thr_is_alive and not self._hide_spin.is_set(): - # set the hidden spinner flag - self._hide_spin.set() + with self._stdout_lock: + # set the hidden spinner flag + self._hide_spin.set() - # clear the current line - sys.stdout.write("\r") - self._clear_line() + # clear the current line + sys.stdout.write("\r") + self._clear_line() - # flush the stdout buffer so the current line can be rewritten to - sys.stdout.flush() + # flush the stdout buffer so the current line + # can be rewritten to + sys.stdout.flush() def show(self): """Show the hidden spinner.""" thr_is_alive = self._spin_thread and self._spin_thread.is_alive() if thr_is_alive and self._hide_spin.is_set(): - # clear the hidden spinner flag - self._hide_spin.clear() + with self._stdout_lock: + # clear the hidden spinner flag + self._hide_spin.clear() - # clear the current line so the spinner is not appended to it - sys.stdout.write("\r") - self._clear_line() + # clear the current line so the spinner is not appended to it + sys.stdout.write("\r") + self._clear_line() def write(self, text): """Write text in the terminal without breaking the spinner.""" # similar to tqdm.write() # https://pypi.python.org/pypi/tqdm#writing-messages - sys.stdout.write("\r") - self._clear_line() + with self._stdout_lock: + sys.stdout.write("\r") + self._clear_line() - _text = to_unicode(text) - if PY2: - _text = _text.encode(ENCODING) + _text = to_unicode(text) + if PY2: + _text = _text.encode(ENCODING) - # Ensure output is bytes for Py2 and Unicode for Py3 - assert isinstance(_text, builtin_str) + # Ensure output is bytes for Py2 and Unicode for Py3 + assert isinstance(_text, builtin_str) - sys.stdout.write("{0}\n".format(_text)) + sys.stdout.write("{0}\n".format(_text)) def ok(self, text="OK"): """Set Ok (success) finalizer to a spinner.""" @@ -312,7 +317,8 @@ class Yaspin(object): # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - sys.stdout.write(self._last_frame) + with self._stdout_lock: + sys.stdout.write(self._last_frame) def _spin(self): while not self._stop_spin.is_set(): @@ -327,13 +333,13 @@ class Yaspin(object): out = self._compose_out(spin_phase) # Write - sys.stdout.write(out) - self._clear_line() - sys.stdout.flush() + with self._stdout_lock: + sys.stdout.write(out) + self._clear_line() + sys.stdout.flush() # Wait time.sleep(self._interval) - sys.stdout.write("\b") def _compose_color_func(self): fn = functools.partial( @@ -530,11 +536,11 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - cursor.hide() + cursor.hide_cursor() @staticmethod def _show_cursor(): - cursor.show() + cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/pipenv/vendor/yaspin/spinners.py b/pipenv/vendor/yaspin/spinners.py index 9c3fa7b8..60822a2c 100644 --- a/pipenv/vendor/yaspin/spinners.py +++ b/pipenv/vendor/yaspin/spinners.py @@ -11,10 +11,7 @@ import codecs import os from collections import namedtuple -try: - import simplejson as json -except ImportError: - import json +import json THIS_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/pipenv/vendor/zipp.LICENSE b/pipenv/vendor/zipp.LICENSE new file mode 100644 index 00000000..5e795a61 --- /dev/null +++ b/pipenv/vendor/zipp.LICENSE @@ -0,0 +1,7 @@ +Copyright Jason R. Coombs + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/pipenv/vendor/zipp.py b/pipenv/vendor/zipp.py new file mode 100644 index 00000000..8ab7d099 --- /dev/null +++ b/pipenv/vendor/zipp.py @@ -0,0 +1,220 @@ +# coding: utf-8 + +from __future__ import division + +import io +import sys +import posixpath +import zipfile +import functools +import itertools + +import more_itertools + +__metaclass__ = type + + +def _parents(path): + """ + Given a path with elements separated by + posixpath.sep, generate all parents of that path. + + >>> list(_parents('b/d')) + ['b'] + >>> list(_parents('/b/d/')) + ['/b'] + >>> list(_parents('b/d/f/')) + ['b/d', 'b'] + >>> list(_parents('b')) + [] + >>> list(_parents('')) + [] + """ + return itertools.islice(_ancestry(path), 1, None) + + +def _ancestry(path): + """ + Given a path with elements separated by + posixpath.sep, generate all elements of that path + + >>> list(_ancestry('b/d')) + ['b/d', 'b'] + >>> list(_ancestry('/b/d/')) + ['/b/d', '/b'] + >>> list(_ancestry('b/d/f/')) + ['b/d/f', 'b/d', 'b'] + >>> list(_ancestry('b')) + ['b'] + >>> list(_ancestry('')) + [] + """ + path = path.rstrip(posixpath.sep) + while path and path != posixpath.sep: + yield path + path, tail = posixpath.split(path) + + +class Path: + """ + A pathlib-compatible interface for zip files. + + Consider a zip file with this structure:: + + . + ├── a.txt + └── b + ├── c.txt + └── d + └── e.txt + + >>> data = io.BytesIO() + >>> zf = zipfile.ZipFile(data, 'w') + >>> zf.writestr('a.txt', 'content of a') + >>> zf.writestr('b/c.txt', 'content of c') + >>> zf.writestr('b/d/e.txt', 'content of e') + >>> zf.filename = 'abcde.zip' + + Path accepts the zipfile object itself or a filename + + >>> root = Path(zf) + + From there, several path operations are available. + + Directory iteration (including the zip file itself): + + >>> a, b = root.iterdir() + >>> a + Path('abcde.zip', 'a.txt') + >>> b + Path('abcde.zip', 'b/') + + name property: + + >>> b.name + 'b' + + join with divide operator: + + >>> c = b / 'c.txt' + >>> c + Path('abcde.zip', 'b/c.txt') + >>> c.name + 'c.txt' + + Read text: + + >>> c.read_text() + 'content of c' + + existence: + + >>> c.exists() + True + >>> (b / 'missing.txt').exists() + False + + Coercion to string: + + >>> str(c) + 'abcde.zip/b/c.txt' + """ + + __repr = "{self.__class__.__name__}({self.root.filename!r}, {self.at!r})" + + def __init__(self, root, at=""): + self.root = ( + root + if isinstance(root, zipfile.ZipFile) + else zipfile.ZipFile(self._pathlib_compat(root)) + ) + self.at = at + + @staticmethod + def _pathlib_compat(path): + """ + For path-like objects, convert to a filename for compatibility + on Python 3.6.1 and earlier. + """ + try: + return path.__fspath__() + except AttributeError: + return str(path) + + @property + def open(self): + return functools.partial(self.root.open, self.at) + + @property + def name(self): + return posixpath.basename(self.at.rstrip("/")) + + def read_text(self, *args, **kwargs): + with self.open() as strm: + return io.TextIOWrapper(strm, *args, **kwargs).read() + + def read_bytes(self): + with self.open() as strm: + return strm.read() + + def _is_child(self, path): + return posixpath.dirname(path.at.rstrip("/")) == self.at.rstrip("/") + + def _next(self, at): + return Path(self.root, at) + + def is_dir(self): + return not self.at or self.at.endswith("/") + + def is_file(self): + return not self.is_dir() + + def exists(self): + return self.at in self._names() + + def iterdir(self): + if not self.is_dir(): + raise ValueError("Can't listdir a file") + subs = map(self._next, self._names()) + return filter(self._is_child, subs) + + def __str__(self): + return posixpath.join(self.root.filename, self.at) + + def __repr__(self): + return self.__repr.format(self=self) + + def joinpath(self, add): + add = self._pathlib_compat(add) + next = posixpath.join(self.at, add) + next_dir = posixpath.join(self.at, add, "") + names = self._names() + return self._next(next_dir if next not in names and next_dir in names else next) + + __truediv__ = joinpath + + @staticmethod + def _implied_dirs(names): + return more_itertools.unique_everseen( + parent + "/" + for name in names + for parent in _parents(name) + if parent + "/" not in names + ) + + @classmethod + def _add_implied_dirs(cls, names): + return names + list(cls._implied_dirs(names)) + + @property + def parent(self): + parent_at = posixpath.dirname(self.at.rstrip('/')) + if parent_at: + parent_at += '/' + return self._next(parent_at) + + def _names(self): + return self._add_implied_dirs(self.root.namelist()) + + if sys.version_info < (3,): + __div__ = __truediv__ diff --git a/pyproject.toml b/pyproject.toml index a799764c..71c94f3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,5 +1,32 @@ [build-system] requires = ["setuptools", "wheel"] +build-backend = "setuptools.build_meta" + +[tool.black] +line-length = 90 +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.github + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist + | pipenv/vendor + | pipenv/patched + | tests/pypi + | tests/pytest-pypi + | tests/test_artifacts + | get-pipenv.py +) +''' [tool.towncrier] package = "pipenv" diff --git a/pytest.ini b/pytest.ini deleted file mode 100644 index ff9847d3..00000000 --- a/pytest.ini +++ /dev/null @@ -1,8 +0,0 @@ -[pytest] -addopts = -ra -n auto -testpaths = tests -; Add vendor and patched in addition to the default list of ignored dirs -norecursedirs = .* build dist CVS _darcs {arch} *.egg vendor patched news tasks docs tests/test_artifacts -filterwarnings = - ignore::DeprecationWarning - ignore::PendingDeprecationWarning diff --git a/run-tests.bat b/run-tests.bat index d20af35f..973425af 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -1,7 +1,15 @@ -rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" +rem If you want to use a ramdisk, use this section: -virtualenv R:\.venv -R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed +rem imdisk -a -s 4G -m R: -p "FS:NTFS /y" +rem if you are using a ram disk, you should comment the following substitution line out +subst R: %TEMP% + +set TMP=R:\\ +set TEMP=R:\\ +set WORKON_HOME=R:\\ +set RAM_DISK=R:\\ + +R:\.venv\Scripts\pip install -e .[test] --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev git submodule sync && git submodule update --init --recursive -SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap +R:\.venv\Scripts\pipenv run pytest -n auto -v tests diff --git a/run-tests.sh b/run-tests.sh index 991c0df7..aef44f63 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -7,6 +7,8 @@ set -eo pipefail export PYTHONIOENCODING="utf-8" export LANG=C.UTF-8 export PIP_PROCESS_DEPENDENCY_LINKS="1" +# Let's use a temporary cache directory +export PIPENV_CACHE_DIR=`mktemp -d 2>/dev/null || mktemp -d -t 'pipenv_cache'` prefix() { sed "s/^/ $1: /" @@ -25,49 +27,27 @@ fi if [[ ! -z "$HOME" ]]; then export PATH="${HOME}/.local/bin:${PATH}" fi -# pip uninstall -y pipenv pip install certifi export GIT_SSL_CAINFO=$(python -m certifi) echo "Path: $PATH" echo "Installing Pipenv…" -PIP_USER="1" python -m pip install --upgrade setuptools -PIP_USER="1" python3 -m pip install --upgrade setuptools -python -m pip install -e "$(pwd)" --upgrade && python3 -m pip install -e "$(pwd)" --upgrade -python3 -m pipenv install --deploy --dev --system - -# Otherwise, we're on a development machine. -# First, try MacOS… -if [[ $(python -c "import sys; print(sys.platform)") == "darwin" ]]; then - - echo "Clearing Caches…" - rm -fr ~/Library/Caches/pip - rm -fr ~/Library/Caches/pipenv - -# Otherwise, assume Linux… -else - echo "Clearing Caches…" - rm -fr ~/.cache/pip - rm -fr ~/.cache/pipenv -fi +python -m pip install --upgrade -e "$(pwd)" setuptools wheel pip +VENV_CMD="python -m pipenv --venv" +RM_CMD="pipenv --rm" +echo "$ PIPENV_PYTHON=2.7 $VENV_CMD && PIPENV_PYTHON=2.7 $RM_CMD" +echo "$ PIPENV_PYTHON=3.7 $VENV_CMD && PIPENV_PYTHON=3.7 $RM_CMD" +{ PIPENV_PYTHON=2.7 $VENV_CMD && PIPENV_PYTHON=2.7 $RM_CMD ; PIPENV_PYTHON=3.7 $VENV_CMD && PIPENV_PYTHON=3.7 $RM_CMD ; } echo "Installing dependencies…" -PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e . -PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e . +INSTALL_CMD="python -m pipenv install --deploy --dev" +echo "$ PIPENV_PYTHON=2.7 $INSTALL_CMD" +echo "$ PIPENV_PYTHON=3.7 $INSTALL_CMD" + +{ ( PIPENV_PYTHON=2.7 $INSTALL_CMD & ); PIPENV_PYTHON=3.7 $INSTALL_CMD ; } echo "$ git submodule sync && git submodule update --init --recursive" git submodule sync && git submodule update --init --recursive -echo "$ pipenv run time pytest -v -n auto tests -m \"$TEST_SUITE\"" -# PIPENV_PYTHON=2.7 pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" | prefix 2.7 & -# PIPENV_PYTHON=3.6 pipenv run time pytest -v -n auto tests -m "$TEST_SUITE" | prefix 3.6 -# Better to run them sequentially. -PIPENV_PYTHON=2.7 python3 -m pipenv run time pytest -PIPENV_PYTHON=3.7 python3 -m pipenv run time pytest - -# test revendoring -pip3 install --upgrade invoke requests parver vistir -python3 -m invoke vendoring.update -# Cleanup junk. -rm -fr .venv +echo "$ pipenv run time pytest" +PIPENV_PYTHON=2.7 python -m pipenv run time pytest +PIPENV_PYTHON=3.7 python -m pipenv run time pytest diff --git a/setup.cfg b/setup.cfg index 78140fa0..f45c0d05 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,7 +4,9 @@ license = MIT license_file = LICENSE [flake8] -exclude = .git,__pycache__,docs/,pipenv/vendor/,get-pipenv.py,setup.py +exclude = + .git,__pycache__,docs/,pipenv/vendor/,pipenv/patched,get-pipenv.py, + .eggs/,setup.py,tests/fixtures/ ignore = # The default ignore list: E121,E123,E126,E226,E24,E704, @@ -17,15 +19,52 @@ ignore = # E402: module level import not at top of file # E501: line too long # W503: line break before binary operator - E127,E128,E129,E222,E231,E402,E501,W503 + E402,E501,W503,E203 [isort] atomic=true lines_after_imports=2 lines_between_types=1 multi_line_output=5 +line_length=80 not_skip=__init__.py known_first_party = pipenv tests ignore_trailing_comma=true + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=3.6 +mypy_path=typeshed/pyi:typeshed/imports + +[tool:pytest] +addopts = -ra -n auto +plugins = xdist +testpaths = tests +; Add vendor and patched in addition to the default list of ignored dirs +; Additionally, ignore tasks, news, test subdirectories and peeps directory +norecursedirs = + .* build + dist + CVS + _darcs + {arch} + *.egg + vendor + patched + news + tasks + docs + tests/test_artifacts + tests/pytest-pypi + tests/pypi + peeps +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + +[bdist_wheel] +universal = 1 diff --git a/setup.py b/setup.py index 4c848c47..33cf3b5b 100644 --- a/setup.py +++ b/setup.py @@ -22,13 +22,28 @@ if sys.argv[-1] == "publish": sys.exit() required = [ - "pip>=9.0.1", + "pip>=18.0", "certifi", "setuptools>=36.2.1", "virtualenv-clone>=0.2.5", "virtualenv", - 'enum34; python_version<"3"' + 'enum34; python_version<"3"', + # LEAVE THIS HERE!!! we have vendored dependencies that require it + 'typing; python_version<"3.5"' ] +extras = { + "dev": [ + "towncrier", + "bs4", + "twine", + "sphinx<2", + "flake8>=3.3.0,<4.0", + "black;python_version>='3.6'", + "parver", + "invoke", + ], + "tests": ["pytest<5.0", "pytest-timeout", "pytest-xdist", "flaky", "mock"], +} # https://pypi.python.org/pypi/stdeb/0.8.5#quickstart-2-just-tell-me-the-fastest-way-to-make-a-deb @@ -103,8 +118,8 @@ setup( description="Python Development Workflow for Humans.", long_description=long_description, long_description_content_type='text/markdown', - author="Kenneth Reitz", - author_email="me@kennethreitz.org", + author="Pipenv maintainer team", + author_email="distutils-sig@python.org", url="https://github.com/pypa/pipenv", packages=find_packages(exclude=["tests", "tests.*", "tasks", "tasks.*"]), entry_points={ @@ -129,9 +144,10 @@ setup( ], }, python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", - setup_requires=["invoke", "parver"], + zip_safe=True, + setup_requires=[], install_requires=required, - extras_require={}, + extras_require=extras, include_package_data=True, license="MIT", classifiers=[ diff --git a/tasks/__init__.py b/tasks/__init__.py index 63fe1388..d81d101d 100644 --- a/tasks/__init__.py +++ b/tasks/__init__.py @@ -1,8 +1,6 @@ # -*- coding=utf-8 -*- # Copied from pip's vendoring process # see https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/tasks/__init__.py -import re - from pathlib import Path import invoke diff --git a/tasks/release.py b/tasks/release.py index 7c2e7aba..ed1575d4 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -3,14 +3,11 @@ import datetime import os import pathlib import re -import sys import invoke from parver import Version -from towncrier._builder import ( - find_fragments, render_fragments, split_fragments -) +from towncrier._builder import find_fragments, render_fragments, split_fragments from towncrier._settings import load_config from pipenv.__version__ import __version__ @@ -19,13 +16,13 @@ from pipenv.vendor.vistir.contextmanagers import temp_environ from .vendoring import _get_git_root, drop_dir -VERSION_FILE = 'pipenv/__version__.py' +VERSION_FILE = "pipenv/__version__.py" ROOT = pathlib.Path(".").parent.parent.absolute() PACKAGE_NAME = "pipenv" def log(msg): - print('[release] %s' % msg) + print("[release] %s" % msg) def get_version_file(ctx): @@ -34,41 +31,41 @@ def get_version_file(ctx): def find_version(ctx): version_file = get_version_file(ctx).read_text() - version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]",version_file, re.M) + version_match = re.search(r"^__version__ = ['\"]([^'\"]*)['\"]", version_file, re.M) if version_match: return version_match.group(1) raise RuntimeError("Unable to find version string.") def get_history_file(ctx): - return _get_git_root(ctx).joinpath('HISTORY.txt') + return _get_git_root(ctx).joinpath("HISTORY.txt") def get_dist_dir(ctx): - return _get_git_root(ctx) / 'dist' + return _get_git_root(ctx) / "dist" def get_build_dir(ctx): - return _get_git_root(ctx) / 'build' + return _get_git_root(ctx) / "build" def _render_log(): """Totally tap into Towncrier internals to get an in-memory result. """ config = load_config(ROOT) - definitions = config['types'] + definitions = config["types"] fragments, fragment_filenames = find_fragments( - pathlib.Path(config['directory']).absolute(), - config['sections'], + pathlib.Path(config["directory"]).absolute(), + config["sections"], None, definitions, ) rendered = render_fragments( - pathlib.Path(config['template']).read_text(encoding='utf-8'), - config['issue_format'], + pathlib.Path(config["template"]).read_text(encoding="utf-8"), + config["issue_format"], split_fragments(fragments, definitions), definitions, - config['underlines'][1:], + config["underlines"][1:], False, # Don't add newlines to wrapped text. ) return rendered @@ -81,20 +78,27 @@ def release(ctx, dry_run=False): version = find_version(ctx) tag_content = _render_log() if dry_run: - ctx.run('towncrier --draft > CHANGELOG.draft.rst') - log('would remove: news/*') - log('would remove: CHANGELOG.draft.rst') + ctx.run("towncrier --draft > CHANGELOG.draft.rst") + log("would remove: news/*") + log("would remove: CHANGELOG.draft.rst") + log("would update: pipenv/pipenv.1") log(f'Would commit with message: "Release v{version}"') else: - ctx.run('towncrier') - ctx.run("git add CHANGELOG.rst news/") + ctx.run("towncrier") + ctx.run( + "git add CHANGELOG.rst news/ {0}".format(get_version_file(ctx).as_posix()) + ) ctx.run("git rm CHANGELOG.draft.rst") + generate_manual(ctx) + ctx.run("git add pipenv/pipenv.1") ctx.run(f'git commit -m "Release v{version}"') tag_content = tag_content.replace('"', '\\"') if dry_run: log(f"Generated tag content: {tag_content}") - markdown = ctx.run("pandoc CHANGELOG.draft.rst -f rst -t markdown", hide=True).stdout.strip() + markdown = ctx.run( + "pandoc CHANGELOG.draft.rst -f rst -t markdown", hide=True + ).stdout.strip() content = clean_mdchangelog(ctx, markdown) log(f"would generate markdown: {content}") else: @@ -104,8 +108,8 @@ def release(ctx, dry_run=False): build_dists(ctx) if dry_run: dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' - artifacts = list(ROOT.joinpath('dist').glob(dist_pattern)) - filename_display = '\n'.join(f' {a}' for a in artifacts) + artifacts = list(ROOT.joinpath("dist").glob(dist_pattern)) + filename_display = "\n".join(f" {a}" for a in artifacts) log(f"Would upload dists: {filename_display}") else: upload_dists(ctx) @@ -113,66 +117,95 @@ def release(ctx, dry_run=False): def drop_dist_dirs(ctx): - log('Dropping Dist dir...') + log("Dropping Dist dir...") drop_dir(get_dist_dir(ctx)) - log('Dropping build dir...') + log("Dropping build dir...") drop_dir(get_build_dir(ctx)) @invoke.task def build_dists(ctx): drop_dist_dirs(ctx) - for py_version in ['3.6', '2.7']: - env = {'PIPENV_PYTHON': py_version} + for py_version in ["3.6", "3.7", "3.8", "2.7"]: + env = {"PIPENV_PYTHON": py_version} with ctx.cd(ROOT.as_posix()), temp_environ(): - executable = ctx.run("python -c 'import sys; print(sys.executable)'", hide=True).stdout.strip() - log('Building sdist using %s ....' % executable) + executable = ctx.run( + "python -c 'import sys; print(sys.executable)'", hide=True + ).stdout.strip() + log("Building sdist using %s ...." % executable) os.environ["PIPENV_PYTHON"] = py_version - ctx.run('pipenv install --dev', env=env) - ctx.run('pipenv run pip install -e . --upgrade --upgrade-strategy=eager', env=env) - log('Building wheel using python %s ....' % py_version) - if py_version == '3.6': - ctx.run('pipenv run python setup.py sdist bdist_wheel', env=env) + ctx.run("pipenv install --dev", env=env) + ctx.run( + "pipenv run pip install -e . --upgrade --upgrade-strategy=eager", env=env + ) + log("Building wheel using python %s ...." % py_version) + tag_arg = "--python-tag py{}".format(py_version.replace(".", "")) + if py_version == "3.8": + ctx.run(f"pipenv run python setup.py sdist bdist_wheel {tag_arg}", env=env) else: - ctx.run('pipenv run python setup.py bdist_wheel', env=env) + ctx.run(f"pipenv run python setup.py bdist_wheel {tag_arg}", env=env) + if py_version in ("3.6", "2.7"): + # generate py2 / py3 generic untagged wheels + ctx.run(f"pipenv run python setup.py sdist bdist_wheel", env=env) + @invoke.task(build_dists) def upload_dists(ctx, repo="pypi"): dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' - artifacts = list(ROOT.joinpath('dist').glob(dist_pattern)) - filename_display = '\n'.join(f' {a}' for a in artifacts) - print(f'[release] Will upload:\n{filename_display}') + artifacts = list(ROOT.joinpath("dist").glob(dist_pattern)) + filename_display = "\n".join(f" {a}" for a in artifacts) + print(f"[release] Will upload:\n{filename_display}") try: - input('[release] Release ready. ENTER to upload, CTRL-C to abort: ') + input("[release] Release ready. ENTER to upload, CTRL-C to abort: ") except KeyboardInterrupt: - print('\nAborted!') + print("\nAborted!") return - arg_display = ' '.join(f'"{n}"' for n in artifacts) + arg_display = " ".join(f'"{n}"' for n in artifacts) ctx.run(f'twine upload --repository="{repo}" {arg_display}') - @invoke.task def generate_markdown(ctx): - log('Generating markdown from changelog...') - ctx.run('pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md') + log("Generating markdown from changelog...") + ctx.run("pandoc CHANGELOG.rst -f rst -t markdown -o CHANGELOG.md") + + +@invoke.task +def generate_manual(ctx, commit=False): + log("Generating manual from reStructuredText source...") + ctx.run("make man -C docs") + ctx.run("cp docs/_build/man/pipenv.1 pipenv/") + if commit: + log("Commiting...") + ctx.run("git add pipenv/pipenv.1") + ctx.run('git commit -m "Update manual page."') + + +@invoke.task +def generate_contributing_md(ctx, commit=False): + log("Generating CONTRIBUTING.md from reStructuredText source...") + ctx.run("pandoc docs/dev/contributing.rst -f rst -t markdown -o CONTRIBUTING.md") + if commit: + log("Commiting...") + ctx.run("git add CONTRIBUTING.md") + ctx.run('git commit -m "Update CONTRIBUTING.md."') @invoke.task def generate_changelog(ctx, commit=False, draft=False): - log('Generating changelog...') + log("Generating changelog...") if draft: commit = False - log('Writing draft to file...') - ctx.run('towncrier --draft > CHANGELOG.draft.rst') + log("Writing draft to file...") + ctx.run("towncrier --draft > CHANGELOG.draft.rst") else: - ctx.run('towncrier') + ctx.run("towncrier") if commit: - log('Committing...') - ctx.run('git add CHANGELOG.rst') - ctx.run('git rm CHANGELOG.draft.rst') + log("Committing...") + ctx.run("git add CHANGELOG.rst") + ctx.run("git rm CHANGELOG.draft.rst") ctx.run('git commit -m "Update changelog."') @@ -182,7 +215,12 @@ def clean_mdchangelog(ctx, content=None): if not content: changelog = _get_git_root(ctx) / "CHANGELOG.md" content = changelog.read_text() - content = re.sub(r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/pypa/[\w\-]+/issues/\d+\)", r"\1 \2", content, flags=re.MULTILINE) + content = re.sub( + r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/pypa/[\w\-]+/issues/\d+\)", + r"\1 \2", + content, + flags=re.MULTILINE, + ) if changelog: changelog.write_text(content) else: @@ -193,12 +231,12 @@ def clean_mdchangelog(ctx, content=None): def tag_version(ctx, push=False): version = find_version(ctx) version = Version.parse(version) - log('Tagging revision: v%s' % version.normalize()) - ctx.run('git tag v%s' % version.normalize()) + log("Tagging revision: v%s" % version.normalize()) + ctx.run("git tag v%s" % version.normalize()) if push: - log('Pushing tags...') - ctx.run('git push origin master') - ctx.run('git push --tags') + log("Pushing tags...") + ctx.run("git push origin master") + ctx.run("git push --tags") @invoke.task @@ -206,32 +244,36 @@ def bump_version(ctx, dry_run=False, dev=False, pre=False, tag=None, commit=Fals current_version = Version.parse(__version__) today = datetime.date.today() tomorrow = today + datetime.timedelta(days=1) - next_month = datetime.date.today().replace(month=today.month+1, day=1) - next_year = datetime.date.today().replace(year=today.year+1, month=1, day=1) if pre and not tag: print('Using "pre" requires a corresponding tag.') return if not (dev or pre or tag): - new_version = current_version.replace(release=today.timetuple()[:3]).clear(pre=True, dev=True) + new_version = current_version.replace(release=today.timetuple()[:3]).clear( + pre=True, dev=True + ) if pre and dev: raise RuntimeError("Can't use 'pre' and 'dev' together!") if dev or pre: - new_version = current_version.replace(release=tomorrow.timetuple()[:3]).clear(pre=True, dev=True) + new_version = current_version.replace(release=tomorrow.timetuple()[:3]).clear( + pre=True, dev=True + ) if dev: new_version = new_version.bump_dev() else: new_version = new_version.bump_pre(tag=tag) - log('Updating version to %s' % new_version.normalize()) + log("Updating version to %s" % new_version.normalize()) version = find_version(ctx) - log('Found current version: %s' % version) + log("Found current version: %s" % version) if dry_run: - log('Would update to: %s' % new_version.normalize()) + log("Would update to: %s" % new_version.normalize()) else: - log('Updating to: %s' % new_version.normalize()) + log("Updating to: %s" % new_version.normalize()) version_file = get_version_file(ctx) file_contents = version_file.read_text() - version_file.write_text(file_contents.replace(version, str(new_version.normalize()))) + version_file.write_text( + file_contents.replace(version, str(new_version.normalize())) + ) if commit: - ctx.run('git add {0}'.format(version_file.as_posix())) - log('Committing...') + ctx.run("git add {0}".format(version_file.as_posix())) + log("Committing...") ctx.run('git commit -s -m "Bumped version."') diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index c9757965..3c0cb22b 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -4,9 +4,12 @@ """"Vendoring script, python 3.5 needed""" import io +import itertools +import json import re import shutil import sys + # from tempfile import TemporaryDirectory import tarfile import zipfile @@ -22,73 +25,74 @@ from urllib3.util import parse_url as urllib3_parse from pipenv.utils import mkdir_p from pipenv.vendor.vistir.compat import NamedTemporaryFile, TemporaryDirectory from pipenv.vendor.vistir.contextmanagers import open_file +from pipenv.vendor.requirementslib.models.lockfile import Lockfile, merge_items +import pipenv.vendor.parse as parse -TASK_NAME = 'update' +TASK_NAME = "update" LIBRARY_DIRNAMES = { - 'requirements-parser': 'requirements', - 'backports.shutil_get_terminal_size': 'backports/shutil_get_terminal_size', - 'backports.weakref': 'backports/weakref', - 'backports.functools_lru_cache': 'backports/functools_lru_cache', - 'shutil_backports': 'backports/shutil_get_terminal_size', - 'python-dotenv': 'dotenv', - 'pip-tools': 'piptools', - 'setuptools': 'pkg_resources', - 'msgpack-python': 'msgpack', - 'attrs': 'attr', - 'enum': 'backports/enum' + "requirements-parser": "requirements", + "backports.shutil_get_terminal_size": "backports/shutil_get_terminal_size", + "backports.weakref": "backports/weakref", + "backports.functools_lru_cache": "backports/functools_lru_cache", + "python-dotenv": "dotenv", + "pip-tools": "piptools", + "setuptools": "pkg_resources", + "msgpack-python": "msgpack", + "attrs": "attr", + "enum": "backports/enum", } -PY2_DOWNLOAD = ['enum34',] +PY2_DOWNLOAD = ["enum34"] # from time to time, remove the no longer needed ones HARDCODED_LICENSE_URLS = { - 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', - 'cursor': 'https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE', - 'delegator.py': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', - 'click-didyoumean': 'https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE', - 'click-completion': 'https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE', - 'blindspin': 'https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE', - 'shutilwhich': 'https://raw.githubusercontent.com/mbr/shutilwhich/master/LICENSE', - 'parse': 'https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE', - 'semver': 'https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt', - 'crayons': 'https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE', - 'pip-tools': 'https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE', - 'pytoml': 'https://github.com/avakar/pytoml/raw/master/LICENSE', - 'webencodings': 'https://github.com/SimonSapin/python-webencodings/raw/' - 'master/LICENSE', - 'requirementslib': 'https://github.com/techalchemy/requirementslib/raw/master/LICENSE', - 'distlib': 'https://github.com/vsajip/distlib/raw/master/LICENSE.txt', - 'pythonfinder': 'https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt', - 'pyparsing': 'https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE', - 'resolvelib': 'https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE' + "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE", + "cursor": "https://raw.githubusercontent.com/GijsTimmers/cursor/master/LICENSE", + "delegator.py": "https://raw.githubusercontent.com/kennethreitz/delegator.py/master/LICENSE", + "click-didyoumean": "https://raw.githubusercontent.com/click-contrib/click-didyoumean/master/LICENSE", + "click-completion": "https://raw.githubusercontent.com/click-contrib/click-completion/master/LICENSE", + "parse": "https://raw.githubusercontent.com/techalchemy/parse/master/LICENSE", + "semver": "https://raw.githubusercontent.com/k-bx/python-semver/master/LICENSE.txt", + "crayons": "https://raw.githubusercontent.com/kennethreitz/crayons/master/LICENSE", + "pip-tools": "https://raw.githubusercontent.com/jazzband/pip-tools/master/LICENSE", + "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE", + "webencodings": "https://github.com/SimonSapin/python-webencodings/raw/" + "master/LICENSE", + "requirementslib": "https://github.com/techalchemy/requirementslib/raw/master/LICENSE", + "distlib": "https://github.com/vsajip/distlib/raw/master/LICENSE.txt", + "pythonfinder": "https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt", + "pyparsing": "https://raw.githubusercontent.com/pyparsing/pyparsing/master/LICENSE", + "resolvelib": "https://raw.githubusercontent.com/sarugaku/resolvelib/master/LICENSE", + "funcsigs": "https://raw.githubusercontent.com/aliles/funcsigs/master/LICENSE", } FILE_WHITE_LIST = ( - 'Makefile', - 'vendor.txt', - 'patched.txt', - '__init__.py', - 'README.rst', - 'README.md', - 'appdirs.py', - 'safety.zip', - 'cacert.pem', - 'vendor_pip.txt', + "Makefile", + "vendor.txt", + "patched.txt", + "__init__.py", + "README.rst", + "README.md", + "appdirs.py", + "safety.zip", + "cacert.pem", + "vendor_pip.txt", ) -PATCHED_RENAMES = { - 'pip': 'notpip' -} +PATCHED_RENAMES = {"pip": "notpip"} LIBRARY_RENAMES = { - 'pip': 'pipenv.patched.notpip', + "pip": "pipenv.patched.notpip", "functools32": "pipenv.vendor.backports.functools_lru_cache", - 'enum34': 'enum', + "enum34": "enum", } +LICENSE_RENAMES = {"pythonfinder/LICENSE": "pythonfinder/pep514tools.LICENSE"} + + def drop_dir(path): if path.exists() and path.is_dir(): shutil.rmtree(str(path), ignore_errors=True) @@ -103,32 +107,32 @@ def remove_all(paths): def log(msg): - print('[vendoring.%s] %s' % (TASK_NAME, msg)) + print("[vendoring.%s] %s" % (TASK_NAME, msg)) def _get_git_root(ctx): - return Path(ctx.run('git rev-parse --show-toplevel', hide=True).stdout.strip()) + return Path(ctx.run("git rev-parse --show-toplevel", hide=True).stdout.strip()) def _get_vendor_dir(ctx): - return _get_git_root(ctx) / 'pipenv' / 'vendor' + return _get_git_root(ctx) / "pipenv" / "vendor" def _get_patched_dir(ctx): - return _get_git_root(ctx) / 'pipenv' / 'patched' + return _get_git_root(ctx) / "pipenv" / "patched" def clean_vendor(ctx, vendor_dir): # Old _vendor cleanup - remove_all(vendor_dir.glob('*.pyc')) - log('Cleaning %s' % vendor_dir) + remove_all(vendor_dir.glob("*.pyc")) + log("Cleaning %s" % vendor_dir) for item in vendor_dir.iterdir(): if item.is_dir(): shutil.rmtree(str(item)) elif item.name not in FILE_WHITE_LIST: item.unlink() else: - log('Skipping %s' % item) + log("Skipping %s" % item) def detect_vendored_libs(vendor_dir): @@ -149,7 +153,7 @@ def rewrite_imports(package_dir, vendored_libs, vendor_dir): for item in package_dir.iterdir(): if item.is_dir(): rewrite_imports(item, vendored_libs, vendor_dir) - elif item.name.endswith('.py'): + elif item.name.endswith(".py"): rewrite_file_imports(item, vendored_libs, vendor_dir) @@ -157,9 +161,9 @@ def rewrite_file_imports(item, vendored_libs, vendor_dir): """Rewrite 'import xxx' and 'from xxx import' for vendored_libs""" # log('Reading file: %s' % item) try: - text = item.read_text(encoding='utf-8') + text = item.read_text(encoding="utf-8") except UnicodeDecodeError: - text = item.read_text(encoding='cp1252') + text = item.read_text(encoding="cp1252") renames = LIBRARY_RENAMES for k in LIBRARY_RENAMES.keys(): if k not in vendored_libs: @@ -169,111 +173,165 @@ def rewrite_file_imports(item, vendored_libs, vendor_dir): if lib in renames: to_lib = renames[lib] text = re.sub( - r'([\n\s]*)import %s([\n\s\.]+)' % lib, - r'\1import %s\2' % to_lib, - text, - ) - text = re.sub( - r'([\n\s]*)from %s([\s\.])+' % lib, - r'\1from %s\2' % to_lib, - text, + r"([\n\s]*)import %s([\n\s\.]+)" % lib, r"\1import %s\2" % to_lib, text, ) + text = re.sub(r"([\n\s]*)from %s([\s\.])+" % lib, r"\1from %s\2" % to_lib, text,) text = re.sub( r"(\n\s*)__import__\('%s([\s'\.])+" % lib, r"\1__import__('%s\2" % to_lib, text, ) - item.write_text(text, encoding='utf-8') + item.write_text(text, encoding="utf-8") def apply_patch(ctx, patch_file_path): - log('Applying patch %s' % patch_file_path.name) - ctx.run('git apply --ignore-whitespace --verbose %s' % patch_file_path) + log("Applying patch %s" % patch_file_path.name) + ctx.run("git apply --ignore-whitespace --verbose %s" % patch_file_path) + + +def _recursive_write_to_zip(zf, path, root=None): + if path == Path(zf.filename): + return + if root is None: + if not path.is_dir(): + raise ValueError('root is required for non-directory path') + root = path + if not path.is_dir(): + zf.write(str(path), str(path.relative_to(root))) + return + for c in path.iterdir(): + _recursive_write_to_zip(zf, c, root) @invoke.task def update_safety(ctx): - ignore_subdeps = ['pip', 'pip-egg-info', 'bin'] - ignore_files = ['pip-delete-this-directory.txt', 'PKG-INFO'] - vendor_dir = _get_patched_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) - log('Downloading safety package files...') - build_dir = vendor_dir / 'build' - download_dir = TemporaryDirectory(prefix='pipenv-', suffix='-safety') - if build_dir.exists() and build_dir.is_dir(): - drop_dir(build_dir) - - ctx.run( - 'pip download -b {0} --no-binary=:all: --no-clean -d {1} safety pyyaml'.format( - str(build_dir), str(download_dir.name), + ignore_subdeps = ["pip", "pip-egg-info", "bin", "pipenv", "virtualenv", "virtualenv-clone", "setuptools",] + ignore_files = ["pip-delete-this-directory.txt", "PKG-INFO", "easy_install.py", "clonevirtualenv.py"] + ignore_patterns = ["*.pyd", "*.so", "**/*.pyc", "*.pyc"] + cmd_envvars = { + "PIPENV_NO_INHERIT": "true", + "PIPENV_IGNORE_VIRTUALENVS": "true", + "PIPENV_VENV_IN_PROJECT": "true" + } + patched_dir = _get_patched_dir(ctx) + vendor_dir = _get_vendor_dir(ctx) + safety_dir = Path(__file__).absolute().parent.joinpath("safety") + log("Using vendor dir: %s" % patched_dir) + log("Downloading safety package files...") + build_dir = patched_dir / "build" + root = _get_git_root(ctx) + with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: + log("generating lockfile...") + packages = "\n".join(["safety", "requests[security]"]) + env = {"PIPENV_PACKAGES": packages} + resolve_cmd = "python {0}".format(root.joinpath("pipenv/resolver.py").as_posix()) + py27_resolve_cmd = "python2.7 {0}".format(root.joinpath("pipenv/resolver.py").as_posix()) + _, _, resolved = ctx.run(resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") + _, _, resolved_py2 = ctx.run(py27_resolve_cmd, hide=True, env=env).stdout.partition("RESULTS:") + resolved = json.loads(resolved.strip()) + resolved_py2 = json.loads(resolved_py2.strip()) + pkg_dict, pkg_dict_py2 = {}, {} + for pkg in resolved: + name = pkg.pop("name") + pkg["version"] = "=={0}".format(pkg["version"]) + pkg_dict[name] = pkg + for pkg in resolved_py2: + name = pkg.pop("name") + pkg["version"] = "=={0}".format(pkg["version"]) + pkg_dict_py2[name] = pkg + merged = merge_items([pkg_dict, pkg_dict_py2]) + lf = Lockfile.create(safety_dir.as_posix()) + lf["default"] = merged + lf.write() + # envvars_no_deps = {"PIP_NO_DEPS": "true"}.update(cmd_envvars) + # ctx.run("python -m pipenv run pip install safety", env=envvars_no_deps) + # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) + # ctx.run("python -m pipenv install safety", env=cmd_envvars) + # ctx.run("python -m pipenv run pip uninstall -y pipenv", env=cmd_envvars) + # ctx.run("python2.7 -m pip install --upgrade --upgrade-strategy=eager -e {}".format(root.as_posix())) + # ctx.run("python2.7 -m pipenv install safety", env=cmd_envvars) + # requirements_txt = ctx.run("python2.7 -m pipenv lock -r", env=cmd_envvars, quiet=True).out + requirements = [ + r.as_line(include_hashes=False, include_markers=False) + for r in lf.requirements + ] + safety_dir.joinpath("requirements.txt").write_text("\n".join(requirements)) + if build_dir.exists() and build_dir.is_dir(): + log("dropping pre-existing build dir at {0}".format(build_dir.as_posix())) + drop_dir(build_dir) + pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( + build_dir.absolute().as_posix(), str(download_dir.name), ) - ) - safety_dir = build_dir / 'safety' - yaml_build_dir = build_dir / 'pyyaml' - main_file = safety_dir / '__main__.py' - main_content = """ -import sys -yaml_lib = 'yaml{0}'.format(sys.version_info[0]) -locals()[yaml_lib] = __import__(yaml_lib) -sys.modules['yaml'] = sys.modules[yaml_lib] -from safety.cli import cli + log("downloading deps via pip: {0}".format(pip_command)) + ctx.run(pip_command) + safety_build_dir = build_dir / "safety" + yaml_build_dir = build_dir / "pyyaml" + lib_dir = safety_dir.joinpath("lib") -# Disable insecure warnings. -import urllib3 -from urllib3.exceptions import InsecureRequestWarning -urllib3.disable_warnings(InsecureRequestWarning) - -cli(prog_name="safety") - """.strip() - with open(str(main_file), 'w') as fh: - fh.write(main_content) - - with ctx.cd(str(safety_dir)): - ctx.run('pip install --no-compile --no-binary=:all: -t . .') - safety_dir = safety_dir.absolute() - yaml_dir = safety_dir / 'yaml' - if yaml_dir.exists(): - version_choices = ['2', '3'] - version_choices.remove(str(sys.version_info[0])) - mkdir_p(str(safety_dir / 'yaml{0}'.format(sys.version_info[0]))) - for fn in yaml_dir.glob('*.py'): - fn.rename(str(safety_dir.joinpath('yaml{0}'.format(sys.version_info[0]), fn.name))) - if version_choices[0] == '2': - lib = yaml_build_dir / 'lib' / 'yaml' - else: - lib = yaml_build_dir / 'lib3' / 'yaml' - shutil.copytree(str(lib.absolute()), str(safety_dir / 'yaml{0}'.format(version_choices[0]))) - requests_dir = safety_dir / 'requests' - cacert = vendor_dir / 'requests' / 'cacert.pem' - if not cacert.exists(): - from pipenv.vendor import requests - cacert = Path(requests.certs.where()) - target_cert = requests_dir / 'cacert.pem' - target_cert.write_bytes(cacert.read_bytes()) - ctx.run("sed -i 's/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers)/r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, verify=False)/g' {0}".format(str(safety_dir / 'safety' / 'safety.py'))) - for egg in safety_dir.glob('*.egg-info'): - drop_dir(egg.absolute()) - for dep in ignore_subdeps: - dep_dir = safety_dir / dep - if dep_dir.exists(): - drop_dir(dep_dir) - for dep in ignore_files: - fn = safety_dir / dep - if fn.exists(): - fn.unlink() - zip_name = '{0}/safety'.format(str(vendor_dir)) - shutil.make_archive(zip_name, format='zip', root_dir=str(safety_dir), base_dir='./') - drop_dir(build_dir) - download_dir.cleanup() + with ctx.cd(str(safety_dir)): + lib_dir.mkdir(exist_ok=True) + install_cmd = "python2.7 -m pip install --ignore-requires-python -t {0} -r {1}".format(lib_dir.as_posix(), safety_dir.joinpath("requirements.txt").as_posix()) + log("installing dependencies: {0}".format(install_cmd)) + ctx.run(install_cmd) + safety_dir = safety_dir.absolute() + yaml_dir = lib_dir / "yaml" + yaml_lib_dir_map = { + "2": { + "current_path": yaml_build_dir / "lib/yaml", + "destination": lib_dir / "yaml2", + }, + "3": { + "current_path": yaml_build_dir / "lib3/yaml", + "destination": lib_dir / "yaml3", + }, + } + if yaml_dir.exists(): + drop_dir(yaml_dir) + log("Mapping yaml paths for python 2 and 3...") + for py_version, path_dict in yaml_lib_dir_map.items(): + path_dict["current_path"].rename(path_dict["destination"]) + log("Ensuring certificates are available...") + requests_dir = lib_dir / "requests" + cacert = vendor_dir / "certifi" / "cacert.pem" + if not cacert.exists(): + from pipenv.vendor import requests + cacert = Path(requests.certs.where()) + target_cert = requests_dir / "cacert.pem" + target_cert.write_bytes(cacert.read_bytes()) + log("dropping ignored files...") + for pattern in ignore_patterns: + for path in lib_dir.rglob(pattern): + log("removing {0!s}".format(path)) + path.unlink() + for dep in ignore_subdeps: + if lib_dir.joinpath(dep).exists(): + log("cleaning up {0}".format(dep)) + drop_dir(lib_dir.joinpath(dep)) + for path in itertools.chain.from_iterable(( + lib_dir.rglob("{0}*.egg-info".format(dep)), + lib_dir.rglob("{0}*.dist-info".format(dep)) + )): + log("cleaning up {0}".format(path)) + drop_dir(path) + for fn in ignore_files: + for path in lib_dir.rglob(fn): + log("cleaning up {0}".format(path)) + path.unlink() + zip_name = "{0}/safety.zip".format(str(patched_dir)) + log("writing zipfile...") + with zipfile.ZipFile(zip_name, 'w', compression=zipfile.ZIP_DEFLATED, compresslevel=6) as zf: + _recursive_write_to_zip(zf, safety_dir) + drop_dir(build_dir) + drop_dir(lib_dir) def rename_if_needed(ctx, vendor_dir, item): - rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES + rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES new_path = None if item.name in rename_dict or item.name in LIBRARY_DIRNAMES: new_name = rename_dict.get(item.name, LIBRARY_DIRNAMES.get(item.name)) new_path = item.parent / new_name - log('Renaming %s => %s' % (item.name, new_path)) + log("Renaming %s => %s" % (item.name, new_path)) # handle existing directories try: item.rename(str(new_path)) @@ -283,101 +341,172 @@ def rename_if_needed(ctx, vendor_dir, item): def write_backport_imports(ctx, vendor_dir): - backport_dir = vendor_dir / 'backports' + backport_dir = vendor_dir / "backports" if not backport_dir.exists(): return - backport_init = backport_dir / '__init__.py' + backport_init = backport_dir / "__init__.py" backport_libs = detect_vendored_libs(backport_dir) init_py_lines = backport_init.read_text().splitlines() for lib in backport_libs: - lib_line = 'from . import {0}'.format(lib) + lib_line = "from . import {0}".format(lib) if lib_line not in init_py_lines: - log('Adding backport %s to __init__.py exports' % lib) + log("Adding backport %s to __init__.py exports" % lib) init_py_lines.append(lib_line) - backport_init.write_text('\n'.join(init_py_lines) + '\n') + backport_init.write_text("\n".join(init_py_lines) + "\n") def _ensure_package_in_requirements(ctx, requirements_file, package): requirement = None - log('using requirements file: %s' % requirements_file) + log("using requirements file: %s" % requirements_file) req_file_lines = [l for l in requirements_file.read_text().splitlines()] if package: match = [r for r in req_file_lines if r.strip().lower().startswith(package)] matched_req = None if match: for m in match: - specifiers = [m.index(s) for s in ['>', '<', '=', '~'] if s in m] - if m.lower() == package or (specifiers and m[:min(specifiers)].lower() == package): + specifiers = [m.index(s) for s in [">", "<", "=", "~"] if s in m] + if m.lower() == package or ( + specifiers and m[: min(specifiers)].lower() == package + ): matched_req = "{0}".format(m) requirement = matched_req log("Matched req: %r" % matched_req) if not matched_req: req_file_lines.append("{0}".format(package)) log("Writing requirements file: %s" % requirements_file) - requirements_file.write_text('\n'.join(req_file_lines)) + requirements_file.write_text("\n".join(req_file_lines)) requirement = "{0}".format(package) return requirement +def install_pyyaml(ctx, vendor_dir): + build_dir = vendor_dir / "build" + if build_dir.exists() and build_dir.is_dir(): + log("dropping pre-existing build dir at {0}".format(build_dir.as_posix())) + drop_dir(build_dir) + with TemporaryDirectory(prefix="pipenv-", suffix="-safety") as download_dir: + pip_command = "pip download -b {0} --no-binary=:all: --no-clean --no-deps -d {1} pyyaml safety".format( + build_dir.absolute().as_posix(), str(download_dir.name), + ) + log("downloading deps via pip: {0}".format(pip_command)) + ctx.run(pip_command) + safety_build_dir = build_dir / "safety" + yaml_build_dir = build_dir / "pyyaml" + yaml_dir = vendor_dir / "yaml" + yaml_lib_dir_map = { + "2": { + "current_path": yaml_build_dir / "lib/yaml", + "destination": vendor_dir / "yaml2", + }, + "3": { + "current_path": yaml_build_dir / "lib3/yaml", + "destination": vendor_dir / "yaml3", + }, + } + if yaml_dir.exists(): + drop_dir(yaml_dir) + log("Mapping yaml paths for python 2 and 3...") + for py_version, path_dict in yaml_lib_dir_map.items(): + path_dict["current_path"].rename(path_dict["destination"]) + path_dict["destination"].joinpath("LICENSE").write_text(yaml_build_dir.joinpath("LICENSE").read_text()) + drop_dir(build_dir) + + def install(ctx, vendor_dir, package=None): requirements_file = vendor_dir / "{0}.txt".format(vendor_dir.name) requirement = "-r {0}".format(requirements_file.as_posix()) - log('Using requirements file: %s' % requirement) + log("Using requirements file: %s" % requirement) if package: requirement = _ensure_package_in_requirements(ctx, requirements_file, package) # We use --no-deps because we want to ensure that all of our dependencies # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} --no-compile --no-deps --upgrade {1}'.format( - vendor_dir.as_posix(), - requirement, + "pip install -t {0} --no-compile --no-deps --upgrade {1}".format( + vendor_dir.as_posix(), requirement, ) ) + # read licenses from distinfo files if possible + for path in vendor_dir.glob("*.dist-info"): + pkg, _, _ = path.stem.rpartition("-") + license_file = path / "LICENSE" + if not license_file.exists(): + continue + if vendor_dir.joinpath(pkg).exists(): + vendor_dir.joinpath(pkg).joinpath("LICENSE").write_text( + license_file.read_text() + ) + elif vendor_dir.joinpath("{0}.py".format(pkg)).exists(): + vendor_dir.joinpath("{0}.py.LICENSE".format(pkg)).write_text( + license_file.read_text() + ) + else: + matched_path = next( + iter(pth for pth in vendor_dir.glob("{0}*".format(pkg))), None + ) + if matched_path is not None: + vendor_dir.joinpath("{0}.LICENSE".format(matched_path)).write_text( + license_file.read_text() + ) def post_install_cleanup(ctx, vendor_dir): - remove_all(vendor_dir.glob('*.dist-info')) - remove_all(vendor_dir.glob('*.egg-info')) + remove_all(vendor_dir.glob("*.dist-info")) + remove_all(vendor_dir.glob("*.egg-info")) # Cleanup setuptools unneeded parts - drop_dir(vendor_dir / 'bin') - drop_dir(vendor_dir / 'tests') - remove_all(vendor_dir.glob('toml.py')) + drop_dir(vendor_dir / "bin") + drop_dir(vendor_dir / "tests") + drop_dir(vendor_dir / "shutil_backports") + remove_all(vendor_dir.glob("toml.py")) + + +@invoke.task +def apply_patches(ctx, patched=False, pre=False): + if patched: + vendor_dir = _get_patched_dir(ctx) + else: + vendor_dir = _get_vendor_dir(ctx) + log("Applying pre-patches...") + patch_dir = Path(__file__).parent / "patches" / vendor_dir.name + if pre: + if not patched: + pass + for patch in patch_dir.glob("*.patch"): + if not patch.name.startswith("_post"): + apply_patch(ctx, patch) + else: + patches = patch_dir.glob("*.patch" if not patched else "_post*.patch") + for patch in patches: + apply_patch(ctx, patch) def vendor(ctx, vendor_dir, package=None, rewrite=True): - log('Reinstalling vendored libraries') - is_patched = vendor_dir.name == 'patched' + log("Reinstalling vendored libraries") + is_patched = vendor_dir.name == "patched" install(ctx, vendor_dir, package=package) - log('Running post-install cleanup...') + log("Running post-install cleanup...") post_install_cleanup(ctx, vendor_dir) # Detect the vendored packages/modules vendored_libs = detect_vendored_libs(_get_vendor_dir(ctx)) - patched_libs = detect_vendored_libs(_get_patched_dir(ctx)) log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) # Apply pre-patches log("Applying pre-patches...") - patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name if is_patched: - for patch in patch_dir.glob('*.patch'): - if not patch.name.startswith('_post'): - apply_patch(ctx, patch) - + apply_patches(ctx, patched=is_patched, pre=True) log("Removing scandir library files...") - remove_all(vendor_dir.glob('*.so')) - drop_dir(vendor_dir / 'setuptools') - drop_dir(vendor_dir / 'pkg_resources' / '_vendor') - drop_dir(vendor_dir / 'pkg_resources' / 'extern') - drop_dir(vendor_dir / 'bin') + for extension in ("*.so", "*.pyd", "*.egg-info", "*.dist-info"): + remove_all(vendor_dir.glob(extension)) + for dirname in ("setuptools", "pkg_resources/_vendor", "pkg_resources/extern", "bin"): + drop_dir(vendor_dir / dirname) # Global import rewrites - log('Renaming specified libs...') + log("Renaming specified libs...") for item in vendor_dir.iterdir(): if item.is_dir(): if rewrite and not package or (package and item.name.lower() in package): - log('Rewriting imports for %s...' % item) + log("Rewriting imports for %s..." % item) rewrite_imports(item, vendored_libs, vendor_dir) rename_if_needed(ctx, vendor_dir, item) elif item.name not in FILE_WHITE_LIST: @@ -385,28 +514,25 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) if not package: - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) + apply_patches(ctx, patched=is_patched, pre=False) if is_patched: - piptools_vendor = vendor_dir / 'piptools' / '_vendored' + piptools_vendor = vendor_dir / "piptools" / "_vendored" if piptools_vendor.exists(): drop_dir(piptools_vendor) - msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' + msgpack = vendor_dir / "notpip" / "_vendor" / "msgpack" if msgpack.exists(): - remove_all(msgpack.glob('*.so')) + remove_all(msgpack.glob("*.so")) @invoke.task def redo_imports(ctx, library): vendor_dir = _get_vendor_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) vendored_libs = detect_vendored_libs(vendor_dir) item = vendor_dir / library - library_name = vendor_dir / '{0}.py'.format(library) + library_name = vendor_dir / "{0}.py".format(library) log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) - log('Rewriting imports for %s...' % item) + log("Rewriting imports for %s..." % item) if item.is_dir(): rewrite_imports(item, vendored_libs, vendor_dir) else: @@ -416,7 +542,7 @@ def redo_imports(ctx, library): @invoke.task def rewrite_all_imports(ctx): vendor_dir = _get_vendor_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) vendored_libs = detect_vendored_libs(vendor_dir) log("Detected vendored libraries: %s" % ", ".join(vendored_libs)) log("Rewriting all imports related to vendored libs") @@ -428,15 +554,21 @@ def rewrite_all_imports(ctx): @invoke.task -def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None): +def packages_missing_licenses( + ctx, vendor_dir=None, requirements_file="vendor.txt", package=None +): if not vendor_dir: vendor_dir = _get_vendor_dir(ctx) requirements = vendor_dir.joinpath(requirements_file).read_text().splitlines() new_requirements = [] - LICENSES = ["LICENSE-MIT", "LICENSE", "LICENSE.txt", "LICENSE.APACHE", "LICENSE.BSD"] + LICENSE_EXTS = ("rst", "txt", "APACHE", "BSD", "md") + LICENSES = [ + ".".join(lic) + for lic in itertools.product(("LICENSE", "LICENSE-MIT"), LICENSE_EXTS) + ] for i, req in enumerate(requirements): pkg = req.strip().split("=")[0] - possible_pkgs = [pkg, pkg.replace('-', '_')] + possible_pkgs = [pkg, pkg.replace("-", "_")] match_found = False if pkg in PY2_DOWNLOAD: match_found = True @@ -445,64 +577,91 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx possible_pkgs.append(LIBRARY_DIRNAMES[pkg]) for pkgpath in possible_pkgs: pkgpath = vendor_dir.joinpath(pkgpath) + py_path = pkgpath.parent / "{0}.py".format(pkgpath.stem) if pkgpath.exists() and pkgpath.is_dir(): - for licensepath in LICENSES: - licensepath = pkgpath.joinpath(licensepath) - if licensepath.exists(): + for license_path in LICENSES: + license_path = pkgpath.joinpath(license_path) + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break - elif (pkgpath.exists() or pkgpath.parent.joinpath("{0}.py".format(pkgpath.stem)).exists()): - for licensepath in LICENSES: - licensepath = pkgpath.parent.joinpath("{0}.{1}".format(pkgpath.stem, licensepath)) - if licensepath.exists(): + elif pkgpath.exists() or py_path.exists(): + for license_path in LICENSES: + license_name = "{0}.{1}".format(pkgpath.stem, license_path) + license_path = pkgpath.parent / license_name + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break if match_found: break if match_found: continue else: - # log("%s: No license found in %s" % (pkg, pkgpath)) + # log("%s: No license found in %s" % (pkg, pkgpath)) new_requirements.append(req) return new_requirements @invoke.task -def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, patched=False): - log('Downloading licenses') +def download_licenses( + ctx, + vendor_dir=None, + requirements_file="vendor.txt", + package=None, + only=False, + patched=False, +): + log("Downloading licenses") if not vendor_dir: if patched: vendor_dir = _get_patched_dir(ctx) - requirements_file = 'patched.txt' + requirements_file = "patched.txt" else: vendor_dir = _get_vendor_dir(ctx) requirements_file = vendor_dir / requirements_file - requirements = packages_missing_licenses(ctx, vendor_dir, requirements_file, package=package) + requirements = packages_missing_licenses( + ctx, vendor_dir, requirements_file, package=package + ) - with NamedTemporaryFile(prefix="pipenv", suffix="vendor-reqs", delete=False, mode="w") as fh: + with NamedTemporaryFile( + prefix="pipenv", suffix="vendor-reqs", delete=False, mode="w" + ) as fh: fh.write("\n".join(requirements)) new_requirements_file = fh.name new_requirements_file = Path(new_requirements_file) log(requirements) - requirement = "-r {0}".format(new_requirements_file.as_posix()) - if package: - if not only: - # for packages we want to add to the requirements file - requirement = _ensure_package_in_requirements(ctx, requirements_file, package) - else: - # for packages we want to get the license for by themselves - requirement = package - tmp_dir = vendor_dir / '__tmp__' + tmp_dir = vendor_dir / "__tmp__" # TODO: Fix this whenever it gets sorted out (see https://github.com/pypa/pip/issues/5739) - ctx.run('pip install flit') # needed for the next step - ctx.run( - 'pip download --no-binary :all: --only-binary requests_download --no-build-isolation --no-deps -d {0} {1}'.format( - tmp_dir.as_posix(), - requirement, - ) - ) + cmd = "pip download --no-binary :all: --only-binary requests_download --no-deps" + enum_cmd = "pip download --no-deps" + ctx.run("pip install flit") # needed for the next step + for req in requirements_file.read_text().splitlines(): + if req.startswith("enum34"): + exe_cmd = "{0} -d {1} {2}".format(enum_cmd, tmp_dir.as_posix(), req) + else: + exe_cmd = "{0} --no-build-isolation -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + try: + ctx.run(exe_cmd) + except invoke.exceptions.UnexpectedExit as e: + if "Disabling PEP 517 processing is invalid" not in e.result.stderr: + log("WARNING: Failed to download license for {0}".format(req)) + continue + parse_target = ( + "Disabling PEP 517 processing is invalid: project specifies a build " + "backend of {backend} in pyproject.toml" + ) + target = parse.parse(parse_target, e.result.stderr.strip()) + backend = target.named.get("backend") + if backend is not None: + if "." in backend: + backend, _, _ = backend.partition(".") + ctx.run("pip install {0}".format(backend)) + ctx.run( + "{0} --no-build-isolation -d {1} {2}".format(cmd, tmp_dir.as_posix(), req) + ) for sdist in tmp_dir.iterdir(): extract_license(vendor_dir, sdist) new_requirements_file.unlink() @@ -510,18 +669,18 @@ def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', pack def extract_license(vendor_dir, sdist): - if sdist.stem.endswith('.tar'): + if sdist.stem.endswith(".tar"): ext = sdist.suffix[1:] - with tarfile.open(sdist, mode='r:{}'.format(ext)) as tar: + with tarfile.open(sdist, mode="r:{}".format(ext)) as tar: found = find_and_extract_license(vendor_dir, tar, tar.getmembers()) - elif sdist.suffix == '.zip': + elif sdist.suffix in (".zip", ".whl"): with zipfile.ZipFile(sdist) as zip: found = find_and_extract_license(vendor_dir, zip, zip.infolist()) else: - raise NotImplementedError('new sdist type!') + raise NotImplementedError("new sdist type!") if not found: - log('License not found in {}, will download'.format(sdist.name)) + log("License not found in {}, will download".format(sdist.name)) license_fallback(vendor_dir, sdist.name) @@ -532,10 +691,10 @@ def find_and_extract_license(vendor_dir, tar, members): name = member.name except AttributeError: # zipfile name = member.filename - if 'LICENSE' in name or 'COPYING' in name: - if '/test' in name: + if "LICENSE" in name or "COPYING" in name: + if "/test" in name: # some testing licenses in hml5lib and distlib - log('Ignoring {}'.format(name)) + log("Ignoring {}".format(name)) continue found = True extract_license_member(vendor_dir, tar, member, name) @@ -546,13 +705,13 @@ def license_fallback(vendor_dir, sdist_name): """Hardcoded license URLs. Check when updating if those are still needed""" libname = libname_from_dir(sdist_name) if libname not in HARDCODED_LICENSE_URLS: - raise ValueError('No hardcoded URL for {} license'.format(libname)) + raise ValueError("No hardcoded URL for {} license".format(libname)) url = HARDCODED_LICENSE_URLS[libname] - _, _, name = url.rpartition('/') + _, _, name = url.rpartition("/") dest = license_destination(vendor_dir, libname, name) r = requests.get(url, allow_redirects=True) - log('Downloading {}'.format(url)) + log("Downloading {}".format(url)) r.raise_for_status() dest.write_bytes(r.content) @@ -560,11 +719,11 @@ def license_fallback(vendor_dir, sdist_name): def libname_from_dir(dirname): """Reconstruct the library name without it's version""" parts = [] - for part in dirname.split('-'): + for part in dirname.split("-"): if part[0].isdigit(): break parts.append(part) - return '-'.join(parts) + return "-".join(parts) def license_destination(vendor_dir, libname, filename): @@ -572,10 +731,10 @@ def license_destination(vendor_dir, libname, filename): normal = vendor_dir / libname if normal.is_dir(): return normal / filename - lowercase = vendor_dir / libname.lower().replace('-', '_') + lowercase = vendor_dir / libname.lower().replace("-", "_") if lowercase.is_dir(): return lowercase / filename - rename_dict = LIBRARY_RENAMES if vendor_dir.name != 'patched' else PATCHED_RENAMES + rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES # Short circuit all logic if we are renaming the whole library if libname in rename_dict: return vendor_dir / rename_dict[libname] / filename @@ -583,12 +742,15 @@ def license_destination(vendor_dir, libname, filename): override = vendor_dir / LIBRARY_DIRNAMES[libname] if not override.exists() and override.parent.exists(): # for flattened subdeps, specifically backports/weakref.py - return ( - vendor_dir / override.parent - ) / '{0}.{1}'.format(override.name, filename) + return (vendor_dir / override.parent) / "{0}.{1}".format( + override.name, filename + ) + license_path = Path(LIBRARY_DIRNAMES[libname]) / filename + if license_path.as_posix() in LICENSE_RENAMES: + return vendor_dir / LICENSE_RENAMES[license_path.as_posix()] return vendor_dir / LIBRARY_DIRNAMES[libname] / filename # fallback to libname.LICENSE (used for nondirs) - return vendor_dir / '{}.{}'.format(libname, filename) + return vendor_dir / "{}.{}".format(libname, filename) def extract_license_member(vendor_dir, tar, member, name): @@ -596,7 +758,7 @@ def extract_license_member(vendor_dir, tar, member, name): dirname = list(mpath.parents)[-2].name # -1 is . libname = libname_from_dir(dirname) dest = license_destination(vendor_dir, libname, mpath.name) - log('Extracting {} into {}'.format(name, dest)) + log("Extracting {} into {}".format(name, dest)) try: fileobj = tar.extractfile(member) dest.write_bytes(fileobj.read()) @@ -605,29 +767,96 @@ def extract_license_member(vendor_dir, tar, member, name): @invoke.task() -def generate_patch(ctx, package_path, patch_description, base='HEAD'): +def generate_patch(ctx, package_path, patch_description, base="HEAD"): pkg = Path(package_path) - if len(pkg.parts) != 2 or pkg.parts[0] not in ('vendor', 'patched'): - raise ValueError('example usage: generate-patch patched/piptools some-description') + if len(pkg.parts) != 2 or pkg.parts[0] not in ("vendor", "patched"): + raise ValueError( + "example usage: generate-patch patched/piptools some-description" + ) if patch_description: - patch_fn = '{0}-{1}.patch'.format(pkg.parts[1], patch_description) + patch_fn = "{0}-{1}.patch".format(pkg.parts[1], patch_description) else: - patch_fn = '{0}.patch'.format(pkg.parts[1]) - command = 'git diff {base} -p {root} > {out}'.format( + patch_fn = "{0}.patch".format(pkg.parts[1]) + command = "git diff {base} -p {root} > {out}".format( base=base, - root=Path('pipenv').joinpath(pkg), - out=Path(__file__).parent.joinpath('patches', pkg.parts[0], patch_fn), + root=Path("pipenv").joinpath(pkg), + out=Path(__file__).parent.joinpath("patches", pkg.parts[0], patch_fn), ) with ctx.cd(str(_get_git_root(ctx))): log(command) ctx.run(command) +@invoke.task() +def update_pip_deps(ctx): + patched_dir = _get_patched_dir(ctx) + base_vendor_dir = _get_vendor_dir(ctx) + base_vendor_file = base_vendor_dir / "vendor_pip.txt" + pip_dir = patched_dir / "notpip" + vendor_dir = pip_dir / "_vendor" + vendor_file = vendor_dir / "vendor.txt" + vendor_file.write_bytes(base_vendor_file.read_bytes()) + download_licenses(ctx, vendor_dir) + + +@invoke.task +def download_all_licenses(ctx, include_pip=False): + vendor_dir = _get_vendor_dir(ctx) + patched_dir = _get_patched_dir(ctx) + download_licenses(ctx, vendor_dir) + download_licenses(ctx, patched_dir, "patched.txt") + if include_pip: + update_pip_deps(ctx) + + +def unpin_file(contents): + requirements = [] + for line in contents.splitlines(): + if "==" in line: + line, _, _ = line.strip().partition("=") + if not line.startswith("#"): + requirements.append(line) + return "\n".join(sorted(requirements)) + + +def unpin_and_copy_requirements(ctx, requirement_file, name="requirements.txt"): + with TemporaryDirectory() as tempdir: + target = Path(tempdir.name).joinpath("requirements.txt") + contents = unpin_file(requirement_file.read_text()) + target.write_text(contents) + env = { + "PIPENV_IGNORE_VIRTUALENVS": "1", + "PIPENV_NOSPIN": "1", + "PIPENV_PYTHON": "2.7", + } + with ctx.cd(tempdir.name): + ctx.run("pipenv install -r {0}".format(target.as_posix()), env=env, hide=True) + result = ctx.run("pipenv lock -r", env=env, hide=True).stdout.strip() + ctx.run("pipenv --rm", env=env, hide=True) + result = list(sorted([line.strip() for line in result.splitlines()[1:]])) + new_requirements = requirement_file.parent.joinpath(name) + requirement_file.rename( + requirement_file.parent.joinpath("{}.bak".format(name)) + ) + new_requirements.write_text("\n".join(result)) + return result + + +@invoke.task +def unpin_and_update_vendored(ctx, vendor=True, patched=False): + if vendor: + vendor_file = _get_vendor_dir(ctx) / "vendor.txt" + unpin_and_copy_requirements(ctx, vendor_file, name="vendor.txt") + if patched: + patched_file = _get_patched_dir(ctx) / "patched.txt" + unpin_and_copy_requirements(ctx, patched_file, name="patched.txt") + + @invoke.task(name=TASK_NAME) def main(ctx, package=None): vendor_dir = _get_vendor_dir(ctx) patched_dir = _get_patched_dir(ctx) - log('Using vendor dir: %s' % vendor_dir) + log("Using vendor dir: %s" % vendor_dir) if package: vendor(ctx, vendor_dir, package=package) download_licenses(ctx, vendor_dir, package=package) @@ -636,20 +865,20 @@ def main(ctx, package=None): clean_vendor(ctx, vendor_dir) clean_vendor(ctx, patched_dir) vendor(ctx, vendor_dir) + install_pyyaml(ctx, patched_dir) vendor(ctx, patched_dir, rewrite=True) - download_licenses(ctx, vendor_dir) - download_licenses(ctx, patched_dir, 'patched.txt') - for pip_dir in [patched_dir / 'notpip']: - _vendor_dir = pip_dir / '_vendor' - vendor_src_file = vendor_dir / 'vendor_pip.txt' - vendor_file = _vendor_dir / 'vendor.txt' - vendor_file.write_bytes(vendor_src_file.read_bytes()) - download_licenses(ctx, _vendor_dir) + download_all_licenses(ctx, include_pip=True) # from .vendor_passa import vendor_passa # log("Vendoring passa...") # vendor_passa(ctx) # update_safety(ctx) - log('Revendoring complete') + log("Revendoring complete") + + +@invoke.task +def install_yaml(ctx): + patched_dir = _get_patched_dir(ctx) + install_pyyaml(ctx, patched_dir) @invoke.task diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch index fce6ae89..792a94fa 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-pep425tags.patch @@ -1,8 +1,19 @@ diff --git a/pipenv/patched/notpip/_internal/pep425tags.py b/pipenv/patched/notpip/_internal/pep425tags.py -index f3b9b5b4..182c1c88 100644 +index 042ba34b..58decc23 100644 --- a/pipenv/patched/notpip/_internal/pep425tags.py +++ b/pipenv/patched/notpip/_internal/pep425tags.py -@@ -159,7 +159,7 @@ def is_manylinux1_compatible(): +@@ -170,8 +170,9 @@ def is_linux_armhf(): + return False + # hard-float ABI can be detected from the ELF header of the running + # process ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + try: +- with open(sys.executable, 'rb') as f: ++ with open(sys_executable, 'rb') as f: + elf_header_raw = f.read(40) # read 40 first bytes of ELF header + except (IOError, OSError, TypeError): + return False +@@ -205,7 +206,7 @@ def is_manylinux1_compatible(): pass # Check glibc version. CentOS 5 uses glibc 2.5. @@ -10,4 +21,22 @@ index f3b9b5b4..182c1c88 100644 + return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 5) + def is_manylinux2010_compatible(): +@@ -223,7 +224,7 @@ def is_manylinux2010_compatible(): + pass + + # Check glibc version. CentOS 6 uses glibc 2.12. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 12) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 12) + + + def is_manylinux2014_compatible(): +@@ -249,7 +250,7 @@ def is_manylinux2014_compatible(): + pass + + # Check glibc version. CentOS 7 uses glibc 2.17. +- return pip._internal.utils.glibc.have_compatible_glibc(2, 17) ++ return pipenv.patched.notpip._internal.utils.glibc.have_compatible_glibc(2, 17) + + def get_darwin_arches(major, minor, machine): diff --git a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch b/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch deleted file mode 100644 index 99269a90..00000000 --- a/tasks/vendoring/patches/patched/_post-pip-update-pypi-uri.patch +++ /dev/null @@ -1,44 +0,0 @@ -diff --git a/pipenv/patched/notpip/_vendor/distlib/index.py b/pipenv/patched/notpip/_vendor/distlib/index.py -index 2406be21..7a87cdcf 100644 ---- a/pipenv/patched/notpip/_vendor/distlib/index.py -+++ b/pipenv/patched/notpip/_vendor/distlib/index.py -@@ -22,7 +22,7 @@ from .util import cached_property, zip_dir, ServerProxy - - logger = logging.getLogger(__name__) - --DEFAULT_INDEX = 'https://pypi.python.org/pypi' -+DEFAULT_INDEX = 'https://pypi.org/pypi' - DEFAULT_REALM = 'pypi' - - class PackageIndex(object): -diff --git a/pipenv/patched/notpip/_vendor/distlib/locators.py b/pipenv/patched/notpip/_vendor/distlib/locators.py -index 11d26361..cb05b184 100644 ---- a/pipenv/patched/notpip/_vendor/distlib/locators.py -+++ b/pipenv/patched/notpip/_vendor/distlib/locators.py -@@ -36,7 +36,7 @@ logger = logging.getLogger(__name__) - HASHER_HASH = re.compile(r'^(\w+)=([a-f0-9]+)') - CHARSET = re.compile(r';\s*charset\s*=\s*(.*)\s*$', re.I) - HTML_CONTENT_TYPE = re.compile('text/html|application/x(ht)?ml') --DEFAULT_INDEX = 'https://pypi.python.org/pypi' -+DEFAULT_INDEX = 'https://pypi.org/pypi' - - def get_all_distribution_names(url=None): - """ -@@ -197,7 +197,7 @@ class Locator(object): - is_downloadable = basename.endswith(self.downloadable_extensions) - if is_wheel: - compatible = is_compatible(Wheel(basename), self.wheel_tags) -- return (t.scheme == 'https', 'pypi.python.org' in t.netloc, -+ return (t.scheme == 'https', 'pypi.org' in t.netloc, - is_downloadable, is_wheel, compatible, basename) - - def prefer_url(self, url1, url2): -@@ -1046,7 +1046,7 @@ class AggregatingLocator(Locator): - # versions which don't conform to PEP 426 / PEP 440. - default_locator = AggregatingLocator( - JSONLocator(), -- SimpleScrapingLocator('https://pypi.python.org/simple/', -+ SimpleScrapingLocator('https://pypi.org/simple/', - timeout=3.0), - scheme='legacy') - diff --git a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch index e2f4f7de..f916579c 100644 --- a/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch +++ b/tasks/vendoring/patches/patched/_post-pip-update-requests-imports.patch @@ -1,5 +1,5 @@ diff --git a/pipenv/patched/notpip/_vendor/requests/packages.py b/pipenv/patched/notpip/_vendor/requests/packages.py -index 9582fa7..928d1bb 100644 +index 9582fa73..5fb6f07d 100644 --- a/pipenv/patched/notpip/_vendor/requests/packages.py +++ b/pipenv/patched/notpip/_vendor/requests/packages.py @@ -4,13 +4,13 @@ import sys @@ -7,7 +7,7 @@ index 9582fa7..928d1bb 100644 for package in ('urllib3', 'idna', 'chardet'): - vendored_package = "pip._vendor." + package -+ vendored_package = "notpip._vendor." + package ++ vendored_package = "pipenv.patched.notpip._vendor." + package locals()[package] = __import__(vendored_package) # This traversal is apparently necessary such that the identities are # preserved (requests.packages.urllib3.* is urllib3.*) @@ -15,7 +15,7 @@ index 9582fa7..928d1bb 100644 if mod == vendored_package or mod.startswith(vendored_package + '.'): - unprefixed_mod = mod[len("pip._vendor."):] - sys.modules['pip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] -+ unprefixed_mod = mod[len("notpip._vendor."):] -+ sys.modules['notpip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] ++ unprefixed_mod = mod[len("pipenv.patched.notpip._vendor."):] ++ sys.modules['pipenv.patched.notpip._vendor.requests.packages.' + unprefixed_mod] = sys.modules[mod] # Kinda cool, though, right? diff --git a/tasks/vendoring/patches/patched/pip18.patch b/tasks/vendoring/patches/patched/pip18.patch deleted file mode 100644 index f4e607c1..00000000 --- a/tasks/vendoring/patches/patched/pip18.patch +++ /dev/null @@ -1,535 +0,0 @@ -diff --git a/pipenv/patched/pip/_internal/download.py b/pipenv/patched/pip/_internal/download.py -index 96f3b65c..cc5b3d15 100644 ---- a/pipenv/patched/pip/_internal/download.py -+++ b/pipenv/patched/pip/_internal/download.py -@@ -19,6 +19,7 @@ from pip._vendor.lockfile import LockError - from pip._vendor.requests.adapters import BaseAdapter, HTTPAdapter - from pip._vendor.requests.auth import AuthBase, HTTPBasicAuth - from pip._vendor.requests.models import CONTENT_CHUNK_SIZE, Response -+from pip._vendor.requests.sessions import Session - from pip._vendor.requests.structures import CaseInsensitiveDict - from pip._vendor.requests.utils import get_netrc_auth - # NOTE: XMLRPC Client is not annotated in typeshed as on 2017-07-17, which is -@@ -69,7 +70,7 @@ def user_agent(): - Return a string representing the user agent. - """ - data = { -- "installer": {"name": "pip", "version": pip.__version__}, -+ "installer": {"name": "pip", "version": pipenv.patched.notpip.__version__}, - "python": platform.python_version(), - "implementation": { - "name": platform.python_implementation(), -diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py -index edc506bf..84d57dac 100644 ---- a/pipenv/patched/pip/_internal/utils/temp_dir.py -+++ b/pipenv/patched/pip/_internal/utils/temp_dir.py -@@ -3,8 +3,10 @@ from __future__ import absolute_import - import logging - import os.path - import tempfile -+import warnings - - from pip._internal.utils.misc import rmtree -+from pipenv.vendor.vistir.compat import finalize, ResourceWarning - - logger = logging.getLogger(__name__) - -@@ -45,6 +47,20 @@ class TempDirectory(object): - self.path = path - self.delete = delete - self.kind = kind -+ self._finalizer = None -+ if path: -+ self._register_finalizer() -+ -+ def _register_finalizer(self): -+ if self.delete and self.path: -+ self._finalizer = finalize( -+ self, -+ self._cleanup, -+ self.path, -+ warn_message=None -+ ) -+ else: -+ self._finalizer = None - - def __repr__(self): - return "<{} {!r}>".format(self.__class__.__name__, self.path) -@@ -72,11 +88,27 @@ class TempDirectory(object): - self.path = os.path.realpath( - tempfile.mkdtemp(prefix="pip-{}-".format(self.kind)) - ) -+ self._register_finalizer() - logger.debug("Created temporary directory: {}".format(self.path)) - -+ @classmethod -+ def _cleanup(cls, name, warn_message=None): -+ try: -+ rmtree(name) -+ except OSError: -+ pass -+ else: -+ if warn_message: -+ warnings.warn(warn_message, ResourceWarning) -+ - def cleanup(self): - """Remove the temporary directory created and reset state - """ -- if self.path is not None and os.path.exists(self.path): -- rmtree(self.path) -- self.path = None -+ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): -+ if os.path.exists(self.path): -+ try: -+ rmtree(self.path) -+ except OSError: -+ pass -+ else: -+ self.path = None -diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py -index 8c2f24f1..cdd48874 100644 ---- a/pipenv/patched/pip/_internal/index.py -+++ b/pipenv/patched/pip/_internal/index.py -@@ -246,6 +246,9 @@ class PackageFinder(object): - # The Session we'll use to make requests - self.session = session - -+ # Kenneth's Hack -+ self.extra = None -+ - # The valid tags to check potential found wheel candidates against - self.valid_tags = get_supported( - versions=versions, -@@ -298,6 +301,25 @@ class PackageFinder(object): - ) - self.dependency_links.extend(links) - -+ @staticmethod -+ def get_extras_links(links): -+ requires = [] -+ extras = {} -+ -+ current_list = requires -+ -+ for link in links: -+ if not link: -+ current_list = requires -+ if link.startswith('['): -+ current_list = [] -+ extras[link[1:-1]] = current_list -+ else: -+ current_list.append(link) -+ -+ return extras -+ -+ - @staticmethod - def _sort_locations(locations, expand_dir=False): - """ -@@ -350,7 +372,7 @@ class PackageFinder(object): - - return files, urls - -- def _candidate_sort_key(self, candidate): -+ def _candidate_sort_key(self, candidate, ignore_compatibility=True): - """ - Function used to generate link sort key for link tuples. - The greater the return value, the more preferred it is. -@@ -370,14 +392,19 @@ class PackageFinder(object): - if candidate.location.is_wheel: - # can raise InvalidWheelFilename - wheel = Wheel(candidate.location.filename) -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - raise UnsupportedWheel( - "%s is not a supported wheel for this platform. It " - "can't be sorted." % wheel.filename - ) - if self.prefer_binary: - binary_preference = 1 -- pri = -(wheel.support_index_min(self.valid_tags)) -+ tags = self.valid_tags if not ignore_compatibility else None -+ try: -+ pri = -(wheel.support_index_min(tags=tags)) -+ except TypeError: -+ pri = -(support_num) -+ - if wheel.build_tag is not None: - match = re.match(r'^(\d+)(.*)$', wheel.build_tag) - build_tag_groups = match.groups() -@@ -528,7 +555,10 @@ class PackageFinder(object): - - page_versions = [] - for page in self._get_pages(url_locations, project_name): -- logger.debug('Analyzing links from page %s', page.url) -+ try: -+ logger.debug('Analyzing links from page %s', page.url) -+ except AttributeError: -+ continue - with indent_log(): - page_versions.extend( - self._package_versions(page.iter_links(), search) -@@ -562,7 +592,7 @@ class PackageFinder(object): - dependency_versions - ) - -- def find_requirement(self, req, upgrade): -+ def find_requirement(self, req, upgrade, ignore_compatibility=False): - """Try to find a Link matching req - - Expects req, an InstallRequirement and upgrade, a boolean -@@ -672,7 +702,10 @@ class PackageFinder(object): - continue - seen.add(location) - -- page = self._get_page(location) -+ try: -+ page = self._get_page(location) -+ except requests.HTTPError: -+ continue - if page is None: - continue - -@@ -709,7 +742,7 @@ class PackageFinder(object): - logger.debug('Skipping link %s; %s', link, reason) - self.logged_links.add(link) - -- def _link_package_versions(self, link, search): -+ def _link_package_versions(self, link, search, ignore_compatibility=True): - """Return an InstallationCandidate or None""" - version = None - if link.egg_fragment: -@@ -725,12 +758,12 @@ class PackageFinder(object): - link, 'unsupported archive format: %s' % ext, - ) - return -- if "binary" not in search.formats and ext == wheel_ext: -+ if "binary" not in search.formats and ext == wheel_ext and not ignore_compatibility: - self._log_skipped_link( - link, 'No binaries permitted for %s' % search.supplied, - ) - return -- if "macosx10" in link.path and ext == '.zip': -+ if "macosx10" in link.path and ext == '.zip' and not ignore_compatibility: - self._log_skipped_link(link, 'macosx10 one') - return - if ext == wheel_ext: -@@ -744,7 +777,7 @@ class PackageFinder(object): - link, 'wrong project name (not %s)' % search.supplied) - return - -- if not wheel.supported(self.valid_tags): -+ if not wheel.supported(self.valid_tags) and not ignore_compatibility: - self._log_skipped_link( - link, 'it is not compatible with this Python') - return -@@ -780,14 +813,14 @@ class PackageFinder(object): - link.filename, link.requires_python) - support_this_python = True - -- if not support_this_python: -+ if not support_this_python and not ignore_compatibility: - logger.debug("The package %s is incompatible with the python" - "version in use. Acceptable python versions are:%s", - link, link.requires_python) - return - logger.debug('Found link %s, version: %s', link, version) - -- return InstallationCandidate(search.supplied, version, link) -+ return InstallationCandidate(search.supplied, version, link, link.requires_python) - - def _get_page(self, link): - return _get_html_page(link, session=self.session) -diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py -index c736de6c..a78566c1 100644 ---- a/pipenv/patched/pip/_internal/models/candidate.py -+++ b/pipenv/patched/pip/_internal/models/candidate.py -@@ -7,10 +7,11 @@ class InstallationCandidate(KeyBasedCompareMixin): - """Represents a potential "candidate" for installation. - """ - -- def __init__(self, project, version, location): -+ def __init__(self, project, version, location, requires_python=None): - self.project = project - self.version = parse_version(version) - self.location = location -+ self.requires_python = requires_python - - super(InstallationCandidate, self).__init__( - key=(self.project, self.version, self.location), -diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py -index 104bea33..ecf78b9a 100644 ---- a/pipenv/patched/pip/_internal/operations/prepare.py -+++ b/pipenv/patched/pip/_internal/operations/prepare.py -@@ -17,7 +17,7 @@ from pip._internal.exceptions import ( - from pip._internal.utils.compat import expanduser - from pip._internal.utils.hashes import MissingHashes - from pip._internal.utils.logging import indent_log --from pip._internal.utils.misc import display_path, normalize_path -+from pip._internal.utils.misc import display_path, normalize_path, rmtree - from pip._internal.vcs import vcs - - logger = logging.getLogger(__name__) -@@ -123,7 +123,11 @@ class IsSDist(DistAbstraction): - " and ".join(map(repr, sorted(missing))) - ) - -- self.req.run_egg_info() -+ try: -+ self.req.run_egg_info() -+ except (OSError, TypeError): -+ self.req._correct_build_location() -+ self.req.run_egg_info() - self.req.assert_source_matches_version() - - -@@ -205,16 +209,8 @@ class RequirementPreparer(object): - # installation. - # FIXME: this won't upgrade when there's an existing - # package unpacked in `req.source_dir` -- # package unpacked in `req.source_dir` - if os.path.exists(os.path.join(req.source_dir, 'setup.py')): -- raise PreviousBuildDirError( -- "pip can't proceed with requirements '%s' due to a" -- " pre-existing build directory (%s). This is " -- "likely due to a previous installation that failed" -- ". pip is being responsible and not assuming it " -- "can delete this. Please delete it and try again." -- % (req, req.source_dir) -- ) -+ rmtree(req.source_dir) - req.populate_link(finder, upgrade_allowed, require_hashes) - - # We can't hit this spot and have populate_link return None. -diff --git a/pipenv/patched/pip/_internal/pep425tags.py b/pipenv/patched/pip/_internal/pep425tags.py -index ab1a0298..763c0a24 100644 ---- a/pipenv/patched/pip/_internal/pep425tags.py -+++ b/pipenv/patched/pip/_internal/pep425tags.py -@@ -10,7 +10,11 @@ import sysconfig - import warnings - from collections import OrderedDict - --import pip._internal.utils.glibc -+try: -+ import pip._internal.utils.glibc -+except ImportError: -+ import pip.utils.glibc -+ - from pip._internal.utils.compat import get_extension_suffixes - - logger = logging.getLogger(__name__) -diff --git a/pipenv/patched/pip/_internal/req/req_install.py b/pipenv/patched/pip/_internal/req/req_install.py -index c2624fee..ee75acd6 100644 ---- a/pipenv/patched/pip/_internal/req/req_install.py -+++ b/pipenv/patched/pip/_internal/req/req_install.py -@@ -452,7 +452,8 @@ class InstallRequirement(object): - - with indent_log(): - script = SETUPTOOLS_SHIM % self.setup_py -- base_cmd = [sys.executable, '-c', script] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ base_cmd = [sys_executable, '-c', script] - if self.isolated: - base_cmd += ["--no-user-cfg"] - egg_info_cmd = base_cmd + ['egg_info'] -@@ -613,10 +614,11 @@ class InstallRequirement(object): - - with indent_log(): - # FIXME: should we do --install-headers here too? -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) - with self.build_env: - call_subprocess( - [ -- sys.executable, -+ sys_executable, - '-c', - SETUPTOOLS_SHIM % self.setup_py - ] + -@@ -834,7 +836,8 @@ class InstallRequirement(object): - - def get_install_args(self, global_options, record_filename, root, prefix, - pycompile): -- install_args = [sys.executable, "-u"] -+ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) -+ install_args = [sys_executable, "-u"] - install_args.append('-c') - install_args.append(SETUPTOOLS_SHIM % self.setup_py) - install_args += list(global_options) + \ -diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py -index b1983171..0bab231d 100644 ---- a/pipenv/patched/pip/_internal/req/req_set.py -+++ b/pipenv/patched/pip/_internal/req/req_set.py -@@ -12,13 +12,16 @@ logger = logging.getLogger(__name__) - - class RequirementSet(object): - -- def __init__(self, require_hashes=False, check_supported_wheels=True): -+ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): - """Create a RequirementSet. - """ - - self.requirements = OrderedDict() - self.require_hashes = require_hashes - self.check_supported_wheels = check_supported_wheels -+ if ignore_compatibility: -+ self.check_supported_wheels = False -+ self.ignore_compatibility = True if (check_supported_wheels is False or ignore_compatibility is True) else False - - # Mapping of alias: real_name - self.requirement_aliases = {} -@@ -171,7 +174,7 @@ class RequirementSet(object): - return self.requirements[name] - if name in self.requirement_aliases: - return self.requirements[self.requirement_aliases[name]] -- raise KeyError("No project with the name %r" % project_name) -+ pass - - def cleanup_files(self): - """Clean up files, remove builds.""" -diff --git a/pipenv/patched/pip/_internal/resolve.py b/pipenv/patched/pip/_internal/resolve.py -index 2d9f1c56..bedc2582 100644 ---- a/pipenv/patched/pip/_internal/resolve.py -+++ b/pipenv/patched/pip/_internal/resolve.py -@@ -35,7 +35,7 @@ class Resolver(object): - - def __init__(self, preparer, session, finder, wheel_cache, use_user_site, - ignore_dependencies, ignore_installed, ignore_requires_python, -- force_reinstall, isolated, upgrade_strategy): -+ force_reinstall, isolated, upgrade_strategy, ignore_compatibility=False): - super(Resolver, self).__init__() - assert upgrade_strategy in self._allowed_strategies - -@@ -55,7 +55,11 @@ class Resolver(object): - self.ignore_dependencies = ignore_dependencies - self.ignore_installed = ignore_installed - self.ignore_requires_python = ignore_requires_python -+ self.ignore_compatibility = ignore_compatibility - self.use_user_site = use_user_site -+ self.requires_python = None -+ if self.ignore_compatibility: -+ self.ignore_requires_python = True - - self._discovered_dependencies = defaultdict(list) - -@@ -237,7 +241,7 @@ class Resolver(object): - - return abstract_dist - -- def _resolve_one(self, requirement_set, req_to_install): -+ def _resolve_one(self, requirement_set, req_to_install, ignore_requires_python=False): - """Prepare a single requirements file. - - :return: A list of additional InstallRequirements to also install. -@@ -260,11 +264,17 @@ class Resolver(object): - try: - check_dist_requires_python(dist) - except UnsupportedPythonVersion as err: -- if self.ignore_requires_python: -+ if self.ignore_requires_python or self.ignore_compatibility: - logger.warning(err.args[0]) - else: - raise - -+ # A huge hack, by Kenneth Reitz. -+ try: -+ self.requires_python = check_dist_requires_python(dist, absorb=False) -+ except TypeError: -+ self.requires_python = None -+ - more_reqs = [] - - def add_req(subreq, extras_requested): -@@ -291,9 +301,13 @@ class Resolver(object): - # can refer to it when adding dependencies. - if not requirement_set.has_requirement(req_to_install.name): - # 'unnamed' requirements will get added here -+ available_requested = sorted( -+ set(dist.extras) & set(req_to_install.extras) -+ ) - req_to_install.is_direct = True - requirement_set.add_requirement( - req_to_install, parent_req_name=None, -+ extras_requested=available_requested, - ) - - if not self.ignore_dependencies: -@@ -317,6 +331,19 @@ class Resolver(object): - for subreq in dist.requires(available_requested): - add_req(subreq, extras_requested=available_requested) - -+ # Hack for deep-resolving extras. -+ for available in available_requested: -+ if hasattr(dist, '_DistInfoDistribution__dep_map'): -+ for req in dist._DistInfoDistribution__dep_map[available]: -+ req = install_req_from_req( -+ str(req), -+ req_to_install, -+ isolated=self.isolated, -+ wheel_cache=self.wheel_cache, -+ ) -+ -+ more_reqs.append(req) -+ - if not req_to_install.editable and not req_to_install.satisfied_by: - # XXX: --no-install leads this to report 'Successfully - # downloaded' for only non-editable reqs, even though we took -diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py -index c43142f0..f241cce0 100644 ---- a/pipenv/patched/pip/_internal/utils/packaging.py -+++ b/pipenv/patched/pip/_internal/utils/packaging.py -@@ -29,7 +29,7 @@ def check_requires_python(requires_python): - requires_python_specifier = specifiers.SpecifierSet(requires_python) - - # We only use major.minor.micro -- python_version = version.parse('.'.join(map(str, sys.version_info[:3]))) -+ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) - return python_version in requires_python_specifier - - -@@ -48,9 +48,11 @@ def get_metadata(dist): - return feed_parser.close() - - --def check_dist_requires_python(dist): -+def check_dist_requires_python(dist, absorb=True): - pkg_info_dict = get_metadata(dist) - requires_python = pkg_info_dict.get('Requires-Python') -+ if absorb: -+ return requires_python - try: - if not check_requires_python(requires_python): - raise exceptions.UnsupportedPythonVersion( -diff --git a/pipenv/patched/pip/_internal/wheel.py b/pipenv/patched/pip/_internal/wheel.py -index 5ce890eb..46c0181c 100644 ---- a/pipenv/patched/pip/_internal/wheel.py -+++ b/pipenv/patched/pip/_internal/wheel.py -@@ -83,7 +83,7 @@ def fix_script(path): - firstline = script.readline() - if not firstline.startswith(b'#!python'): - return False -- exename = sys.executable.encode(sys.getfilesystemencoding()) -+ exename = os.environ.get('PIP_PYTHON_PATH', sys.executable).encode(sys.getfilesystemencoding()) - firstline = b'#!' + exename + os.linesep.encode("ascii") - rest = script.read() - with open(path, 'wb') as script: -@@ -167,7 +167,8 @@ def message_about_scripts_not_on_PATH(scripts): - ] - # If an executable sits with sys.executable, we don't warn for it. - # This covers the case of venv invocations without activating the venv. -- not_warn_dirs.append(os.path.normcase(os.path.dirname(sys.executable))) -+ executable_loc = os.environ.get("PIP_PYTHON_PATH", sys.executable) -+ not_warn_dirs.append(os.path.normcase(os.path.dirname(executable_loc))) - warn_for = { - parent_dir: scripts for parent_dir, scripts in grouped_by_dir.items() - if os.path.normcase(parent_dir) not in not_warn_dirs -@@ -667,8 +668,9 @@ class WheelBuilder(object): - # isolating. Currently, it breaks Python in virtualenvs, because it - # relies on site.py to find parts of the standard library outside the - # virtualenv. -+ executable_loc = os.environ.get('PIP_PYTHON_PATH', sys.executable) - return [ -- sys.executable, '-u', '-c', -+ executable_loc, '-u', '-c', - SETUPTOOLS_SHIM % req.setup_py - ] + list(self.global_options) - diff --git a/tasks/vendoring/patches/patched/pip19.patch b/tasks/vendoring/patches/patched/pip19.patch new file mode 100644 index 00000000..fa0a7dea --- /dev/null +++ b/tasks/vendoring/patches/patched/pip19.patch @@ -0,0 +1,593 @@ +diff --git a/pipenv/patched/pip/_internal/build_env.py b/pipenv/patched/pip/_internal/build_env.py +index 5e6dc460..0412f635 100644 +--- a/pipenv/patched/pip/_internal/build_env.py ++++ b/pipenv/patched/pip/_internal/build_env.py +@@ -169,8 +169,9 @@ class BuildEnvironment(object): + prefix.setup = True + if not requirements: + return ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) + args = [ +- sys.executable, os.path.dirname(pip_location), 'install', ++ sys_executable, os.path.dirname(pip_location), 'install', + '--ignore-installed', '--no-user', '--prefix', prefix.path, + '--no-warn-script-location', + ] # type: List[str] +diff --git a/pipenv/patched/pip/_internal/commands/install.py b/pipenv/patched/pip/_internal/commands/install.py +index 5842d18d..4e56d0bb 100644 +--- a/pipenv/patched/pip/_internal/commands/install.py ++++ b/pipenv/patched/pip/_internal/commands/install.py +@@ -388,7 +388,7 @@ class InstallCommand(RequirementCommand): + else: + # If we're not replacing an already installed pip, + # we're not modifying it. +- modifying_pip = pip_req.satisfied_by is None ++ modifying_pip = getattr(pip_req, "satisfied_by", None) is None + protect_pip_from_modification_on_windows( + modifying_pip=modifying_pip + ) +diff --git a/pipenv/patched/pip/_internal/index.py b/pipenv/patched/pip/_internal/index.py +index 897444aa..4c61043c 100644 +--- a/pipenv/patched/pip/_internal/index.py ++++ b/pipenv/patched/pip/_internal/index.py +@@ -119,6 +119,7 @@ class LinkEvaluator(object): + target_python, # type: TargetPython + allow_yanked, # type: bool + ignore_requires_python=None, # type: Optional[bool] ++ ignore_compatibility=None, # type: Optional[bool] + ): + # type: (...) -> None + """ +@@ -137,15 +138,20 @@ class LinkEvaluator(object): + :param ignore_requires_python: Whether to ignore incompatible + PEP 503 "data-requires-python" values in HTML links. Defaults + to False. ++ :param Optional[bool] ignore_compatibility: Whether to ignore ++ compatibility of python versions and allow all versions of packages. + """ + if ignore_requires_python is None: + ignore_requires_python = False ++ if ignore_compatibility is None: ++ ignore_compatibility = True + + self._allow_yanked = allow_yanked + self._canonical_name = canonical_name + self._ignore_requires_python = ignore_requires_python + self._formats = formats + self._target_python = target_python ++ self._ignore_compatibility = ignore_compatibility + + self.project_name = project_name + +@@ -176,10 +182,10 @@ class LinkEvaluator(object): + return (False, 'not a file') + if ext not in SUPPORTED_EXTENSIONS: + return (False, 'unsupported archive format: %s' % ext) +- if "binary" not in self._formats and ext == WHEEL_EXTENSION: ++ if "binary" not in self._formats and ext == WHEEL_EXTENSION and not self._ignore_compatibility: + reason = 'No binaries permitted for %s' % self.project_name + return (False, reason) +- if "macosx10" in link.path and ext == '.zip': ++ if "macosx10" in link.path and ext == '.zip' and not self._ignore_compatibility: + return (False, 'macosx10 one') + if ext == WHEEL_EXTENSION: + try: +@@ -191,7 +197,7 @@ class LinkEvaluator(object): + return (False, reason) + + supported_tags = self._target_python.get_tags() +- if not wheel.supported(supported_tags): ++ if not wheel.supported(supported_tags) and not self._ignore_compatibility: + # Include the wheel's tags in the reason string to + # simplify troubleshooting compatibility issues. + file_tags = wheel.get_formatted_file_tags() +@@ -228,7 +234,7 @@ class LinkEvaluator(object): + link, version_info=self._target_python.py_version_info, + ignore_requires_python=self._ignore_requires_python, + ) +- if not supports_python: ++ if not supports_python and not self._ignore_compatibility: + # Return None for the reason text to suppress calling + # _log_skipped_link(). + return (False, None) +@@ -479,8 +485,8 @@ class CandidateEvaluator(object): + project_name=self._project_name, + ) + +- def _sort_key(self, candidate): +- # type: (InstallationCandidate) -> CandidateSortingKey ++ def _sort_key(self, candidate, ignore_compatibility=True): ++ # type: (InstallationCandidate, bool) -> CandidateSortingKey + """ + Function to pass as the `key` argument to a call to sorted() to sort + InstallationCandidates by preference. +@@ -518,14 +524,18 @@ class CandidateEvaluator(object): + if link.is_wheel: + # can raise InvalidWheelFilename + wheel = Wheel(link.filename) +- if not wheel.supported(valid_tags): ++ if not wheel.supported(valid_tags) and not ignore_compatibility: + raise UnsupportedWheel( + "%s is not a supported wheel for this platform. It " + "can't be sorted." % wheel.filename + ) + if self._prefer_binary: + binary_preference = 1 +- pri = -(wheel.support_index_min(valid_tags)) ++ tags = self.valid_tags if not ignore_compatibility else None ++ try: ++ pri = -(wheel.support_index_min(tags=tags)) ++ except TypeError: ++ pri = -(support_num) + if wheel.build_tag is not None: + match = re.match(r'^(\d+)(.*)$', wheel.build_tag) + build_tag_groups = match.groups() +@@ -603,6 +613,7 @@ class PackageFinder(object): + format_control=None, # type: Optional[FormatControl] + candidate_prefs=None, # type: CandidatePreferences + ignore_requires_python=None, # type: Optional[bool] ++ ignore_compatibility=None, # type: Optional[bool] + ): + # type: (...) -> None + """ +@@ -617,6 +628,8 @@ class PackageFinder(object): + """ + if candidate_prefs is None: + candidate_prefs = CandidatePreferences() ++ if ignore_compatibility is None: ++ ignore_compatibility = False + + format_control = format_control or FormatControl(set(), set()) + +@@ -625,12 +638,16 @@ class PackageFinder(object): + self._ignore_requires_python = ignore_requires_python + self._link_collector = link_collector + self._target_python = target_python ++ self._ignore_compatibility = ignore_compatibility + + self.format_control = format_control + + # These are boring links that have already been logged somehow. + self._logged_links = set() # type: Set[Link] + ++ # Kenneth's Hack ++ self.extra = None ++ + # Don't include an allow_yanked default value to make sure each call + # site considers whether yanked releases are allowed. This also causes + # that decision to be made explicit in the calling code, which helps +@@ -668,6 +685,23 @@ class PackageFinder(object): + ignore_requires_python=selection_prefs.ignore_requires_python, + ) + ++ @staticmethod ++ def get_extras_links(links): ++ requires = [] ++ extras = {} ++ ++ current_list = requires ++ ++ for link in links: ++ if not link: ++ current_list = requires ++ if link.startswith('['): ++ current_list = [] ++ extras[link[1:-1]] = current_list ++ else: ++ current_list.append(link) ++ return extras ++ + @property + def search_scope(self): + # type: () -> SearchScope +@@ -715,6 +749,7 @@ class PackageFinder(object): + target_python=self._target_python, + allow_yanked=self._allow_yanked, + ignore_requires_python=self._ignore_requires_python, ++ ignore_compatibility=self._ignore_compatibility + ) + + def _sort_links(self, links): +@@ -763,6 +798,7 @@ class PackageFinder(object): + # Convert the Text result to str since InstallationCandidate + # accepts str. + version=str(result), ++ requires_python=getattr(link, "requires_python", None) + ) + + def evaluate_links(self, link_evaluator, links): +diff --git a/pipenv/patched/pip/_internal/legacy_resolve.py b/pipenv/patched/pip/_internal/legacy_resolve.py +index c24158f4..37c3197f 100644 +--- a/pipenv/patched/pip/_internal/legacy_resolve.py ++++ b/pipenv/patched/pip/_internal/legacy_resolve.py +@@ -126,6 +126,7 @@ class Resolver(object): + force_reinstall, # type: bool + upgrade_strategy, # type: str + py_version_info=None, # type: Optional[Tuple[int, ...]] ++ ignore_compatibility=False, # type: bool + ): + # type: (...) -> None + super(Resolver, self).__init__() +@@ -152,6 +153,10 @@ class Resolver(object): + self.ignore_requires_python = ignore_requires_python + self.use_user_site = use_user_site + self._make_install_req = make_install_req ++ self.ignore_compatibility = ignore_compatibility ++ self.requires_python = None ++ if self.ignore_compatibility: ++ self.ignore_requires_python = True + + self._discovered_dependencies = \ + defaultdict(list) # type: DefaultDict[str, List] +@@ -344,7 +349,8 @@ class Resolver(object): + def _resolve_one( + self, + requirement_set, # type: RequirementSet +- req_to_install # type: InstallRequirement ++ req_to_install, # type: InstallRequirement ++ ignore_requires_python=False, # type: bool + ): + # type: (...) -> List[InstallRequirement] + """Prepare a single requirements file. +@@ -368,11 +374,21 @@ class Resolver(object): + dist = abstract_dist.get_pkg_resources_distribution() + # This will raise UnsupportedPythonVersion if the given Python + # version isn't compatible with the distribution's Requires-Python. ++ ignore_requires_python = ( ++ ignore_requires_python or self.ignore_requires_python or ++ self.ignore_compatibility ++ ) + _check_dist_requires_python( + dist, version_info=self._py_version_info, +- ignore_requires_python=self.ignore_requires_python, ++ ignore_requires_python=ignore_requires_python, + ) +- ++ # Patched in - lets get the python version on here then ++ # FIXME: Does this patch even work? it puts the python version ++ # on the resolver... why? ++ try: ++ self.requires_python = get_requires_python(dist) ++ except TypeError: ++ self.requires_python = None + more_reqs = [] # type: List[InstallRequirement] + + def add_req(subreq, extras_requested): +@@ -397,9 +413,13 @@ class Resolver(object): + # can refer to it when adding dependencies. + if not requirement_set.has_requirement(req_to_install.name): + # 'unnamed' requirements will get added here ++ available_requested = sorted( ++ set(dist.extras) & set(req_to_install.extras) ++ ) + req_to_install.is_direct = True + requirement_set.add_requirement( + req_to_install, parent_req_name=None, ++ extras_requested=available_requested, + ) + + if not self.ignore_dependencies: +diff --git a/pipenv/patched/pip/_internal/models/candidate.py b/pipenv/patched/pip/_internal/models/candidate.py +index 4d49604d..cdfe65aa 100644 +--- a/pipenv/patched/pip/_internal/models/candidate.py ++++ b/pipenv/patched/pip/_internal/models/candidate.py +@@ -16,11 +16,12 @@ class InstallationCandidate(KeyBasedCompareMixin): + """Represents a potential "candidate" for installation. + """ + +- def __init__(self, project, version, link): +- # type: (Any, str, Link) -> None ++ def __init__(self, project, version, link, requires_python=None): ++ # type: (Any, str, Link, Any) -> None + self.project = project + self.version = parse_version(version) # type: _BaseVersion + self.link = link ++ self.requires_python = requires_python + + super(InstallationCandidate, self).__init__( + key=(self.project, self.version, self.link), +diff --git a/pipenv/patched/pip/_internal/operations/prepare.py b/pipenv/patched/pip/_internal/operations/prepare.py +index d0930458..91527ae8 100644 +--- a/pipenv/patched/pip/_internal/operations/prepare.py ++++ b/pipenv/patched/pip/_internal/operations/prepare.py +@@ -140,14 +140,7 @@ class RequirementPreparer(object): + # FIXME: this won't upgrade when there's an existing + # package unpacked in `req.source_dir` + if os.path.exists(os.path.join(req.source_dir, 'setup.py')): +- raise PreviousBuildDirError( +- "pip can't proceed with requirements '%s' due to a" +- " pre-existing build directory (%s). This is " +- "likely due to a previous installation that failed" +- ". pip is being responsible and not assuming it " +- "can delete this. Please delete it and try again." +- % (req, req.source_dir) +- ) ++ rmtree(req.source_dir) + + # Now that we have the real link, we can tell what kind of + # requirements we have and raise some more informative errors +diff --git a/pipenv/patched/pip/_internal/req/req_set.py b/pipenv/patched/pip/_internal/req/req_set.py +index b34a2bb1..afcd2e4f 100644 +--- a/pipenv/patched/pip/_internal/req/req_set.py ++++ b/pipenv/patched/pip/_internal/req/req_set.py +@@ -24,8 +24,8 @@ logger = logging.getLogger(__name__) + + class RequirementSet(object): + +- def __init__(self, require_hashes=False, check_supported_wheels=True): +- # type: (bool, bool) -> None ++ def __init__(self, require_hashes=False, check_supported_wheels=True, ignore_compatibility=True): ++ # type: (bool) -> None + """Create a RequirementSet. + """ + +@@ -36,6 +36,9 @@ class RequirementSet(object): + self.unnamed_requirements = [] # type: List[InstallRequirement] + self.successfully_downloaded = [] # type: List[InstallRequirement] + self.reqs_to_cleanup = [] # type: List[InstallRequirement] ++ if ignore_compatibility: ++ self.check_supported_wheels = False ++ self.ignore_compatibility = (check_supported_wheels is False or ignore_compatibility is True) + + def __str__(self): + # type: () -> str +@@ -199,7 +202,7 @@ class RequirementSet(object): + if project_name in self.requirements: + return self.requirements[project_name] + +- raise KeyError("No project with the name %r" % name) ++ pass + + def cleanup_files(self): + # type: () -> None +diff --git a/pipenv/patched/pip/_internal/utils/logging.py b/pipenv/patched/pip/_internal/utils/logging.py +index 7767111a..52738e16 100644 +--- a/pipenv/patched/pip/_internal/utils/logging.py ++++ b/pipenv/patched/pip/_internal/utils/logging.py +@@ -314,8 +314,8 @@ def setup_logging(verbosity, no_color, user_log_file): + "stderr": "ext://sys.stderr", + } + handler_classes = { +- "stream": "pip._internal.utils.logging.ColorizedStreamHandler", +- "file": "pip._internal.utils.logging.BetterRotatingFileHandler", ++ "stream": "pipenv.patched.notpip._internal.utils.logging.ColorizedStreamHandler", ++ "file": "pipenv.patched.notpip._internal.utils.logging.BetterRotatingFileHandler", + } + handlers = ["console", "console_errors", "console_subprocess"] + ( + ["user_log"] if include_user_log else [] +@@ -326,7 +326,7 @@ def setup_logging(verbosity, no_color, user_log_file): + "disable_existing_loggers": False, + "filters": { + "exclude_warnings": { +- "()": "pip._internal.utils.logging.MaxLevelFilter", ++ "()": "pipenv.patched.notpip._internal.utils.logging.MaxLevelFilter", + "level": logging.WARNING, + }, + "restrict_to_subprocess": { +@@ -334,7 +334,7 @@ def setup_logging(verbosity, no_color, user_log_file): + "name": subprocess_logger.name, + }, + "exclude_subprocess": { +- "()": "pip._internal.utils.logging.ExcludeLoggerFilter", ++ "()": "pipenv.patched.notpip._internal.utils.logging.ExcludeLoggerFilter", + "name": subprocess_logger.name, + }, + }, +diff --git a/pipenv/patched/pip/_internal/utils/misc.py b/pipenv/patched/pip/_internal/utils/misc.py +index b8482635..2fae4e08 100644 +--- a/pipenv/patched/pip/_internal/utils/misc.py ++++ b/pipenv/patched/pip/_internal/utils/misc.py +@@ -136,8 +136,8 @@ def get_prog(): + @retry(stop_max_delay=3000, wait_fixed=500) + def rmtree(dir, ignore_errors=False): + # type: (str, bool) -> None +- shutil.rmtree(dir, ignore_errors=ignore_errors, +- onerror=rmtree_errorhandler) ++ from pipenv.vendor.vistir.path import rmtree as vistir_rmtree, handle_remove_readonly ++ vistir_rmtree(dir, onerror=handle_remove_readonly, ignore_errors=ignore_errors) + + + def rmtree_errorhandler(func, path, exc_info): +diff --git a/pipenv/patched/pip/_internal/utils/packaging.py b/pipenv/patched/pip/_internal/utils/packaging.py +index 68aa86ed..8577d387 100644 +--- a/pipenv/patched/pip/_internal/utils/packaging.py ++++ b/pipenv/patched/pip/_internal/utils/packaging.py +@@ -1,6 +1,7 @@ + from __future__ import absolute_import + + import logging ++import sys + from email.parser import FeedParser + + from pip._vendor import pkg_resources +@@ -37,7 +38,7 @@ def check_requires_python(requires_python, version_info): + return True + requires_python_specifier = specifiers.SpecifierSet(requires_python) + +- python_version = version.parse('.'.join(map(str, version_info))) ++ python_version = version.parse('{0}.{1}.{2}'.format(*sys.version_info[:3])) + return python_version in requires_python_specifier + + +diff --git a/pipenv/patched/pip/_internal/utils/setuptools_build.py b/pipenv/patched/pip/_internal/utils/setuptools_build.py +index 12d866e0..28649a4d 100644 +--- a/pipenv/patched/pip/_internal/utils/setuptools_build.py ++++ b/pipenv/patched/pip/_internal/utils/setuptools_build.py +@@ -1,3 +1,4 @@ ++import os + import sys + + from pip._internal.utils.typing import MYPY_CHECK_RUNNING +@@ -36,7 +37,8 @@ def make_setuptools_shim_args( + :param unbuffered_output: If True, adds the unbuffered switch to the + argument list. + """ +- args = [sys.executable] ++ sys_executable = os.environ.get('PIP_PYTHON_PATH', sys.executable) ++ args = [sys_executable] + if unbuffered_output: + args.append('-u') + args.extend(['-c', _SETUPTOOLS_SHIM.format(setup_py_path)]) +diff --git a/pipenv/patched/pip/_internal/utils/temp_dir.py b/pipenv/patched/pip/_internal/utils/temp_dir.py +index 77d40be6..8a32cf2d 100644 +--- a/pipenv/patched/pip/_internal/utils/temp_dir.py ++++ b/pipenv/patched/pip/_internal/utils/temp_dir.py +@@ -8,9 +8,11 @@ import itertools + import logging + import os.path + import tempfile ++import warnings + + from pip._internal.utils.misc import rmtree + from pip._internal.utils.typing import MYPY_CHECK_RUNNING ++from pipenv.vendor.vistir.compat import finalize, ResourceWarning + + if MYPY_CHECK_RUNNING: + from typing import Optional +@@ -60,6 +62,20 @@ class TempDirectory(object): + self._deleted = False + self.delete = delete + self.kind = kind ++ self._finalizer = None ++ if self._path: ++ self._register_finalizer() ++ ++ def _register_finalizer(self): ++ if self.delete and self._path: ++ self._finalizer = finalize( ++ self, ++ self._cleanup, ++ self._path, ++ warn_message = None ++ ) ++ else: ++ self._finalizer = None + + @property + def path(self): +@@ -92,12 +108,28 @@ class TempDirectory(object): + logger.debug("Created temporary directory: {}".format(path)) + return path + ++ @classmethod ++ def _cleanup(cls, name, warn_message=None): ++ if not os.path.exists(name): ++ return ++ try: ++ rmtree(name) ++ except OSError: ++ pass ++ else: ++ if warn_message: ++ warnings.warn(warn_message, ResourceWarning) ++ + def cleanup(self): + """Remove the temporary directory created and reset state + """ +- self._deleted = True +- if os.path.exists(self._path): +- rmtree(self._path) ++ if getattr(self._finalizer, "detach", None) and self._finalizer.detach(): ++ if os.path.exists(self._path): ++ self._deleted = True ++ try: ++ rmtree(self._path) ++ except OSError: ++ pass + + + class AdjacentTempDirectory(TempDirectory): +@@ -169,4 +201,4 @@ class AdjacentTempDirectory(TempDirectory): + ) + + logger.debug("Created temporary directory: {}".format(path)) +- return path ++ return path +\ No newline at end of file +diff --git a/pipenv/patched/pip/_internal/commands/__init__.py b/pipenv/patched/pip/_internal/commands/__init__.py +index abcafa55..ca155a94 100644 +--- a/pipenv/patched/pip/_internal/commands/__init__.py ++++ b/pipenv/patched/pip/_internal/commands/__init__.py +@@ -21,7 +21,7 @@ CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary') + + # The ordering matters for help display. + # Also, even though the module path starts with the same +-# "pip._internal.commands" prefix in each case, we include the full path ++# "pipenv.patched.notpip._internal.commands" prefix in each case, we include the full path + # because it makes testing easier (specifically when modifying commands_dict + # in test setup / teardown by adding info for a FakeCommand class defined + # in a test-related module). +@@ -29,59 +29,59 @@ CommandInfo = namedtuple('CommandInfo', 'module_path, class_name, summary') + # so that the ordering won't be lost when using Python 2.7. + commands_dict = OrderedDict([ + ('install', CommandInfo( +- 'pip._internal.commands.install', 'InstallCommand', ++ 'pipenv.patched.notpip._internal.commands.install', 'InstallCommand', + 'Install packages.', + )), + ('download', CommandInfo( +- 'pip._internal.commands.download', 'DownloadCommand', ++ 'pipenv.patched.notpip._internal.commands.download', 'DownloadCommand', + 'Download packages.', + )), + ('uninstall', CommandInfo( +- 'pip._internal.commands.uninstall', 'UninstallCommand', ++ 'pipenv.patched.notpip._internal.commands.uninstall', 'UninstallCommand', + 'Uninstall packages.', + )), + ('freeze', CommandInfo( +- 'pip._internal.commands.freeze', 'FreezeCommand', ++ 'pipenv.patched.notpip._internal.commands.freeze', 'FreezeCommand', + 'Output installed packages in requirements format.', + )), + ('list', CommandInfo( +- 'pip._internal.commands.list', 'ListCommand', ++ 'pipenv.patched.notpip._internal.commands.list', 'ListCommand', + 'List installed packages.', + )), + ('show', CommandInfo( +- 'pip._internal.commands.show', 'ShowCommand', ++ 'pipenv.patched.notpip._internal.commands.show', 'ShowCommand', + 'Show information about installed packages.', + )), + ('check', CommandInfo( +- 'pip._internal.commands.check', 'CheckCommand', ++ 'pipenv.patched.notpip._internal.commands.check', 'CheckCommand', + 'Verify installed packages have compatible dependencies.', + )), + ('config', CommandInfo( +- 'pip._internal.commands.configuration', 'ConfigurationCommand', ++ 'pipenv.patched.notpip._internal.commands.configuration', 'ConfigurationCommand', + 'Manage local and global configuration.', + )), + ('search', CommandInfo( +- 'pip._internal.commands.search', 'SearchCommand', ++ 'pipenv.patched.notpip._internal.commands.search', 'SearchCommand', + 'Search PyPI for packages.', + )), + ('wheel', CommandInfo( +- 'pip._internal.commands.wheel', 'WheelCommand', ++ 'pipenv.patched.notpip._internal.commands.wheel', 'WheelCommand', + 'Build wheels from your requirements.', + )), + ('hash', CommandInfo( +- 'pip._internal.commands.hash', 'HashCommand', ++ 'pipenv.patched.notpip._internal.commands.hash', 'HashCommand', + 'Compute hashes of package archives.', + )), + ('completion', CommandInfo( +- 'pip._internal.commands.completion', 'CompletionCommand', ++ 'pipenv.patched.notpip._internal.commands.completion', 'CompletionCommand', + 'A helper command used for command completion.', + )), + ('debug', CommandInfo( +- 'pip._internal.commands.debug', 'DebugCommand', ++ 'pipenv.patched.notpip._internal.commands.debug', 'DebugCommand', + 'Show information useful for debugging.', + )), + ('help', CommandInfo( +- 'pip._internal.commands.help', 'HelpCommand', ++ 'pipenv.patched.notpip._internal.commands.help', 'HelpCommand', + 'Show help for commands.', + )), + ]) # type: OrderedDict[str, CommandInfo] diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 3799ccf4..45eb963b 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -1,175 +1,219 @@ diff --git a/pipenv/patched/piptools/_compat/__init__.py b/pipenv/patched/piptools/_compat/__init__.py -index 1fa3805..c0ecec8 100644 +index eccbf36..fd8ecdd 100644 --- a/pipenv/patched/piptools/_compat/__init__.py +++ b/pipenv/patched/piptools/_compat/__init__.py -@@ -27,6 +27,8 @@ from .pip_compat import ( - cmdoptions, - get_installed_distributions, - PyPI, -+ SafeFileCache, +@@ -11,6 +11,7 @@ from .pip_compat import ( + FormatControl, + InstallationCandidate, + InstallCommand, + InstallationError, - install_req_from_line, - install_req_from_editable, - ) + InstallRequirement, + Link, + PackageFinder, +@@ -18,6 +19,8 @@ from .pip_compat import ( + RequirementSet, + RequirementTracker, + Resolver, ++ SafeFileCache, ++ VcsSupport, + Wheel, + WheelCache, + cmdoptions, +@@ -29,6 +32,7 @@ from .pip_compat import ( + is_vcs_url, + parse_requirements, + path_to_url, ++ pip_version, + stdlib_pkgs, + url_to_path, + user_cache_dir, diff --git a/pipenv/patched/piptools/_compat/pip_compat.py b/pipenv/patched/piptools/_compat/pip_compat.py -index 28da51f..c466ef0 100644 +index 67da307..765bd49 100644 --- a/pipenv/patched/piptools/_compat/pip_compat.py +++ b/pipenv/patched/piptools/_compat/pip_compat.py -@@ -1,45 +1,55 @@ +@@ -1,26 +1,24 @@ # -*- coding=utf-8 -*- --import importlib - + from __future__ import absolute_import +- + import importlib +-from contextlib import contextmanager +- -import pip --import pkg_resources -+__all__ = [ -+ "InstallRequirement", -+ "parse_requirements", -+ "RequirementSet", -+ "user_cache_dir", -+ "FAVORITE_HASH", -+ "is_file_url", -+ "url_to_path", -+ "PackageFinder", -+ "FormatControl", -+ "Wheel", -+ "Command", -+ "cmdoptions", -+ "get_installed_distributions", -+ "PyPI", -+ "SafeFileCache", -+ "InstallationError", -+ "parse_version", -+ "pip_version", -+ "install_req_from_editable", -+ "install_req_from_line", -+ "user_cache_dir" -+] +-from pip._vendor.packaging.version import parse as parse_version +- +-PIP_VERSION = tuple(map(int, parse_version(pip.__version__).base_version.split("."))) ++import os ++from appdirs import user_cache_dir ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pipenv.patched.notpip") ++import pip_shims.shims ++from pip_shims.models import ShimmedPathCollection, ImportTypes --def do_import(module_path, subimport=None, old_path=None): -- old_path = old_path or module_path +-try: +- from pip._internal.req.req_tracker import RequirementTracker +-except ImportError: ++InstallationCandidate = ShimmedPathCollection("InstallationCandidate", ImportTypes.CLASS) ++InstallationCandidate.create_path("models.candidate", "18.0", "9999") ++InstallationCandidate.create_path("index", "7.0.3", "10.9.9") + +- @contextmanager +- def RequirementTracker(): +- yield ++PIP_VERSION = tuple(map(int, pip_shims.shims.parsed_pip_version.parsed_version.base_version.split("."))) + ++RequirementTracker = pip_shims.shims.RequirementTracker + + def do_import(module_path, subimport=None, old_path=None): + old_path = old_path or module_path - prefixes = ["pip._internal", "pip"] -- paths = [module_path, old_path] -- search_order = ["{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None] -- package = subimport if subimport else None -- for to_import in search_order: -- if not subimport: -- to_import, _, package = to_import.rpartition(".") -- try: -- imported = importlib.import_module(to_import) -- except ImportError: -- continue -- else: -- return getattr(imported, package) -- -- --InstallRequirement = do_import('req.req_install', 'InstallRequirement') --parse_requirements = do_import('req.req_file', 'parse_requirements') --RequirementSet = do_import('req.req_set', 'RequirementSet') --user_cache_dir = do_import('utils.appdirs', 'user_cache_dir') --FAVORITE_HASH = do_import('utils.hashes', 'FAVORITE_HASH') --is_file_url = do_import('download', 'is_file_url') --url_to_path = do_import('download', 'url_to_path') --PackageFinder = do_import('index', 'PackageFinder') --FormatControl = do_import('index', 'FormatControl') --Wheel = do_import('wheel', 'Wheel') --Command = do_import('cli.base_command', 'Command', old_path='basecommand') --cmdoptions = do_import('cli.cmdoptions', old_path='cmdoptions') --get_installed_distributions = do_import('utils.misc', 'get_installed_distributions', old_path='utils') --PyPI = do_import('models.index', 'PyPI') -+from pipenv.vendor.appdirs import user_cache_dir -+from pip_shims.shims import ( -+ InstallRequirement, -+ parse_requirements, -+ RequirementSet, -+ FAVORITE_HASH, -+ is_file_url, -+ url_to_path, -+ PackageFinder, -+ FormatControl, -+ Wheel, -+ Command, -+ cmdoptions, -+ get_installed_distributions, -+ PyPI, -+ SafeFileCache, -+ InstallationError, -+ parse_version, -+ pip_version, -+) ++ pip_path = os.environ.get("PIP_SHIMS_BASE_MODULE", "pip") ++ prefixes = ["{}._internal".format(pip_path), pip_path] + paths = [module_path, old_path] + search_order = [ + "{0}.{1}".format(p, pth) for p in prefixes for pth in paths if pth is not None +@@ -37,31 +35,29 @@ def do_import(module_path, subimport=None, old_path=None): + return getattr(imported, package) + + +-InstallRequirement = do_import("req.req_install", "InstallRequirement") +-InstallationCandidate = do_import( +- "models.candidate", "InstallationCandidate", old_path="index" +-) +-parse_requirements = do_import("req.req_file", "parse_requirements") +-RequirementSet = do_import("req.req_set", "RequirementSet") +-user_cache_dir = do_import("utils.appdirs", "user_cache_dir") +-FAVORITE_HASH = do_import("utils.hashes", "FAVORITE_HASH") +-path_to_url = do_import("utils.urls", "path_to_url", old_path="download") +-url_to_path = do_import("utils.urls", "url_to_path", old_path="download") +-PackageFinder = do_import("index.package_finder", "PackageFinder", old_path="index") +-FormatControl = do_import("models.format_control", "FormatControl", old_path="index") +-InstallCommand = do_import("commands.install", "InstallCommand") +-Wheel = do_import("wheel", "Wheel") +-cmdoptions = do_import("cli.cmdoptions", old_path="cmdoptions") +-get_installed_distributions = do_import( +- "utils.misc", "get_installed_distributions", old_path="utils" +-) +-PyPI = do_import("models.index", "PyPI") +-stdlib_pkgs = do_import("utils.compat", "stdlib_pkgs", old_path="compat") +-DEV_PKGS = do_import("commands.freeze", "DEV_PKGS") +-Link = do_import("models.link", "Link", old_path="index") ++InstallRequirement = pip_shims.shims.InstallRequirement ++InstallationError = pip_shims.shims.InstallationError ++parse_requirements = pip_shims.shims.parse_requirements ++RequirementSet = pip_shims.shims.RequirementSet ++SafeFileCache = pip_shims.shims.SafeFileCache ++FAVORITE_HASH = pip_shims.shims.FAVORITE_HASH ++path_to_url = pip_shims.shims.path_to_url ++url_to_path = pip_shims.shims.url_to_path ++PackageFinder = pip_shims.shims.PackageFinder ++FormatControl = pip_shims.shims.FormatControl ++InstallCommand = pip_shims.shims.InstallCommand ++Wheel = pip_shims.shims.Wheel ++cmdoptions = pip_shims.shims.cmdoptions ++get_installed_distributions = pip_shims.shims.get_installed_distributions ++PyPI = pip_shims.shims.PyPI ++stdlib_pkgs = pip_shims.shims.stdlib_pkgs ++DEV_PKGS = pip_shims.shims.DEV_PKGS ++Link = pip_shims.shims.Link + Session = do_import("_vendor.requests.sessions", "Session") +-Resolver = do_import("legacy_resolve", "Resolver", old_path="resolve") +-WheelCache = do_import("cache", "WheelCache", old_path="wheel") ++Resolver = pip_shims.shims.Resolver ++VcsSupport = pip_shims.shims.VcsSupport ++WheelCache = pip_shims.shims.WheelCache ++pip_version = pip_shims.shims.pip_version # pip 18.1 has refactored InstallRequirement constructors use by pip-tools. --if pkg_resources.parse_version(pip.__version__) < pkg_resources.parse_version('18.1'): -+if parse_version(pip_version) < parse_version('18.1'): - install_req_from_line = InstallRequirement.from_line - install_req_from_editable = InstallRequirement.from_editable - else: -- install_req_from_line = do_import('req.constructors', 'install_req_from_line') -- install_req_from_editable = do_import('req.constructors', 'install_req_from_editable') -+ from pip_shims.shims import ( -+ install_req_from_editable, install_req_from_line -+ ) + if PIP_VERSION < (18, 1): +diff --git a/pipenv/patched/piptools/locations.py b/pipenv/patched/piptools/locations.py +index fb66cf3..bb199f6 100644 +--- a/pipenv/patched/piptools/locations.py ++++ b/pipenv/patched/piptools/locations.py +@@ -5,7 +5,10 @@ from ._compat import user_cache_dir + from .click import secho + + # The user_cache_dir helper comes straight from pip itself +-CACHE_DIR = user_cache_dir("pip-tools") ++try: ++ from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR ++except ImportError: ++ CACHE_DIR = user_cache_dir("pipenv") + + # NOTE + # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/local.py b/pipenv/patched/piptools/repositories/local.py -index 08dabe1..480ad1e 100644 +index f389784..c1bcf9d 100644 --- a/pipenv/patched/piptools/repositories/local.py +++ b/pipenv/patched/piptools/repositories/local.py -@@ -56,7 +56,7 @@ class LocalRequirementsRepository(BaseRepository): +@@ -61,7 +61,8 @@ class LocalRequirementsRepository(BaseRepository): if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin): project, version, _ = as_tuple(existing_pin) return make_install_requirement( - project, version, ireq.extras, constraint=ireq.constraint -+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers ++ project, version, ireq.extras, ireq.markers, ++ constraint=ireq.constraint ) else: return self.repository.find_best_match(ireq, prereleases) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index bf69803..31b85b9 100644 +index acbd680..13378ae 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -1,7 +1,7 @@ - # coding: utf-8 - from __future__ import (absolute_import, division, print_function, - unicode_literals) -- +@@ -2,21 +2,29 @@ + from __future__ import absolute_import, division, print_function, unicode_literals + + import collections +import copy import hashlib import os from contextlib import contextmanager -@@ -15,13 +15,22 @@ from .._compat import ( - Wheel, - FAVORITE_HASH, - TemporaryDirectory, -- PyPI -+ PyPI, -+ InstallRequirement, -+ SafeFileCache - ) -+os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") -+from pip_shims.shims import do_import, VcsSupport, WheelCache -+from packaging.requirements import Requirement -+from packaging.specifiers import SpecifierSet, Specifier -+InstallationError = do_import(("exceptions.InstallationError", "7.0", "9999")) -+from pip._internal.resolve import Resolver as PipResolver -+ + from functools import partial + from shutil import rmtree --from ..cache import CACHE_DIR -+from pipenv.environments import PIPENV_CACHE_DIR as CACHE_DIR ++from packaging.requirements import Requirement ++from packaging.specifiers import Specifier, SpecifierSet ++ + from .._compat import ( + FAVORITE_HASH, + PIP_VERSION, ++ InstallationError, ++ InstallRequirement, + Link, + PyPI, + RequirementSet, + RequirementTracker, + Resolver as PipResolver, ++ SafeFileCache, + TemporaryDirectory, ++ VcsSupport, + Wheel, + WheelCache, + contextlib, +@@ -24,6 +32,7 @@ from .._compat import ( + is_file_url, + is_vcs_url, + path_to_url, ++ pip_version, + url_to_path, + ) + from ..cache import CACHE_DIR +@@ -31,6 +40,8 @@ from ..click import progressbar from ..exceptions import NoCandidateFound --from ..utils import (fs_str, is_pinned_requirement, lookup_table, -- make_install_requirement) -+from ..utils import (fs_str, is_pinned_requirement, lookup_table, dedup, -+ make_install_requirement, clean_requires_python) + from ..logging import log + from ..utils import ( ++ dedup, ++ clean_requires_python, + create_install_command, + fs_str, + is_pinned_requirement, +@@ -40,10 +51,50 @@ from ..utils import ( + ) from .base import BaseRepository - try: -@@ -31,10 +40,44 @@ except ImportError: - def RequirementTracker(): - yield ++os.environ["PIP_SHIMS_BASE_MODULE"] = str("pip") + FILE_CHUNK_SIZE = 4096 + FileStream = collections.namedtuple("FileStream", "stream size") + --try: -- from pip._internal.cache import WheelCache --except ImportError: -- from pip.wheel import WheelCache -+ +class HashCache(SafeFileCache): + """Caches hashes of PyPI artifacts so we do not need to re-download them + @@ -203,25 +247,38 @@ index bf69803..31b85b9 100644 + + def _get_file_hash(self, location): + h = hashlib.new(FAVORITE_HASH) -+ with open_local_or_remote_file(location, self.session) as fp: ++ with open_local_or_remote_file(location, self.session) as (fp, size): + for chunk in iter(lambda: fp.read(8096), b""): + h.update(chunk) + return ":".join([FAVORITE_HASH, h.hexdigest()]) - - ++ ++ class PyPIRepository(BaseRepository): -@@ -46,8 +89,9 @@ class PyPIRepository(BaseRepository): - config), but any other PyPI mirror can be used if index_urls is + DEFAULT_INDEX_URL = PyPI.simple_url + +@@ -54,8 +105,9 @@ class PyPIRepository(BaseRepository): changed/configured on the Finder. """ -- def __init__(self, pip_options, session): -+ def __init__(self, pip_options, session, use_json=False): - self.session = session -+ self.use_json = use_json - self.pip_options = pip_options - index_urls = [pip_options.index_url] + pip_options.extra_index_urls -@@ -73,6 +117,10 @@ class PyPIRepository(BaseRepository): +- def __init__(self, pip_args, build_isolation=False): ++ def __init__(self, pip_args, session=None, build_isolation=False, use_json=False): + self.build_isolation = build_isolation ++ self.use_json = use_json + + # Use pip's parser for pip.conf management and defaults. + # General options (find_links, index_url, extra_index_url, trusted_host, +@@ -63,7 +115,9 @@ class PyPIRepository(BaseRepository): + command = create_install_command() + self.options, _ = command.parse_args(pip_args) + +- self.session = command._build_session(self.options) ++ if session is None: ++ session = command._build_session(self.options) ++ self.session = session + self.finder = command._build_package_finder( + options=self.options, session=self.session + ) +@@ -78,6 +132,10 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -232,30 +289,47 @@ index bf69803..31b85b9 100644 # Setup file paths self.freshen_build_caches() -@@ -113,10 +161,13 @@ class PyPIRepository(BaseRepository): - if ireq.editable: +@@ -118,13 +176,15 @@ class PyPIRepository(BaseRepository): + if ireq.editable or is_url_requirement(ireq): return ireq # return itself as the best match - all_candidates = self.find_all_candidates(ireq.name) + all_candidates = clean_requires_python(self.find_all_candidates(ireq.name)) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True) -- matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), + candidates_by_version = lookup_table( + all_candidates, key=lambda c: c.version, unique=True + ) +- matching_versions = ireq.specifier.filter( +- (candidate.version for candidate in all_candidates), prereleases=prereleases +- ) + try: + matching_versions = ireq.specifier.filter((candidate.version for candidate in all_candidates), - prereleases=prereleases) ++ prereleases=prereleases) + except TypeError: + matching_versions = [candidate.version for candidate in all_candidates] # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,25 +177,87 @@ class PyPIRepository(BaseRepository): +@@ -153,11 +213,74 @@ class PyPIRepository(BaseRepository): + best_candidate.project, + best_candidate.version, + ireq.extras, ++ ireq.markers, + constraint=ireq.constraint, + ) - # Turn the candidate into a pinned InstallRequirement - return make_install_requirement( -- best_candidate.project, best_candidate.version, ireq.extras, constraint=ireq.constraint -- ) -+ best_candidate.project, best_candidate.version, ireq.extras, ireq.markers, constraint=ireq.constraint -+ ) ++ def get_dependencies(self, ireq): ++ json_results = set() ++ ++ if self.use_json: ++ try: ++ json_results = self.get_json_dependencies(ireq) ++ except TypeError: ++ json_results = set() ++ ++ legacy_results = self.get_legacy_dependencies(ireq) ++ json_results.update(legacy_results) ++ ++ return json_results + + def get_json_dependencies(self, ireq): + @@ -294,297 +368,218 @@ index bf69803..31b85b9 100644 + try: + if ireq not in self._json_dep_cache: + self._json_dep_cache[ireq] = [g for g in gen(ireq)] - -- def resolve_reqs(self, download_dir, ireq, wheel_cache): ++ + return set(self._json_dep_cache[ireq]) + except Exception: + return set() + -+ def get_dependencies(self, ireq): -+ json_results = set() -+ -+ if self.use_json: -+ try: -+ json_results = self.get_json_dependencies(ireq) -+ except TypeError: -+ json_results = set() -+ -+ legacy_results = self.get_legacy_dependencies(ireq) -+ json_results.update(legacy_results) -+ -+ return json_results -+ -+ def resolve_reqs(self, download_dir, ireq, wheel_cache, setup_requires={}, dist=None): + def resolve_reqs(self, download_dir, ireq, wheel_cache): results = None -+ setup_requires = {} -+ dist = None -+ ireq.isolated = False ++ ireq.isolated = self.build_isolation + ireq._wheel_cache = wheel_cache -+ - try: - from pip._internal.operations.prepare import RequirementPreparer -- from pip._internal.resolve import Resolver as PipResolver - except ImportError: -- # Pip 9 and below -+ # Pip 9 and below ++ if ireq and not ireq.link: ++ ireq.populate_link(self.finder, False, False) ++ if ireq.link and not ireq.link.is_wheel: ++ ireq.ensure_has_source_dir(self.source_dir) + + if PIP_VERSION < (10,): reqset = RequirementSet( - self.build_dir, - self.source_dir, +@@ -166,11 +289,13 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, + ignore_installed=True, + ignore_compatibility=False, - wheel_cache=wheel_cache + wheel_cache=wheel_cache, ) - results = reqset._prepare_file(self.finder, ireq) + results = reqset._prepare_file(self.finder, ireq, ignore_requires_python=True) else: - # pip >= 10 +- from pip._internal.operations.prepare import RequirementPreparer ++ from pip_shims.shims import RequirementPreparer + preparer_kwargs = { -@@ -159,13 +272,14 @@ class PyPIRepository(BaseRepository): - 'finder': self.finder, - 'session': self.session, - 'upgrade_strategy': "to-satisfy-only", -- 'force_reinstall': False, -+ 'force_reinstall': True, - 'ignore_dependencies': False, -- 'ignore_requires_python': False, -+ 'ignore_requires_python': True, - 'ignore_installed': True, - 'isolated': False, - 'wheel_cache': wheel_cache, -- 'use_user_site': False -+ 'use_user_site': False, -+ 'ignore_compatibility': False + "build_dir": self.build_dir, +@@ -186,21 +311,24 @@ class PyPIRepository(BaseRepository): + "upgrade_strategy": "to-satisfy-only", + "force_reinstall": False, + "ignore_dependencies": False, +- "ignore_requires_python": False, ++ "ignore_requires_python": True, + "ignore_installed": True, + "use_user_site": False, ++ "ignore_compatibility": False, ++ "use_pep517": True, } + make_install_req_kwargs = {"isolated": False, "wheel_cache": wheel_cache} + + if PIP_VERSION < (19, 3): + resolver_kwargs.update(**make_install_req_kwargs) + else: +- from pip._internal.req.constructors import install_req_from_req_string ++ from pipenv.vendor.pip_shims.shims import install_req_from_req_string + + make_install_req = partial( + install_req_from_req_string, **make_install_req_kwargs + ) + resolver_kwargs["make_install_req"] = make_install_req ++ del resolver_kwargs["use_pep517"] + + if PIP_VERSION >= (20,): + preparer_kwargs["session"] = self.session +@@ -208,6 +336,7 @@ class PyPIRepository(BaseRepository): + resolver = None preparer = None -@@ -177,15 +291,109 @@ class PyPIRepository(BaseRepository): - resolver_kwargs['preparer'] = preparer ++ reqset = None + with RequirementTracker() as req_tracker: + # Pip 18 uses a requirement tracker to prevent fork bombs + if req_tracker: +@@ -216,7 +345,6 @@ class PyPIRepository(BaseRepository): + resolver_kwargs["preparer"] = preparer reqset = RequirementSet() ireq.is_direct = True - reqset.add_requirement(ireq) -+ # reqset.add_requirement(ireq) + resolver = PipResolver(**resolver_kwargs) - resolver.require_hashes = False - results = resolver._resolve_one(reqset, ireq) + require_hashes = False +@@ -225,12 +353,16 @@ class PyPIRepository(BaseRepository): + results = resolver._resolve_one(reqset, ireq) + else: + results = resolver._resolve_one(reqset, ireq, require_hashes) ++ try: ++ reqset.cleanup_files() ++ except (AttributeError, OSError): ++ pass + - reqset.cleanup_files() ++ results = set(results) if results else set() - return set(results) -+ cleanup_fn = getattr(reqset, "cleanup_files", None) -+ if cleanup_fn is not None: -+ try: -+ cleanup_fn() -+ except OSError: -+ pass -+ -+ if ireq.editable and (not ireq.source_dir or not os.path.exists(ireq.source_dir)): -+ if ireq.editable: -+ self._source_dir = TemporaryDirectory(fs_str("source")) -+ ireq.ensure_has_source_dir(self.source_dir) -+ -+ if ireq.editable and (ireq.source_dir and os.path.exists(ireq.source_dir)): -+ # Collect setup_requires info from local eggs. -+ # Do this after we call the preparer on these reqs to make sure their -+ # egg info has been created -+ from pipenv.utils import chdir -+ with chdir(ireq.setup_py_dir): -+ try: -+ from setuptools.dist import distutils -+ dist = distutils.core.run_setup(ireq.setup_py) -+ except InstallationError: -+ ireq.run_egg_info() -+ except (TypeError, ValueError, AttributeError): -+ pass -+ if not dist: -+ try: -+ dist = ireq.get_dist() -+ except (ImportError, ValueError, TypeError, AttributeError): -+ pass -+ if ireq.editable and dist: -+ setup_requires = getattr(dist, "extras_require", None) -+ if not setup_requires: -+ setup_requires = {"setup_requires": getattr(dist, "setup_requires", None)} -+ if not getattr(ireq, 'req', None): -+ try: -+ ireq.req = dist.as_requirement() if dist else None -+ except (ValueError, TypeError) as e: -+ pass ++ return results, ireq - def get_dependencies(self, ireq): -+ # Convert setup_requires dict into a somewhat usable form. -+ if setup_requires: -+ for section in setup_requires: -+ python_version = section -+ not_python = not (section.startswith('[') and ':' in section) -+ -+ # This is for cleaning up :extras: formatted markers -+ # by adding them to the results of the resolver -+ # since any such extra would have been returned as a result anyway -+ for value in setup_requires[section]: -+ # This is a marker. -+ if value.startswith('[') and ':' in value: -+ python_version = value[1:-1] -+ not_python = False -+ # Strip out other extras. -+ if value.startswith('[') and ':' not in value: -+ not_python = True -+ -+ if ':' not in value: -+ try: -+ if not not_python: -+ results.add(InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))) -+ # Anything could go wrong here -- can't be too careful. -+ except Exception: -+ pass -+ -+ # this section properly creates 'python_version' markers for cross-python -+ # virtualenv creation and for multi-python compatibility. -+ requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else resolver.requires_python -+ if requires_python: -+ marker_str = '' -+ # This corrects a logic error from the previous code which said that if -+ # we Encountered any 'requires_python' attributes, basically only create a -+ # single result no matter how many we resolved. This should fix -+ # a majority of the remaining non-deterministic resolution issues. -+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()): -+ # We are checking first if we have leading specifier operator -+ # if not, we can assume we should be doing a == comparison -+ specifierset = SpecifierSet(requires_python) -+ # for multiple specifiers, the correct way to represent that in -+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` -+ from passa.internals.specifiers import cleanup_pyspecs -+ marker_str = str(Marker(" and ".join(dedup([ -+ "python_version {0[0]} '{0[1]}'".format(spec) -+ for spec in cleanup_pyspecs(specifierset) -+ ])))) -+ # The best way to add markers to a requirement is to make a separate requirement -+ # with only markers on it, and then to transfer the object istelf -+ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker -+ if ireq in results: -+ results.remove(ireq) -+ print(marker_to_add) -+ ireq.req.marker = marker_to_add -+ -+ results = set(results) if results else set() -+ return results, ireq -+ + def get_legacy_dependencies(self, ireq): """ - Given a pinned or an editable InstallRequirement, returns a set of + Given a pinned, URL, or editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -200,6 +408,7 @@ class PyPIRepository(BaseRepository): - # If a download_dir is passed, pip will unnecessarely - # archive the entire source directory - download_dir = None -+ - elif ireq.link and not ireq.link.is_artifact: - # No download_dir for VCS sources. This also works around pip - # using git-checkout-index, which gets rid of the .git dir. -@@ -214,7 +423,8 @@ class PyPIRepository(BaseRepository): - wheel_cache = WheelCache(CACHE_DIR, self.pip_options.format_control) - prev_tracker = os.environ.get('PIP_REQ_TRACKER') +@@ -265,9 +397,8 @@ class PyPIRepository(BaseRepository): + wheel_cache = WheelCache(CACHE_DIR, self.options.format_control) + prev_tracker = os.environ.get("PIP_REQ_TRACKER") try: -- self._dependencies_cache[ireq] = self.resolve_reqs(download_dir, ireq, wheel_cache) +- self._dependencies_cache[ireq] = self.resolve_reqs( +- download_dir, ireq, wheel_cache +- ) + results, ireq = self.resolve_reqs(download_dir, ireq, wheel_cache) + self._dependencies_cache[ireq] = results finally: - if 'PIP_REQ_TRACKER' in os.environ: + if "PIP_REQ_TRACKER" in os.environ: if prev_tracker: -@@ -236,6 +446,10 @@ class PyPIRepository(BaseRepository): - if ireq.editable: - return set() +@@ -305,7 +436,7 @@ class PyPIRepository(BaseRepository): + cached_link = Link(path_to_url(cached_path)) + else: + cached_link = link +- return {self._get_file_hash(cached_link)} ++ return {self._hash_cache._get_file_hash(cached_link)} -+ vcs = VcsSupport() -+ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: -+ return set() -+ if not is_pinned_requirement(ireq): - raise TypeError( - "Expected pinned requirement, got {}".format(ireq)) -@@ -243,24 +457,22 @@ class PyPIRepository(BaseRepository): + raise TypeError("Expected pinned requirement, got {}".format(ireq)) +@@ -313,12 +444,10 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. - all_candidates = self.find_all_candidates(ireq.name) - candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) - matching_versions = list( -- ireq.specifier.filter((candidate.version for candidate in all_candidates))) -- matching_candidates = candidates_by_version[matching_versions[0]] -+ ### Modification -- this is much more efficient.... -+ ### modification again -- still more efficient +- ireq.specifier.filter((candidate.version for candidate in all_candidates)) + matching_candidates = ( + c for c in clean_requires_python(self.find_all_candidates(ireq.name)) + if c.version in ireq.specifier -+ ) -+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version) -+ # matching_versions = list( -+ # ireq.specifier.filter((candidate.version for candidate in all_candidates))) -+ # matching_candidates = candidates_by_version[matching_versions[0]] + ) +- matching_candidates = candidates_by_version[matching_versions[0]] + + log.debug(" {}".format(ireq.name)) + +@@ -328,30 +457,11 @@ class PyPIRepository(BaseRepository): + return candidate.link return { -- self._get_file_hash(candidate.location) +- self._get_file_hash(get_candidate_link(candidate)) - for candidate in matching_candidates -+ h for h in map(lambda c: self._hash_cache.get_hash(c.location), -+ matching_candidates) if h is not None ++ h for h in ++ map(lambda c: self._hash_cache.get_hash(get_candidate_link(c)), matching_candidates) ++ if h is not None } -- def _get_file_hash(self, location): +- def _get_file_hash(self, link): +- log.debug(" Hashing {}".format(link.url_without_fragment)) - h = hashlib.new(FAVORITE_HASH) -- with open_local_or_remote_file(location, self.session) as fp: -- for chunk in iter(lambda: fp.read(8096), b""): -- h.update(chunk) +- with open_local_or_remote_file(link, self.session) as f: +- # Chunks to iterate +- chunks = iter(lambda: f.stream.read(FILE_CHUNK_SIZE), b"") +- +- # Choose a context manager depending on verbosity +- if log.verbosity >= 1: +- iter_length = f.size / FILE_CHUNK_SIZE if f.size else None +- context_manager = progressbar(chunks, length=iter_length, label=" ") +- else: +- context_manager = contextlib.nullcontext(chunks) +- +- # Iterate over the chosen context manager +- with context_manager as bar: +- for chunk in bar: +- h.update(chunk) - return ":".join([FAVORITE_HASH, h.hexdigest()]) - @contextmanager def allow_all_wheels(self): """ diff --git a/pipenv/patched/piptools/resolver.py b/pipenv/patched/piptools/resolver.py -index c2d323c..d5a471d 100644 +index fc53f18..7e856fe 100644 --- a/pipenv/patched/piptools/resolver.py +++ b/pipenv/patched/piptools/resolver.py -@@ -13,7 +13,7 @@ from . import click - from .cache import DependencyCache - from .exceptions import UnsupportedConstraint - from .logging import log --from .utils import (format_requirement, format_specifier, full_groupby, -+from .utils import (format_requirement, format_specifier, full_groupby, dedup, simplify_markers, - is_pinned_requirement, key_from_ireq, key_from_req, UNSAFE_PACKAGES) +@@ -6,6 +6,9 @@ import os + from functools import partial + from itertools import chain, count - green = partial(click.style, fg='green') -@@ -27,6 +27,7 @@ class RequirementSummary(object): - def __init__(self, ireq): ++from pipenv.vendor.requirementslib.models.markers import normalize_marker_str ++from packaging.markers import Marker ++ + from . import click + from ._compat import install_req_from_line + from .cache import DependencyCache +@@ -34,6 +37,7 @@ class RequirementSummary(object): self.req = ireq.req self.key = key_from_req(ireq.req) -+ self.markers = ireq.markers self.extras = str(sorted(ireq.extras)) ++ self.markers = ireq.markers self.specifier = str(ireq.specifier) -@@ -119,7 +120,7 @@ class Resolver(object): - @staticmethod - def check_constraints(constraints): - for constraint in constraints: -- if constraint.link is not None and not constraint.editable: -+ if constraint.link is not None and not constraint.editable and not constraint.is_wheel: - msg = ('pip-compile does not support URLs as packages, unless they are editable. ' - 'Perhaps add -e option?') - raise UnsupportedConstraint(msg, constraint) -@@ -155,6 +156,13 @@ class Resolver(object): - # NOTE we may be losing some info on dropped reqs here - combined_ireq.req.specifier &= ireq.req.specifier - combined_ireq.constraint &= ireq.constraint -+ if not combined_ireq.markers: -+ combined_ireq.markers = ireq.markers + def __eq__(self, other): +@@ -63,6 +67,17 @@ def combine_install_requirements(ireqs): + # NOTE we may be losing some info on dropped reqs here + combined_ireq.req.specifier &= ireq.req.specifier + combined_ireq.constraint &= ireq.constraint ++ if ireq.markers and not combined_ireq.markers: ++ combined_ireq.markers = copy.deepcopy(ireq.markers) ++ elif ireq.markers and combined_ireq.markers: ++ _markers = [] # type: List[Marker] ++ for marker in [ireq.markers, combined_ireq.markers]: ++ if isinstance(marker, str): ++ _markers.append(Marker(marker)) + else: -+ _markers = combined_ireq.markers._markers -+ if not isinstance(_markers[0], (tuple, list)): -+ combined_ireq.markers._markers = [_markers, 'and', ireq.markers._markers] -+ - # Return a sorted, de-duped tuple of extras - combined_ireq.extras = tuple(sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras)))) - yield combined_ireq -@@ -272,6 +280,15 @@ class Resolver(object): ++ _markers.append(marker) ++ marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) ++ combined_ireq.markers = Marker(marker_str) + # Return a sorted, de-duped tuple of extras + combined_ireq.extras = tuple( + sorted(set(tuple(combined_ireq.extras) + tuple(ireq.extras))) +@@ -335,10 +350,19 @@ class Resolver(object): + Editable requirements will never be looked up, as they may have + changed at any time. + """ +- if ireq.editable or is_url_requirement(ireq): ++ if ireq.editable or (is_url_requirement(ireq) and not ireq.link.is_wheel): for dependency in self.repository.get_dependencies(ireq): yield dependency return @@ -598,39 +593,52 @@ index c2d323c..d5a471d 100644 + ireq.extras = ireq.extra + elif not is_pinned_requirement(ireq): - raise TypeError('Expected pinned or editable requirement, got {}'.format(ireq)) - -@@ -282,7 +299,7 @@ class Resolver(object): - if ireq not in self.dependency_cache: - log.debug(' {} not in cache, need to check index'.format(format_requirement(ireq)), fg='yellow') + raise TypeError( + "Expected pinned or editable requirement, got {}".format(ireq) +@@ -356,7 +380,7 @@ class Resolver(object): + fg="yellow", + ) dependencies = self.repository.get_dependencies(ireq) - self.dependency_cache[ireq] = sorted(str(ireq.req) for ireq in dependencies) + self.dependency_cache[ireq] = sorted(set(format_requirement(ireq) for ireq in dependencies)) # Example: ['Werkzeug>=0.9', 'Jinja2>=2.4'] dependency_strings = self.dependency_cache[ireq] +@@ -372,7 +396,8 @@ class Resolver(object): + ) + + def reverse_dependencies(self, ireqs): ++ is_non_wheel_url = lambda r: is_url_requirement(r) and not r.link.is_wheel + non_editable = [ +- ireq for ireq in ireqs if not (ireq.editable or is_url_requirement(ireq)) ++ ireq for ireq in ireqs if not (ireq.editable or is_non_wheel_url(ireq)) + ] + return self.dependency_cache.reverse_dependencies(non_editable) diff --git a/pipenv/patched/piptools/utils.py b/pipenv/patched/piptools/utils.py -index 2360a04..6f62eb9 100644 +index 8727f1e..c9f53f7 100644 --- a/pipenv/patched/piptools/utils.py +++ b/pipenv/patched/piptools/utils.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, +@@ -1,6 +1,7 @@ + # coding: utf-8 + from __future__ import absolute_import, division, print_function, unicode_literals - import os ++import os import sys -+import six - from itertools import chain, groupby from collections import OrderedDict - from contextlib import contextmanager -@@ -11,11 +12,78 @@ from contextlib import contextmanager - from ._compat import install_req_from_line + from itertools import chain, groupby +@@ -8,6 +9,10 @@ from itertools import chain, groupby + import six + from click.utils import LazyFile + from six.moves import shlex_quote ++from pipenv.vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier ++from pipenv.vendor.packaging.version import Version, InvalidVersion, parse as parse_version ++from pipenv.vendor.packaging.markers import Marker, Op, Value, Variable ++ + from ._compat import PIP_VERSION, InstallCommand, install_req_from_line from .click import style -+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier -+from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable - - - UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'} +@@ -23,6 +28,70 @@ COMPILE_EXCLUDE_OPTIONS = { + } +def simplify_markers(ireq): @@ -681,11 +689,11 @@ index 2360a04..6f62eb9 100644 + all_candidates = [] + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3])))) + for c in candidates: -+ if c.requires_python: ++ if getattr(c, "requires_python", None): + # Old specifications had people setting this to single digits + # which is effectively the same as '>=digit,<digit+1' -+ if c.requires_python.isdigit(): -+ c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1) ++ if len(c.requires_python) == 1 and c.requires_python in ("2", "3"): ++ c.requires_python = '>={0},<{1!s}'.format(c.requires_python, int(c.requires_python) + 1) + try: + specifierset = SpecifierSet(c.requires_python) + except InvalidSpecifier: @@ -700,8 +708,8 @@ index 2360a04..6f62eb9 100644 def key_from_ireq(ireq): """Get a standardized key for an InstallRequirement.""" if ireq.req is None and ireq.link is not None: -@@ -41,30 +109,61 @@ def comment(text): - return style(text, fg='green') +@@ -48,16 +117,51 @@ def comment(text): + return style(text, fg="green") -def make_install_requirement(name, version, extras, constraint=False): @@ -713,8 +721,8 @@ index 2360a04..6f62eb9 100644 extras_string = "[{}]".format(",".join(sorted(extras))) - return install_req_from_line( -- str('{}{}=={}'.format(name, extras_string, version)), -- constraint=constraint) +- str("{}{}=={}".format(name, extras_string, version)), constraint=constraint +- ) + if not markers: + return install_req_from_line( + str('{}{}=={}'.format(name, extras_string, version)), @@ -723,13 +731,10 @@ index 2360a04..6f62eb9 100644 + return install_req_from_line( + str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), + constraint=constraint) - - --def format_requirement(ireq, marker=None): ++ ++ +def _requirement_to_str_lowercase_name(requirement): - """ -- Generic formatter for pretty printing InstallRequirements to the terminal -- in a less verbose way than using its `__str__` method. ++ """ + Formats a packaging.requirements.Requirement with a lowercase name. + + This is simply a copy of @@ -740,7 +745,7 @@ index 2360a04..6f62eb9 100644 + lowercasing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lowercased (such as the marker). See + this issue for more information: https://github.com/pypa/pipenv/issues/2113. - """ ++ """ + parts = [requirement.name.lower()] + + if requirement.extras: @@ -756,19 +761,20 @@ index 2360a04..6f62eb9 100644 + parts.append("; {0}".format(requirement.marker)) + + return "".join(parts) -+ -+ -+def format_requirement(ireq, marker=None): - if ireq.editable: - line = '-e {}'.format(ireq.link) + + + def is_url_requirement(ireq): +@@ -78,10 +182,10 @@ def format_requirement(ireq, marker=None, hashes=None): + elif is_url_requirement(ireq): + line = ireq.link.url else: - line = str(ireq.req).lower() + line = _requirement_to_str_lowercase_name(ireq.req) - if marker: -- line = '{} ; {}'.format(line, marker) +- line = "{} ; {}".format(line, marker) + if marker and ';' not in line: -+ line = '{}; {}'.format(line, marker) - - return line ++ line = "{}; {}".format(line, marker) + if hashes: + for hash_ in sorted(hashes): diff --git a/tasks/vendoring/patches/patched/safety-main.patch b/tasks/vendoring/patches/patched/safety-main.patch new file mode 100644 index 00000000..3d214863 --- /dev/null +++ b/tasks/vendoring/patches/patched/safety-main.patch @@ -0,0 +1,57 @@ +diff --git a/pipenv/patched/safety/__main__.py b/pipenv/patched/safety/__main__.py +index d9a0bdab..f905408a 100644 +--- a/pipenv/patched/safety/__main__.py ++++ b/pipenv/patched/safety/__main__.py +@@ -1,8 +1,51 @@ + """Allow safety to be executable through `python -m safety`.""" + from __future__ import absolute_import + +-from .cli import cli ++import os ++import sys ++import sysconfig ++ ++ ++PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) ++PIPENV_DIR = os.path.dirname(PATCHED_DIR) ++VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor") ++ ++ ++def get_site_packages(): ++ prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} ++ try: ++ prefixes.add(sys.real_prefix) ++ except AttributeError: ++ pass ++ form = sysconfig.get_path('purelib', expand=False) ++ py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) ++ return { ++ form.format(base=prefix, py_version_short=py_version_short) ++ for prefix in prefixes ++ } ++ ++ ++def insert_before_site_packages(*paths): ++ site_packages = get_site_packages() ++ index = None ++ for i, path in enumerate(sys.path): ++ if path in site_packages: ++ index = i ++ break ++ if index is None: ++ sys.path += list(paths) ++ else: ++ sys.path = sys.path[:index] + list(paths) + sys.path[index:] ++ ++ ++def insert_pipenv_dirs(): ++ insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR) + + + if __name__ == "__main__": # pragma: no cover ++ insert_pipenv_dirs() ++ yaml_lib = "pipenv.patched.yaml{0}".format(sys.version_info[0]) ++ locals()[yaml_lib] = __import__(yaml_lib) ++ sys.modules["yaml"] = sys.modules[yaml_lib] ++ from safety.cli import cli + cli(prog_name="safety") diff --git a/tasks/vendoring/patches/vendor/dotenv-typing-imports.patch b/tasks/vendoring/patches/vendor/dotenv-typing-imports.patch new file mode 100644 index 00000000..b0fcac2e --- /dev/null +++ b/tasks/vendoring/patches/vendor/dotenv-typing-imports.patch @@ -0,0 +1,161 @@ +diff --git a/pipenv/vendor/dotenv/__init__.py b/pipenv/vendor/dotenv/__init__.py +index 105a32a..b88d9bc 100644 +--- a/pipenv/vendor/dotenv/__init__.py ++++ b/pipenv/vendor/dotenv/__init__.py +@@ -1,6 +1,9 @@ +-from typing import Any, Optional # noqa ++from .compat import IS_TYPE_CHECKING + from .main import load_dotenv, get_key, set_key, unset_key, find_dotenv, dotenv_values + ++if IS_TYPE_CHECKING: ++ from typing import Any, Optional ++ + + def load_ipython_extension(ipython): + # type: (Any) -> None +diff --git a/pipenv/vendor/dotenv/cli.py b/pipenv/vendor/dotenv/cli.py +index 235f329..d2a021a 100644 +--- a/pipenv/vendor/dotenv/cli.py ++++ b/pipenv/vendor/dotenv/cli.py +@@ -1,7 +1,6 @@ + import os + import sys + from subprocess import Popen +-from typing import Any, Dict, List # noqa + + try: + import click +@@ -10,9 +9,13 @@ except ImportError: + 'Run pip install "python-dotenv[cli]" to fix this.') + sys.exit(1) + ++from .compat import IS_TYPE_CHECKING + from .main import dotenv_values, get_key, set_key, unset_key + from .version import __version__ + ++if IS_TYPE_CHECKING: ++ from typing import Any, List, Dict ++ + + @click.group() + @click.option('-f', '--file', default=os.path.join(os.getcwd(), '.env'), +diff --git a/pipenv/vendor/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py +index 394d3a3..61f555d 100644 +--- a/pipenv/vendor/dotenv/compat.py ++++ b/pipenv/vendor/dotenv/compat.py +@@ -1,5 +1,4 @@ + import sys +-from typing import Text # noqa + + PY2 = sys.version_info[0] == 2 # type: bool + +@@ -9,6 +8,22 @@ else: + from io import StringIO # noqa + + ++def is_type_checking(): ++ # type: () -> bool ++ try: ++ from typing import TYPE_CHECKING ++ except ImportError: # pragma: no cover ++ return False ++ return TYPE_CHECKING ++ ++ ++IS_TYPE_CHECKING = is_type_checking() ++ ++ ++if IS_TYPE_CHECKING: ++ from typing import Text ++ ++ + def to_env(text): + # type: (Text) -> str + """ +diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py +index 04d2241..06a210e 100644 +--- a/pipenv/vendor/dotenv/main.py ++++ b/pipenv/vendor/dotenv/main.py +@@ -7,16 +7,17 @@ import re + import shutil + import sys + import tempfile +-from typing import (Dict, Iterator, List, Match, Optional, # noqa +- Pattern, Union, TYPE_CHECKING, Text, IO, Tuple) + import warnings + from collections import OrderedDict + from contextlib import contextmanager + +-from .compat import StringIO, PY2, to_env ++from .compat import StringIO, PY2, to_env, IS_TYPE_CHECKING + from .parser import parse_stream + +-if TYPE_CHECKING: # pragma: no cover ++if IS_TYPE_CHECKING: ++ from typing import ( ++ Dict, Iterator, Match, Optional, Pattern, Union, Text, IO, Tuple ++ ) + if sys.version_info >= (3, 6): + _PathLike = os.PathLike + else: +@@ -273,6 +274,14 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): + + def load_dotenv(dotenv_path=None, stream=None, verbose=False, override=False, **kwargs): + # type: (Union[Text, _PathLike, None], Optional[_StringIO], bool, bool, Union[None, Text]) -> bool ++ """Parse a .env file and then load all the variables found as environment variables. ++ ++ - *dotenv_path*: absolute or relative path to .env file. ++ - *stream*: `StringIO` object with .env content. ++ - *verbose*: whether to output the warnings related to missing .env file etc. Defaults to `False`. ++ - *override*: where to override the system environment variables with the variables in `.env` file. ++ Defaults to `False`. ++ """ + f = dotenv_path or stream or find_dotenv() + return DotEnv(f, verbose=verbose, **kwargs).set_as_environment_variables(override=override) + +diff --git a/pipenv/vendor/dotenv/parser.py b/pipenv/vendor/dotenv/parser.py +index b63cb3a..034ebfd 100644 +--- a/pipenv/vendor/dotenv/parser.py ++++ b/pipenv/vendor/dotenv/parser.py +@@ -1,9 +1,14 @@ + import codecs + import re +-from typing import (IO, Iterator, Match, NamedTuple, Optional, Pattern, # noqa +- Sequence, Text) + +-from .compat import to_text ++from .compat import to_text, IS_TYPE_CHECKING ++ ++ ++if IS_TYPE_CHECKING: ++ from typing import ( # noqa:F401 ++ IO, Iterator, Match, NamedTuple, Optional, Pattern, Sequence, Text, ++ Tuple ++ ) + + + def make_regex(string, extra_flags=0): +@@ -25,9 +30,20 @@ _rest_of_line = make_regex(r"[^\r\n]*(?:\r|\n|\r\n)?") + _double_quote_escapes = make_regex(r"\\[\\'\"abfnrtv]") + _single_quote_escapes = make_regex(r"\\[\\']") + +-Binding = NamedTuple("Binding", [("key", Optional[Text]), +- ("value", Optional[Text]), +- ("original", Text)]) ++ ++try: ++ # this is necessary because we only import these from typing ++ # when we are type checking, and the linter is upset if we ++ # re-import ++ import typing ++ Binding = typing.NamedTuple("Binding", [("key", typing.Optional[typing.Text]), ++ ("value", typing.Optional[typing.Text]), ++ ("original", typing.Text)]) ++except ImportError: # pragma: no cover ++ from collections import namedtuple ++ Binding = namedtuple("Binding", ["key", # type: ignore ++ "value", ++ "original"]) # type: Tuple[Optional[Text], Optional[Text], Text] + + + class Error(Exception): diff --git a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch b/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch deleted file mode 100644 index c090e885..00000000 --- a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py -index 3d1bd72f..75f49c4a 100644 ---- a/pipenv/vendor/dotenv/main.py -+++ b/pipenv/vendor/dotenv/main.py -@@ -94,6 +94,13 @@ class DotEnv(): - for k, v in self.dict().items(): - if k in os.environ and not override: - continue -+ # With Python 2 on Windows, ensuree environment variables are -+ # system strings to avoid "TypeError: environment can only contain -+ # strings" in Python's subprocess module. -+ if sys.version_info.major < 3 and sys.platform == 'win32': -+ from pipenv.utils import fs_str -+ k = fs_str(k) -+ v = fs_str(v) - os.environ[k] = v - - return True diff --git a/tasks/vendoring/patches/vendor/dparse-configparser.patch b/tasks/vendoring/patches/vendor/dparse-configparser.patch new file mode 100644 index 00000000..bc61a75e --- /dev/null +++ b/tasks/vendoring/patches/vendor/dparse-configparser.patch @@ -0,0 +1,13 @@ +diff --git a/pipenv/vendor/dparse/parser.py b/pipenv/vendor/dparse/parser.py +index c01ebab4..9b2a0728 100644 +--- a/pipenv/vendor/dparse/parser.py ++++ b/pipenv/vendor/dparse/parser.py +@@ -6,7 +6,7 @@ import yaml + + from io import StringIO + +-from configparser import SafeConfigParser, NoOptionError ++from six.moves.configparser import SafeConfigParser, NoOptionError + + + from .regex import URL_REGEX, HASH_REGEX diff --git a/tasks/vendoring/patches/vendor/pip_shims_module_names.patch b/tasks/vendoring/patches/vendor/pip_shims_module_names.patch index 8658dcaa..7c49f82a 100644 --- a/tasks/vendoring/patches/vendor/pip_shims_module_names.patch +++ b/tasks/vendoring/patches/vendor/pip_shims_module_names.patch @@ -1,17 +1,42 @@ diff --git a/pipenv/vendor/pip_shims/__init__.py b/pipenv/vendor/pip_shims/__init__.py -index 1342f793..70fb0d58 100644 +index 2af4166e..598b9ad8 100644 --- a/pipenv/vendor/pip_shims/__init__.py +++ b/pipenv/vendor/pip_shims/__init__.py -@@ -8,10 +8,10 @@ __version__ = '0.3.1' - from . import shims - - --old_module = sys.modules["pip_shims"] -+old_module = sys.modules[__name__] +@@ -11,10 +11,13 @@ __version__ = "0.5.1" + if "pip_shims" in sys.modules: + # mainly to keep a reference to the old module on hand so it doesn't get + # weakref'd away +- old_module = sys.modules["pip_shims"] ++ if __name__ != "pip_shims": ++ del sys.modules["pip_shims"] ++if __name__ in sys.modules: ++ old_module = sys.modules[__name__] -module = sys.modules["pip_shims"] = shims._new() -+module = sys.modules[__name__] = shims._new() ++module = sys.modules[__name__] = sys.modules["pip_shims"] = shims._new() module.shims = shims - module.__dict__.update({ - '__file__': __file__, + module.__dict__.update( + { +diff --git a/pipenv/vendor/pip_shims/compat.py b/pipenv/vendor/pip_shims/compat.py +index ed99d970..63061a6a 100644 +--- a/pipenv/vendor/pip_shims/compat.py ++++ b/pipenv/vendor/pip_shims/compat.py +@@ -25,14 +25,14 @@ from .utils import ( + ) + + if sys.version_info[:2] < (3, 5): +- from backports.tempfile import TemporaryDirectory ++ from pipenv.vendor.vistir.compat import TemporaryDirectory + else: + from tempfile import TemporaryDirectory + + if six.PY3: + from contextlib import ExitStack + else: +- from contextlib2 import ExitStack ++ from pipenv.vendor.contextlib2 import ExitStack + + + if MYPY_RUNNING: + diff --git a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch index d479ebfa..5d5a381f 100644 --- a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch +++ b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch @@ -7,7 +7,7 @@ index 7820aa5..2082fc8 100644 from ordereddict import OrderedDict -try: -- from pipenv.patched.notpip._internal import get_installed_distributions +- from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - from pipenv.patched.notpip._internal.operations.freeze import FrozenRequirement -except ImportError: - from pipenv.patched.notpip import get_installed_distributions, FrozenRequirement @@ -17,3 +17,4 @@ index 7820aa5..2082fc8 100644 import pkg_resources # inline: + diff --git a/tasks/vendoring/patches/vendor/pythonfinder-pathlib-import.patch b/tasks/vendoring/patches/vendor/pythonfinder-pathlib-import.patch new file mode 100644 index 00000000..89cad037 --- /dev/null +++ b/tasks/vendoring/patches/vendor/pythonfinder-pathlib-import.patch @@ -0,0 +1,13 @@ +diff --git a/pipenv/vendor/pythonfinder/compat.py b/pipenv/vendor/pythonfinder/compat.py +index 6fb4542f..d76c4efc 100644 +--- a/pipenv/vendor/pythonfinder/compat.py ++++ b/pipenv/vendor/pythonfinder/compat.py +@@ -4,7 +4,7 @@ import sys + import six + + if sys.version_info[:2] <= (3, 4): +- from pathlib2 import Path # type: ignore # noqa ++ from pipenv.vendor.pathlib2 import Path # type: ignore # noqa + else: + from pathlib import Path + diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch index 36e2f808..49931e1f 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-fix.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -11,11 +11,12 @@ index e541c20c..0ac26752 100644 from .container import Container from .items import AoT diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py -index cb8af1d5..9b5db5cb 100644 +index a7d4fe90..96415901 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -1,13 +1,5 @@ - from __future__ import unicode_literals +@@ -2,14 +2,6 @@ from __future__ import unicode_literals + + import copy -from typing import Any -from typing import Dict @@ -28,7 +29,7 @@ index cb8af1d5..9b5db5cb 100644 from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey -@@ -17,6 +9,7 @@ from .items import Item +@@ -19,6 +11,7 @@ from .items import Item from .items import Key from .items import Null from .items import Table @@ -36,7 +37,7 @@ index cb8af1d5..9b5db5cb 100644 from .items import Whitespace from .items import item as _item -@@ -221,7 +214,12 @@ class Container(dict): +@@ -226,7 +219,12 @@ class Container(dict): for i in idx: self._body[i] = (None, Null()) else: @@ -61,7 +62,7 @@ index 4fbc667b..c1a4e620 100644 class TOMLKitError(Exception): diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py -index 375b5f02..cccfd4a1 100644 +index 8399d0c3..68c47a6d 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -6,14 +6,6 @@ import string @@ -78,8 +79,8 @@ index 375b5f02..cccfd4a1 100644 - from ._compat import PY2 - from ._compat import decode -@@ -22,9 +14,12 @@ from ._compat import unicode + from ._compat import PY38 +@@ -23,9 +15,12 @@ from ._compat import unicode from ._utils import escape_string if PY2: @@ -92,11 +93,10 @@ index 375b5f02..cccfd4a1 100644 def item(value, _parent=None): -@@ -40,7 +35,10 @@ def item(value, _parent=None): - elif isinstance(value, float): +@@ -42,6 +37,10 @@ def item(value, _parent=None): return Float(value, Trivia(), str(value)) elif isinstance(value, dict): -- val = Table(Container(), Trivia(), False) + val = Table(Container(), Trivia(), False) + if isinstance(value, InlineTableDict): + val = InlineTable(Container(), Trivia()) + else: diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index 673efad8..725e8a56 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -1,8 +1,8 @@ diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py -index 483a479a..43470a6e 100644 +index f5594a2d..a3d7f3df 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py -@@ -13,7 +13,7 @@ import six +@@ -12,7 +12,7 @@ import six try: from weakref import finalize except ImportError: @@ -10,31 +10,35 @@ index 483a479a..43470a6e 100644 + from pipenv.vendor.backports.weakref import finalize - __all__ = ["finalize", "NamedTemporaryFile"] + def fs_encode(path): diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py -index 9ae33fdc..ec3b65cb 100644 +index b5904bc7..a44aafbe 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): - from functools import lru_cache - else: - from pathlib2 import Path +@@ -43,7 +43,7 @@ __all__ = [ + if sys.version_info >= (3, 5): # pragma: no cover + from pathlib import Path + else: # pragma: no cover +- from pathlib2 import Path ++ from pipenv.vendor.pathlib2 import Path + + if six.PY3: # pragma: no cover + # Only Python 3.4+ is supported +@@ -53,14 +53,14 @@ if six.PY3: # pragma: no cover + from weakref import finalize + else: # pragma: no cover + # Only Python 2.7 is supported - from backports.functools_lru_cache import lru_cache + from pipenv.vendor.backports.functools_lru_cache import lru_cache - - from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile - if sys.version_info < (3, 3): + from .backports.functools import partialmethod # type: ignore - from backports.shutil_get_terminal_size import get_terminal_size + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + from .backports.surrogateescape import register_surrogateescape + + register_surrogateescape() NamedTemporaryFile = _NamedTemporaryFile - else: - from tempfile import NamedTemporaryFile -@@ -44,7 +44,7 @@ else: - try: - from weakref import finalize - except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize +- from backports.weakref import finalize # type: ignore ++ from pipenv.vendor.backports.weakref import finalize # type: ignore try: - from functools import partialmethod + # Introduced Python 3.5 diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch index a1d27cd3..c6144f40 100644 --- a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -7,7 +7,7 @@ index d01fb98e..06b8b621 100644 import time +import colorama -+import cursor ++from pipenv.vendor.vistir import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -48,15 +48,31 @@ index d01fb98e..06b8b621 100644 def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() -+ cursor.hide() ++ cursor.hide_cursor() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() -+ cursor.show() ++ cursor.show_cursor() @staticmethod def _clear_line(): - sys.stdout.write("\033[K") + sys.stdout.write(chr(27) + "[K") +diff --git a/pipenv/vendor/yaspin/spinners.py b/pipenv/vendor/yaspin/spinners.py +index 9c3fa7b8..60822a2c 100644 +--- a/pipenv/vendor/yaspin/spinners.py ++++ b/pipenv/vendor/yaspin/spinners.py +@@ -11,10 +11,7 @@ import codecs + import os + from collections import namedtuple + +-try: +- import simplejson as json +-except ImportError: +- import json ++import json + + + THIS_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tasks/vendoring/safety/Pipfile b/tasks/vendoring/safety/Pipfile new file mode 100644 index 00000000..95769f0d --- /dev/null +++ b/tasks/vendoring/safety/Pipfile @@ -0,0 +1,9 @@ +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + +[dev-packages] + +[packages] +safety = "*" diff --git a/tasks/vendoring/safety/__main__.py b/tasks/vendoring/safety/__main__.py new file mode 100644 index 00000000..9c6bbba2 --- /dev/null +++ b/tasks/vendoring/safety/__main__.py @@ -0,0 +1,45 @@ +"""Allow safety to be executable through `python -m safety`.""" +from __future__ import absolute_import + +import os +import sys +import sysconfig + +LIBPATH = os.path.join(os.path.dirname(os.path.abspath(__file__)), "lib") + + +def get_site_packages(): + prefixes = {sys.prefix, sysconfig.get_config_var('prefix')} + try: + prefixes.add(sys.real_prefix) + except AttributeError: + pass + form = sysconfig.get_path('purelib', expand=False) + py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info) + return { + form.format(base=prefix, py_version_short=py_version_short) + for prefix in prefixes + } + + +def insert_before_site_packages(*paths): + site_packages = get_site_packages() + index = None + for i, path in enumerate(sys.path): + if path in site_packages: + index = i + break + if index is None: + sys.path += list(paths) + else: + sys.path = sys.path[:index] + list(paths) + sys.path[index:] + + + +if __name__ == "__main__": + insert_before_site_packages(LIBPATH) + yaml_lib = 'yaml{0}'.format(sys.version_info[0]) + locals()[yaml_lib] = __import__(yaml_lib) + sys.modules['yaml'] = sys.modules[yaml_lib] + from safety.cli import cli + cli(prog_name="safety") diff --git a/tasks/vendoring/safety/requirements.txt b/tasks/vendoring/safety/requirements.txt new file mode 100644 index 00000000..a521ea82 --- /dev/null +++ b/tasks/vendoring/safety/requirements.txt @@ -0,0 +1,29 @@ +pipenv==2018.11.26 +packaging==20.3 +safety==1.8.7 +pyyaml==5.3.1 +virtualenv==20.0.18 +requests==2.23.0 +idna==2.9 +pyparsing==2.4.7 +appdirs==1.4.3 +filelock==3.0.12 +distlib==0.3.0 +certifi==2020.4.5.1 +click==7.1.1 +chardet==3.0.4 +dparse==0.5.0 +virtualenv-clone==0.5.4 +urllib3==1.25.9 +six==1.14.0 +toml==0.10.0 +singledispatch==3.4.0.3 +configparser==4.0.2 +importlib-metadata==1.6.0 +typing==3.7.4.1 +pathlib2==2.3.5 +importlib-resources==1.4.0 +zipp==1.2.0 +contextlib2==0.6.0.post1 +enum34==1.1.10 +scandir==1.10.0 \ No newline at end of file diff --git a/tasks/vendoring/vendor_passa.py b/tasks/vendoring/vendor_passa.py index f2c58745..2da91259 100644 --- a/tasks/vendoring/vendor_passa.py +++ b/tasks/vendoring/vendor_passa.py @@ -2,7 +2,7 @@ import invoke from pipenv._compat import TemporaryDirectory -from . import _get_git_root, _get_vendor_dir, log +from . import _get_vendor_dir, log @invoke.task diff --git a/tests/fixtures/fake-package/.coveragerc b/tests/fixtures/fake-package/.coveragerc new file mode 100644 index 00000000..1b3a0571 --- /dev/null +++ b/tests/fixtures/fake-package/.coveragerc @@ -0,0 +1,27 @@ +[run] +branch = True +parallel = True +source = src/fake_package/ + +[report] +# Regexes for lines to exclude from consideration +exclude_lines = + # Have to re-enable the standard pragma + pragma: no cover + + # Don't complain about missing debug-only code: + def __repr__ + if self\.debug + + # Don't complain if tests don't hit defensive assertion code: + raise AssertionError + raise NotImplementedError + # Don't complain if non-runnable code isn't run: + if 0: + if __name__ == .__main__.: + +[html] +directory = htmlcov + +[xml] +output = coverage.xml diff --git a/tests/fixtures/fake-package/.editorconfig b/tests/fixtures/fake-package/.editorconfig new file mode 100644 index 00000000..7470e9db --- /dev/null +++ b/tests/fixtures/fake-package/.editorconfig @@ -0,0 +1,27 @@ +root = true + +[*] +indent_style = space +indent_size = 4 +end_of_line = lf +charset = utf-8 +trim_trailing_whitespace = true +insert_final_newline = true + +[*.md] +trim_trailing_whitespace = false + +[*.toml] +indent_size = 2 + +[*.{yaml,yml}] +indent_size = 2 + +# Makefiles always use tabs for indentation. +[Makefile] +indent_style = tab + +# Batch files use tabs for indentation, and old Notepad hates LF. +[*.bat] +indent_style = tab +end_of_line = crlf diff --git a/tests/fixtures/fake-package/.gitignore b/tests/fixtures/fake-package/.gitignore new file mode 100644 index 00000000..ab621d86 --- /dev/null +++ b/tests/fixtures/fake-package/.gitignore @@ -0,0 +1,108 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.typeshed + +.vscode/ + +pip-wheel-metadata diff --git a/tests/fixtures/fake-package/.pre-commit-config.yaml b/tests/fixtures/fake-package/.pre-commit-config.yaml new file mode 100644 index 00000000..7ecca7dc --- /dev/null +++ b/tests/fixtures/fake-package/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/ambv/black + rev: stable + hooks: + - id: black + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.0.0 + hooks: + - id: flake8 + + - repo: https://github.com/asottile/seed-isort-config + rev: v1.7.0 + hooks: + - id: seed-isort-config + + - repo: https://github.com/pre-commit/mirrors-isort + rev: v4.3.9 + hooks: + - id: isort diff --git a/tests/fixtures/fake-package/.travis.yml b/tests/fixtures/fake-package/.travis.yml new file mode 100644 index 00000000..84bf6c03 --- /dev/null +++ b/tests/fixtures/fake-package/.travis.yml @@ -0,0 +1,38 @@ +language: python +sudo: false +cache: pip +dist: trusty + +matrix: + fast_finish: true + +install: + - "python -m pip install --upgrade pip pytest-timeout" + - "python -m pip install -e .[tests]" +script: + - "python -m pytest -v -n 8 tests/" + +jobs: + include: + - stage: test + - python: "3.7" + dist: xenial + sudo: required + - python: "3.6" + - python: "2.7" + - python: "3.5" + - python: "3.4" + - stage: packaging + python: "3.6" + install: + - "pip install --upgrade twine readme-renderer[md]" + script: + - "python setup.py sdist" + - "twine check dist/*" + - stage: coverage + python: "3.6" + install: + - "python -m pip install --upgrade pip pytest-timeout pytest-cov" + - "python -m pip install --upgrade -e .[tests]" + script: + - "python -m pytest -n auto --timeout 300 --cov=fake_package --cov-report=term-missing --cov-report=xml --cov-report=html tests" diff --git a/tests/fixtures/fake-package/LICENSE b/tests/fixtures/fake-package/LICENSE new file mode 100644 index 00000000..0beb71e0 --- /dev/null +++ b/tests/fixtures/fake-package/LICENSE @@ -0,0 +1,13 @@ +Copyright (c) 2019, Dan Ryan <dan@danryan.co> + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. diff --git a/tests/fixtures/fake-package/MANIFEST.in b/tests/fixtures/fake-package/MANIFEST.in new file mode 100644 index 00000000..7ab57a21 --- /dev/null +++ b/tests/fixtures/fake-package/MANIFEST.in @@ -0,0 +1,19 @@ +include LICENSE* README* +include CHANGELOG.rst +include pyproject.toml + +exclude .editorconfig +exclude .coveragerc +exclude .travis.yml +exclude tox.ini +exclude appveyor.yml +exclude Pipfile* + +recursive-include docs Makefile *.rst *.py *.bat +recursive-exclude docs requirements*.txt + +prune .github +prune docs/build +prune news +prune tasks +prune tests diff --git a/tests/fixtures/fake-package/Pipfile b/tests/fixtures/fake-package/Pipfile new file mode 100644 index 00000000..284b7798 --- /dev/null +++ b/tests/fixtures/fake-package/Pipfile @@ -0,0 +1,14 @@ +[packages] +fake_package = { path = '.', editable = true, extras = ["dev", "tests"] } + +[dev-packages] +towncrier = '*' +sphinx = '*' +sphinx-rtd-theme = '*' + +[scripts] +release = 'inv release' +tests = "pytest -v tests" +draft = "towncrier --draft" +changelog = "towncrier" +build = "setup.py sdist bdist_wheel" diff --git a/tests/fixtures/fake-package/README.rst b/tests/fixtures/fake-package/README.rst new file mode 100644 index 00000000..4256cd1f --- /dev/null +++ b/tests/fixtures/fake-package/README.rst @@ -0,0 +1,3 @@ +=============================================================================== +fake_package: A fake python package. +=============================================================================== diff --git a/tests/fixtures/fake-package/appveyor.yml b/tests/fixtures/fake-package/appveyor.yml new file mode 100644 index 00000000..758f4cfc --- /dev/null +++ b/tests/fixtures/fake-package/appveyor.yml @@ -0,0 +1,61 @@ +build: off +version: 1.0.{build} +skip_branch_with_pr: true + +init: +- ps: >- + + git config --global core.sharedRepository true + + git config --global core.longpaths true + + git config --global core.autocrlf input + + if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod ` + https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | ` + Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { ` + Write-Host "There are newer queued builds for this pull request, skipping build." + Exit-AppveyorBuild + } + + If (($env:SKIP_NOTAG -eq "true") -and ($env:APPVEYOR_REPO_TAG -ne "true")) { + Write-Host "Skipping build, not at a tag." + Exit-AppveyorBuild + } + +environment: + GIT_ASK_YESNO: 'false' + APPVEYOR_SAVE_CACHE_ON_ERROR: 'true' + APPVEYOR_SKIP_FINALIZE_ON_EXIT: 'true' + SHELL: 'windows' + PYTHON_ARCH: '64' + PYTHONIOENCODING: 'utf-8' + + matrix: + # Unit and integration tests. + - PYTHON: "C:\\Python27" + RUN_INTEGRATION_TESTS: "True" + - PYTHON: "C:\\Python37-x64" + RUN_INTEGRATION_TESTS: "True" + # Unit tests only. + - PYTHON: "C:\\Python36-x64" + - PYTHON: "C:\\Python34-x64" + - PYTHON: "C:\\Python35-x64" + +cache: +- '%LocalAppData%\pip\cache' +- '%LocalAppData%\pipenv\cache' + +install: + - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%" + - "python --version" + - "python -m pip install --upgrade pip pytest-timeout" + - "python -m pip install -e .[tests]" + + +test_script: + # Shorten paths, workaround https://bugs.python.org/issue18199 + - "subst T: %TEMP%" + - "set TEMP=T:\\" + - "set TMP=T:\\" + - "python -m pytest -n auto -v tests" diff --git a/tests/fixtures/fake-package/docs/conf.py b/tests/fixtures/fake-package/docs/conf.py new file mode 100644 index 00000000..3aded982 --- /dev/null +++ b/tests/fixtures/fake-package/docs/conf.py @@ -0,0 +1,208 @@ +# -*- coding: utf-8 -*- +# +# Configuration file for the Sphinx documentation builder. +# +# This file does only contain a selection of the most common options. For a +# full list see the documentation: +# http://www.sphinx-doc.org/en/master/config + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +PACKAGE_DIR = os.path.join(ROOT, "src/fake_package") +sys.path.insert(0, PACKAGE_DIR) + + +# -- Project information ----------------------------------------------------- + +project = 'fake_package' +copyright = '2019, Dan Ryan <dan@danryan.co>' +author = 'Dan Ryan <dan@danryan.co>' + +# The short X.Y version +version = '0.0' +# The full version, including alpha/beta/rc tags +release = '0.0.0.dev0' + + +# -- General configuration --------------------------------------------------- + +# If your documentation needs a minimal Sphinx version, state it here. +# +# needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'sphinx.ext.intersphinx', + 'sphinx.ext.autosummary' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The master toctree document. +master_doc = 'index' + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = 'en' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path . +exclude_patterns = ['_build', '_man', 'Thumbs.db', '.DS_Store'] + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' +autosummary_generate = True + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +# +# html_theme_options = {} + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + +# Custom sidebar templates, must be a dictionary that maps document names +# to template names. +# +# The default sidebars (for documents that don't match any pattern) are +# defined by theme itself. Builtin themes are using these templates by +# default: ``['localtoc.html', 'relations.html', 'sourcelink.html', +# 'searchbox.html']``. +# +# html_sidebars = {} + + +# -- Options for HTMLHelp output --------------------------------------------- + +# Output file base name for HTML help builder. +htmlhelp_basename = 'fake_packagedoc' +extlinks = { + 'issue': ('https://github.com/sarugaku/fake_package/issues/%s', '#'), + 'pull': ('https://github.com/sarugaku/fake_package/pull/%s', 'PR #'), +} +html_theme_options = { + 'display_version': True, + 'prev_next_buttons_location': 'bottom', + 'style_external_links': True, + 'vcs_pageview_mode': '', + # Toc options + 'collapse_navigation': True, + 'sticky_navigation': True, + 'navigation_depth': 4, + 'includehidden': True, + 'titles_only': False +} + +# -- Options for LaTeX output ------------------------------------------------ + +latex_elements = { + # The paper size ('letterpaper' or 'a4paper'). + # + # 'papersize': 'letterpaper', + + # The font size ('10pt', '11pt' or '12pt'). + # + # 'pointsize': '10pt', + + # Additional stuff for the LaTeX preamble. + # + # 'preamble': '', + + # Latex figure (float) alignment + # + # 'figure_align': 'htbp', +} + +# Grouping the document tree into LaTeX files. List of tuples +# (source start file, target name, title, +# author, documentclass [howto, manual, or own class]). +latex_documents = [ + (master_doc, 'fake_package.tex', 'fake_package Documentation', + 'Dan Ryan \\textless{}dan@danryan.co\\textgreater{}', 'manual'), +] + + +# -- Options for manual page output ------------------------------------------ + +# One entry per manual page. List of tuples +# (source start file, name, description, authors, manual section). +man_pages = [ + (master_doc, 'fake_package', 'fake_package Documentation', + [author], 1) +] + + +# -- Options for Texinfo output ---------------------------------------------- + +# Grouping the document tree into Texinfo files. List of tuples +# (source start file, target name, title, author, +# dir menu entry, description, category) +texinfo_documents = [ + (master_doc, 'fake_package', 'fake_package Documentation', + author, 'fake_package', 'A fake python package.', + 'Miscellaneous'), +] + + +# -- Options for Epub output ------------------------------------------------- + +# Bibliographic Dublin Core info. +epub_title = project +epub_author = author +epub_publisher = author +epub_copyright = copyright + +# The unique identifier of the text. This can be a ISBN number +# or the project homepage. +# +# epub_identifier = '' + +# A unique identification for the text. +# +# epub_uid = '' + +# A list of files that should not be packed into the epub file. +epub_exclude_files = ['search.html'] + + +# -- Extension configuration ------------------------------------------------- + +# -- Options for todo extension ---------------------------------------------- + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = True +intersphinx_mapping = {'python': ('https://docs.python.org/3', None)} diff --git a/tests/fixtures/fake-package/docs/requirements.txt b/tests/fixtures/fake-package/docs/requirements.txt new file mode 100644 index 00000000..82133027 --- /dev/null +++ b/tests/fixtures/fake-package/docs/requirements.txt @@ -0,0 +1,2 @@ +sphinx +sphinx_rtd_theme diff --git a/tests/fixtures/fake-package/news/.gitignore b/tests/fixtures/fake-package/news/.gitignore new file mode 100644 index 00000000..f935021a --- /dev/null +++ b/tests/fixtures/fake-package/news/.gitignore @@ -0,0 +1 @@ +!.gitignore diff --git a/tests/fixtures/fake-package/pyproject.toml b/tests/fixtures/fake-package/pyproject.toml new file mode 100644 index 00000000..e157956b --- /dev/null +++ b/tests/fixtures/fake-package/pyproject.toml @@ -0,0 +1,50 @@ +[build-system] +requires = ['setuptools>=40.8.0', 'wheel>=0.33.0'] + +[tool.black] +line-length = 90 +include = '\.pyi?$' +exclude = ''' +/( + \.eggs + | \.git + | \.hg + | \.mypy_cache + | \.tox + | \.pyre_configuration + | \.venv + | _build + | buck-out + | build + | dist +) +''' + +[tool.towncrier] +package = 'fake-package' +package_dir = 'src' +filename = 'CHANGELOG.rst' +directory = 'news/' +title_format = '{version} ({project_date})' +issue_format = '`#{issue} <https://github.com/sarugaku/fake_package/issues/{issue}>`_' +template = 'tasks/CHANGELOG.rst.jinja2' + + [[tool.towncrier.type]] + directory = 'feature' + name = 'Features' + showcontent = true + + [[tool.towncrier.type]] + directory = 'bugfix' + name = 'Bug Fixes' + showcontent = true + + [[tool.towncrier.type]] + directory = 'trivial' + name = 'Trivial Changes' + showcontent = false + + [[tool.towncrier.type]] + directory = 'removal' + name = 'Removals and Deprecations' + showcontent = true diff --git a/tests/fixtures/fake-package/setup.cfg b/tests/fixtures/fake-package/setup.cfg new file mode 100644 index 00000000..c357cea9 --- /dev/null +++ b/tests/fixtures/fake-package/setup.cfg @@ -0,0 +1,120 @@ +[metadata] +name = fake_package +description = A fake python package. +url = https://github.com/sarugaku/fake_package +author = Dan Ryan +author_email = dan@danryan.co +long_description = file: README.rst +license = ISC License +keywords = fake package test +classifier = + Development Status :: 1 - Planning + License :: OSI Approved :: ISC License (ISCL) + Operating System :: OS Independent + Programming Language :: Python :: 2 + Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 + Programming Language :: Python :: 3.7 + Topic :: Software Development :: Libraries :: Python Modules + +[options.extras_require] +tests = + pytest + pytest-xdist + pytest-cov + pytest-timeout + readme-renderer[md] + twine +dev = + black;python_version>="3.6" + flake8 + flake8-bugbear;python_version>="3.5" + invoke + isort + mypy;python_version>="3.5" + parver + pre-commit + rope + wheel + +[options] +zip_safe = true +python_requires = >=2.6,!=3.0,!=3.1,!=3.2,!=3.3 +setup_requires = setuptools>=40.8.0 +install_requires = + invoke + attrs + +[bdist_wheel] +universal = 1 + +[tool:pytest] +strict = true +plugins = cov flake8 +addopts = -ra +testpaths = tests/ +norecursedirs = .* build dist news tasks docs +flake8-ignore = + docs/source/* ALL + tests/*.py ALL + setup.py ALL +filterwarnings = + ignore::DeprecationWarning + ignore::PendingDeprecationWarning + +[isort] +atomic = true +not_skip = __init__.py +line_length = 90 +indent = ' ' +multi_line_output = 3 +known_third_party = invoke,parver,pytest,setuptools,towncrier +known_first_party = + fake_package + tests +combine_as_imports=True +include_trailing_comma = True +force_grid_wrap=0 + +[flake8] +max-line-length = 90 +select = C,E,F,W,B,B950 +ignore = + # The default ignore list: + D203,F401,E123,E203,W503,E501,E402 + #E121,E123,E126,E226,E24,E704, + # Our additions: + # E123: closing bracket does not match indentation of opening bracket’s line + # E203: whitespace before ‘:’ + # E129: visually indented line with same indent as next logical line + # E222: multiple spaces after operator + # E231: missing whitespace after ',' + # D203: 1 blank line required before class docstring + # E402: module level import not at top of file + # E501: line too long (using B950 from flake8-bugbear) + # F401: Module imported but unused + # W503: line break before binary operator (not a pep8 issue, should be ignored) +exclude = + .tox, + .git, + __pycache__, + docs/source/*, + build, + dist, + tests/*, + *.pyc, + *.egg-info, + .cache, + .eggs, + setup.py, +max-complexity=13 + +[mypy] +ignore_missing_imports=true +follow_imports=skip +html_report=mypyhtml +python_version=2.7 diff --git a/tests/fixtures/fake-package/setup.py b/tests/fixtures/fake-package/setup.py new file mode 100644 index 00000000..1d2c88f8 --- /dev/null +++ b/tests/fixtures/fake-package/setup.py @@ -0,0 +1,35 @@ +import ast +import os + +from setuptools import find_packages, setup + + +ROOT = os.path.dirname(__file__) + +PACKAGE_NAME = 'fake_package' + +VERSION = None + +with open(os.path.join(ROOT, 'src', PACKAGE_NAME.replace("-", "_"), '__init__.py')) as f: + for line in f: + if line.startswith('__version__ = '): + VERSION = ast.literal_eval(line[len('__version__ = '):].strip()) + break +if VERSION is None: + raise EnvironmentError('failed to read version') + + +# Put everything in setup.cfg, except those that don't actually work? +setup( + # These really don't work. + package_dir={'': 'src'}, + packages=find_packages('src'), + + # I don't know how to specify an empty key in setup.cfg. + package_data={ + '': ['LICENSE*', 'README*'], + }, + + # I need this to be dynamic. + version=VERSION, +) diff --git a/tests/fixtures/fake-package/src/fake_package/__init__.py b/tests/fixtures/fake-package/src/fake_package/__init__.py new file mode 100644 index 00000000..b8023d8b --- /dev/null +++ b/tests/fixtures/fake-package/src/fake_package/__init__.py @@ -0,0 +1 @@ +__version__ = '0.0.1' diff --git a/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 b/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 new file mode 100644 index 00000000..8aff2057 --- /dev/null +++ b/tests/fixtures/fake-package/tasks/CHANGELOG.rst.jinja2 @@ -0,0 +1,40 @@ +{% for section in sections %} +{% set underline = "-" %} +{% if section %} +{{section}} +{{ underline * section|length }}{% set underline = "~" %} + +{% endif %} +{% if sections[section] %} +{% for category, val in definitions.items() if category in sections[section] and category != 'trivial' %} + +{{ definitions[category]['name'] }} +{{ underline * definitions[category]['name']|length }} + +{% if definitions[category]['showcontent'] %} +{% for text, values in sections[section][category]|dictsort(by='value') %} +- {{ text }}{% if category != 'process' %} + {{ values|sort|join(',\n ') }} + {% endif %} + +{% endfor %} +{% else %} +- {{ sections[section][category]['']|sort|join(', ') }} + + +{% endif %} +{% if sections[section][category]|length == 0 %} + +No significant changes. + + +{% else %} +{% endif %} +{% endfor %} +{% else %} + +No significant changes. + + +{% endif %} +{% endfor %} diff --git a/tests/fixtures/fake-package/tasks/__init__.py b/tests/fixtures/fake-package/tasks/__init__.py new file mode 100644 index 00000000..a8cedab4 --- /dev/null +++ b/tests/fixtures/fake-package/tasks/__init__.py @@ -0,0 +1,175 @@ +import pathlib +import shutil +import subprocess + +import invoke +import parver + +from towncrier._builder import ( + find_fragments, render_fragments, split_fragments, +) +from towncrier._settings import load_config + + +ROOT = pathlib.Path(__file__).resolve().parent.parent + +PACKAGE_NAME = 'fake_package' + +INIT_PY = ROOT.joinpath('src', PACKAGE_NAME, '__init__.py') + + +@invoke.task() +def typecheck(ctx): + src_dir = ROOT / "src" / PACKAGE_NAME + src_dir = src_dir.as_posix() + config_file = ROOT / "setup.cfg" + env = {"MYPYPATH": src_dir} + ctx.run(f"mypy {src_dir} --config-file={config_file}", env=env) + + +@invoke.task() +def clean(ctx): + """Clean previously built package artifacts. + """ + ctx.run(f'python setup.py clean') + dist = ROOT.joinpath('dist') + print(f'[clean] Removing {dist}') + if dist.exists(): + shutil.rmtree(str(dist)) + + +def _read_version(): + out = subprocess.check_output(['git', 'tag'], encoding='ascii') + try: + version = max(parver.Version.parse(v).normalize() for v in ( + line.strip() for line in out.split('\n') + ) if v) + except ValueError: + version = parver.Version.parse('0.0.0') + return version + + +def _write_version(v): + lines = [] + with INIT_PY.open() as f: + for line in f: + if line.startswith("__version__ = "): + line = f"__version__ = {repr(str(v))}\n".replace("'", '"') + lines.append(line) + with INIT_PY.open("w", newline="\n") as f: + f.write("".join(lines)) + + +def _render_log(): + """Totally tap into Towncrier internals to get an in-memory result. + """ + config = load_config(ROOT) + definitions = config["types"] + fragments, fragment_filenames = find_fragments( + pathlib.Path(config["directory"]).absolute(), + config["sections"], + None, + definitions, + ) + rendered = render_fragments( + pathlib.Path(config["template"]).read_text(encoding="utf-8"), + config["issue_format"], + split_fragments(fragments, definitions), + definitions, + config["underlines"][1:], + False, # Don't add newlines to wrapped text. + ) + return rendered + + +REL_TYPES = ("major", "minor", "patch", "post") + + +def _bump_release(version, type_): + if type_ not in REL_TYPES: + raise ValueError(f"{type_} not in {REL_TYPES}") + index = REL_TYPES.index(type_) + next_version = version.base_version().bump_release(index=index) + print(f"[bump] {version} -> {next_version}") + return next_version + + +def _prebump(version, prebump): + next_version = version.bump_release(index=prebump).bump_dev() + print(f"[bump] {version} -> {next_version}") + return next_version + + +PREBUMP = 'patch' + + +@invoke.task(pre=[clean]) +def release(ctx, type_, repo, prebump=PREBUMP): + """Make a new release. + """ + if prebump not in REL_TYPES: + raise ValueError(f'{type_} not in {REL_TYPES}') + prebump = REL_TYPES.index(prebump) + + version = _read_version() + version = _bump_release(version, type_) + _write_version(version) + + # Needs to happen before Towncrier deletes fragment files. + tag_content = _render_log() + + ctx.run('towncrier') + + ctx.run(f'git commit -am "Release {version}"') + + tag_content = tag_content.replace('"', '\\"') + ctx.run(f'git tag -a {version} -m "Version {version}\n\n{tag_content}"') + + ctx.run(f'python setup.py sdist bdist_wheel') + + dist_pattern = f'{PACKAGE_NAME.replace("-", "[-_]")}-*' + artifacts = list(ROOT.joinpath('dist').glob(dist_pattern)) + filename_display = '\n'.join(f' {a}' for a in artifacts) + print(f'[release] Will upload:\n{filename_display}') + try: + input('[release] Release ready. ENTER to upload, CTRL-C to abort: ') + except KeyboardInterrupt: + print('\nAborted!') + return + + arg_display = ' '.join(f'"{n}"' for n in artifacts) + ctx.run(f'twine upload --repository="{repo}" {arg_display}') + + version = _prebump(version, prebump) + _write_version(version) + + ctx.run(f'git commit -am "Prebump to {version}"') + + +@invoke.task +def build_docs(ctx): + _current_version = _read_version() + minor = [str(i) for i in _current_version.release[:2]] + docs_folder = (ROOT / 'docs').as_posix() + if not docs_folder.endswith('/'): + docs_folder = '{0}/'.format(docs_folder) + args = ["--ext-autodoc", "--ext-viewcode", "-o", docs_folder] + args.extend(["-A", "'Dan Ryan <dan@danryan.co>'"]) + args.extend(["-R", str(_current_version)]) + args.extend(["-V", ".".join(minor)]) + args.extend(["-e", "-M", "-F", f"src/{PACKAGE_NAME}"]) + print("Building docs...") + ctx.run("sphinx-apidoc {0}".format(" ".join(args))) + + +@invoke.task +def clean_mdchangelog(ctx): + changelog = ROOT / "CHANGELOG.md" + content = changelog.read_text() + content = re.sub( + r"([^\n]+)\n?\s+\[[\\]+(#\d+)\]\(https://github\.com/sarugaku/[\w\-]+/issues/\d+\)", + r"\1 \2", + content, + flags=re.MULTILINE, + ) + changelog.write_text(content) diff --git a/tests/fixtures/fake-package/tox.ini b/tests/fixtures/fake-package/tox.ini new file mode 100644 index 00000000..2bc8e1d2 --- /dev/null +++ b/tests/fixtures/fake-package/tox.ini @@ -0,0 +1,37 @@ +[tox] +envlist = + docs, packaging, py27, py35, py36, py37, coverage-report + +[testenv] +passenv = CI GIT_SSL_CAINFO +setenv = + LC_ALL = en_US.UTF-8 +deps = + coverage + -e .[tests] +commands = coverage run --parallel -m pytest --timeout 300 [] +install_command = python -m pip install {opts} {packages} +usedevelop = True + +[testenv:coverage-report] +deps = coverage +skip_install = true +commands = + coverage combine + coverage report + +[testenv:docs] +deps = + -r{toxinidir}/docs/requirements.txt + -e .[tests] +commands = + sphinx-build -d {envtmpdir}/doctrees -b html docs docs/build/html + sphinx-build -d {envtmpdir}/doctrees -b man docs docs/build/man + +[testenv:packaging] +deps = + check-manifest + readme_renderer +commands = + check-manifest + python setup.py check -m -r -s diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b5b5bc85..18e38878 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,37 +1,64 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import errno import json +import logging import os +import shutil import sys import warnings +from shutil import rmtree as _rmtree + import pytest +import requests -from vistir.compat import ResourceWarning, fs_str -from vistir.path import mkdir_p - -from pipenv._compat import Path, TemporaryDirectory +from pipenv._compat import Path from pipenv.exceptions import VirtualenvActivationException -from pipenv.utils import temp_environ -from pipenv.vendor import delegator, requests, toml, tomlkit +from pipenv.vendor import delegator, toml, tomlkit +from pipenv.vendor.vistir.compat import ( + FileNotFoundError, PermissionError, ResourceWarning, TemporaryDirectory, + fs_encode, fs_str +) +from pipenv.vendor.vistir.contextmanagers import temp_environ +from pipenv.vendor.vistir.misc import run +from pipenv.vendor.vistir.path import ( + create_tracked_tempdir, handle_remove_readonly, mkdir_p +) from pytest_pypi.app import prepare_fixtures from pytest_pypi.app import prepare_packages as prepare_pypi_packages +log = logging.getLogger(__name__) warnings.simplefilter("default", category=ResourceWarning) HAS_WARNED_GITHUB = False +def try_internet(url="http://httpbin.org/ip", timeout=1.5): + resp = requests.get(url, timeout=timeout) + resp.raise_for_status() + + def check_internet(): - try: - # Kenneth represents the Internet LGTM. - resp = requests.get('http://httpbin.org/ip', timeout=1.0) - resp.raise_for_status() - except Exception: - warnings.warn('Cannot connect to HTTPBin...', RuntimeWarning) - warnings.warn('Will skip tests requiring Internet', RuntimeWarning) - return False - return True + has_internet = False + for url in ("http://httpbin.org/ip", "http://clients3.google.com/generate_204"): + try: + try_internet(url) + except KeyboardInterrupt: + warnings.warn( + "Skipped connecting to internet: {0}".format(url), RuntimeWarning + ) + except Exception: + warnings.warn( + "Failed connecting to internet: {0}".format(url), RuntimeWarning + ) + else: + has_internet = True + break + return has_internet def check_github_ssh(): @@ -42,8 +69,12 @@ def check_github_ssh(): # GitHub does not provide shell access.' if ssh keys are available and # registered with GitHub. Otherwise, the command will fail with # return_code=255 and say 'Permission denied (publickey).' - c = delegator.run('ssh -T git@github.com') + c = delegator.run('ssh -o StrictHostKeyChecking=accept-new -o CheckHostIP=no -T git@github.com', timeout=30) res = True if c.return_code == 1 else False + except KeyboardInterrupt: + warnings.warn( + "KeyboardInterrupt while checking GitHub ssh access", RuntimeWarning + ) except Exception: pass global HAS_WARNED_GITHUB @@ -69,17 +100,39 @@ def check_for_mercurial(): TESTS_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) PYPI_VENDOR_DIR = os.path.join(TESTS_ROOT, 'pypi') WE_HAVE_HG = check_for_mercurial() -prepare_pypi_packages(PYPI_VENDOR_DIR) prepare_fixtures(os.path.join(PYPI_VENDOR_DIR, "fixtures")) +prepare_pypi_packages(PYPI_VENDOR_DIR) def pytest_runtest_setup(item): - if item.get_marker('needs_internet') is not None and not WE_HAVE_INTERNET: + if item.get_closest_marker('needs_internet') is not None and not WE_HAVE_INTERNET: pytest.skip('requires internet') - if item.get_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: + if item.get_closest_marker('needs_github_ssh') is not None and not WE_HAVE_GITHUB_SSH_KEYS: pytest.skip('requires github ssh') - if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: + if item.get_closest_marker('needs_hg') is not None and not WE_HAVE_HG: pytest.skip('requires mercurial') + if item.get_closest_marker('skip_py27_win') is not None and ( + sys.version_info[:2] <= (2, 7) and os.name == "nt" + ): + pytest.skip('must use python > 2.7 on windows') + if item.get_closest_marker('skip_py38') is not None and ( + sys.version_info[:2] == (3, 8) + ): + pytest.skip('test not applicable on python 3.8') + if item.get_closest_marker('py3_only') is not None and ( + sys.version_info < (3, 0) + ): + pytest.skip('test only runs on python 3') + if item.get_closest_marker('skip_osx') is not None and sys.platform == 'darwin': + pytest.skip('test does not apply on OSX') + if item.get_closest_marker('lte_py36') is not None and ( + sys.version_info >= (3, 7) + ): + pytest.skip('test only runs on python < 3.7') + if item.get_closest_marker('skip_py36') is not None and ( + sys.version_info[:2] == (3, 6) + ): + pytest.skip('test is skipped on python 3.6') @pytest.fixture @@ -91,9 +144,49 @@ def pathlib_tmpdir(request, tmpdir): pass +def _create_tracked_dir(): + tmp_location = os.environ.get("TEMP", os.environ.get("TMP")) + temp_args = {"prefix": "pipenv-", "suffix": "-test"} + if tmp_location is not None: + temp_args["dir"] = tmp_location + temp_path = create_tracked_tempdir(**temp_args) + return temp_path + + +@pytest.fixture +def vistir_tmpdir(): + temp_path = _create_tracked_dir() + yield Path(temp_path) + + +@pytest.fixture() +def local_tempdir(request): + old_temp = os.environ.get("TEMP", "") + new_temp = Path(os.getcwd()).absolute() / "temp" + new_temp.mkdir(parents=True, exist_ok=True) + os.environ["TEMP"] = new_temp.as_posix() + + def finalize(): + os.environ['TEMP'] = fs_str(old_temp) + _rmtree_func(new_temp.as_posix()) + + request.addfinalizer(finalize) + with TemporaryDirectory(dir=new_temp.as_posix()) as temp_dir: + yield Path(temp_dir.name) + + +@pytest.fixture(name='create_tmpdir') +def vistir_tmpdir_factory(): + + def create_tmpdir(): + return Path(_create_tracked_dir()) + + yield create_tmpdir + + # Borrowed from pip's test runner filesystem isolation @pytest.fixture(autouse=True) -def isolate(pathlib_tmpdir): +def isolate(create_tmpdir): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -102,18 +195,23 @@ def isolate(pathlib_tmpdir): """ # Create a directory to use as our home location. - home_dir = os.path.join(str(pathlib_tmpdir), "home") + home_dir = os.path.join(str(create_tmpdir()), "home") os.makedirs(home_dir) mkdir_p(os.path.join(home_dir, ".config", "git")) - with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: + git_config_file = os.path.join(home_dir, ".config", "git", "config") + with open(git_config_file, "wb") as fp: fp.write( b"[user]\n\tname = pipenv\n\temail = pipenv@pipenv.org\n" ) + # os.environ["GIT_CONFIG"] = fs_str(git_config_file) os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") - mkdir_p(os.path.join(home_dir, ".virtualenvs")) - os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + os.environ["GIT_ASK_YESNO"] = fs_str("false") + workon_home = create_tmpdir() + os.environ["WORKON_HOME"] = fs_str(str(workon_home)) + os.environ["HOME"] = home_dir + mkdir_p(os.path.join(home_dir, "projects")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. os.environ.pop("PIPENV_ACTIVE", None) os.environ.pop("VIRTUAL_ENV", None) @@ -128,11 +226,15 @@ WE_HAVE_GITHUB_SSH_KEYS = check_github_ssh() class _Pipfile(object): def __init__(self, path): self.path = path - self.document = tomlkit.document() - self.document["source"] = tomlkit.aot() - self.document["requires"] = tomlkit.table() - self.document["packages"] = tomlkit.table() - self.document["dev_packages"] = tomlkit.table() + if self.path.exists(): + self.loads() + else: + self.document = tomlkit.document() + self.document["source"] = self.document.get("source", tomlkit.aot()) + self.document["requires"] = self.document.get("requires", tomlkit.table()) + self.document["packages"] = self.document.get("packages", tomlkit.table()) + self.document["dev_packages"] = self.document.get("dev_packages", tomlkit.table()) + super(_Pipfile, self).__init__() def install(self, package, value, dev=False): section = "packages" if not dev else "dev_packages" @@ -144,15 +246,27 @@ class _Pipfile(object): self.document[section][package] = value self.write() + def remove(self, package, dev=False): + section = "packages" if not dev else "dev_packages" + if not dev and package not in self.document[section]: + if package in self.document["dev_packages"]: + section = "dev_packages" + del self.document[section][package] + self.write() + def add(self, package, value, dev=False): self.install(package, value, dev=dev) + def update(self, package, value, dev=False): + self.install(package, value, dev=dev) + def loads(self): self.document = tomlkit.loads(self.path.read_text()) def dumps(self): source_table = tomlkit.table() - source_table["url"] = os.environ.get("PIPENV_TEST_INDEX") + pypi_url = os.environ.get("PIPENV_PYPI_URL", "https://pypi.org/simple") + source_table["url"] = os.environ.get("PIPENV_TEST_INDEX", pypi_url) source_table["verify_ssl"] = False source_table["name"] = "pipenv_test_index" self.document["source"].append(source_table) @@ -175,11 +289,12 @@ class _Pipfile(object): file_path = os.path.join(pkg, filename) if filename and not pkg: pkg = os.path.basename(filename) - if pypi: + fixture_pypi = os.getenv("ARTIFACT_PYPI_URL") + if fixture_pypi: if pkg and not filename: - url = "{0}/artifacts/{1}".format(pypi, pkg) + url = "{0}/artifacts/{1}".format(fixture_pypi, pkg) else: - url = "{0}/artifacts/{1}/{2}".format(pypi, pkg, filename) + url = "{0}/artifacts/{1}/{2}".format(fixture_pypi, pkg, filename) return url if pkg and not filename: return cls.get_fixture_path(file_path).as_uri() @@ -187,22 +302,47 @@ class _Pipfile(object): class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): - self.pypi = pypi - self.original_umask = os.umask(0o007) + def __init__( + self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None, + venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None + ): + self.index_url = os.getenv("PIPENV_TEST_INDEX") + self.pypi = None + if pypi: + self.pypi = pypi.url + elif self.index_url is not None: + self.pypi, _, _ = self.index_url.rpartition("/") if self.index_url else "" + self.index = os.getenv("PIPENV_PYPI_INDEX") + os.environ["PYTHONWARNINGS"] = "ignore:DEPRECATION" + if ignore_virtualenvs: + os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") + if venv_root: + os.environ["VIRTUAL_ENV"] = venv_root + if venv_in_project: + os.environ["PIPENV_VENV_IN_PROJECT"] = fs_str("1") + else: + os.environ.pop("PIPENV_VENV_IN_PROJECT", None) + self.original_dir = os.path.abspath(os.curdir) - os.environ["PIPENV_NOSPIN"] = fs_str("1") - os.environ["CI"] = fs_str("1") - warnings.simplefilter("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>") path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) + if name is not None: + path = Path(os.environ["HOME"]) / "projects" / name + path.mkdir(exist_ok=True) if not path: - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + if isinstance(path, TemporaryDirectory): + self._path = path path = Path(self._path.name) try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) + elif isinstance(path, Path): + self._path = path + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) else: self._path = path self.path = path @@ -210,9 +350,10 @@ class _PipenvInstance(object): self.pipfile_path = None self.chdir = chdir - if self.pypi: - os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi.url)) - os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) + if self.pypi and "PIPENV_PYPI_URL" not in os.environ: + os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi)) + # os.environ['PIPENV_PYPI_URL'] = fs_str('{0}'.format(self.pypi.url)) + # os.environ['PIPENV_TEST_INDEX'] = fs_str('{0}/simple'.format(self.pypi.url)) if pipfile: p_path = os.sep.join([self.path, 'Pipfile']) @@ -224,10 +365,6 @@ class _PipenvInstance(object): self._pipfile = _Pipfile(Path(p_path)) def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') - os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') - os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -237,13 +374,12 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - if self._path: + if self._path and getattr(self._path, "cleanup", None): try: self._path.cleanup() except OSError as e: _warn_msg = warn_msg.format(e) warnings.warn(_warn_msg, ResourceWarning) - os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): @@ -252,8 +388,10 @@ class _PipenvInstance(object): with TemporaryDirectory(prefix='pipenv-', suffix='-cache') as tempdir: os.environ['PIPENV_CACHE_DIR'] = fs_str(tempdir.name) - c = delegator.run('pipenv {0}'.format(cmd), block=block, - cwd=os.path.abspath(self.path)) + c = delegator.run( + 'pipenv {0}'.format(cmd), block=block, + cwd=os.path.abspath(self.path), env=os.environ.copy() + ) if 'PIPENV_CACHE_DIR' in os.environ: del os.environ['PIPENV_CACHE_DIR'] @@ -264,7 +402,7 @@ class _PipenvInstance(object): if block: print('$ pipenv {0}'.format(cmd)) print(c.out) - print(c.err) + print(c.err, file=sys.stderr) if c.return_code != 0: print("Command failed...") @@ -288,15 +426,23 @@ class _PipenvInstance(object): return os.sep.join([self.path, 'Pipfile.lock']) +def _rmtree_func(path, ignore_errors=True, onerror=None): + directory = fs_encode(path) + shutil_rmtree = _rmtree + if onerror is None: + onerror = handle_remove_readonly + try: + shutil_rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) + except (IOError, OSError, FileNotFoundError, PermissionError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno != errno.ENOENT: + raise + + @pytest.fixture() -def PipenvInstance(): - yield _PipenvInstance - - -@pytest.fixture(autouse=True) -def pip_src_dir(request, pathlib_tmpdir): +def pip_src_dir(request, vistir_tmpdir): old_src_dir = os.environ.get('PIP_SRC', '') - os.environ['PIP_SRC'] = pathlib_tmpdir.as_posix() + os.environ['PIP_SRC'] = vistir_tmpdir.as_posix() def finalize(): os.environ['PIP_SRC'] = fs_str(old_src_dir) @@ -305,25 +451,100 @@ def pip_src_dir(request, pathlib_tmpdir): return request +@pytest.fixture() +def PipenvInstance(pip_src_dir, monkeypatch, pypi): + with temp_environ(), monkeypatch.context() as m: + m.setattr(shutil, "rmtree", _rmtree_func) + original_umask = os.umask(0o007) + m.setenv("PIPENV_NOSPIN", fs_str("1")) + m.setenv("CI", fs_str("1")) + m.setenv('PIPENV_DONT_USE_PYENV', fs_str('1')) + m.setenv("PIPENV_TEST_INDEX", "{0}/simple".format(pypi.url)) + m.setenv("PIPENV_PYPI_INDEX", "simple") + m.setenv("ARTIFACT_PYPI_URL", pypi.url) + m.setenv("PIPENV_PYPI_URL", pypi.url) + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) + + +@pytest.fixture() +def PipenvInstance_NoPyPI(monkeypatch, pip_src_dir, pypi): + with temp_environ(), monkeypatch.context() as m: + m.setattr(shutil, "rmtree", _rmtree_func) + original_umask = os.umask(0o007) + m.setenv("PIPENV_NOSPIN", fs_str("1")) + m.setenv("CI", fs_str("1")) + m.setenv('PIPENV_DONT_USE_PYENV', fs_str('1')) + m.setenv("PIPENV_TEST_INDEX", "{0}/simple".format(pypi.url)) + m.setenv("ARTIFACT_PYPI_URL", pypi.url) + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*<ssl.SSLSocket.*>") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) + + @pytest.fixture() def testsroot(): return TESTS_ROOT -@pytest.fixture() -def virtualenv(pathlib_tmpdir): - virtualenv_path = pathlib_tmpdir / "venv" - with temp_environ(): - c = delegator.run("virtualenv {}".format(virtualenv_path), block=True) - assert c.return_code == 0 - for name in ("bin", "Scripts"): - activate_this = virtualenv_path / name / "activate_this.py" - if activate_this.exists(): - with open(str(activate_this)) as f: - code = compile(f.read(), str(activate_this), "exec") - exec(code, dict(__file__=str(activate_this))) - break +class VirtualEnv(object): + def __init__(self, name="venv", base_dir=None): + if base_dir is None: + base_dir = Path(_create_tracked_dir()) + self.base_dir = base_dir + self.name = name + self.path = base_dir / name + + def __enter__(self): + self._old_environ = os.environ.copy() + self.create() + return self.activate() + + def __exit__(self, *args, **kwargs): + os.environ = self._old_environ + + def create(self): + python = Path(sys.executable).absolute().as_posix() + cmd = [ + python, "-m", "virtualenv", self.path.absolute().as_posix() + ] + c = run( + cmd, verbose=False, return_object=True, write_to_stdout=False, + combine_stderr=False, block=True, nospin=True, + ) + # cmd = "{0} -m virtualenv {1}".format(python, self.path.as_posix()) + # c = delegator.run(cmd, block=True) + assert c.returncode == 0 + + def activate(self): + script_path = "Scripts" if os.name == "nt" else "bin" + activate_this = self.path / script_path / "activate_this.py" + if activate_this.exists(): + with open(str(activate_this)) as f: + code = compile(f.read(), str(activate_this), "exec") + exec(code, dict(__file__=str(activate_this))) + os.environ["VIRTUAL_ENV"] = str(self.path) + try: + return self.path.absolute().resolve() + except OSError: + return self.path.absolute() else: raise VirtualenvActivationException("Can't find the activate_this.py script.") - os.environ["VIRTUAL_ENV"] = str(virtualenv_path) - yield virtualenv_path + + +@pytest.fixture() +def virtualenv(vistir_tmpdir): + with temp_environ(), VirtualEnv(base_dir=vistir_tmpdir) as venv: + yield venv + + +@pytest.fixture() +def raw_venv(): + yield VirtualEnv diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index a38883e0..04253fc5 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function """Tests to ensure `pipenv --option` works. """ @@ -12,8 +14,8 @@ from pipenv.utils import normalize_drive @pytest.mark.cli -def test_pipenv_where(PipenvInstance, pypi_secure): - with PipenvInstance(pypi=pypi_secure) as p: +def test_pipenv_where(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("--where") assert c.ok assert normalize_drive(p.path) in c.out @@ -47,7 +49,7 @@ def test_pipenv_site_packages(PipenvInstance): c = p.pipenv('--python python --site-packages') assert c.return_code == 0 assert 'Making site-packages available' in c.err - + # no-global-site-packages.txt under stdlib dir should not exist. c = p.pipenv('run python -c "import sysconfig; print(sysconfig.get_path(\'stdlib\'))"') assert c.return_code == 0 @@ -80,54 +82,68 @@ def test_pipenv_rm(PipenvInstance): @pytest.mark.cli -def test_pipenv_graph(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests') +def test_pipenv_graph(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install tablib') assert c.ok graph = p.pipenv("graph") assert graph.ok - assert "requests" in graph.out + assert "tablib" in graph.out graph_json = p.pipenv("graph --json") assert graph_json.ok - assert "requests" in graph_json.out + assert "tablib" in graph_json.out graph_json_tree = p.pipenv("graph --json-tree") assert graph_json_tree.ok - assert "requests" in graph_json_tree.out + assert "tablib" in graph_json_tree.out @pytest.mark.cli -def test_pipenv_graph_reverse(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests==2.18.4') +def test_pipenv_graph_reverse(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install tablib==0.13.0') assert c.ok c = p.pipenv('graph --reverse') assert c.ok output = c.out - requests_dependency = [ - ('certifi', 'certifi>=2017.4.17'), - ('chardet', 'chardet(>=3.0.2,<3.1.0|<3.1.0,>=3.0.2)'), - ('idna', 'idna(>=2.5,<2.7|<2.7,>=2.5)'), - ('urllib3', 'urllib3(>=1.21.1,<1.23|<1.23,>=1.21.1)') - ] - - for dep_name, dep_constraint in requests_dependency: - dep_match = re.search(r'^{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) - dep_requests_match = re.search(r'^ - requests==2.18.4 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) - assert dep_match is not None - assert dep_requests_match is not None - assert dep_requests_match.start() > dep_match.start() - c = p.pipenv('graph --reverse --json') assert c.return_code == 1 assert 'Warning: Using both --reverse and --json together is not supported.' in c.err + requests_dependency = [ + ('backports.csv', 'backports.csv'), + ('odfpy', 'odfpy'), + ('openpyxl', 'openpyxl>=2.4.0'), + ('pyyaml', 'pyyaml'), + ('xlrd', 'xlrd'), + ('xlwt', 'xlwt'), + ] + + for dep_name, dep_constraint in requests_dependency: + pat = r'^[ -]*{}==[\d.]+'.format(dep_name) + dep_match = re.search(pat, output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found in {}'.format(pat, output) + + # openpyxl should be indented + if dep_name == 'openpyxl': + openpyxl_dep = re.search(r'^openpyxl', output, flags=re.MULTILINE) + assert openpyxl_dep is None, 'openpyxl should not appear at begining of lines in {}'.format(output) + + assert ' - openpyxl==2.5.4 [requires: et-xmlfile]' in output + else: + dep_match = re.search(r'^[ -]*{}==[\d.]+$'.format(dep_name), output, flags=re.MULTILINE) + assert dep_match is not None, '{} not found at beginning of line in {}'.format(dep_name, output) + + dep_requests_match = re.search(r'^ +- tablib==0.13.0 \[requires: {}\]$'.format(dep_constraint), output, flags=re.MULTILINE) + assert dep_requests_match is not None, 'constraint {} not found in {}'.format(dep_constraint, output) + assert dep_requests_match.start() > dep_match.start() + @pytest.mark.cli @pytest.mark.needs_internet(reason='required by check') @flaky -def test_pipenv_check(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_pipenv_check(PipenvInstance): + with PipenvInstance() as p: p.pipenv('install requests==1.0.0') c = p.pipenv('check') assert c.return_code != 0 @@ -195,8 +211,8 @@ def test_man(PipenvInstance): @pytest.mark.cli -def test_install_parse_error(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_install_parse_error(PipenvInstance): + with PipenvInstance() as p: # Make sure unparseable packages don't wind up in the pipfile # Escape $ for shell input @@ -207,7 +223,7 @@ def test_install_parse_error(PipenvInstance, pypi): [dev-packages] """.strip() f.write(contents) - c = p.pipenv('install requests u/\\/p@r\$34b13+pkg') + c = p.pipenv('install requests u/\\/p@r\\$34b13+pkg') assert c.return_code != 0 assert 'u/\\/p@r$34b13+pkg' not in p.pipfile['packages'] @@ -215,20 +231,50 @@ def test_install_parse_error(PipenvInstance, pypi): @pytest.mark.code @pytest.mark.check @pytest.mark.unused -@pytest.mark.skip(reason="non-deterministic") -def test_check_unused(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.skip_osx +@pytest.mark.needs_internet(reason='required by check') +def test_check_unused(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open('__init__.py', 'w') as f: contents = """ -import tablib +import click import records +import flask """.strip() f.write(contents) - p.pipenv('install requests') - p.pipenv('install tablib') - p.pipenv('install records') + p.pipenv('install requests click flask') - assert all(pkg in p.pipfile['packages'] for pkg in ['requests', 'tablib', 'records']) + assert all(pkg in p.pipfile['packages'] for pkg in ['requests', 'click', 'flask']), p.pipfile["packages"] c = p.pipenv('check --unused .') - assert 'tablib' not in c.out + assert 'click' not in c.out + assert 'flask' not in c.out + + +@pytest.mark.cli +def test_pipenv_clear(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--clear') + assert c.return_code == 0 + assert 'Clearing caches' in c.out + + +@pytest.mark.cli +def test_pipenv_three(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('--three') + assert c.return_code == 0 + assert 'Successfully created virtual environment' in c.err + + +@pytest.mark.outdated +def test_pipenv_outdated_prerelease(PipenvInstance): + with PipenvInstance(chdir=True) as p: + with open(p.pipfile_path, "w") as f: + contents = """ +[packages] +sqlalchemy = "<=1.2.3" + """.strip() + f.write(contents) + c = p.pipenv('update --pre --outdated') + assert c.return_code == 0 diff --git a/tests/integration/test_dot_venv.py b/tests/integration/test_dot_venv.py index db61d321..3cbc88db 100644 --- a/tests/integration/test_dot_venv.py +++ b/tests/integration/test_dot_venv.py @@ -1,18 +1,18 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import pytest from pipenv._compat import Path, TemporaryDirectory -from pipenv.project import Project -from pipenv.utils import get_windows_path, normalize_drive, temp_environ -from pipenv.vendor import delegator +from pipenv.utils import normalize_drive, temp_environ @pytest.mark.dotvenv -def test_venv_in_project(PipenvInstance, pypi): +def test_venv_in_project(PipenvInstance): with temp_environ(): os.environ['PIPENV_VENV_IN_PROJECT'] = '1' - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: c = p.pipenv('install requests') assert c.return_code == 0 assert normalize_drive(p.path) in p.pipenv('--venv').out @@ -34,8 +34,8 @@ def test_venv_at_project_root(PipenvInstance): @pytest.mark.dotvenv -def test_reuse_previous_venv(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_reuse_previous_venv(PipenvInstance): + with PipenvInstance(chdir=True) as p: os.mkdir('.venv') c = p.pipenv('install requests') assert c.return_code == 0 @@ -44,11 +44,11 @@ def test_reuse_previous_venv(PipenvInstance, pypi): @pytest.mark.dotvenv @pytest.mark.parametrize('venv_name', ('test-venv', os.path.join('foo', 'test-venv'))) -def test_venv_file(venv_name, PipenvInstance, pypi): +def test_venv_file(venv_name, PipenvInstance): """Tests virtualenv creation when a .venv file exists at the project root and contains a venv name. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: file_path = os.path.join(p.path, '.venv') with open(file_path, 'w') as f: f.write(venv_name) @@ -68,20 +68,20 @@ def test_venv_file(venv_name, PipenvInstance, pypi): venv_loc = Path(c.out.strip()).absolute() assert venv_loc.exists() assert venv_loc.joinpath('.project').exists() - venv_path = venv_loc.as_posix() + venv_path = normalize_drive(venv_loc.as_posix()) if os.path.sep in venv_name: venv_expected_path = Path(p.path).joinpath(venv_name).absolute().as_posix() else: venv_expected_path = Path(workon_home.name).joinpath(venv_name).absolute().as_posix() - assert venv_path == venv_expected_path + assert venv_path == normalize_drive(venv_expected_path) @pytest.mark.dotvenv -def test_venv_file_with_path(PipenvInstance, pypi): +def test_venv_file_with_path(PipenvInstance): """Tests virtualenv creation when a .venv file exists at the project root and contains an absolute path. """ - with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: + with temp_environ(), PipenvInstance(chdir=True) as p: with TemporaryDirectory( prefix='pipenv-', suffix='-test_venv' ) as venv_path: diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f858f8ac..9d3247fe 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import pytest @@ -5,15 +7,15 @@ import pytest from flaky import flaky from pipenv._compat import Path, TemporaryDirectory -from pipenv.project import Project from pipenv.utils import temp_environ from pipenv.vendor import delegator -@pytest.mark.install @pytest.mark.setup -def test_basic_setup(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.basic +@pytest.mark.install +def test_basic_setup(PipenvInstance): + with PipenvInstance() as p: with PipenvInstance(pipfile=False) as p: c = p.pipenv("install requests") assert c.return_code == 0 @@ -26,10 +28,12 @@ def test_basic_setup(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @flaky -def test_basic_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.basic +@pytest.mark.install +@pytest.mark.skip_osx +def test_basic_install(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install requests") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -40,10 +44,11 @@ def test_basic_install(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @flaky -def test_mirror_install(PipenvInstance, pypi): - with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.basic +@pytest.mark.install +def test_mirror_install(PipenvInstance): + with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop( "PIPENV_TEST_INDEX", "https://pypi.python.org/simple" ) @@ -66,10 +71,11 @@ def test_mirror_install(PipenvInstance, pypi): assert "certifi" in p.lockfile["default"] +@flaky +@pytest.mark.basic @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_bad_mirror_install(PipenvInstance, pypi): +def test_bad_mirror_install(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: # This demonstrates that the mirror parameter is being used os.environ.pop("PIPENV_TEST_INDEX", None) @@ -77,11 +83,11 @@ def test_bad_mirror_install(PipenvInstance, pypi): assert c.return_code != 0 -@pytest.mark.complex @pytest.mark.lock +@pytest.mark.complex @pytest.mark.skip(reason="Does not work unless you can explicitly install into py2") -def test_complex_lock(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_complex_lock(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install apscheduler") assert c.return_code == 0 assert "apscheduler" in p.pipfile["packages"] @@ -89,11 +95,11 @@ def test_complex_lock(PipenvInstance, pypi): assert "futures" in p.lockfile[u"default"] +@flaky @pytest.mark.dev @pytest.mark.run -@flaky -def test_basic_dev_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_basic_dev_install(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install requests --dev") assert c.return_code == 0 assert "requests" in p.pipfile["dev-packages"] @@ -107,37 +113,39 @@ def test_basic_dev_install(PipenvInstance, pypi): assert c.return_code == 0 -@pytest.mark.dev -@pytest.mark.install @flaky -def test_install_without_dev(PipenvInstance, pypi): +@pytest.mark.dev +@pytest.mark.basic +@pytest.mark.install +def test_install_without_dev(PipenvInstance): """Ensure that running `pipenv install` doesn't install dev packages""" - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] six = "*" [dev-packages] -pytz = "*" +tablib = "*" """.strip() f.write(contents) c = p.pipenv("install") assert c.return_code == 0 assert "six" in p.pipfile["packages"] - assert "pytz" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["dev-packages"] assert "six" in p.lockfile["default"] - assert "pytz" in p.lockfile["develop"] - c = p.pipenv('run python -c "import pytz"') + assert "tablib" in p.lockfile["develop"] + c = p.pipenv('run python -c "import tablib"') assert c.return_code != 0 c = p.pipenv('run python -c "import six"') assert c.return_code == 0 -@pytest.mark.install @flaky -def test_install_without_dev_section(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.basic +@pytest.mark.install +def test_install_without_dev_section(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -154,11 +162,12 @@ six = "*" assert c.return_code == 0 +@flaky +@pytest.mark.lock @pytest.mark.extras @pytest.mark.install -@flaky -def test_extras_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_extras_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install requests[socks]") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -171,11 +180,12 @@ def test_extras_install(PipenvInstance, pypi): assert "pysocks" in p.lockfile["default"] -@pytest.mark.install -@pytest.mark.pin @flaky -def test_windows_pinned_pipfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.pin +@pytest.mark.basic +@pytest.mark.install +def test_windows_pinned_pipfile(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -188,12 +198,13 @@ requests = "==2.19.1" assert "requests" in p.lockfile["default"] +@flaky +@pytest.mark.basic @pytest.mark.install @pytest.mark.resolver @pytest.mark.backup_resolver -@flaky -def test_backup_resolver(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_backup_resolver(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -206,11 +217,11 @@ def test_backup_resolver(PipenvInstance, pypi): assert "ibm-db-sa-py3" in p.lockfile["default"] +@flaky @pytest.mark.run @pytest.mark.alt -@flaky -def test_alternative_version_specifier(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_alternative_version_specifier(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -231,11 +242,11 @@ requests = {version = "*"} assert c.return_code == 0 +@flaky @pytest.mark.run @pytest.mark.alt -@flaky -def test_outline_table_specifier(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_outline_table_specifier(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ [packages.requests] @@ -257,20 +268,22 @@ version = "*" @pytest.mark.bad +@pytest.mark.basic @pytest.mark.install -def test_bad_packages(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_bad_packages(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("install NotAPackage") assert c.return_code > 0 +@pytest.mark.lock @pytest.mark.extras @pytest.mark.install @pytest.mark.requirements @pytest.mark.skip(reason="Not mocking this.") -def test_requirements_to_pipfile(PipenvInstance, pypi): +def test_requirements_to_pipfile(PipenvInstance): - with PipenvInstance(pipfile=False, chdir=True, pypi=pypi) as p: + with PipenvInstance(pipfile=False, chdir=True) as p: # Write a requirements file with open("requirements.txt", "w") as f: @@ -294,15 +307,17 @@ def test_requirements_to_pipfile(PipenvInstance, pypi): assert "pysocks" in p.lockfile["default"] +@pytest.mark.basic @pytest.mark.install +@pytest.mark.skip_osx @pytest.mark.requirements -def test_skip_requirements_when_pipfile(PipenvInstance, pypi): +def test_skip_requirements_when_pipfile(PipenvInstance): """Ensure requirements.txt is NOT imported when 1. We do `pipenv install [package]` 2. A Pipfile already exists when we run `pipenv install`. """ - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: with open("requirements.txt", "w") as f: f.write("requests==2.18.1\n") c = p.pipenv("install six") @@ -311,12 +326,13 @@ def test_skip_requirements_when_pipfile(PipenvInstance, pypi): contents = """ [packages] six = "*" -tablib = "<0.12" +fake_package = "<0.12" """.strip() f.write(contents) c = p.pipenv("install") - assert "tablib" in p.pipfile["packages"] - assert "tablib" in p.lockfile["default"] + assert c.ok + assert "fake_package" in p.pipfile["packages"] + assert "fake-package" in p.lockfile["default"] assert "six" in p.pipfile["packages"] assert "six" in p.lockfile["default"] assert "requests" not in p.pipfile["packages"] @@ -325,18 +341,20 @@ tablib = "<0.12" @pytest.mark.cli @pytest.mark.clean -def test_clean_on_empty_venv(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_clean_on_empty_venv(PipenvInstance): + with PipenvInstance() as p: c = p.pipenv("clean") assert c.return_code == 0 +@pytest.mark.basic @pytest.mark.install -def test_install_does_not_extrapolate_environ(PipenvInstance, pypi): +def test_install_does_not_extrapolate_environ(PipenvInstance): """Ensure environment variables are not expanded in lock file. """ - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ["PYPI_URL"] = pypi.url + with temp_environ(), PipenvInstance(chdir=True) as p: + # os.environ["PYPI_URL"] = pypi.url + os.environ["PYPI_URL"] = p.pypi with open(p.pipfile_path, "w") as f: f.write( @@ -361,6 +379,7 @@ name = 'mockpi' assert p.lockfile["_meta"]["sources"][0]["url"] == "${PYPI_URL}/simple" +@pytest.mark.basic @pytest.mark.editable @pytest.mark.badparameter @pytest.mark.install @@ -371,12 +390,13 @@ def test_editable_no_args(PipenvInstance): assert "Error: -e option requires an argument" in c.err +@pytest.mark.basic @pytest.mark.install @pytest.mark.virtualenv -def test_install_venv_project_directory(PipenvInstance, pypi): +def test_install_venv_project_directory(PipenvInstance): """Test the project functionality during virtualenv creation. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with temp_environ(), TemporaryDirectory( prefix="pipenv-", suffix="temp_workon_home" ) as workon_home: @@ -395,10 +415,11 @@ def test_install_venv_project_directory(PipenvInstance, pypi): assert venv_loc.joinpath(".project").exists() +@pytest.mark.cli @pytest.mark.deploy @pytest.mark.system -def test_system_and_deploy_work(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_system_and_deploy_work(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install six requests") assert c.return_code == 0 c = p.pipenv("--rm") @@ -419,6 +440,7 @@ requests assert c.return_code == 0 +@pytest.mark.basic @pytest.mark.install def test_install_creates_pipfile(PipenvInstance): with PipenvInstance(chdir=True) as p: @@ -432,39 +454,43 @@ def test_install_creates_pipfile(PipenvInstance): assert os.path.isfile(p.pipfile_path) +@pytest.mark.basic @pytest.mark.install -def test_install_non_exist_dep(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_install_non_exist_dep(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install dateutil") assert not c.ok assert "dateutil" not in p.pipfile["packages"] +@pytest.mark.basic @pytest.mark.install -def test_install_package_with_dots(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_install_package_with_dots(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install backports.html") assert c.ok assert "backports.html" in p.pipfile["packages"] +@pytest.mark.basic @pytest.mark.install -def test_rewrite_outline_table(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_rewrite_outline_table(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -six = {version = "*", editable = true} +six = {version = "*"} [packages.requests] version = "*" +extras = ["socks"] """.strip() f.write(contents) - c = p.pipenv("install -e click") + c = p.pipenv("install flask") assert c.return_code == 0 with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents - assert 'six = {version = "*", editable = true}' in contents - assert 'requests = {version = "*"}' in contents - assert 'click = {' in contents + assert 'six = {version = "*"}' in contents + assert 'requests = {version = "*"' in contents + assert 'flask = "*"' in contents diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index bee1d994..84237eb5 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import sys @@ -10,37 +12,34 @@ from pipenv.project import Project from pipenv.utils import temp_environ -py3_only = pytest.mark.skipif(sys.version_info < (3, 0), reason="requires Python3") -skip_py37 = pytest.mark.skipif(sys.version_info >= (3, 7), reason="Skip for python 3.7") - - -@pytest.mark.markers @flaky -def test_package_environment_markers(PipenvInstance, pypi): +@pytest.mark.markers +def test_package_environment_markers(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -tablib = {version = "*", markers="os_name=='splashwear'"} +fake_package = {version = "*", markers="os_name=='splashwear'"} """.strip() f.write(contents) c = p.pipenv('install') assert c.return_code == 0 assert 'Ignoring' in c.out - assert 'markers' in p.lockfile['default']['tablib'] + assert 'markers' in p.lockfile['default']['fake-package'], p.lockfile["default"] - c = p.pipenv('run python -c "import tablib;"') + c = p.pipenv('run python -c "import fake_package;"') assert c.return_code == 1 -@pytest.mark.markers + @flaky -def test_platform_python_implementation_marker(PipenvInstance, pypi): +@pytest.mark.markers +def test_platform_python_implementation_marker(PipenvInstance): """Markers should be converted during locking to help users who input this incorrectly. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -59,17 +58,17 @@ depends-on-marked-package = "*" "platform_python_implementation == 'CPython'" -@pytest.mark.run -@pytest.mark.alt -@pytest.mark.install @flaky -def test_specific_package_environment_markers(PipenvInstance, pypi): +@pytest.mark.alt +@pytest.mark.markers +@pytest.mark.install +def test_specific_package_environment_markers(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -requests = {version = "*", os_name = "== 'splashwear'"} +fake-package = {version = "*", os_name = "== 'splashwear'"} """.strip() f.write(contents) @@ -77,18 +76,18 @@ requests = {version = "*", os_name = "== 'splashwear'"} assert c.return_code == 0 assert 'Ignoring' in c.out - assert 'markers' in p.lockfile['default']['requests'] + assert 'markers' in p.lockfile['default']['fake-package'] - c = p.pipenv('run python -c "import requests;"') + c = p.pipenv('run python -c "import fake_package;"') assert c.return_code == 1 -@pytest.mark.markers @flaky -def test_top_level_overrides_environment_markers(PipenvInstance, pypi): +@pytest.mark.markers +def test_top_level_overrides_environment_markers(PipenvInstance): """Top-level environment markers should take precedence. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -99,21 +98,21 @@ funcsigs = {version = "*", os_name = "== 'splashwear'"} c = p.pipenv('install') assert c.return_code == 0 - - assert p.lockfile['default']['funcsigs']['markers'] == "os_name == 'splashwear'" + assert "markers" in p.lockfile['default']['funcsigs'], p.lockfile['default']['funcsigs'] + assert p.lockfile['default']['funcsigs']['markers'] == "os_name == 'splashwear'", p.lockfile['default']['funcsigs'] +@flaky @pytest.mark.markers @pytest.mark.install -@flaky -def test_global_overrides_environment_markers(PipenvInstance, pypi): +def test_global_overrides_environment_markers(PipenvInstance): """Empty (unconditional) dependency should take precedence. If a dependency is specified without environment markers, it should override dependencies with environment markers. In this example, APScheduler requires funcsigs only on Python 2, but since funcsigs is also specified as an unconditional dep, its markers should be empty. """ - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -128,12 +127,12 @@ funcsigs = "*" assert p.lockfile['default']['funcsigs'].get('markers', '') == '' -@pytest.mark.lock -@pytest.mark.complex @flaky -@py3_only -@skip_py37 -def test_resolver_unique_markers(PipenvInstance, pypi): +@pytest.mark.markers +@pytest.mark.complex +@pytest.mark.py3_only +@pytest.mark.skip("test is flaky on CI") +def test_resolver_unique_markers(PipenvInstance): """vcrpy has a dependency on `yarl` which comes with a marker of 'python version in "3.4, 3.5, 3.6" - this marker duplicates itself: @@ -141,8 +140,8 @@ def test_resolver_unique_markers(PipenvInstance, pypi): This verifies that we clean that successfully. """ - with PipenvInstance(chdir=True, pypi=pypi) as p: - c = p.pipenv('install vcrpy==1.11.0') + with PipenvInstance(chdir=True) as p: + c = p.pipenv('install vcrpy==2.0.1') assert c.return_code == 0 c = p.pipenv('lock') assert c.return_code == 0 @@ -150,13 +149,17 @@ def test_resolver_unique_markers(PipenvInstance, pypi): yarl = p.lockfile['default']['yarl'] assert 'markers' in yarl # Two possible marker sets are ok here - assert yarl['markers'] in ["python_version in '3.4, 3.5, 3.6'", "python_version >= '3.4.1'"] + assert yarl['markers'] in [ + "python_version in '3.4, 3.5, 3.6'", + "python_version >= '3.4'", + "python_version >= '3.5'", # yarl 1.3.0 requires python 3.5.3 + ] -@pytest.mark.project @flaky -def test_environment_variable_value_does_not_change_hash(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +@pytest.mark.project +def test_environment_variable_value_does_not_change_hash(PipenvInstance): + with PipenvInstance(chdir=True) as p: with temp_environ(): with open(p.pipfile_path, 'w') as f: f.write(""" diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 0879a265..90726446 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -1,12 +1,14 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import shutil +import sys import pytest from flaky import flaky from pipenv._compat import Path -from pipenv.project import Project from pipenv.utils import mkdir_p, temp_environ from pipenv.vendor import delegator @@ -14,31 +16,29 @@ from pipenv.vendor import delegator @pytest.mark.extras @pytest.mark.install @pytest.mark.local -def test_local_extras_install(PipenvInstance, pypi): +def test_local_extras_install(PipenvInstance): """Ensure -e .[extras] installs. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: setup_py = os.path.join(p.path, "setup.py") with open(setup_py, "w") as fh: contents = """ from setuptools import setup, find_packages setup( -name='testpipenv', -version='0.1', -description='Pipenv Test Package', -author='Pipenv Test', -author_email='test@pipenv.package', -license='MIT', -packages=find_packages(), -install_requires=[], -extras_require={'dev': ['six']}, -zip_safe=False + name='testpipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='MIT', + packages=find_packages(), + install_requires=[], + extras_require={'dev': ['six']}, + zip_safe=False ) """.strip() fh.write(contents) line = "-e .[dev]" - # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} - project = Project() with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] @@ -52,9 +52,9 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "testpipenv" in p.lockfile["default"] assert p.lockfile["default"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] - c = p.pipenv("--rm") + c = p.pipenv("uninstall --all") assert c.return_code == 0 - project.write_toml({"packages": {}, "dev-packages": {}}) + print("Current directory: {0}".format(os.getcwd()), file=sys.stderr) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 assert "testpipenv" in p.pipfile["packages"] @@ -63,11 +63,11 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "six" in p.lockfile["default"] -@pytest.mark.install @pytest.mark.local +@pytest.mark.install @pytest.mark.needs_internet @flaky -class TestDependencyLinks(object): +class TestDirectDependencies(object): """Ensure dependency_links are parsed and installed. This is needed for private repo dependencies. @@ -85,45 +85,44 @@ setup( version='0.1', packages=[], install_requires=[ - 'test-private-dependency' - ], - dependency_links=[ '{0}' - ] + ], ) """.strip().format(deplink) fh.write(contents) @staticmethod def helper_dependency_links_install_test(pipenv_instance, deplink): - TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink) + TestDirectDependencies.helper_dependency_links_install_make_setup(pipenv_instance, deplink) c = pipenv_instance.pipenv("install -v -e .") assert c.return_code == 0 assert "test-private-dependency" in pipenv_instance.lockfile["default"] assert "version" in pipenv_instance.lockfile["default"]["test-private-dependency"] assert "0.1" in pipenv_instance.lockfile["default"]["test-private-dependency"]["version"] - def test_https_dependency_links_install(self, PipenvInstance, pypi): + def test_https_dependency_links_install(self, PipenvInstance): """Ensure dependency_links are parsed and installed (needed for private repo dependencies). """ - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + with temp_environ(), PipenvInstance(chdir=True) as p: + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.needs_github_ssh - def test_ssh_dependency_links_install(self, PipenvInstance, pypi): - with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: + def test_ssh_dependency_links_install(self, PipenvInstance): + with temp_environ(), PipenvInstance(chdir=True) as p: os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.e +@pytest.mark.local @pytest.mark.install @pytest.mark.skip(reason="this doesn't work on windows") def test_e_dot(PipenvInstance, pip_src_dir): @@ -137,14 +136,14 @@ def test_e_dot(PipenvInstance, pip_src_dir): assert "path" in p.pipfile["dev-packages"][key] assert "requests" in p.lockfile["develop"] - @pytest.mark.install +@pytest.mark.multiprocessing @flaky -def test_multiprocess_bug_and_install(PipenvInstance, pypi): +def test_multiprocess_bug_and_install(PipenvInstance): with temp_environ(): os.environ["PIPENV_MAX_SUBPROCESS"] = "2" - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -165,12 +164,12 @@ urllib3 = "*" assert c.return_code == 0 -@pytest.mark.sequential @pytest.mark.install +@pytest.mark.sequential @flaky -def test_sequential_mode(PipenvInstance, pypi): +def test_sequential_mode(PipenvInstance): - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, "w") as f: contents = """ [packages] @@ -191,10 +190,10 @@ pytz = "*" assert c.return_code == 0 -@pytest.mark.install @pytest.mark.run -def test_normalize_name_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.install +def test_normalize_name_install(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ # Pre comment @@ -222,18 +221,19 @@ Requests = "==2.14.0" # Inline comment assert "# Inline comment" in contents -@pytest.mark.files -@pytest.mark.resolver -@pytest.mark.eggs @flaky -def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot): +@pytest.mark.eggs +@pytest.mark.files +@pytest.mark.local +@pytest.mark.resolver +def test_local_package(PipenvInstance, pip_src_dir, testsroot): """This test ensures that local packages (directories with a setup.py) installed in editable mode have their dependencies resolved as well""" file_name = "requests-2.19.1.tar.gz" package = "requests-2.19.1" # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: # This tests for a bug when installing a zipfile in the current dir copy_to = os.path.join(p.path, file_name) shutil.copy(source_path, copy_to) @@ -250,13 +250,14 @@ def test_local_package(PipenvInstance, pip_src_dir, pypi, testsroot): @pytest.mark.files +@pytest.mark.local @flaky -def test_local_zipfiles(PipenvInstance, pypi, testsroot): +def test_local_zipfiles(PipenvInstance, testsroot): file_name = "requests-2.19.1.tar.gz" # Not sure where travis/appveyor run tests from source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: # This tests for a bug when installing a zipfile in the current dir shutil.copy(source_path, os.path.join(p.path, file_name)) @@ -268,19 +269,20 @@ def test_local_zipfiles(PipenvInstance, pypi, testsroot): assert "file" in dep or "path" in dep assert c.return_code == 0 - key = [k for k in p.lockfile["default"].keys()][0] - dep = p.lockfile["default"][key] + # This now gets resolved to its name correctly + dep = p.lockfile["default"]["requests"] assert "file" in dep or "path" in dep +@pytest.mark.local @pytest.mark.files @flaky -def test_relative_paths(PipenvInstance, pypi, testsroot): +def test_relative_paths(PipenvInstance, testsroot): file_name = "requests-2.19.1.tar.gz" source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: artifact_dir = "artifacts" artifact_path = os.path.join(p.path, artifact_dir) mkdir_p(artifact_path) @@ -297,10 +299,11 @@ def test_relative_paths(PipenvInstance, pypi, testsroot): @pytest.mark.install +@pytest.mark.local @pytest.mark.local_file @flaky -def test_install_local_file_collision(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_install_local_file_collision(PipenvInstance): + with PipenvInstance() as p: target_package = "alembic" fake_file = os.path.join(p.path, target_package) with open(fake_file, "w") as f: @@ -312,7 +315,7 @@ def test_install_local_file_collision(PipenvInstance, pypi): assert target_package in p.lockfile["default"] -@pytest.mark.url +@pytest.mark.urls @pytest.mark.install def test_install_local_uri_special_character(PipenvInstance, testsroot): file_name = "six-1.11.0+mkl-py2.py3-none-any.whl" @@ -336,10 +339,10 @@ six = {{path = "./artifacts/{}"}} assert "six" in p.lockfile["default"] +@pytest.mark.run @pytest.mark.files @pytest.mark.install -@pytest.mark.run -def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, testsroot): +def test_multiple_editable_packages_should_not_race(PipenvInstance, testsroot): """Test for a race condition that can occur when installing multiple 'editable' packages at once, and which causes some of them to not be importable. @@ -356,7 +359,7 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, testsr [packages] """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: for pkg_name in pkgs: source_path = p._pipfile.get_fixture_path("git/{0}/".format(pkg_name)).as_posix() c = delegator.run("git clone {0} ./{1}".format(source_path, pkg_name)) @@ -372,3 +375,18 @@ def test_multiple_editable_packages_should_not_race(PipenvInstance, pypi, testsr c = p.pipenv('run python -c "import requests, flask, six, jinja2"') assert c.return_code == 0, c.err + + +@pytest.mark.outdated +@pytest.mark.py3_only +def test_outdated_should_compare_postreleases_without_failing(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install ibm-db-sa-py3==0.3.0") + assert c.return_code == 0 + c = p.pipenv("update --outdated") + assert c.return_code == 0 + assert "Skipped Update" in c.err + p._pipfile.update("ibm-db-sa-py3", "*") + c = p.pipenv("update --outdated") + assert c.return_code != 0 + assert "out-of-date" in c.out diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 4dd35882..8772df54 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,5 +1,5 @@ -import os - +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import pytest from flaky import flaky @@ -9,12 +9,12 @@ import delegator from pipenv._compat import Path +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_basic_vcs_install(PipenvInstance): # ! This is failing + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+https://github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 # edge case where normal package starts with VCS name shouldn't be flagged as vcs @@ -25,16 +25,17 @@ def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "https://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } assert "gitdb2" in p.lockfile["default"] +@flaky @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_git_vcs_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+git://github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 assert "six" in p.pipfile["packages"] @@ -42,16 +43,17 @@ def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "git://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } +@flaky @pytest.mark.vcs @pytest.mark.install -@pytest.mark.needs_github_ssh @pytest.mark.needs_internet -@flaky -def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +@pytest.mark.needs_github_ssh +def test_ssh_vcs_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv("install git+ssh://git@github.com/benjaminp/six.git@1.11.0#egg=six") assert c.return_code == 0 assert "six" in p.pipfile["packages"] @@ -59,15 +61,17 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "ssh://git@github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } -@pytest.mark.files -@pytest.mark.urls -@pytest.mark.needs_internet @flaky -def test_urls_work(PipenvInstance, pypi, pip_src_dir): - with PipenvInstance(pypi=pypi, chdir=True) as p: +@pytest.mark.urls +@pytest.mark.files +@pytest.mark.needs_internet +def test_urls_work(PipenvInstance): + with PipenvInstance(chdir=True) as p: + # the library this installs is "django-cms" path = p._pipfile.get_url("django", "3.4.x.zip") c = p.pipenv( "install {0}".format(path) @@ -77,12 +81,13 @@ def test_urls_work(PipenvInstance, pypi, pip_src_dir): dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - dep = list(p.lockfile["default"].values())[0] + # now that we handle resolution with requirementslib, this will resolve to a name + dep = p.lockfile["default"]["django-cms"] assert "file" in dep, p.lockfile -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files def test_file_urls_work(PipenvInstance, pip_src_dir): with PipenvInstance(chdir=True) as p: whl = Path(__file__).parent.parent.joinpath( @@ -99,13 +104,13 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): assert "file" in p.pipfile["packages"]["six"] -@pytest.mark.files @pytest.mark.urls +@pytest.mark.files @pytest.mark.needs_internet -def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): +def test_local_vcs_urls_work(PipenvInstance, tmpdir): six_dir = tmpdir.join("six") six_path = Path(six_dir.strpath) - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = delegator.run( "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath) ) @@ -118,13 +123,13 @@ def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): @pytest.mark.e @pytest.mark.vcs +@pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_editable_vcs_install(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( - "install -e git+https://github.com/requests/requests.git#egg=requests" + "install -e git+https://github.com/kennethreitz/requests.git#egg=requests" ) assert c.return_code == 0 assert "requests" in p.pipfile["packages"] @@ -137,15 +142,14 @@ def test_editable_vcs_install(PipenvInstance, pip_src_dir, pypi): assert "certifi" in p.lockfile["default"] -@pytest.mark.install @pytest.mark.vcs -@pytest.mark.tablib +@pytest.mark.urls +@pytest.mark.install @pytest.mark.needs_internet -@flaky -def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): +def test_install_editable_git_tag(PipenvInstance_NoPyPI): # This uses the real PyPI since we need Internet to access the Git # dependency anyway. - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( "install -e git+https://github.com/benjaminp/six.git@1.11.0#egg=six" ) @@ -160,11 +164,12 @@ def test_install_editable_git_tag(PipenvInstance, pip_src_dir, pypi): assert "ref" in p.lockfile["default"]["six"] -@pytest.mark.install +@pytest.mark.urls @pytest.mark.index +@pytest.mark.install @pytest.mark.needs_internet -def test_install_named_index_alias(PipenvInstance): - with PipenvInstance() as p: +def test_install_named_index_alias(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, "w") as f: contents = """ [[source]] @@ -188,9 +193,10 @@ six = "*" @pytest.mark.vcs +@pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): +def test_install_local_vcs_not_in_lockfile(PipenvInstance): with PipenvInstance(chdir=True) as p: # six_path = os.path.join(p.path, "six") six_path = p._pipfile.get_fixture_path("git/six/").as_posix() @@ -204,10 +210,11 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): @pytest.mark.vcs +@pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_get_vcs_refs(PipenvInstance, pip_src_dir): - with PipenvInstance(chdir=True) as p: +def test_get_vcs_refs(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI(chdir=True) as p: c = p.pipenv( "install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six" ) @@ -219,8 +226,8 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): == "5efb522b0647f7467248273ec1b893d06b984a59" ) pipfile = Path(p.pipfile_path) - new_content = pipfile.read_bytes().replace(b"1.9.0", b"1.11.0") - pipfile.write_bytes(new_content) + new_content = pipfile.read_text().replace(u"1.9.0", u"1.11.0") + pipfile.write_text(new_content) c = p.pipenv("lock") assert c.return_code == 0 assert ( @@ -232,15 +239,19 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): @pytest.mark.vcs +@pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir): +@pytest.mark.skip_py27_win +@pytest.mark.skip_py38 +def test_vcs_entry_supersedes_non_vcs(PipenvInstance): """See issue #2181 -- non-editable VCS dep was specified, but not showing up in the lockfile -- due to not running pip install before locking and not locking the resolution graph of non-editable vcs dependencies. """ with PipenvInstance(chdir=True) as p: - pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + # pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + pyinstaller_uri = "https://github.com/pyinstaller/pyinstaller.git" with open(p.pipfile_path, "w") as f: f.write( """ @@ -251,8 +262,8 @@ name = "pypi" [packages] PyUpdater = "*" -PyInstaller = {{ref = "develop", git = "{0}"}} - """.format(pyinstaller_path.as_uri()).strip() +PyInstaller = {{ref = "v3.6", git = "{0}"}} + """.format(pyinstaller_uri).strip() ) c = p.pipenv("install") assert c.return_code == 0 @@ -263,15 +274,16 @@ PyInstaller = {{ref = "develop", git = "{0}"}} assert p.lockfile["default"]["pyinstaller"].get("ref") is not None assert ( p.lockfile["default"]["pyinstaller"]["git"] - == pyinstaller_path.as_uri() + == pyinstaller_uri ) @pytest.mark.vcs +@pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_vcs_can_use_markers(PipenvInstance, pip_src_dir, pypi): - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_vcs_can_use_markers(PipenvInstance): + with PipenvInstance(chdir=True) as p: path = p._pipfile.get_fixture_path("git/six/.git") p._pipfile.install("six", {"git": "{0}".format(path.as_uri()), "markers": "sys_platform == 'linux'"}) assert "six" in p.pipfile["packages"] diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index d993b1a2..4f202bea 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -1,17 +1,21 @@ +# -*- coding: utf-8 -*- + +import json import os import sys import pytest from flaky import flaky - +from vistir.compat import Path +from vistir.misc import to_text from pipenv.utils import temp_environ @pytest.mark.lock @pytest.mark.requirements -def test_lock_handle_eggs(PipenvInstance, pypi): - """Ensure locking works with packages provoding egg formats. +def test_lock_handle_eggs(PipenvInstance): + """Ensure locking works with packages providing egg formats. """ with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: @@ -27,9 +31,9 @@ RandomWords = "*" @pytest.mark.lock @pytest.mark.requirements -def test_lock_requirements_file(PipenvInstance, pypi): +def test_lock_requirements_file(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -56,9 +60,10 @@ flask = "==0.12.2" @pytest.mark.lock -def test_lock_keep_outdated(PipenvInstance, pypi): +@pytest.mark.keep_outdated +def test_lock_keep_outdated(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -92,20 +97,85 @@ PyTest = "*" assert lock['default']['pytest']['version'] == "==3.1.0" +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_remove_lockfile_entries(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("requests", "==2.18.4") + p._pipfile.add("colorama", {"version": "*", "markers": "os_name=='FakeOS'"}) + p.pipenv("install") + p._pipfile.add("six", "*") + p.pipenv("lock --keep-outdated") + assert "colorama" in p.lockfile["default"] + assert p.lockfile["default"]["colorama"]["markers"] == "os_name == 'FakeOS'" + + +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("urllib3", "==1.21.1") + c = p.pipenv("install") + assert c.ok + p._pipfile.add("requests", "==2.18.4") + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert "requests" in p.lockfile["default"] + assert "urllib3" in p.lockfile["default"] + assert p.lockfile["default"]["requests"]["version"] == "==2.18.4" + assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1" + + +@pytest.mark.lock +def test_keep_outdated_keeps_markers_not_removed(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv("install six click") + assert c.ok + lockfile = Path(p.lockfile_path) + lockfile_content = lockfile.read_text() + lockfile_json = json.loads(lockfile_content) + assert "six" in lockfile_json["default"] + lockfile_json["default"]["six"]["markers"] = "python_version >= '2.7'" + lockfile.write_text(to_text(json.dumps(lockfile_json))) + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert p.lockfile["default"]["six"].get("markers", "") == "python_version >= '2.7'" + + +@pytest.mark.lock +@pytest.mark.keep_outdated +def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance): + with PipenvInstance(chdir=True) as p: + p._pipfile.add("requests", "==2.18.4") + c = p.pipenv("install") + assert c.ok + p._pipfile.add("requests", "*") + assert p.pipfile["packages"]["requests"] == "*" + c = p.pipenv("lock --keep-outdated") + assert c.ok + assert "requests" in p.lockfile["default"] + assert "urllib3" in p.lockfile["default"] + # ensure this didn't update requests + assert p.lockfile["default"]["requests"]["version"] == "==2.18.4" + c = p.pipenv("lock") + assert c.ok + assert p.lockfile["default"]["requests"]["version"] != "==2.18.4" + + @pytest.mark.lock @pytest.mark.complex @pytest.mark.needs_internet -def test_complex_lock_with_vcs_deps(PipenvInstance, pip_src_dir): +def test_complex_lock_with_vcs_deps(local_tempdir, PipenvInstance, pip_src_dir): # This uses the real PyPI since we need Internet to access the Git # dependency anyway. - with PipenvInstance() as p: + with PipenvInstance() as p, local_tempdir: with open(p.pipfile_path, 'w') as f: contents = """ [packages] click = "==6.7" [dev-packages] -requests = {git = "https://github.com/requests/requests.git"} +requests = {git = "https://github.com/psf/requests.git"} """.strip() f.write(contents) @@ -129,9 +199,9 @@ requests = {git = "https://github.com/requests/requests.git"} @pytest.mark.lock @pytest.mark.requirements -def test_lock_with_prereleases(PipenvInstance, pypi): +def test_lock_with_prereleases(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -148,13 +218,13 @@ allow_prereleases = true @pytest.mark.lock -@pytest.mark.complex @pytest.mark.maya +@pytest.mark.complex @pytest.mark.needs_internet @flaky -def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir, pypi): +def test_complex_deps_lock_and_install_properly(PipenvInstance, pip_src_dir): # This uses the real PyPI because Maya has too many dependencies... - with PipenvInstance(chdir=True, pypi=pypi) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -169,10 +239,10 @@ maya = "*" assert c.return_code == 0 -@pytest.mark.extras @pytest.mark.lock -def test_lock_extras_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +@pytest.mark.extras +def test_lock_extras_without_install(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -191,16 +261,16 @@ requests = {version = "*", extras = ["socks"]} assert "extra == 'socks'" not in c.out.strip() -@pytest.mark.extras @pytest.mark.lock +@pytest.mark.extras @pytest.mark.complex -@pytest.mark.skip(reason='Needs numpy to be mocked') @pytest.mark.needs_internet -def test_complex_lock_deep_extras(PipenvInstance, pypi): +@pytest.mark.skip(reason='Needs numpy to be mocked') +def test_complex_lock_deep_extras(PipenvInstance): # records[pandas] requires tablib[pandas] which requires pandas. # This uses the real PyPI; Pandas has too many requirements to mock. - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] @@ -216,12 +286,12 @@ records = {extras = ["pandas"], version = "==0.5.2"} assert 'pandas' in p.lockfile['default'] -@pytest.mark.skip_lock @pytest.mark.index -@pytest.mark.needs_internet @pytest.mark.install # private indexes need to be uncached for resolution -def test_private_index_skip_lock(PipenvInstance): - with PipenvInstance() as p: +@pytest.mark.skip_lock +@pytest.mark.needs_internet +def test_private_index_skip_lock(PipenvInstance_NoPyPI): + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -243,14 +313,14 @@ requests = "*" assert c.return_code == 0 -@pytest.mark.requirements @pytest.mark.lock @pytest.mark.index @pytest.mark.install # private indexes need to be uncached for resolution +@pytest.mark.requirements @pytest.mark.needs_internet -def test_private_index_lock_requirements(PipenvInstance): +def test_private_index_lock_requirements(PipenvInstance_NoPyPI): # Don't use the local fake pypi - with PipenvInstance() as p: + with PipenvInstance_NoPyPI() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -276,14 +346,14 @@ requests = "*" assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip() -@pytest.mark.requirements @pytest.mark.lock @pytest.mark.index @pytest.mark.install # private indexes need to be uncached for resolution +@pytest.mark.requirements @pytest.mark.needs_internet -def test_private_index_mirror_lock_requirements(PipenvInstance): +def test_private_index_mirror_lock_requirements(PipenvInstance_NoPyPI): # Don't use the local fake pypi - with temp_environ(), PipenvInstance(chdir=True) as p: + with temp_environ(), PipenvInstance_NoPyPI(chdir=True) as p: # Using pypi.python.org as pipenv-test-public-package is not # included in the local pypi mirror mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") @@ -302,25 +372,23 @@ name = "testpypi" [packages] six = {version = "*", index = "testpypi"} -requests = "*" +fake-package = "*" """.strip() f.write(contents) c = p.pipenv('install --pypi-mirror {0}'.format(mirror_url)) assert c.return_code == 0 c = p.pipenv('lock -r --pypi-mirror {0}'.format(mirror_url)) assert c.return_code == 0 - assert '-i https://pypi.org/simple' in c.out.strip() + assert '-i {0}'.format(mirror_url) in c.out.strip() assert '--extra-index-url https://test.pypi.org/simple' in c.out.strip() - # Mirror url should not have replaced source URLs - assert '-i {0}'.format(mirror_url) not in c.out.strip() assert '--extra-index-url {}'.format(mirror_url) not in c.out.strip() -@pytest.mark.install @pytest.mark.index -def test_lock_updated_source(PipenvInstance, pypi): +@pytest.mark.install +def test_lock_updated_source(PipenvInstance): - with PipenvInstance(pypi=pypi) as p: + with PipenvInstance() as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -328,7 +396,7 @@ url = "{url}/${{MY_ENV_VAR}}" [packages] requests = "==2.14.0" - """.strip().format(url=pypi.url) + """.strip().format(url=p.pypi) f.write(contents) with temp_environ(): @@ -344,7 +412,7 @@ url = "{url}/simple" [packages] requests = "==2.14.0" - """.strip().format(url=pypi.url) + """.strip().format(url=p.pypi) f.write(contents) c = p.pipenv('lock') @@ -352,15 +420,15 @@ requests = "==2.14.0" assert 'requests' in p.lockfile['default'] -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true} +requests = {git = "https://github.com/psf/requests.git", ref = "master", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -371,52 +439,52 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_ref_in_git(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_ref_in_git(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git@883caaf", editable = true} +requests = {git = "https://github.com/psf/requests.git@883caaf", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 - assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['git'] == 'https://github.com/psf/requests.git' assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' c = p.pipenv('install') assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_ref(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_ref(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "883caaf", editable = true} +requests = {git = "https://github.com/psf/requests.git", ref = "883caaf", editable = true} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 - assert p.lockfile['default']['requests']['git'] == 'https://github.com/requests/requests.git' + assert p.lockfile['default']['requests']['git'] == 'https://github.com/psf/requests.git' assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' c = p.pipenv('install') assert c.return_code == 0 +@pytest.mark.vcs +@pytest.mark.lock @pytest.mark.extras -@pytest.mark.lock -@pytest.mark.vcs @pytest.mark.needs_internet -def test_lock_editable_vcs_with_extras_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_extras_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", editable = true, extras = ["socks"]} +requests = {git = "https://github.com/psf/requests.git", editable = true, extras = ["socks"]} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -428,15 +496,15 @@ requests = {git = "https://github.com/requests/requests.git", editable = true, e assert c.return_code == 0 -@pytest.mark.lock @pytest.mark.vcs +@pytest.mark.lock @pytest.mark.needs_internet -def test_lock_editable_vcs_with_markers_without_install(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_editable_vcs_with_markers_without_install(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] -requests = {git = "https://github.com/requests/requests.git", ref = "master", editable = true, markers = "python_version >= '2.6'"} +requests = {git = "https://github.com/psf/requests.git", ref = "master", editable = true, markers = "python_version >= '2.6'"} """.strip()) c = p.pipenv('lock') assert c.return_code == 0 @@ -449,8 +517,8 @@ requests = {git = "https://github.com/requests/requests.git", ref = "master", ed @pytest.mark.lock @pytest.mark.skip(reason="This doesn't work for some reason.") -def test_lock_respecting_python_version(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_respecting_python_version(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -491,10 +559,10 @@ def test_lockfile_with_empty_dict(PipenvInstance): @pytest.mark.lock -@pytest.mark.skip_lock @pytest.mark.install -def test_lock_with_incomplete_source(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +@pytest.mark.skip_lock +def test_lock_with_incomplete_source(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [[source]] @@ -512,8 +580,8 @@ requests = "*" @pytest.mark.lock @pytest.mark.install -def test_lock_no_warnings(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_no_warnings(PipenvInstance): + with PipenvInstance(chdir=True) as p: os.environ["PYTHONWARNINGS"] = str("once") c = p.pipenv("install six") assert c.return_code == 0 @@ -527,18 +595,20 @@ def test_lock_no_warnings(PipenvInstance, pypi): @pytest.mark.lock @pytest.mark.install @pytest.mark.skipif(sys.version_info >= (3, 5), reason="scandir doesn't get installed on python 3.5+") -def test_lock_missing_cache_entries_gets_all_hashes(monkeypatch, PipenvInstance, pypi, tmpdir): +def test_lock_missing_cache_entries_gets_all_hashes(PipenvInstance, tmpdir): """ Test locking pathlib2 on python2.7 which needs `scandir`, but fails to resolve when using a fresh dependency cache. """ - with monkeypatch.context() as m: - monkeypatch.setattr("pipenv.patched.piptools.locations.CACHE_DIR", tmpdir.strpath) - with PipenvInstance(pypi=pypi, chdir=True) as p: + with temp_environ(): + os.environ["PIPENV_CACHE_DIR"] = str(tmpdir.strpath) + with PipenvInstance(chdir=True) as p: p._pipfile.add("pathlib2", "*") assert "pathlib2" in p.pipfile["packages"] c = p.pipenv("install") + assert c.return_code == 0, (c.err, ("\n".join(["{0}: {1}\n".format(k, v) for k, v in os.environ.items()]))) + c = p.pipenv("lock --clear") assert c.return_code == 0, c.err assert "pathlib2" in p.lockfile["default"] assert "scandir" in p.lockfile["default"] @@ -546,12 +616,12 @@ def test_lock_missing_cache_entries_gets_all_hashes(monkeypatch, PipenvInstance, assert len(p.lockfile["default"]["scandir"]["hashes"]) > 1 -@pytest.mark.lock @pytest.mark.vcs -def test_vcs_lock_respects_top_level_pins(PipenvInstance, pypi): +@pytest.mark.lock +def test_vcs_lock_respects_top_level_pins(PipenvInstance): """Test that locking VCS dependencies respects top level packages pinned in Pipfiles""" - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: requests_uri = p._pipfile.get_fixture_path("git/requests").as_uri() p._pipfile.add("requests", { "editable": True, "git": "{0}".format(requests_uri), @@ -567,8 +637,8 @@ def test_vcs_lock_respects_top_level_pins(PipenvInstance, pypi): @pytest.mark.lock -def test_lock_after_update_source_name(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_lock_after_update_source_name(PipenvInstance): + with PipenvInstance(chdir=True) as p: contents = """ [[source]] url = "https://test.pypi.org/simple" @@ -585,6 +655,7 @@ six = "*" assert p.lockfile["default"]["six"]["index"] == "test" with open(p.pipfile_path, 'w') as f: f.write(contents.replace('name = "test"', 'name = "custom"')) - c = p.pipenv("lock") + c = p.pipenv("lock --clear") assert c.return_code == 0 - assert p.lockfile["default"]["six"]["index"] == "custom" + assert "index" in p.lockfile["default"]["six"] + assert p.lockfile["default"]["six"]["index"] == "custom", Path(p.lockfile_path).read_text() # p.lockfile["default"]["six"] diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index deacc495..a8c2b5e2 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function """Misc. tests that don't fit anywhere. XXX: Try our best to reduce tests in this file. @@ -5,12 +7,8 @@ XXX: Try our best to reduce tests in this file. import os -from tempfile import mkdtemp - -import mock import pytest -from pipenv._compat import Path from pipenv.project import Project from pipenv.utils import temp_environ from pipenv.vendor import delegator @@ -29,18 +27,17 @@ def test_code_import_manual(PipenvInstance): @pytest.mark.lock @pytest.mark.deploy -@pytest.mark.cli -def test_deploy_works(PipenvInstance, pypi): +def test_deploy_works(PipenvInstance): - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [packages] -requests = "==2.14.0" -flask = "==0.12.2" +requests = "==2.19.1" +flask = "==1.1.2" [dev-packages] -pytest = "==3.1.1" +pytest = "==4.6.9" """.strip() f.write(contents) c = p.pipenv('install --verbose') @@ -53,7 +50,7 @@ pytest = "==3.1.1" with open(p.pipfile_path, 'w') as f: contents = """ [packages] -requests = "==2.14.0" +requests = "==2.19.1" """.strip() f.write(contents) @@ -63,29 +60,30 @@ requests = "==2.14.0" @pytest.mark.update @pytest.mark.lock -def test_update_locks(PipenvInstance, pypi): - - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv('install requests==2.14.0') +def test_update_locks(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv('install jdcal==1.3') assert c.return_code == 0 + assert p.lockfile['default']['jdcal']['version'] == '==1.3' with open(p.pipfile_path, 'r') as fh: pipfile_contents = fh.read() - pipfile_contents = pipfile_contents.replace('==2.14.0', '*') + assert '==1.3' in pipfile_contents + pipfile_contents = pipfile_contents.replace('==1.3', '*') with open(p.pipfile_path, 'w') as fh: fh.write(pipfile_contents) - c = p.pipenv('update requests') + c = p.pipenv('update jdcal') assert c.return_code == 0 - assert p.lockfile['default']['requests']['version'] == '==2.19.1' + assert p.lockfile['default']['jdcal']['version'] == '==1.4' c = p.pipenv('run pip freeze') assert c.return_code == 0 lines = c.out.splitlines() - assert 'requests==2.19.1' in [l.strip() for l in lines] + assert 'jdcal==1.4' in [l.strip() for l in lines] @pytest.mark.project @pytest.mark.proper_names -def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): - with PipenvInstance(chdir=True, pypi=pypi): +def test_proper_names_unamanged_virtualenv(PipenvInstance): + with PipenvInstance(chdir=True): c = delegator.run('python -m virtualenv .venv') assert c.return_code == 0 project = Project() @@ -93,17 +91,16 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli -def test_directory_with_leading_dash(PipenvInstance): - def mocked_mkdtemp(suffix, prefix, dir): - if suffix == '-project': - prefix = '-dir-with-leading-dash' - return mkdtemp(suffix, prefix, dir) - - with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp): - with temp_environ(), PipenvInstance(chdir=True) as p: - del os.environ['PIPENV_VENV_IN_PROJECT'] - p.pipenv('--python python') - venv_path = p.pipenv('--venv').out.strip() +def test_directory_with_leading_dash(raw_venv, PipenvInstance): + with temp_environ(): + with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() assert os.path.isdir(venv_path) # Manually clean up environment, since PipenvInstance assumes that # the virutalenv is in the project directory. diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 6a4a858a..ad38d74a 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import io import os import tarfile @@ -8,6 +9,8 @@ import pytest from pipenv.patched import pipfile from pipenv.project import Project from pipenv.utils import temp_environ +from pipenv.vendor.vistir.path import is_in_path, normalize_path +from pipenv.vendor.delegator import run as delegator_run @pytest.mark.project @@ -35,8 +38,8 @@ pytz = "*" @pytest.mark.project @pytest.mark.sources @pytest.mark.parametrize('lock_first', [True, False]) -def test_get_source(PipenvInstance, pypi, lock_first): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_get_source(PipenvInstance, lock_first): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -82,8 +85,8 @@ six = {{version = "*", index = "pypi"}} @pytest.mark.install @pytest.mark.project @pytest.mark.parametrize('newlines', [u'\n', u'\r\n']) -def test_maintain_file_line_endings(PipenvInstance, pypi, newlines): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_maintain_file_line_endings(PipenvInstance, newlines): + with PipenvInstance(chdir=True) as p: # Initial pipfile + lockfile generation c = p.pipenv('install pytz') assert c.return_code == 0 @@ -118,8 +121,8 @@ def test_maintain_file_line_endings(PipenvInstance, pypi, newlines): @pytest.mark.project @pytest.mark.sources -def test_many_indexes(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_many_indexes(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: contents = """ [[source]] @@ -150,17 +153,17 @@ six = {{version = "*", index = "pypi"}} @pytest.mark.install @pytest.mark.project -def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpdir): - file_name = "requests-2.19.1.tar.gz" - package = pathlib_tmpdir.joinpath("requests-2.19.1") - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) - with PipenvInstance(chdir=True, pypi=pypi) as p: +def test_include_editable_packages(PipenvInstance, testsroot, pathlib_tmpdir): + file_name = "tablib-0.12.1.tar.gz" + package = pathlib_tmpdir.joinpath("tablib-0.12.1") + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) + with PipenvInstance(chdir=True) as p: with tarfile.open(source_path, "r:gz") as tarinfo: tarinfo.extractall(path=str(pathlib_tmpdir)) - c = p.pipenv('install -e {}'.format(package)) + c = p.pipenv('install -e {0}'.format(package.as_posix())) assert c.return_code == 0 project = Project() - assert "requests" in [ + assert "tablib" in [ package.project_name for package in project.environment.get_installed_packages() ] @@ -168,17 +171,59 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.project @pytest.mark.virtualenv -def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): - with PipenvInstance(chdir=True, pypi=pypi) as p: - os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) +def test_run_in_virtualenv_with_global_context(PipenvInstance, virtualenv): + with PipenvInstance(chdir=True, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p: + c = delegator_run( + "pipenv run pip freeze", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert 'Creating a virtualenv' not in c.err, c.err + project = Project() + assert project.virtualenv_location == virtualenv.as_posix(), ( + project.virtualenv_location, virtualenv.as_posix() + ) + c = delegator_run( + "pipenv run pip install click", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert "Courtesy Notice" in c.err, (c.out, c.err) + c = delegator_run( + "pipenv install six", cwd=os.path.abspath(p.path), env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + c = delegator_run( + 'pipenv run python -c "import click;print(click.__file__)"', + cwd=os.path.abspath(p.path), env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert is_in_path(c.out.strip(), str(virtualenv)), (c.out.strip(), str(virtualenv)) + c = delegator_run( + "pipenv clean --dry-run", cwd=os.path.abspath(p.path), + env=os.environ.copy() + ) + assert c.return_code == 0, (c.out, c.err) + assert "click" in c.out, c.out + + +@pytest.mark.project +@pytest.mark.virtualenv +def test_run_in_virtualenv(PipenvInstance): + with PipenvInstance(chdir=True) as p: + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + assert 'Creating a virtualenv' in c.err project = Project() - assert project.virtualenv_location == str(virtualenv) c = p.pipenv("run pip install click") assert c.return_code == 0 - assert "Courtesy Notice" in c.err + c = p.pipenv("install six") + assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 - assert c.out.strip().startswith(str(virtualenv)) + assert normalize_path(c.out.strip()).startswith( + normalize_path(str(project.virtualenv_location)) + ) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 8cd39115..28f97e48 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import pytest diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 8b8745e7..1c5b3e96 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -1,3 +1,6 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function +import json import os import pytest @@ -5,9 +8,10 @@ import pytest from pipenv.utils import temp_environ +@pytest.mark.lock @pytest.mark.sync -def test_sync_error_without_lockfile(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_sync_error_without_lockfile(PipenvInstance): + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -20,12 +24,17 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): @pytest.mark.sync @pytest.mark.lock -def test_mirror_lock_sync(PipenvInstance, pypi): +def test_mirror_lock_sync(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") assert 'pypi.org' not in mirror_url with open(p.pipfile_path, 'w') as f: f.write(""" +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + [packages] six = "*" """.strip()) @@ -37,10 +46,10 @@ six = "*" @pytest.mark.sync @pytest.mark.lock -def test_sync_should_not_lock(PipenvInstance, pypi): +def test_sync_should_not_lock(PipenvInstance): """Sync should not touch the lock file, even if Pipfile is changed. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: with open(p.pipfile_path, 'w') as f: f.write(""" [packages] @@ -61,3 +70,46 @@ six = "*" c = p.pipenv('sync') assert c.return_code == 0 assert lockfile_content == p.lockfile + + +@pytest.mark.sync +@pytest.mark.lock +def test_sync_sequential_detect_errors(PipenvInstance): + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +requests = "*" + """.strip() + f.write(contents) + + c = p.pipenv('lock') + assert c.return_code == 0 + + # Force hash mismatch when installing `requests` + lock = p.lockfile + lock['default']['requests']['hashes'] = ['sha256:' + '0' * 64] + with open(p.lockfile_path, 'w') as f: + json.dump(lock, f) + + c = p.pipenv('sync --sequential') + assert c.return_code != 0 + + +@pytest.mark.sync +@pytest.mark.lock +def test_sync_sequential_verbose(PipenvInstance): + with PipenvInstance() as p: + with open(p.pipfile_path, 'w') as f: + contents = """ +[packages] +requests = "*" + """.strip() + f.write(contents) + + c = p.pipenv('lock') + assert c.return_code == 0 + + c = p.pipenv('sync --sequential --verbose') + for package in p.lockfile['default']: + assert 'Successfully installed {}'.format(package) in c.out diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index e366861b..c87c702f 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -1,3 +1,5 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import shutil @@ -6,37 +8,54 @@ import pytest from pipenv.utils import temp_environ -@pytest.mark.run @pytest.mark.uninstall @pytest.mark.install -def test_uninstall(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_uninstall_requests(PipenvInstance): + # Uninstalling requests can fail even when uninstall Django below + # succeeds, if requests was de-vendored. + # See https://github.com/pypa/pipenv/issues/3644 for problems + # caused by devendoring + with PipenvInstance() as p: c = p.pipenv("install requests") assert c.return_code == 0 assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + + c = p.pipenv("run python -m requests.help") + assert c.return_code == 0 c = p.pipenv("uninstall requests") assert c.return_code == 0 assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] c = p.pipenv("run python -m requests.help") assert c.return_code > 0 -@pytest.mark.run @pytest.mark.uninstall +def test_uninstall_django(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv("install Django==1.11.13") + assert c.return_code == 0 + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] + + c = p.pipenv("run python -m django --version") + assert c.return_code == 0 + + c = p.pipenv("uninstall Django") + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} + + c = p.pipenv("run python -m django --version") + assert c.return_code > 0 + + @pytest.mark.install -def test_mirror_uninstall(PipenvInstance, pypi): +@pytest.mark.uninstall +def test_mirror_uninstall(PipenvInstance): with temp_environ(), PipenvInstance(chdir=True) as p: mirror_url = os.environ.pop( @@ -44,97 +63,94 @@ def test_mirror_uninstall(PipenvInstance, pypi): ) assert "pypi.org" not in mirror_url - c = p.pipenv("install requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("install Django==1.11.13 --pypi-mirror {0}".format(mirror_url)) assert c.return_code == 0 - assert "requests" in p.pipfile["packages"] - assert "requests" in p.lockfile["default"] - assert "chardet" in p.lockfile["default"] - assert "idna" in p.lockfile["default"] - assert "urllib3" in p.lockfile["default"] - assert "certifi" in p.lockfile["default"] + assert "django" in p.pipfile["packages"] + assert "django" in p.lockfile["default"] + assert "pytz" in p.lockfile["default"] # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("uninstall requests --pypi-mirror {0}".format(mirror_url)) + c = p.pipenv("run python -m django --version") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] - assert "chardet" not in p.lockfile["develop"] - assert "idna" not in p.lockfile["develop"] - assert "urllib3" not in p.lockfile["develop"] - assert "certifi" not in p.lockfile["develop"] + + c = p.pipenv("uninstall Django --pypi-mirror {0}".format(mirror_url)) + assert c.return_code == 0 + assert "django" not in p.pipfile["dev-packages"] + assert "django" not in p.lockfile["develop"] + assert p.lockfile["develop"] == {} # Ensure the --pypi-mirror parameter hasn't altered the Pipfile or Pipfile.lock sources assert len(p.pipfile["source"]) == 1 assert len(p.lockfile["_meta"]["sources"]) == 1 assert "https://pypi.org/simple" == p.pipfile["source"][0]["url"] assert "https://pypi.org/simple" == p.lockfile["_meta"]["sources"][0]["url"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv("run python -m django --version") assert c.return_code > 0 @pytest.mark.files -@pytest.mark.uninstall @pytest.mark.install +@pytest.mark.uninstall def test_uninstall_all_local_files(PipenvInstance, testsroot): - file_name = "requests-2.19.1.tar.gz" + file_name = "tablib-0.12.1.tar.gz" # Not sure where travis/appveyor run tests from - source_path = os.path.abspath(os.path.join(testsroot, "test_artifacts", file_name)) + source_path = os.path.abspath(os.path.join(testsroot, "pypi", "tablib", file_name)) with PipenvInstance(chdir=True) as p: shutil.copy(source_path, os.path.join(p.path, file_name)) - os.mkdir(os.path.join(p.path, "requests")) + os.mkdir(os.path.join(p.path, "tablib")) c = p.pipenv("install {}".format(file_name)) assert c.return_code == 0 c = p.pipenv("uninstall --all") assert c.return_code == 0 - assert "requests" in c.out + assert "tablib" in c.out # Uninstall --all is not supposed to remove things from the pipfile # Note that it didn't before, but that instead local filenames showed as hashes - assert "requests" in p.pipfile["packages"] + assert "tablib" in p.pipfile["packages"] -@pytest.mark.run -@pytest.mark.uninstall @pytest.mark.install -def test_uninstall_all_dev(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: - c = p.pipenv("install --dev requests six") +@pytest.mark.uninstall +def test_uninstall_all_dev(PipenvInstance): + with PipenvInstance() as p: + c = p.pipenv("install --dev Django==1.11.13 six") assert c.return_code == 0 - c = p.pipenv("install pytz") + c = p.pipenv("install tablib") assert c.return_code == 0 - assert "pytz" in p.pipfile["packages"] - assert "requests" in p.pipfile["dev-packages"] + assert "tablib" in p.pipfile["packages"] + assert "django" in p.pipfile["dev-packages"] assert "six" in p.pipfile["dev-packages"] - assert "pytz" in p.lockfile["default"] - assert "requests" in p.lockfile["develop"] + assert "tablib" in p.lockfile["default"] + assert "django" in p.lockfile["develop"] assert "six" in p.lockfile["develop"] + c = p.pipenv('run python -c "import django"') + assert c.return_code == 0 + c = p.pipenv("uninstall --all-dev") assert c.return_code == 0 - assert "requests" not in p.pipfile["dev-packages"] - assert "six" not in p.pipfile["dev-packages"] - assert "requests" not in p.lockfile["develop"] + assert p.pipfile["dev-packages"] == {} + assert "django" not in p.lockfile["develop"] assert "six" not in p.lockfile["develop"] - assert "pytz" in p.pipfile["packages"] - assert "pytz" in p.lockfile["default"] + assert "tablib" in p.pipfile["packages"] + assert "tablib" in p.lockfile["default"] - c = p.pipenv("run python -m requests.help") + c = p.pipenv('run python -c "import django"') assert c.return_code > 0 - c = p.pipenv('run python -c "import pytz"') + c = p.pipenv('run python -c "import tablib"') assert c.return_code == 0 @pytest.mark.uninstall -@pytest.mark.run -def test_normalize_name_uninstall(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi) as p: +def test_normalize_name_uninstall(PipenvInstance): + with PipenvInstance() as p: with open(p.pipfile_path, "w") as f: contents = """ # Pre comment diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index 9c2b7074..7355bbd0 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -1,9 +1,10 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function import os import pytest from pipenv._compat import Path -from pipenv.project import Project # This module is run only on Windows. @@ -11,10 +12,10 @@ pytestmark = pytest.mark.skipif(os.name != 'nt', reason="only relevant on window @pytest.mark.project -def test_case_changes_windows(PipenvInstance, pypi): +def test_case_changes_windows(PipenvInstance): """Test project matching for case changes on Windows. """ - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install pytz') assert c.return_code == 0 @@ -38,7 +39,8 @@ def test_case_changes_windows(PipenvInstance, pypi): @pytest.mark.files -def test_local_path_windows(PipenvInstance, pypi): +@pytest.mark.local +def test_local_path_windows(PipenvInstance): whl = ( Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') @@ -47,13 +49,14 @@ def test_local_path_windows(PipenvInstance, pypi): whl = whl.resolve() except OSError: whl = whl.absolute() - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl)) assert c.return_code == 0 +@pytest.mark.local @pytest.mark.files -def test_local_path_windows_forward_slash(PipenvInstance, pypi): +def test_local_path_windows_forward_slash(PipenvInstance): whl = ( Path(__file__).parent.parent .joinpath('pypi', 'six', 'six-1.11.0-py2.py3-none-any.whl') @@ -62,14 +65,14 @@ def test_local_path_windows_forward_slash(PipenvInstance, pypi): whl = whl.resolve() except OSError: whl = whl.absolute() - with PipenvInstance(pypi=pypi, chdir=True) as p: + with PipenvInstance(chdir=True) as p: c = p.pipenv('install "{0}"'.format(whl.as_posix())) assert c.return_code == 0 @pytest.mark.cli -def test_pipenv_clean_windows(PipenvInstance, pypi): - with PipenvInstance(pypi=pypi, chdir=True) as p: +def test_pipenv_clean_windows(PipenvInstance): + with PipenvInstance(chdir=True) as p: c = p.pipenv('install requests') assert c.return_code == 0 c = p.pipenv('run pip install click') diff --git a/tests/pypi b/tests/pypi new file mode 160000 index 00000000..6faddf97 --- /dev/null +++ b/tests/pypi @@ -0,0 +1 @@ +Subproject commit 6faddf97c2a0220870da0a1409a196667b06c9cc diff --git a/tests/pypi/alembic/alembic-0.9.8.tar.gz b/tests/pypi/alembic/alembic-0.9.8.tar.gz deleted file mode 100644 index a23b9200..00000000 Binary files a/tests/pypi/alembic/alembic-0.9.8.tar.gz and /dev/null differ diff --git a/tests/pypi/alembic/alembic-0.9.9.tar.gz b/tests/pypi/alembic/alembic-0.9.9.tar.gz deleted file mode 100644 index 0c77f5df..00000000 Binary files a/tests/pypi/alembic/alembic-0.9.9.tar.gz and /dev/null differ diff --git a/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl b/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl deleted file mode 100644 index faf7dbf5..00000000 Binary files a/tests/pypi/apscheduler/APScheduler-3.5.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz b/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz deleted file mode 100644 index 672b4d13..00000000 Binary files a/tests/pypi/apscheduler/APScheduler-3.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/backports-html/backports.html-1.1.0.tar.gz b/tests/pypi/backports-html/backports.html-1.1.0.tar.gz deleted file mode 100644 index de553eba..00000000 Binary files a/tests/pypi/backports-html/backports.html-1.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl b/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl deleted file mode 100644 index 3348f725..00000000 Binary files a/tests/pypi/certifi/certifi-2018.1.18-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.1.18.tar.gz b/tests/pypi/certifi/certifi-2018.1.18.tar.gz deleted file mode 100644 index a7f32456..00000000 Binary files a/tests/pypi/certifi/certifi-2018.1.18.tar.gz and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl b/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl deleted file mode 100644 index 37d13a39..00000000 Binary files a/tests/pypi/certifi/certifi-2018.4.16-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/certifi/certifi-2018.4.16.tar.gz b/tests/pypi/certifi/certifi-2018.4.16.tar.gz deleted file mode 100644 index 47b37025..00000000 Binary files a/tests/pypi/certifi/certifi-2018.4.16.tar.gz and /dev/null differ diff --git a/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl b/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl deleted file mode 100644 index d276977d..00000000 Binary files a/tests/pypi/chardet/chardet-3.0.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/chardet/chardet-3.0.4.tar.gz b/tests/pypi/chardet/chardet-3.0.4.tar.gz deleted file mode 100644 index 13028bfc..00000000 Binary files a/tests/pypi/chardet/chardet-3.0.4.tar.gz and /dev/null differ diff --git a/tests/pypi/click/click-6.7-py2.py3-none-any.whl b/tests/pypi/click/click-6.7-py2.py3-none-any.whl deleted file mode 100644 index 56c7ff34..00000000 Binary files a/tests/pypi/click/click-6.7-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/click/click-6.7.tar.gz b/tests/pypi/click/click-6.7.tar.gz deleted file mode 100644 index b0790fc0..00000000 Binary files a/tests/pypi/click/click-6.7.tar.gz and /dev/null differ diff --git a/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl b/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl deleted file mode 100644 index 29b83f06..00000000 Binary files a/tests/pypi/colorama/colorama-0.3.9-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl b/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl deleted file mode 100644 index 14f7d7e1..00000000 Binary files a/tests/pypi/dateparser/dateparser-0.7.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/dateparser/dateparser-0.7.0.tar.gz b/tests/pypi/dateparser/dateparser-0.7.0.tar.gz deleted file mode 100644 index a37b5536..00000000 Binary files a/tests/pypi/dateparser/dateparser-0.7.0.tar.gz and /dev/null differ diff --git a/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl b/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl deleted file mode 100644 index d320deb3..00000000 Binary files a/tests/pypi/depends-on-marked-package/depends_on_marked_package-0.0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz b/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz deleted file mode 100644 index b88f48ae..00000000 Binary files a/tests/pypi/django-classy-tags/django-classy-tags-0.8.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl b/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl deleted file mode 100644 index 47ac7f85..00000000 Binary files a/tests/pypi/django-classy-tags/django_classy_tags-0.8.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-formtools/django-formtools-2.1.tar.gz b/tests/pypi/django-formtools/django-formtools-2.1.tar.gz deleted file mode 100644 index 2b8ca348..00000000 Binary files a/tests/pypi/django-formtools/django-formtools-2.1.tar.gz and /dev/null differ diff --git a/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl b/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl deleted file mode 100644 index 1086751a..00000000 Binary files a/tests/pypi/django-formtools/django_formtools-2.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz b/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz deleted file mode 100644 index b11067dc..00000000 Binary files a/tests/pypi/django-sekizai/django-sekizai-0.10.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl b/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl deleted file mode 100644 index 20c107cc..00000000 Binary files a/tests/pypi/django-sekizai/django_sekizai-0.10.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz b/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz deleted file mode 100644 index 3d92cb7e..00000000 Binary files a/tests/pypi/django-treebeard/django-treebeard-4.2.0.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl b/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl deleted file mode 100644 index 8d515c4a..00000000 Binary files a/tests/pypi/django/Django-1.11.10-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.10.tar.gz b/tests/pypi/django/Django-1.11.10.tar.gz deleted file mode 100644 index 2d69623e..00000000 Binary files a/tests/pypi/django/Django-1.11.10.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl b/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl deleted file mode 100644 index 38dfe813..00000000 Binary files a/tests/pypi/django/Django-1.11.13-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-1.11.13.tar.gz b/tests/pypi/django/Django-1.11.13.tar.gz deleted file mode 100644 index ba99816e..00000000 Binary files a/tests/pypi/django/Django-1.11.13.tar.gz and /dev/null differ diff --git a/tests/pypi/django/Django-2.0.6-py3-none-any.whl b/tests/pypi/django/Django-2.0.6-py3-none-any.whl deleted file mode 100644 index b966b5d4..00000000 Binary files a/tests/pypi/django/Django-2.0.6-py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/django/Django-2.0.6.tar.gz b/tests/pypi/django/Django-2.0.6.tar.gz deleted file mode 100644 index bc69557b..00000000 Binary files a/tests/pypi/django/Django-2.0.6.tar.gz and /dev/null differ diff --git a/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz b/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz deleted file mode 100644 index f3afc45b..00000000 Binary files a/tests/pypi/djangocms-admin-style/djangocms-admin-style-1.2.7.tar.gz and /dev/null differ diff --git a/tests/pypi/docopt/docopt-0.6.2.tar.gz b/tests/pypi/docopt/docopt-0.6.2.tar.gz deleted file mode 100644 index 153ce415..00000000 Binary files a/tests/pypi/docopt/docopt-0.6.2.tar.gz and /dev/null differ diff --git a/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz b/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz deleted file mode 100644 index 73a5894e..00000000 Binary files a/tests/pypi/et-xmlfile/et_xmlfile-1.0.1.tar.gz and /dev/null differ diff --git a/tests/pypi/fixtures/django/3.4.x.zip b/tests/pypi/fixtures/django/3.4.x.zip deleted file mode 100644 index 6c2ae5f6..00000000 Binary files a/tests/pypi/fixtures/django/3.4.x.zip and /dev/null differ diff --git a/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl b/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl deleted file mode 100644 index 98e55415..00000000 Binary files a/tests/pypi/flask/Flask-0.12.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/flask/Flask-0.12.2.tar.gz b/tests/pypi/flask/Flask-0.12.2.tar.gz deleted file mode 100644 index 278c2108..00000000 Binary files a/tests/pypi/flask/Flask-0.12.2.tar.gz and /dev/null differ diff --git a/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl b/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl deleted file mode 100644 index ec5973f4..00000000 Binary files a/tests/pypi/funcsigs/funcsigs-1.0.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz b/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz deleted file mode 100644 index c53bf6aa..00000000 Binary files a/tests/pypi/funcsigs/funcsigs-1.0.2.tar.gz and /dev/null differ diff --git a/tests/pypi/futures/futures-3.2.0-py2-none-any.whl b/tests/pypi/futures/futures-3.2.0-py2-none-any.whl deleted file mode 100644 index c0659c1a..00000000 Binary files a/tests/pypi/futures/futures-3.2.0-py2-none-any.whl and /dev/null differ diff --git a/tests/pypi/futures/futures-3.2.0.tar.gz b/tests/pypi/futures/futures-3.2.0.tar.gz deleted file mode 100644 index 5e896bfa..00000000 Binary files a/tests/pypi/futures/futures-3.2.0.tar.gz and /dev/null differ diff --git a/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl b/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl deleted file mode 100644 index 26ffc716..00000000 Binary files a/tests/pypi/gitdb2/gitdb2-2.0.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz b/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz deleted file mode 100644 index de6ee8af..00000000 Binary files a/tests/pypi/gitdb2/gitdb2-2.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/humanize/humanize-0.5.1.tar.gz b/tests/pypi/humanize/humanize-0.5.1.tar.gz deleted file mode 100644 index b293ff05..00000000 Binary files a/tests/pypi/humanize/humanize-0.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz b/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz deleted file mode 100644 index dd4e4a88..00000000 Binary files a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.0.tar.gz and /dev/null differ diff --git a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz b/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz deleted file mode 100644 index 7770d23e..00000000 Binary files a/tests/pypi/ibm-db-sa-py3/ibm-db-sa-py3-0.3.1-1.tar.gz and /dev/null differ diff --git a/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl b/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl deleted file mode 100644 index 11cb4140..00000000 Binary files a/tests/pypi/idna/idna-2.6-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/idna/idna-2.6.tar.gz b/tests/pypi/idna/idna-2.6.tar.gz deleted file mode 100644 index d38e1f29..00000000 Binary files a/tests/pypi/idna/idna-2.6.tar.gz and /dev/null differ diff --git a/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl b/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl deleted file mode 100644 index 9d1a3285..00000000 Binary files a/tests/pypi/idna/idna-2.7-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/idna/idna-2.7.tar.gz b/tests/pypi/idna/idna-2.7.tar.gz deleted file mode 100644 index 8f0c5f27..00000000 Binary files a/tests/pypi/idna/idna-2.7.tar.gz and /dev/null differ diff --git a/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz b/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz deleted file mode 100644 index d0416330..00000000 Binary files a/tests/pypi/itsdangerous/itsdangerous-0.24.tar.gz and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.3.tar.gz b/tests/pypi/jdcal/jdcal-1.3.tar.gz deleted file mode 100644 index 2f3ba8ce..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.3.tar.gz and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl b/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl deleted file mode 100644 index bac3f040..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/jdcal/jdcal-1.4.tar.gz b/tests/pypi/jdcal/jdcal-1.4.tar.gz deleted file mode 100644 index e82fdae7..00000000 Binary files a/tests/pypi/jdcal/jdcal-1.4.tar.gz and /dev/null differ diff --git a/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl b/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl deleted file mode 100644 index 7bc4e35f..00000000 Binary files a/tests/pypi/jinja2/Jinja2-2.10-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/jinja2/Jinja2-2.10.tar.gz b/tests/pypi/jinja2/Jinja2-2.10.tar.gz deleted file mode 100644 index c311087a..00000000 Binary files a/tests/pypi/jinja2/Jinja2-2.10.tar.gz and /dev/null differ diff --git a/tests/pypi/mako/Mako-1.0.7.tar.gz b/tests/pypi/mako/Mako-1.0.7.tar.gz deleted file mode 100644 index 9e5825f6..00000000 Binary files a/tests/pypi/mako/Mako-1.0.7.tar.gz and /dev/null differ diff --git a/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz b/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz deleted file mode 100644 index 606021ae..00000000 Binary files a/tests/pypi/markupsafe/MarkupSafe-1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl b/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl deleted file mode 100644 index 7dcd32bd..00000000 Binary files a/tests/pypi/maya/maya-0.3.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/maya/maya-0.3.4.tar.gz b/tests/pypi/maya/maya-0.3.4.tar.gz deleted file mode 100644 index 8a20fb58..00000000 Binary files a/tests/pypi/maya/maya-0.3.4.tar.gz and /dev/null differ diff --git a/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl b/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl deleted file mode 100644 index b36d48e5..00000000 Binary files a/tests/pypi/maya/maya-0.5.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/maya/maya-0.5.0.tar.gz b/tests/pypi/maya/maya-0.5.0.tar.gz deleted file mode 100644 index 5f4016f0..00000000 Binary files a/tests/pypi/maya/maya-0.5.0.tar.gz and /dev/null differ diff --git a/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl b/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl deleted file mode 100644 index 6f8c38f3..00000000 Binary files a/tests/pypi/multidict/multidict-4.1.0-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/multidict/multidict-4.1.0.tar.gz b/tests/pypi/multidict/multidict-4.1.0.tar.gz deleted file mode 100644 index 89adbb22..00000000 Binary files a/tests/pypi/multidict/multidict-4.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg b/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg deleted file mode 100644 index ccb4eeed..00000000 Binary files a/tests/pypi/odfpy/odfpy-1.3.6-py2.7.egg and /dev/null differ diff --git a/tests/pypi/odfpy/odfpy-1.3.6.tar.gz b/tests/pypi/odfpy/odfpy-1.3.6.tar.gz deleted file mode 100644 index c8861245..00000000 Binary files a/tests/pypi/odfpy/odfpy-1.3.6.tar.gz and /dev/null differ diff --git a/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz b/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz deleted file mode 100644 index 52f84550..00000000 Binary files a/tests/pypi/openpyxl/openpyxl-2.5.0.tar.gz and /dev/null differ diff --git a/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz b/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz deleted file mode 100644 index b257fba5..00000000 Binary files a/tests/pypi/openpyxl/openpyxl-2.5.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl deleted file mode 100644 index 6230a732..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl deleted file mode 100644 index bf879e90..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl deleted file mode 100644 index d01d7706..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 878bb1bd..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl deleted file mode 100644 index 19a474d9..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-macosx_10_6_intel.macosx_10_9_intel.macosx_10_9_x86_64.macosx_10_10_intel.macosx_10_10_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 6ef88537..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl b/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl deleted file mode 100644 index 871d5094..00000000 Binary files a/tests/pypi/pandas/pandas-0.22.0-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/parse/parse-1.8.2.tar.gz b/tests/pypi/parse/parse-1.8.2.tar.gz deleted file mode 100644 index d24aa0d1..00000000 Binary files a/tests/pypi/parse/parse-1.8.2.tar.gz and /dev/null differ diff --git a/tests/pypi/pathlib2/pathlib2-2.3.2-py2.py3-none-any.whl b/tests/pypi/pathlib2/pathlib2-2.3.2-py2.py3-none-any.whl deleted file mode 100644 index 0cd52d85..00000000 Binary files a/tests/pypi/pathlib2/pathlib2-2.3.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pathlib2/pathlib2-2.3.2.tar.gz b/tests/pypi/pathlib2/pathlib2-2.3.2.tar.gz deleted file mode 100644 index 86e784d1..00000000 Binary files a/tests/pypi/pathlib2/pathlib2-2.3.2.tar.gz and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index b38c9d13..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl deleted file mode 100644 index 72ef97b4..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-macosx_10_13_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 31d50199..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.4.4.tar.gz b/tests/pypi/pendulum/pendulum-1.4.4.tar.gz deleted file mode 100644 index 826bc090..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.4.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl deleted file mode 100644 index 0d22ef79..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index d29ce7a8..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl deleted file mode 100644 index 53eaa9c3..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index af4c30b9..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/pendulum/pendulum-1.5.1.tar.gz b/tests/pypi/pendulum/pendulum-1.5.1.tar.gz deleted file mode 100644 index 77aff89d..00000000 Binary files a/tests/pypi/pendulum/pendulum-1.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl b/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl deleted file mode 100644 index 127d886e..00000000 Binary files a/tests/pypi/py/py-1.5.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/py/py-1.5.3.tar.gz b/tests/pypi/py/py-1.5.3.tar.gz deleted file mode 100644 index a0592b18..00000000 Binary files a/tests/pypi/py/py-1.5.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pysocks/PySocks-1.6.8.tar.gz b/tests/pypi/pysocks/PySocks-1.6.8.tar.gz deleted file mode 100644 index e6366dcf..00000000 Binary files a/tests/pypi/pysocks/PySocks-1.6.8.tar.gz and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl b/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl deleted file mode 100644 index dfaf980a..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.0.tar.gz b/tests/pypi/pytest/pytest-3.1.0.tar.gz deleted file mode 100644 index bb1e3573..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl b/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl deleted file mode 100644 index aaab63cd..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytest/pytest-3.1.1.tar.gz b/tests/pypi/pytest/pytest-3.1.1.tar.gz deleted file mode 100644 index 5b5f6dc4..00000000 Binary files a/tests/pypi/pytest/pytest-3.1.1.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz b/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz deleted file mode 100644 index 6b86c6d3..00000000 Binary files a/tests/pypi/python-dateutil/python-dateutil-2.6.1.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz b/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz deleted file mode 100644 index 8f1a68d9..00000000 Binary files a/tests/pypi/python-dateutil/python-dateutil-2.7.3.tar.gz and /dev/null differ diff --git a/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl b/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl deleted file mode 100644 index 97b3947a..00000000 Binary files a/tests/pypi/python-dateutil/python_dateutil-2.6.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl b/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl deleted file mode 100644 index 55da69c3..00000000 Binary files a/tests/pypi/python-dateutil/python_dateutil-2.7.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/python-editor/python-editor-1.0.3.tar.gz b/tests/pypi/python-editor/python-editor-1.0.3.tar.gz deleted file mode 100644 index 8149cccc..00000000 Binary files a/tests/pypi/python-editor/python-editor-1.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl b/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl deleted file mode 100644 index d274bd36..00000000 Binary files a/tests/pypi/pytz/pytz-2018.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.3.tar.gz b/tests/pypi/pytz/pytz-2018.3.tar.gz deleted file mode 100644 index c8045799..00000000 Binary files a/tests/pypi/pytz/pytz-2018.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl b/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl deleted file mode 100644 index 53b5bd48..00000000 Binary files a/tests/pypi/pytz/pytz-2018.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytz/pytz-2018.4.tar.gz b/tests/pypi/pytz/pytz-2018.4.tar.gz deleted file mode 100644 index 58526bf0..00000000 Binary files a/tests/pypi/pytz/pytz-2018.4.tar.gz and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl b/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl deleted file mode 100644 index af5f8bf7..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz b/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz deleted file mode 100644 index fd4e397d..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.3.tar.gz and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl b/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl deleted file mode 100644 index 73740fb2..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.5-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz b/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz deleted file mode 100644 index 506cb247..00000000 Binary files a/tests/pypi/pytzdata/pytzdata-2018.5.tar.gz and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl deleted file mode 100644 index d9f5e245..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win32.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl deleted file mode 100644 index 2eb7094f..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl deleted file mode 100644 index f58c2f69..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win32.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl b/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl deleted file mode 100644 index c3d36a8b..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12-cp35-cp35m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/pyyaml/PyYAML-3.12.tar.gz b/tests/pypi/pyyaml/PyYAML-3.12.tar.gz deleted file mode 100644 index aabee39f..00000000 Binary files a/tests/pypi/pyyaml/PyYAML-3.12.tar.gz and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg deleted file mode 100644 index ef607e11..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py2.7.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg deleted file mode 100644 index e8fa3f78..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py3.5.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg b/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg deleted file mode 100644 index bd0b78c7..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1-py3.6.egg and /dev/null differ diff --git a/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz b/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz deleted file mode 100644 index 3791a446..00000000 Binary files a/tests/pypi/randomwords/RandomWords-0.2.1.tar.gz and /dev/null differ diff --git a/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl b/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl deleted file mode 100644 index 0516a923..00000000 Binary files a/tests/pypi/records/records-0.5.2-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/records/records-0.5.2.tar.gz b/tests/pypi/records/records-0.5.2.tar.gz deleted file mode 100644 index f3690c19..00000000 Binary files a/tests/pypi/records/records-0.5.2.tar.gz and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl b/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl deleted file mode 100644 index 31d8b2bc..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl b/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl deleted file mode 100644 index ce6a00f3..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21-cp36-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.02.21.tar.gz b/tests/pypi/regex/regex-2018.02.21.tar.gz deleted file mode 100644 index 92e23a09..00000000 Binary files a/tests/pypi/regex/regex-2018.02.21.tar.gz and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl deleted file mode 100644 index e715e787..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl deleted file mode 100644 index ef5e3a24..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp36-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl b/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl deleted file mode 100644 index 4b1ccfad..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21-cp37-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/regex/regex-2018.06.21.tar.gz b/tests/pypi/regex/regex-2018.06.21.tar.gz deleted file mode 100644 index e4696f18..00000000 Binary files a/tests/pypi/regex/regex-2018.06.21.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-1.0.0.tar.gz b/tests/pypi/requests/requests-1.0.0.tar.gz deleted file mode 100644 index d6184496..00000000 Binary files a/tests/pypi/requests/requests-1.0.0.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl deleted file mode 100644 index 1fb0b95a..00000000 Binary files a/tests/pypi/requests/requests-2.14.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.14.0.tar.gz b/tests/pypi/requests/requests-2.14.0.tar.gz deleted file mode 100644 index 33631280..00000000 Binary files a/tests/pypi/requests/requests-2.14.0.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl deleted file mode 100644 index cf3bdc07..00000000 Binary files a/tests/pypi/requests/requests-2.18.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.18.4.tar.gz b/tests/pypi/requests/requests-2.18.4.tar.gz deleted file mode 100644 index fce54795..00000000 Binary files a/tests/pypi/requests/requests-2.18.4.tar.gz and /dev/null differ diff --git a/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl b/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl deleted file mode 100644 index bc342f1d..00000000 Binary files a/tests/pypi/requests/requests-2.19.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/requests/requests-2.19.1.tar.gz b/tests/pypi/requests/requests-2.19.1.tar.gz deleted file mode 100644 index 5d4ad446..00000000 Binary files a/tests/pypi/requests/requests-2.19.1.tar.gz and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index d6919701..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 26e63a8e..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl deleted file mode 100644 index b2b6bcd4..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9-cp27-none-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz b/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz deleted file mode 100644 index 0a6b0769..00000000 Binary files a/tests/pypi/ruamel-ordereddict/ruamel.ordereddict-0.4.9.tar.gz and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl deleted file mode 100644 index 26694f18..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl deleted file mode 100644 index 362db9cb..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp27-cp27mu-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index 73912720..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz b/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz deleted file mode 100644 index 9e607d32..00000000 Binary files a/tests/pypi/ruamel-yaml/ruamel.yaml-0.15.9.tar.gz and /dev/null differ diff --git a/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win32.whl b/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win32.whl deleted file mode 100644 index fd2f1047..00000000 Binary files a/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win32.whl and /dev/null differ diff --git a/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win_amd64.whl b/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win_amd64.whl deleted file mode 100644 index 7c718c67..00000000 Binary files a/tests/pypi/scandir/scandir-1.9.0-cp27-cp27m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win32.whl b/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win32.whl deleted file mode 100644 index 39bff799..00000000 Binary files a/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win32.whl and /dev/null differ diff --git a/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win_amd64.whl b/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win_amd64.whl deleted file mode 100644 index 10a8d7ef..00000000 Binary files a/tests/pypi/scandir/scandir-1.9.0-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pypi/scandir/scandir-1.9.0.tar.gz b/tests/pypi/scandir/scandir-1.9.0.tar.gz deleted file mode 100644 index f134aace..00000000 Binary files a/tests/pypi/scandir/scandir-1.9.0.tar.gz and /dev/null differ diff --git a/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl b/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl deleted file mode 100644 index edc3ca2d..00000000 Binary files a/tests/pypi/setuptools/setuptools-39.0.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl b/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl deleted file mode 100644 index 59960239..00000000 Binary files a/tests/pypi/six/six-1.11.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/six/six-1.11.0.tar.gz b/tests/pypi/six/six-1.11.0.tar.gz deleted file mode 100644 index 353f31f9..00000000 Binary files a/tests/pypi/six/six-1.11.0.tar.gz and /dev/null differ diff --git a/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl b/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl deleted file mode 100644 index 72599115..00000000 Binary files a/tests/pypi/smmap2/smmap2-2.0.3-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/smmap2/smmap2-2.0.3.tar.gz b/tests/pypi/smmap2/smmap2-2.0.3.tar.gz deleted file mode 100644 index e959ad02..00000000 Binary files a/tests/pypi/smmap2/smmap2-2.0.3.tar.gz and /dev/null differ diff --git a/tests/pypi/snaptime/snaptime-0.2.4.tar.gz b/tests/pypi/snaptime/snaptime-0.2.4.tar.gz deleted file mode 100644 index df152642..00000000 Binary files a/tests/pypi/snaptime/snaptime-0.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz deleted file mode 100644 index 40eafa66..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.0b3.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz deleted file mode 100644 index e965a280..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz b/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz deleted file mode 100644 index d624e540..00000000 Binary files a/tests/pypi/sqlalchemy/SQLAlchemy-1.2.8.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.11.5.tar.gz b/tests/pypi/tablib/tablib-0.11.5.tar.gz deleted file mode 100644 index 460cd0a7..00000000 Binary files a/tests/pypi/tablib/tablib-0.11.5.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.12.0.tar.gz b/tests/pypi/tablib/tablib-0.12.0.tar.gz deleted file mode 100644 index 36ab89f2..00000000 Binary files a/tests/pypi/tablib/tablib-0.12.0.tar.gz and /dev/null differ diff --git a/tests/pypi/tablib/tablib-0.12.1.tar.gz b/tests/pypi/tablib/tablib-0.12.1.tar.gz deleted file mode 100644 index 825ee278..00000000 Binary files a/tests/pypi/tablib/tablib-0.12.1.tar.gz and /dev/null differ diff --git a/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl b/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl deleted file mode 100644 index a1e64899..00000000 Binary files a/tests/pypi/tpfd/tpfd-0.2.4-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/tpfd/tpfd-0.2.4.tar.gz b/tests/pypi/tpfd/tpfd-0.2.4.tar.gz deleted file mode 100644 index c13d45c0..00000000 Binary files a/tests/pypi/tpfd/tpfd-0.2.4.tar.gz and /dev/null differ diff --git a/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz b/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz deleted file mode 100644 index 3f5704a8..00000000 Binary files a/tests/pypi/tzlocal/tzlocal-1.5.1.tar.gz and /dev/null differ diff --git a/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz b/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz deleted file mode 100644 index a7858b00..00000000 Binary files a/tests/pypi/unicodecsv/unicodecsv-0.14.1.tar.gz and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.21.1-py2.py3-none-any.whl b/tests/pypi/urllib3/urllib3-1.21.1-py2.py3-none-any.whl deleted file mode 100644 index 9061cfdf..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.21.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.21.1.tar.gz b/tests/pypi/urllib3/urllib3-1.21.1.tar.gz deleted file mode 100644 index d2e458ec..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.21.1.tar.gz and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl b/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl deleted file mode 100644 index d4464993..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.22-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.22.tar.gz b/tests/pypi/urllib3/urllib3-1.22.tar.gz deleted file mode 100644 index 3d9ee9f1..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.22.tar.gz and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl b/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl deleted file mode 100644 index a385e3c9..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.23-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/urllib3/urllib3-1.23.tar.gz b/tests/pypi/urllib3/urllib3-1.23.tar.gz deleted file mode 100644 index 401dc252..00000000 Binary files a/tests/pypi/urllib3/urllib3-1.23.tar.gz and /dev/null differ diff --git a/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl b/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl deleted file mode 100644 index 2853aafc..00000000 Binary files a/tests/pypi/vcrpy/vcrpy-1.11.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz b/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz deleted file mode 100644 index 605bce97..00000000 Binary files a/tests/pypi/vcrpy/vcrpy-1.11.0.tar.gz and /dev/null differ diff --git a/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl b/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl deleted file mode 100644 index 865d5248..00000000 Binary files a/tests/pypi/werkzeug/Werkzeug-0.14.1-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz b/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz deleted file mode 100644 index 27e7b2d7..00000000 Binary files a/tests/pypi/werkzeug/Werkzeug-0.14.1.tar.gz and /dev/null differ diff --git a/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl b/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl deleted file mode 100644 index 8cfdda05..00000000 Binary files a/tests/pypi/wheel/wheel-0.31.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz b/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz deleted file mode 100644 index dfb77887..00000000 Binary files a/tests/pypi/win-inet-pton/win_inet_pton-1.0.1.tar.gz and /dev/null differ diff --git a/tests/pypi/wrapt/wrapt-1.10.11.tar.gz b/tests/pypi/wrapt/wrapt-1.10.11.tar.gz deleted file mode 100644 index 7f6870e2..00000000 Binary files a/tests/pypi/wrapt/wrapt-1.10.11.tar.gz and /dev/null differ diff --git a/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl b/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl deleted file mode 100644 index fd326f8a..00000000 Binary files a/tests/pypi/xlrd/xlrd-1.1.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/xlrd/xlrd-1.1.0.tar.gz b/tests/pypi/xlrd/xlrd-1.1.0.tar.gz deleted file mode 100644 index 6f8d5666..00000000 Binary files a/tests/pypi/xlrd/xlrd-1.1.0.tar.gz and /dev/null differ diff --git a/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl b/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl deleted file mode 100644 index ab824454..00000000 Binary files a/tests/pypi/xlwt/xlwt-1.3.0-py2.py3-none-any.whl and /dev/null differ diff --git a/tests/pypi/xlwt/xlwt-1.3.0.tar.gz b/tests/pypi/xlwt/xlwt-1.3.0.tar.gz deleted file mode 100644 index f9a65e06..00000000 Binary files a/tests/pypi/xlwt/xlwt-1.3.0.tar.gz and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl deleted file mode 100644 index ab6c0e0c..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_i686.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl deleted file mode 100644 index baeeb2c1..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-manylinux1_x86_64.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl deleted file mode 100644 index 45fe89ed..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win32.whl and /dev/null differ diff --git a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl b/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl deleted file mode 100644 index e11d5c62..00000000 Binary files a/tests/pypi/yarl/yarl-1.1.1-cp36-cp36m-win_amd64.whl and /dev/null differ diff --git a/tests/pytest-pypi/DESCRIPTION.rst b/tests/pytest-pypi/DESCRIPTION.rst index 4e0cba87..823e4733 100644 --- a/tests/pytest-pypi/DESCRIPTION.rst +++ b/tests/pytest-pypi/DESCRIPTION.rst @@ -1,30 +1,5 @@ -pytest-httpbin -============== +pytest-pypi +=========== -httpbin is an amazing web service for testing HTTP libraries. It has several -great endpoints that can test pretty much everything you need in a HTTP -library. The only problem is: maybe you don't want to wait for your tests to -travel across the Internet and back to make assertions against a remote web -service. - -Enter pytest-httpbin. Pytest-httpbin creates a pytest "fixture" that is -dependency-injected into your tests. It automatically starts up a HTTP server -in a separate thread running httpbin and provides your test with the URL in the -fixture. Check out this example: - -.. code-block:: python - - def test_that_my_library_works_kinda_ok(httpbin): - assert requests.get(httpbin.url + '/get/').status_code == 200 - -This replaces a test that might have looked like this before: - -.. code-block:: python - - def test_that_my_library_works_kinda_ok(): - assert requests.get('http://httpbin.org/get').status_code == 200 - -pytest-httpbin also supports https and includes its own CA cert you can use. -Check out `the full documentation`_ on the github page. - -.. _the full documentation: https://github.com/kevin1024/pytest-httpbin +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/README.md b/tests/pytest-pypi/README.md index a9ba37c2..31b33807 100644 --- a/tests/pytest-pypi/README.md +++ b/tests/pytest-pypi/README.md @@ -1,194 +1,4 @@ -# pytest-httpbin +# pytest-pypi -[![Build Status](https://travis-ci.org/kevin1024/pytest-httpbin.svg?branch=master)](https://travis-ci.org/kevin1024/pytest-httpbin) - -[httpbin](https://httpbin.org/) is an amazing web service for testing HTTP libraries. It has several great endpoints that can test pretty much everything you need in a HTTP library. The only problem is: maybe you don't want to wait for your tests to travel across the Internet and back to make assertions against a remote web service (speed), and maybe you want to work offline (convenience). - -Enter **pytest-httpbin**. Pytest-httpbin creates a [pytest fixture](http://pytest.org/latest/fixture.html) that is dependency-injected into your tests. It automatically starts up a HTTP server in a separate thread running httpbin and provides your test with the URL in the fixture. Check out this example: - -```python -def test_that_my_library_works_kinda_ok(httpbin): - assert requests.get(httpbin.url + '/get').status_code == 200 -``` - -This replaces a test that might have looked like this before: - -```python -def test_that_my_library_works_kinda_ok(): - assert requests.get('http://httpbin.org/get').status_code == 200 -``` - -If you're making a lot of requests to httpbin, it can radically speed up your tests. - -![demo](http://i.imgur.com/heNOQLP.gif) - - -# HTTPS support - -pytest-httpbin also supports HTTPS: - -```python -def test_that_my_library_works_kinda_ok(httpbin_secure): - assert requests.get(httpbin_secure.url + '/get/').status_code == 200 -``` - -It's actually starting 2 web servers in separate threads in the background: one HTTP and one HTTPS. The servers are started on a random port (see below for fixed port support), on the loopback interface on your machine. Pytest-httpbin includes a self-signed certificate. If your library verifies certificates against a CA (and it should), you'll have to add the CA from pytest-httpbin. The path to the pytest-httpbin CA bundle can by found like this `python -m pytest_httpbin.certs`. - -For example in requests, you can set the `REQUESTS_CA_BUNDLE` python path. You can run your tests like this: - -```bash -REQUESTS_CA_BUNDLE=`python -m pytest_httpbin.certs` py.test tests/ -``` - -# API of the injected object - -The injected object has the following attributes: - - * url - * port - * host - -and the following methods: - - * join(string): Returns the results of calling `urlparse.urljoin` with the url from the injected server automatically applied as the first argument. You supply the second argument - -Also, I defined `__add__` on the object to append to `httpbin.url`. This means you can do stuff like `httpbin + '/get'` instead of `httpbin.url + '/get'`. - -## Testing both HTTP and HTTPS endpoints with one test - -If you ever find yourself needing to test both the http and https version of and endpoint, you can use the `httpbin_both` funcarg like this: - - -```python -def test_that_my_library_works_kinda_ok(httpbin_both): - assert requests.get(httpbin_both.url + '/get/').status_code == 200 -``` - -Through the magic of pytest parametrization, this function will actually execute twice: once with an http url and once with an https url. - -## Using pytest-httpbin with unittest-style test cases - -I have provided 2 additional fixtures to make testing with class-based tests easier. I have also provided a couple decorators that provide some syntactic sugar around the pytest method of adding the fixtures to class-based tests. Just add the `use_class_based_httpbin` and/or `use_class_based_httpbin_secure` class decorators to your class, and then you can access httpbin using self.httpbin and self.httpbin_secure. - -```python -import pytest_httpbin - -@pytest_httpbin.use_class_based_httpbin -@pytest_httpbin.use_class_based_httpbin_secure -class TestClassBassedTests(unittest.TestCase): - def test_http(self): - assert requests.get(self.httpbin.url + '/get').response - - def test_http_secure(self): - assert requests.get(self.httpbin_secure.url + '/get').response -``` - -## Running the server on fixed port - -Sometimes a randomized port can be a problem. Worry not, you can fix the port number to a desired value with the `HTTPBIN_HTTP_PORT` and `HTTPBIN_HTTPS_PORT` environment variables. If those are defined during pytest plugins are loaded, `httbin` and `httpbin_secure` fixtures will run on given ports. You can run your tests like this: - -```bash -HTTPBIN_HTTP_PORT=8080 HTTPBIN_HTTPS_PORT=8443 py.test tests/ -``` - -## Installation - -All you need to do is this: - -```bash -pip install pytest-httpbin -``` - -and your tests executed by pytest all will have access to the `httpbin` and `httpbin_secure` funcargs. Cool right? - -## Support and dependencies - -pytest-httpbin supports Python 2.6, 2.7, 3.4, and pypy. It will automatically install httpbin and flask when you install it from pypi. - -[httpbin](https://github.com/kennethreitz/httpbin) itself does not support python 2.6 as of version 0.6.0, when the Flask-common dependency was added. If you need python 2.6 support pin the httpbin version to 0.5.0 - -## Running the pytest-httpbin test suite - -If you want to run pytest-httpbin's test suite, you'll need to install requests and pytest, and then use the ./runtests.sh script. - -```bash -pip install pytest -/.runtests.sh -``` - -Also, you can use tox to run the tests on all supported python versions: - -```bash -pip install tox -tox -``` - -## Changelog - -* 0.3.0 - * Allow to run httpbin on fixed port using environment variables (thanks @hroncok) - * Allow server to be thread.join()ed (thanks @graingert) - * Add support for Python 3.6 (thanks @graingert) -* 0.2.3: - * Another attempt to fix #32 (Rare bug, only happens on Travis) -* 0.2.2: - * Fix bug with python3 -* 0.2.1: - * Attempt to fix strange, impossible-to-reproduce bug with broken SSL certs - that only happens on Travis (#32) [Bad release, breaks py3] -* 0.2.0: - * Remove threaded HTTP server. I built it for Requests, but they deleted - their threaded test since it didn't really work very well. The threaded - server seems to cause some strange problems with HTTP chunking, so I'll - just remove it since nobody is using it (I hope) -* 0.1.1: - * Fix weird hang with SSL on pypy (again) -* 0.1.0: - * Update server to use multithreaded werkzeug server -* 0.0.7: - * Update the certificates (they expired) -* 0.0.6: - * Fix an issue where pypy was hanging when a request was made with an invalid - certificate -* 0.0.5: - * Fix broken version parsing in 0.0.4 -* 0.0.4: - * **Bad release: Broken version parsing** - * Fix `BadStatusLine` error that occurs when sending multiple requests - in a single session (PR #16). Thanks @msabramo! - * Fix #9 ("Can't be installed at the same time than pytest?") (PR - #14). Thanks @msabramo! - * Add `httpbin_ca_bundle` pytest fixture. With this fixture there is - no need to specify the bundle on every request, as it will - automatically set `REQUESTS_CA_BUNDLE` if using - [requests](http://docs.python-requests.org/). And you don't have to - care about where it is located (PR #8). Thanks @t-8ch! -* 0.0.3: Add a couple test fixtures to make testing old class-based test suites - easier -* 0.0.2: Fixed a couple bugs with the wsgiref server to bring behavior in line - with httpbin.org, thanks @jakubroztocil for the bug reports -* 0.0.1: Initial release - -## License - -The MIT License (MIT) - -Copyright (c) 2014-2015 Kevin McCarthy - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. +Easily test your HTTP library against a local copy of PyPI. +This is an internal pytest plugin of pipenv. diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 4e013ab5..6f829819 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -1,11 +1,22 @@ -import os +# -*- coding: utf-8 -*- +from __future__ import absolute_import, print_function +import collections +import contextlib +import io import json -import sys +import os -import requests -from flask import Flask, redirect, abort, render_template, send_file, jsonify -from zipfile import is_zipfile from tarfile import is_tarfile +from zipfile import is_zipfile + +import distlib.wheel +import requests +from six.moves import xmlrpc_client + +from flask import Flask, redirect, abort, render_template, send_file, jsonify + + +ReleaseTuple = collections.namedtuple("ReleaseTuple", ["path", "requires_python"]) app = Flask(__name__) session = requests.Session() @@ -13,6 +24,24 @@ session = requests.Session() packages = {} ARTIFACTS = {} + +@contextlib.contextmanager +def xml_pypi_server(server): + transport = xmlrpc_client.Transport() + client = xmlrpc_client.ServerProxy(server, transport) + try: + yield client + finally: + transport.close() + + +def get_pypi_package_names(): + pypi_packages = set() + with xml_pypi_server("https://pypi.org/pypi") as client: + pypi_packages = set(client.list_packages()) + return pypi_packages + + class Package(object): """Package represents a collection of releases from one or more directories""" @@ -24,12 +53,31 @@ class Package(object): @property def json(self): - for path in self._package_dirs: + for path, _ in self._package_dirs: try: with open(os.path.join(path, 'api.json')) as f: return json.load(f) except FileNotFoundError: - pass + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir.path) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response def __repr__(self): return "<Package name={0!r} releases={1!r}".format(self.name, len(self.releases)) @@ -37,8 +85,15 @@ class Package(object): def add_release(self, path_to_binary): path_to_binary = os.path.abspath(path_to_binary) path, release = os.path.split(path_to_binary) - self.releases[release] = path_to_binary - self._package_dirs.add(path) + requires_python = "" + if path_to_binary.endswith(".whl"): + pkg = distlib.wheel.Wheel(path_to_binary) + md_dict = pkg.metadata.todict() + requires_python = md_dict.get("requires_python", "") + if requires_python.count(".") > 1: + requires_python, _, _ = requires_python.rpartition(".") + self.releases[release] = ReleaseTuple(path_to_binary, requires_python) + self._package_dirs.add(ReleaseTuple(path, requires_python)) class Artifact(object): @@ -87,16 +142,22 @@ def prepare_packages(path): if not (os.path.exists(path) and os.path.isdir(path)): raise ValueError("{} is not a directory!".format(path)) for root, dirs, files in os.walk(path): + if all([setup_file in list(files) for setup_file in ("setup.py", "setup.cfg")]): + continue for file in files: if not file.startswith('.') and not file.endswith('.json'): package_name = os.path.basename(root) if package_name and package_name == "fixtures": prepare_fixtures(root) continue + package_name = package_name.replace("_", "-") if package_name not in packages: packages[package_name] = Package(package_name) packages[package_name].add_release(os.path.join(root, file)) + remaining = get_pypi_package_names() - set(list(packages.keys())) + for pypi_pkg in remaining: + packages[pypi_pkg] = Package(pypi_pkg) @app.route('/') @@ -116,10 +177,18 @@ def artifacts(): @app.route('/simple/<package>/') def simple_package(package): - if package in packages: + if package in packages and packages[package].releases: return render_template('package.html', package=packages[package]) else: - abort(404) + try: + r = requests.get("https://pypi.org/simple/{0}".format(package)) + r.raise_for_status() + except Exception: + abort(404) + else: + return render_template( + 'package_pypi.html', package_contents=r.text + ) @app.route('/artifacts/<artifact>/') @@ -136,7 +205,7 @@ def serve_package(package, release): package = packages[package] if release in package.releases: - return send_file(package.releases[release]) + return send_file(package.releases[release].path) abort(404) @@ -152,13 +221,11 @@ def serve_artifact(artifact, fn): @app.route('/pypi/<package>/json') def json_for_package(package): - try: - return jsonify(packages[package].json) - except Exception: - pass - - r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) - return jsonify(r.json()) + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) if __name__ == '__main__': diff --git a/tests/pytest-pypi/pytest_pypi/certs.py b/tests/pytest-pypi/pytest_pypi/certs.py index f9e33870..b73fc63a 100644 --- a/tests/pytest-pypi/pytest_pypi/certs.py +++ b/tests/pytest-pypi/pytest_pypi/certs.py @@ -17,5 +17,6 @@ def where(): # vendored bundle inside Requests return os.path.join(os.path.abspath(os.path.dirname(__file__)), 'certs', 'cacert.pem') + if __name__ == '__main__': print(where()) diff --git a/tests/pytest-pypi/pytest_pypi/plugin.py b/tests/pytest-pypi/pytest_pypi/plugin.py index a17fcb24..83cd73fb 100644 --- a/tests/pytest-pypi/pytest_pypi/plugin.py +++ b/tests/pytest-pypi/pytest_pypi/plugin.py @@ -3,6 +3,7 @@ import pytest from .app import app as pypi_app from . import serve, certs + @pytest.fixture(scope='session') def pypi(request): server = serve.Server(application=pypi_app) @@ -31,6 +32,7 @@ def pypi_both(request, pypi, pypi_secure): def class_based_pypi(request, pypi): request.cls.pypi = pypi + @pytest.fixture(scope='class') def class_based_pypi_secure(request, pypi_secure): request.cls.pypi_secure = pypi_secure diff --git a/tests/pytest-pypi/pytest_pypi/templates/package.html b/tests/pytest-pypi/pytest_pypi/templates/package.html index 26ba9eca..3d364517 100644 --- a/tests/pytest-pypi/pytest_pypi/templates/package.html +++ b/tests/pytest-pypi/pytest_pypi/templates/package.html @@ -6,8 +6,8 @@ </head> <body> <h1>Links for {{ package.name }}</h1> - {% for release in package.releases %} - <a href="/{{ package.name }}/{{ release }}">{{ release }}</a> + {% for release, value in package.releases.items() %} + <a href="/{{ package.name }}/{{ release }}"{%- if value.requires_python %} data-requires-python="{{ value.requires_python }}"{% endif %}>{{ release }}</a> <br> {% endfor %} </body> diff --git a/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html new file mode 100644 index 00000000..217d8aa0 --- /dev/null +++ b/tests/pytest-pypi/pytest_pypi/templates/package_pypi.html @@ -0,0 +1,4 @@ + +{% autoescape false %} + {{ package_contents }} +{% endautoescape %} diff --git a/tests/pytest-pypi/setup.py b/tests/pytest-pypi/setup.py index 87f6b752..558dd3f8 100644 --- a/tests/pytest-pypi/setup.py +++ b/tests/pytest-pypi/setup.py @@ -62,7 +62,7 @@ setup( long_description=long_description, # The project URL. - url='https://github.com/kennethreitz/pytest-pypi', + url='https://github.com/pypa/pipenv/tree/master/tests/pytest-pypi', # Author details author='Kenneth Reitz', diff --git a/tests/test_artifacts/git/requests b/tests/test_artifacts/git/requests index 57d7284c..4983a9bd 160000 --- a/tests/test_artifacts/git/requests +++ b/tests/test_artifacts/git/requests @@ -1 +1 @@ -Subproject commit 57d7284c1a245cf9fbcecb594f50471d86e879f7 +Subproject commit 4983a9bde39c6320aa4f3e34e50dac6e263dab6f diff --git a/tests/test_artifacts/git/six b/tests/test_artifacts/git/six index e114efce..aa4e90bc 160000 --- a/tests/test_artifacts/git/six +++ b/tests/test_artifacts/git/six @@ -1 +1 @@ -Subproject commit e114efceea962fb143c909c904157ca994246fd2 +Subproject commit aa4e90bcd7b7bc13a71dfaebcb2021f4caaa8432 diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 61d318c3..8eca202c 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -20,40 +20,55 @@ def test_suppress_nested_venv_warning(capsys): @pytest.mark.core -def test_load_dot_env_from_environment_variable_location(capsys): - with temp_environ(), TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: +def test_load_dot_env_from_environment_variable_location(monkeypatch, capsys): + with temp_environ(), monkeypatch.context() as m, TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: + if os.name == "nt": + import click + is_console = False + m.setattr(click._winconsole, "_is_console", lambda x: is_console) dotenv_path = os.path.join(tempdir.name, 'test.env') key, val = 'SOME_KEY', 'some_value' with open(dotenv_path, 'w') as f: f.write('{}={}'.format(key, val)) - with mock.patch('pipenv.environments.PIPENV_DOTENV_LOCATION', dotenv_path): - load_dot_env() + m.setenv("PIPENV_DOTENV_LOCATION", str(dotenv_path)) + m.setattr("pipenv.environments.PIPENV_DOTENV_LOCATION", str(dotenv_path)) + load_dot_env() assert os.environ[key] == val @pytest.mark.core -def test_doesnt_load_dot_env_if_disabled(capsys): - with temp_environ(), TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: +def test_doesnt_load_dot_env_if_disabled(monkeypatch, capsys): + with temp_environ(), monkeypatch.context() as m, TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: + if os.name == "nt": + import click + is_console = False + m.setattr(click._winconsole, "_is_console", lambda x: is_console) dotenv_path = os.path.join(tempdir.name, 'test.env') key, val = 'SOME_KEY', 'some_value' with open(dotenv_path, 'w') as f: f.write('{}={}'.format(key, val)) - with mock.patch('pipenv.environments.PIPENV_DOTENV_LOCATION', dotenv_path): - with mock.patch('pipenv.environments.PIPENV_DONT_LOAD_ENV', '1'): - load_dot_env() - assert key not in os.environ - - load_dot_env() - assert key in os.environ + m.setenv("PIPENV_DOTENV_LOCATION", str(dotenv_path)) + m.setattr("pipenv.environments.PIPENV_DOTENV_LOCATION", str(dotenv_path)) + m.setattr("pipenv.environments.PIPENV_DONT_LOAD_ENV", True) + load_dot_env() + assert key not in os.environ + m.setattr("pipenv.environments.PIPENV_DONT_LOAD_ENV", False) + load_dot_env() + assert key in os.environ @pytest.mark.core -def test_load_dot_env_warns_if_file_doesnt_exist(capsys): - with temp_environ(), TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: +def test_load_dot_env_warns_if_file_doesnt_exist(monkeypatch, capsys): + with temp_environ(), monkeypatch.context() as m, TemporaryDirectory(prefix='pipenv-', suffix='') as tempdir: + if os.name == "nt": + import click + is_console = False + m.setattr(click._winconsole, "_is_console", lambda x: is_console) dotenv_path = os.path.join(tempdir.name, 'does-not-exist.env') - with mock.patch('pipenv.environments.PIPENV_DOTENV_LOCATION', dotenv_path): - load_dot_env() + m.setenv("PIPENV_DOTENV_LOCATION", str(dotenv_path)) + m.setattr("pipenv.environments.PIPENV_DOTENV_LOCATION", str(dotenv_path)) + load_dot_env() output, err = capsys.readouterr() assert 'Warning' in err diff --git a/tests/unit/test_help.py b/tests/unit/test_help.py index 2432e969..5179e4d3 100644 --- a/tests/unit/test_help.py +++ b/tests/unit/test_help.py @@ -2,10 +2,29 @@ import os import subprocess import sys +import pytest + +@pytest.mark.cli +@pytest.mark.help def test_help(): output = subprocess.check_output( [sys.executable, '-m', 'pipenv.help'], stderr=subprocess.STDOUT, env=os.environ.copy(), ) assert output + + +@pytest.mark.cli +@pytest.mark.help +def test_count_of_description_pre_option(): + test_command = 'pipenv install --help' + test_line = '--pre Allow pre-releases.' + out = subprocess.Popen(test_command.split(), stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout, _ = out.communicate() + lines = stdout.decode().split('\n') + count = 0 + for line in lines: + if line.strip().split() == test_line.split(): + count += 1 + assert count == 1 diff --git a/tests/unit/test_patched.py b/tests/unit/test_patched.py index 249292b4..f3581915 100644 --- a/tests/unit/test_patched.py +++ b/tests/unit/test_patched.py @@ -122,6 +122,8 @@ get_extras_links_scenarios = { ), } + +@pytest.mark.patched @pytest.mark.parametrize( 'scenarios,expected', list(get_extras_links_scenarios.values()), diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac5eb29b..4b9cd75c 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -3,11 +3,7 @@ import os import pytest -from first import first -from mock import Mock, patch - import pipenv.utils -import pythonfinder.utils from pipenv.exceptions import PipenvUsageError @@ -55,11 +51,11 @@ DEP_PIP_PAIRS = [ # Extras in url { "discord.py": { - "file": "https://github.com/Rapptz/discord.py/archive/rewrite.zip", + "file": "https://github.com/Rapptz/discord.py/archive/async.zip", "extras": ["voice"], } }, - "https://github.com/Rapptz/discord.py/archive/rewrite.zip#egg=discord.py[voice]", + "https://github.com/Rapptz/discord.py/archive/async.zip#egg=discord.py[voice]", ), ( { @@ -75,12 +71,20 @@ DEP_PIP_PAIRS = [ ] +def mock_unpack(link, source_dir, download_dir, only_download=False, session=None, + hashes=None, progress_bar="off"): + return + + @pytest.mark.utils @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) -def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] +def test_convert_deps_to_pip(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + if expected.startswith("Django"): + expected = expected.lower() + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] @pytest.mark.utils @@ -207,25 +211,48 @@ class TestUtils: @pytest.mark.windows @pytest.mark.skipif(os.name != "nt", reason="Windows test only") def test_windows_shellquote(self): - test_path = "C:\Program Files\Python36\python.exe" + test_path = r"C:\Program Files\Python36\python.exe" expected_path = '"C:\\\\Program Files\\\\Python36\\\\python.exe"' assert pipenv.utils.escape_grouped_arguments(test_path) == expected_path @pytest.mark.utils def test_is_valid_url(self): - url = "https://github.com/kennethreitz/requests.git" + url = "https://github.com/psf/requests.git" not_url = "something_else" assert pipenv.utils.is_valid_url(url) assert pipenv.utils.is_valid_url(not_url) is False @pytest.mark.utils def test_download_file(self): - url = "https://github.com/kennethreitz/pipenv/blob/master/README.md" + url = "https://github.com/pypa/pipenv/blob/master/README.md" output = "test_download.md" pipenv.utils.download_file(url, output) assert os.path.exists(output) os.remove(output) + @pytest.mark.utils + @pytest.mark.parametrize('line, expected', [ + ("python", True), + ("python3.7", True), + ("python2.7", True), + ("python2", True), + ("python3", True), + ("pypy3", True), + ("anaconda3-5.3.0", True), + ("which", False), + ("vim", False), + ("miniconda", True), + ("micropython", True), + ("ironpython", True), + ("jython3.5", True), + ("2", True), + ("2.7", True), + ("3.7", True), + ("3", True) + ]) + def test_is_python_command(self, line, expected): + assert pipenv.utils.is_python_command(line) == expected + @pytest.mark.utils def test_new_line_end_of_toml_file(this): # toml file that needs clean up @@ -298,6 +325,15 @@ twine = "*" "test.example.com", ], ), + ( + [{"url": "https://test.example.com:12345/simple", "verify_ssl": False}], + [ + "-i", + "https://test.example.com:12345/simple", + "--trusted-host", + "test.example.com:12345", + ], + ), ( [ {"url": "https://pypi.org/simple"}, @@ -324,6 +360,20 @@ twine = "*" "custom.example.com", ], ), + ( + [ + {"url": "https://pypi.org/simple"}, + {"url": "https://custom.example.com:12345/simple", "verify_ssl": False}, + ], + [ + "-i", + "https://pypi.org/simple", + "--extra-index-url", + "https://custom.example.com:12345/simple", + "--trusted-host", + "custom.example.com:12345", + ], + ), ( [ {"url": "https://pypi.org/simple"}, @@ -375,6 +425,7 @@ twine = "*" == expected_args ) + @pytest.mark.utils def test_invalid_prepare_pip_source_args(self): sources = [{}] with pytest.raises(PipenvUsageError): diff --git a/tests/unit/test_utils_windows_executable.py b/tests/unit/test_utils_windows_executable.py index b74cfbf3..22f20b35 100644 --- a/tests/unit/test_utils_windows_executable.py +++ b/tests/unit/test_utils_windows_executable.py @@ -13,6 +13,7 @@ pytestmark = pytest.mark.skipif( ) +@pytest.mark.utils @mock.patch('os.path.isfile') @mock.patch('pipenv.utils.find_executable') def test_find_windows_executable(mocked_find_executable, mocked_isfile):