diff --git a/.azure-pipelines/jobs/run-tests.yml b/.azure-pipelines/jobs/run-tests.yml index 3544af41..2602cba6 100644 --- a/.azure-pipelines/jobs/run-tests.yml +++ b/.azure-pipelines/jobs/run-tests.yml @@ -16,8 +16,9 @@ steps: export PIP_PROCESS_DEPENDENCY_LINKS="1" echo "Path $PATH" echo "Installing Pipenv…" - pip install -e "$(pwd)" --upgrade + pip install -e "$(pwd)[test]" --upgrade pipenv install --deploy --dev + pipenv run pip install -e "$(pwd)[test]" --upgrade echo pipenv --venv && echo pipenv --py && echo pipenv run python --version displayName: Make Virtualenv diff --git a/.azure-pipelines/jobs/run-vendor-scripts.yml b/.azure-pipelines/jobs/run-vendor-scripts.yml index 5af6f866..a419d941 100644 --- a/.azure-pipelines/jobs/run-vendor-scripts.yml +++ b/.azure-pipelines/jobs/run-vendor-scripts.yml @@ -10,9 +10,8 @@ jobs: maxParallel: 4 matrix: ${{ if eq(parameters.vmImage, 'vs2017-win2016') }}: - # TODO remove once vs2017-win2016 has Python 3.7 Python37: - python.version: '>= 3.7.0-b2' + python.version: '>= 3.7.2' python.architecture: x64 ${{ if ne(parameters.vmImage, 'vs2017-win2016' )}}: Python37: @@ -33,7 +32,7 @@ jobs: pip install certifi export GIT_SSL_CAINFO=$(python -m certifi) export LANG=C.UTF-8 - python -m pip install --upgrade invoke requests parver bs4 vistir towncrier + python -m pip install --upgrade invoke requests parver bs4 vistir towncrier pip setuptools wheel --upgrade-strategy=eager python -m invoke vendoring.update - template: ./run-manifest-check.yml diff --git a/.azure-pipelines/steps/create-virtualenv.yml b/.azure-pipelines/steps/create-virtualenv.yml index 9d1a6903..60ade40b 100644 --- a/.azure-pipelines/steps/create-virtualenv.yml +++ b/.azure-pipelines/steps/create-virtualenv.yml @@ -1,6 +1,6 @@ steps: - script: | virtualenv D:\.venv - D:\.venv\Scripts\pip.exe install -e . && D:\.venv\Scripts\pipenv install --dev + D:\.venv\Scripts\pip.exe install -e .[test] && D:\.venv\Scripts\pipenv install --dev && D:\.venv\Scripts\pipenv run pip install -e .[test] echo D:\.venv\Scripts\pipenv --venv && echo D:\.venv\Scripts\pipenv --py && echo D:\.venv\Scripts\pipenv run python --version displayName: Make Virtualenv diff --git a/.azure-pipelines/steps/install-dependencies.yml b/.azure-pipelines/steps/install-dependencies.yml index dfe733b6..16b3d6b8 100644 --- a/.azure-pipelines/steps/install-dependencies.yml +++ b/.azure-pipelines/steps/install-dependencies.yml @@ -1,3 +1,3 @@ steps: -- script: 'python -m pip install --upgrade pip && python -m pip install -e .' +- script: 'python -m pip install --upgrade pip && python -m pip install -e .[test]' displayName: Upgrade Pip & Install Pipenv diff --git a/.gitignore b/.gitignore index 766ffe3a..44abed16 100644 --- a/.gitignore +++ b/.gitignore @@ -153,3 +153,8 @@ venv.bak/ # Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) .vs/slnx.sqlite + +# mypy/typing section +typeshed/ +.dmypy.json +mypyhtml/ diff --git a/Pipfile b/Pipfile index db952be7..a8ddd5e6 100644 --- a/Pipfile +++ b/Pipfile @@ -1,16 +1,11 @@ [dev-packages] -pipenv = {path = ".", editable = true} +pipenv = {path = ".", editable = true, extras = ["test"]} "flake8" = ">=3.3.0,<4" -pytest = "*" -mock = "*" sphinx = "<=1.5.5" twine = "*" sphinx-click = "*" -pytest-xdist = "*" click = "*" -pytest-pypy = {path = "./tests/pytest-pypi", editable = true} -pytest-tap = "*" -flaky = "*" +pytest-pypi = {path = "./tests/pytest-pypi", editable = true} stdeb = {version="*", markers="sys_platform == 'linux'"} black = {version="*", markers="python_version >= '3.6'"} pytz = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 46115fdc..1bdc1c21 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "f0a57ba6f180e7312f799a460a9d2790c9c26c3556eff2b5d1fe7d9b9174d05b" + "sha256": "d7119fe8fa7be8224ff46509352efbd76dd17accf6a57580dbaf5762e613468b" }, "pipfile-spec": 6, "requires": {}, @@ -36,14 +36,6 @@ ], "version": "==1.4.3" }, - "argparse": { - "hashes": [ - "sha256:62b089a55be1d8949cd2bc7e0df0bddb9e028faefc8c32038cc84862aefdd6e4", - "sha256:c31647edb69fd3d465a847ea3157d37bed1f95f19760b11a47aa91c04b666314" - ], - "markers": "python_version == '2.6'", - "version": "==1.4.0" - }, "arpeggio": { "hashes": [ "sha256:a5258b84f76661d558492fa87e42db634df143685a0e51802d59cae7daad8732", @@ -53,10 +45,10 @@ }, "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ @@ -72,13 +64,37 @@ ], "version": "==2.6.0" }, + "backports.functools-lru-cache": { + "hashes": [ + "sha256:9d98697f088eb1b0fa451391f91afb5e3ebde16bbdb272819fd091151fda4f1a", + "sha256:f0b0e4eba956de51238e17573b7087e852dfe9854afd2e9c873f73fc0ca0a6dd" + ], + "markers": "python_version < '3'", + "version": "==1.5" + }, + "backports.shutil-get-terminal-size": { + "hashes": [ + "sha256:0975ba55054c15e346944b38956a4c9cbee9009391e41b86c68990effb8c1f64", + "sha256:713e7a8228ae80341c70586d1cc0a8caa5207346927e23d09dcbcaf18eadec80" + ], + "markers": "python_version < '3.3'", + "version": "==1.0.0" + }, + "backports.weakref": { + "hashes": [ + "sha256:81bc9b51c0abc58edc76aefbbc68c62a787918ffe943a37947e162c3f8e19e82", + "sha256:bc4170a29915f8b22c9e7c4939701859650f2eb84184aee80da329ac0b9825c2" + ], + "markers": "python_version < '3.3'", + "version": "==1.0.post1" + }, "beautifulsoup4": { "hashes": [ - "sha256:194ec62a25438adcb3fdb06378b26559eda1ea8a747367d34c33cef9c7f48d57", - "sha256:90f8e61121d6ae58362ce3bed8cd997efb00c914eae0ff3d363c32f9a9822d10", - "sha256:f0abd31228055d698bb392a826528ea08ebb9959e6bea17c606fd9c9009db938" + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" ], - "version": "==4.6.3" + "version": "==4.7.1" }, "black": { "hashes": [ @@ -91,10 +107,10 @@ }, "bleach": { "hashes": [ - "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", - "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" ], - "version": "==3.0.2" + "version": "==3.1.0" }, "bs4": { "hashes": [ @@ -111,47 +127,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" - }, - "cffi": { - "hashes": [ - "sha256:151b7eefd035c56b2b2e1eb9963c90c6302dc15fbd8c1c0a83a163ff2c7d7743", - "sha256:1553d1e99f035ace1c0544050622b7bc963374a00c467edafac50ad7bd276aef", - "sha256:1b0493c091a1898f1136e3f4f991a784437fac3673780ff9de3bcf46c80b6b50", - "sha256:2ba8a45822b7aee805ab49abfe7eec16b90587f7f26df20c71dd89e45a97076f", - "sha256:3bb6bd7266598f318063e584378b8e27c67de998a43362e8fce664c54ee52d30", - "sha256:3c85641778460581c42924384f5e68076d724ceac0f267d66c757f7535069c93", - "sha256:3eb6434197633b7748cea30bf0ba9f66727cdce45117a712b29a443943733257", - "sha256:495c5c2d43bf6cebe0178eb3e88f9c4aa48d8934aa6e3cddb865c058da76756b", - "sha256:4c91af6e967c2015729d3e69c2e51d92f9898c330d6a851bf8f121236f3defd3", - "sha256:57b2533356cb2d8fac1555815929f7f5f14d68ac77b085d2326b571310f34f6e", - "sha256:770f3782b31f50b68627e22f91cb182c48c47c02eb405fd689472aa7b7aa16dc", - "sha256:79f9b6f7c46ae1f8ded75f68cf8ad50e5729ed4d590c74840471fc2823457d04", - "sha256:7a33145e04d44ce95bcd71e522b478d282ad0eafaf34fe1ec5bbd73e662f22b6", - "sha256:857959354ae3a6fa3da6651b966d13b0a8bed6bbc87a0de7b38a549db1d2a359", - "sha256:87f37fe5130574ff76c17cab61e7d2538a16f843bb7bca8ebbc4b12de3078596", - "sha256:95d5251e4b5ca00061f9d9f3d6fe537247e145a8524ae9fd30a2f8fbce993b5b", - "sha256:9d1d3e63a4afdc29bd76ce6aa9d58c771cd1599fbba8cf5057e7860b203710dd", - "sha256:a36c5c154f9d42ec176e6e620cb0dd275744aa1d804786a71ac37dc3661a5e95", - "sha256:a6a5cb8809091ec9ac03edde9304b3ad82ad4466333432b16d78ef40e0cce0d5", - "sha256:ae5e35a2c189d397b91034642cb0eab0e346f776ec2eb44a49a459e6615d6e2e", - "sha256:b0f7d4a3df8f06cf49f9f121bead236e328074de6449866515cea4907bbc63d6", - "sha256:b75110fb114fa366b29a027d0c9be3709579602ae111ff61674d28c93606acca", - "sha256:ba5e697569f84b13640c9e193170e89c13c6244c24400fc57e88724ef610cd31", - "sha256:be2a9b390f77fd7676d80bc3cdc4f8edb940d8c198ed2d8c0be1319018c778e1", - "sha256:ca1bd81f40adc59011f58159e4aa6445fc585a32bb8ac9badf7a2c1aa23822f2", - "sha256:d5d8555d9bfc3f02385c1c37e9f998e2011f0db4f90e250e5bc0c0a85a813085", - "sha256:e55e22ac0a30023426564b1059b035973ec82186ddddbac867078435801c7801", - "sha256:e90f17980e6ab0f3c2f3730e56d1fe9bcba1891eeea58966e89d352492cc74f4", - "sha256:ecbb7b01409e9b782df5ded849c178a0aa7c906cf8c5a67368047daab282b184", - "sha256:ed01918d545a38998bfa5902c7c00e0fee90e957ce036a4000a88e3fe2264917", - "sha256:edabd457cd23a02965166026fd9bfd196f4324fe6032e866d0f3bd0301cd486f", - "sha256:fdf1c1dc5bafc32bc5d08b054f94d659422b05aba244d6be4ddc1c72d9aa70fb" - ], - "version": "==1.11.5" + "version": "==2018.11.29" }, "chardet": { "hashes": [ @@ -168,39 +147,6 @@ "index": "pypi", "version": "==7.0" }, - "cmarkgfm": { - "hashes": [ - "sha256:0186dccca79483e3405217993b83b914ba4559fe9a8396efc4eea56561b74061", - "sha256:1a625afc6f62da428df96ec325dc30866cc5781520cbd904ff4ec44cf018171c", - "sha256:207b7673ff4e177374c572feeae0e4ef33be620ec9171c08fd22e2b796e03e3d", - "sha256:275905bb371a99285c74931700db3f0c078e7603bed383e8cf1a09f3ee05a3de", - "sha256:50098f1c4950722521f0671e54139e0edc1837d63c990cf0f3d2c49607bb51a2", - "sha256:50ed116d0b60a07df0dc7b180c28569064b9d37d1578d4c9021cff04d725cb63", - "sha256:61a72def110eed903cd1848245897bcb80d295cd9d13944d4f9f30cba5b76655", - "sha256:64186fb75d973a06df0e6ea12879533b71f6e7ba1ab01ffee7fc3e7534758889", - "sha256:665303d34d7f14f10d7b0651082f25ebf7107f29ef3d699490cac16cdc0fc8ce", - "sha256:70b18f843aec58e4e64aadce48a897fe7c50426718b7753aaee399e72df64190", - "sha256:761ee7b04d1caee2931344ac6bfebf37102ffb203b136b676b0a71a3f0ea3c87", - "sha256:811527e9b7280b136734ed6cb6845e5fbccaeaa132ddf45f0246cbe544016957", - "sha256:987b0e157f70c72a84f3c2f9ef2d7ab0f26c08f2bf326c12c087ff9eebcb3ff5", - "sha256:9fc6a2183d0a9b0974ec7cdcdad42bd78a3be674cc3e65f87dd694419b3b0ab7", - "sha256:a3d17ee4ae739fe16f7501a52255c2e287ac817cfd88565b9859f70520afffea", - "sha256:ba5b5488719c0f2ced0aa1986376f7baff1a1653a8eb5fdfcf3f84c7ce46ef8d", - "sha256:c573ea89dd95d41b6d8cf36799c34b6d5b1eac4aed0212dee0f0a11fb7b01e8f", - "sha256:c5f1b9e8592d2c448c44e6bc0d91224b16ea5f8293908b1561de1f6d2d0658b1", - "sha256:cbe581456357d8f0674d6a590b1aaf46c11d01dd0a23af147a51a798c3818034", - "sha256:cf219bec69e601fe27e3974b7307d2f06082ab385d42752738ad2eb630a47d65", - "sha256:cf5014eb214d814a83a7a47407272d5db10b719dbeaf4d3cfe5969309d0fcf4b", - "sha256:d08bad67fa18f7e8ff738c090628ee0cbf0505d74a991c848d6d04abfe67b697", - "sha256:d6f716d7b1182bf35862b5065112f933f43dd1aa4f8097c9bcfb246f71528a34", - "sha256:e08e479102627641c7cb4ece421c6ed4124820b1758765db32201136762282d9", - "sha256:e20ac21418af0298437d29599f7851915497ce9f2866bc8e86b084d8911ee061", - "sha256:e25f53c37e319241b9a412382140dffac98ca756ba8f360ac7ab5e30cad9670a", - "sha256:e8932bddf159064f04e946fbb64693753488de21586f20e840b3be51745c8c09", - "sha256:f20900f16377f2109783ae9348d34bc80530808439591c3d3df73d5c7ef1a00c" - ], - "version": "==0.4.2" - }, "colorama": { "hashes": [ "sha256:05eed71e2e327246ad6b38c540c4a3117230b19679b875190486ddd2d721422d", @@ -210,16 +156,18 @@ }, "configparser": { "hashes": [ - "sha256:5308b47021bc2340965c371f0f058cc6971a04502638d4244225c49d80db273a" + "sha256:5bd5fa2a491dc3cfe920a3f2a107510d65eceae10e9c6e547b90261a4710df32", + "sha256:c114ff90ee2e762db972fa205f02491b1f5cf3ff950decd8542c62970c9bedac", + "sha256:df28e045fbff307a28795b18df6ac8662be3219435560ddb068c283afab1ea7a" ], "markers": "python_version < '3.2'", - "version": "==3.5.0" + "version": "==3.7.1" }, "cursor": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:7e728934f555a84a1c8b0850b66efcb580d092acc927b7d15dd43eb27dd4c4c5" ], - "version": "==1.2.0" + "version": "==1.3.1" }, "distlib": { "hashes": [ @@ -235,6 +183,13 @@ ], "version": "==0.14" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, "enum34": { "hashes": [ "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", @@ -242,7 +197,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version < '3.4'", + "markers": "python_version < '3'", "version": "==1.1.6" }, "execnet": { @@ -261,19 +216,18 @@ }, "flake8": { "hashes": [ - "sha256:7253265f7abd8b313e3892944044a365e3f4ac3fcdcfb4298f55ee9ddf188ba0", - "sha256:c7841163e2b576d435799169b78703ad6ac1bbb0f199994fc05f700b2a90ea37" + "sha256:c3ba1e130c813191db95c431a18cb4d20a468e98af7a77e2181b68574481ad36", + "sha256:fd9ddf503110bf3d8b1d270e8c673aab29ccb3dd6abf29bae1f54e5116ab4a91" ], "index": "pypi", - "version": "==3.5.0" + "version": "==3.7.5" }, "flaky": { "hashes": [ - "sha256:4ad7880aef8c35a34ddb394d4fa33047765bca1e3d67d182bf6eba9c8eabf3a2", - "sha256:d0533f473a46b916e6db6e84e20b06d8a70656600a0c14e819b0760b63f70226" + "sha256:12bd5e41f372b2190e8d754b6e5829c2f11dbc764e10b30f57e59f829c9ca1da", + "sha256:a94931c46a33469ec26f09b652bc88f55a8f5cc77807b90ca7bbafef1108fd7d" ], - "index": "pypi", - "version": "==3.4.0" + "version": "==3.5.3" }, "flask": { "hashes": [ @@ -287,29 +241,31 @@ "sha256:330cc27ccbf7f1e992e69fef78261dc7c6569012cf397db8d3de0234e6c937ca", "sha256:a7bb0f2cf3a3fd1ab2732cb49eba4252c2af4240442415b4abce3b87022a8f50" ], - "markers": "python_version < '3.3'", + "markers": "python_version < '3.0'", "version": "==1.0.2" }, - "future": { + "functools32": { "hashes": [ - "sha256:e39ced1ab767b5936646cedba8bcce582398233d6a627067d4c6a454c90cfedb" + "sha256:89d824aa6c358c421a234d7f9ee0bd75933a67c29588ce50aaa3acdf4d403fa0", + "sha256:f6253dfbe0538ad2e387bd8fdfd9293c925d63553f5813c4e587745416501e6d" ], - "version": "==0.16.0" + "markers": "python_version >= '2.7' and python_version < '2.8'", + "version": "==3.2.3.post2" }, "futures": { "hashes": [ "sha256:9ec02aa7d674acb8618afb127e27fde7fc68994c0437ad759fa094a574adb265", "sha256:ec0a6cb848cc212002b9828c3e34c675e0c9ff6741dc445cab6fdd4e1085d1f1" ], - "markers": "python_version < '3' and python_version >= '2.6'", + "markers": "python_version < '3.0'", "version": "==3.2.0" }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "imagesize": { "hashes": [ @@ -352,11 +308,11 @@ }, "jedi": { "hashes": [ - "sha256:0191c447165f798e6a730285f2eee783fff81b0d3df261945ecb80983b5c3ca7", - "sha256:b7493f73a2febe0dc33d51c99b474547f7f6c0b2c8fb2b21f453eef204c12148" + "sha256:571702b5bd167911fe9036e5039ba67f820d6502832285cde8c881ab2b2149fd", + "sha256:c8481b5e59d34a5c7c42e98f6625e633f6ef59353abea6437472c7ec2093f191" ], "index": "pypi", - "version": "==0.13.1" + "version": "==0.13.2" }, "jinja2": { "hashes": [ @@ -410,57 +366,66 @@ "sha256:5ce3c71c5545b472da17b72268978914d0252980348636840bd34a00b5cc96c1", "sha256:b158b6df76edd239b8208d481dc46b6afd45a846b7812ff0ce58971cf5bc8bba" ], - "index": "pypi", "version": "==2.0.0" }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:38a936c0a6d98a38bcc2d03fdaaedaba9f412879461dd2ceff8d37564d6522e4", + "sha256:c0a5785b1109a6bd7fac76d6837fd1feca158e54e521ccd2ae8bfe393cc9d4fc", + "sha256:fe7a7cae1ccb57d33952113ff4fa1bc5f879963600ed74918f1236e212ee50b9" ], - "version": "==4.3.0" + "markers": "python_version <= '2.7'", + "version": "==5.0.0" }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" ], - "version": "==18.0" + "version": "==19.0" }, "parso": { "hashes": [ - "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2", - "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24" + "sha256:4580328ae3f548b358f4901e38c0578229186835f0fa0846e47369796dd5bcc9", + "sha256:68406ebd7eafe17f8e40e15a84b56848eccbf27d7c1feb89e93d8fca395706db" ], - "version": "==0.3.1" + "version": "==0.3.4" }, "parver": { "hashes": [ - "sha256:ac4afff688d19d5e1876bb68d4bccc1a1b6a5cc8bd6a646939a14d366695ba15", - "sha256:f025fba8f88a9c776971df6d62b6cf7f37d1108f84c163bda91e157d7d527075" + "sha256:1b37a691af145a3a193eff269d53ba5b2ab16dfbb65d47d85360755919f5fe4b", + "sha256:72d056b8f8883ac90eef5554a9c8a47fac39d3b66479f3d2c8d5bc21b849cdba" ], "index": "pypi", - "version": "==0.1.1" + "version": "==0.2.1" }, "passa": { "editable": true, "git": "https://github.com/sarugaku/passa.git", - "ref": "4f3b8102f122cf0b75e5d7c513a2e61b0b093dcd" + "ref": "a2ba0b30c86339cae5ef3a03046fc9c583452c40", + "version": "==0.3.1.dev0" + }, + "pathlib2": { + "hashes": [ + "sha256:25199318e8cc3c25dcb45cbe084cc061051336d5a9ea2a12448d3d8cb748f742", + "sha256:5887121d7f7df3603bca2f710e7219f3eca0eb69e0b7cc6e0a022e155ac931a7" + ], + "markers": "python_version < '3.5'", + "version": "==2.3.3" }, "pbr": { "hashes": [ - "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", - "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" + "sha256:a7953f66e1f82e4b061f43096a4bcc058f7d3d41de9b94ac871770e8bdd831a2", + "sha256:d717573351cfe09f49df61906cd272abaa759b3e91744396b804965ff7bff38b" ], - "version": "==5.1.1" + "version": "==5.1.2" }, "pep517": { "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + "sha256:43a7aa3902efd305a605c1028e4045968cd012831233ecab633a31d3ba4860a5", + "sha256:cb5ca55450b64e80744cd5c32f7d5b8928004042dfea50fdc3f96ad7f27cba96" ], - "version": "==0.3" + "version": "==0.5.0" }, "pip-shims": { "hashes": [ @@ -471,14 +436,15 @@ }, "pipenv": { "editable": true, - "path": "." + "path": ".", + "version": "==2018.11.27.dev0" }, "pkginfo": { "hashes": [ - "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", - "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" ], - "version": "==1.4.2" + "version": "==1.5.0.1" }, "plette": { "extras": [ @@ -492,10 +458,10 @@ }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", + "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a" ], - "version": "==0.8.0" + "version": "==0.8.1" }, "py": { "hashes": [ @@ -506,72 +472,64 @@ }, "pycodestyle": { "hashes": [ - "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" ], - "version": "==2.4.0" - }, - "pycparser": { - "hashes": [ - "sha256:a988718abfad80b6b157acce7bf130a30876d27603738ac39f140993246b25b3" - ], - "version": "==2.19" + "version": "==2.5.0" }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", + "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" ], - "version": "==2.0.0" + "version": "==2.1.0" }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", + "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", + "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pytest": { "hashes": [ - "sha256:7e258ee50338f4e46957f9e09a0f10fb1c2d05493fa901d113a8dafd0790de4e", - "sha256:9332147e9af2dcf46cd7ceb14d5acadb6564744ddff1fe8c17f0ce60ece7d9a2" + "sha256:3f193df1cfe1d1609d4c583838bea3d532b18d6160fd3f55c9447fdca30848ec", + "sha256:e246cf173c01169b9617fc07264b7b1316e78d7a650055235d6d897bc80d9660" ], - "index": "pypi", - "version": "==3.8.2" + "version": "==3.10.1" }, "pytest-forked": { "hashes": [ - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" + "sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f", + "sha256:d352aaced2ebd54d42a65825722cb433004b4446ab5d2044851d9cc7a00c9e38" ], - "version": "==0.2" + "version": "==1.0.2" }, - "pytest-pypy": { + "pytest-pypi": { "editable": true, - "path": "./tests/pytest-pypi" + "path": "./tests/pytest-pypi", + "version": "==0.1.1" }, "pytest-tap": { "hashes": [ "sha256:3b05ec931424bbe44e944726b68f7ef185bb6d25ce9ce21ac52c9af7ffa9b506", "sha256:ca063de56298034302f3cbce55c87a27d7bfa7af7de591cdb9ec6ce45fea5467" ], - "index": "pypi", "version": "==2.3" }, "pytest-xdist": { "hashes": [ - "sha256:06aa39361694c9365baaa03bec71159b59ad06c9826c6279ebba368cb3571561", - "sha256:1ef0d05c905cfa0c5442c90e9e350e65c6ada120e33a00a066ca51c89f5f869a" + "sha256:4a201bb3ee60f5dd6bb40c5209d4e491cecc4d5bafd656cfb10f86178786e568", + "sha256:d03d1ff1b008458ed04fa73e642d840ac69b4107c168e06b71037c62d7813dd4" ], - "index": "pypi", - "version": "==1.23.2" + "version": "==1.26.1" }, "pytoml": { "hashes": [ @@ -581,11 +539,11 @@ }, "pytz": { "hashes": [ - "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", - "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + "sha256:32b0891edff07e28efe91284ed9c31e123d84bea3fd98e1f72be2508f43ef8d9", + "sha256:d5f05e487007e29e03409f9398d074e158d920d36eb82eaf66fb1136b0c5374c" ], "index": "pypi", - "version": "==2018.5" + "version": "==2018.9" }, "readme-renderer": { "hashes": [ @@ -596,24 +554,24 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], - "version": "==2.20.1" + "version": "==2.21.0" }, "requests-toolbelt": { "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" ], - "version": "==0.8.0" + "version": "==0.9.1" }, "requirementslib": { "hashes": [ - "sha256:c2c00c7bd3bd4984c97d10cd4d143efbe33b5ed9e55961bea30ca7a9a4927289", - "sha256:dc6b692e8dee03d6e90c29db1e337b0bf8152cce84a57f0fb4765e596afde4e0" + "sha256:c26feee79853dedddab550cf79fb2fa83b4bc1a16eab58f2c870e8314caa6cc5", + "sha256:d302b780afbd1d60f49d368b535929d8ff4b6d972797f3777c9560d48abdded7" ], - "version": "==1.3.3" + "version": "==1.4.0" }, "resolvelib": { "hashes": [ @@ -624,17 +582,34 @@ }, "rope": { "hashes": [ - "sha256:a108c445e1cd897fe19272ab7877d172e7faf3d4148c80e7d20faba42ea8f7b2" + "sha256:031eb54b3eeec89f4304ede816995ed2b93a21e6fba16bd02aff10a0d6c257b7" ], "index": "pypi", - "version": "==0.11.0" + "version": "==0.12.0" + }, + "scandir": { + "hashes": [ + "sha256:04b8adb105f2ed313a7c2ef0f1cf7aff4871aa7a1883fa4d8c44b5551ab052d6", + "sha256:1444134990356c81d12f30e4b311379acfbbcd03e0bab591de2696a3b126d58e", + "sha256:1b5c314e39f596875e5a95dd81af03730b338c277c54a454226978d5ba95dbb6", + "sha256:346619f72eb0ddc4cf355ceffd225fa52506c92a2ff05318cfabd02a144e7c4e", + "sha256:44975e209c4827fc18a3486f257154d34ec6eaec0f90fef0cca1caa482db7064", + "sha256:61859fd7e40b8c71e609c202db5b0c1dbec0d5c7f1449dec2245575bdc866792", + "sha256:a5e232a0bf188362fa00123cc0bb842d363a292de7126126df5527b6a369586a", + "sha256:c14701409f311e7a9b7ec8e337f0815baf7ac95776cc78b419a1e6d49889a383", + "sha256:c7708f29d843fc2764310732e41f0ce27feadde453261859ec0fca7865dfc41b", + "sha256:c9009c527929f6e25604aec39b0a43c3f831d2947d89d6caaab22f057b7055c8", + "sha256:f5c71e29b4e2af7ccdc03a020c626ede51da471173b4a6ad1e904f2b2e04b4bd" + ], + "markers": "python_version < '3.5'", + "version": "==1.9.0" }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "snowballstemmer": { "hashes": [ @@ -643,6 +618,13 @@ ], "version": "==1.2.1" }, + "soupsieve": { + "hashes": [ + "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", + "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" + ], + "version": "==1.8" + }, "sphinx": { "hashes": [ "sha256:11f271e7a9398385ed730e90f0bb41dc3815294bdcd395b46ed2d033bc2e7d87", @@ -653,18 +635,11 @@ }, "sphinx-click": { "hashes": [ - "sha256:0550d3e5dcd6244847bd0861ebe64101a2ef302913866e0ccd9095b2aa230051", - "sha256:404784f724504e3da2cb056767ba64955c4bfb9bfca8cfedd7142a962bafd70f" + "sha256:926da1a7c677ae1b35cf255269ff84fec65d0f92e4863acfa77b92cf8ae32275", + "sha256:f0c03d6ea0e4258c9c09646b6f745090ea8dd13e7e045903e4b789dfc02f7846" ], "index": "pypi", - "version": "==1.3.0" - }, - "sphinxcontrib-websupport": { - "hashes": [ - "sha256:68ca7ff70785cbe1e7bccc71a48b5b6d965d79ca50629606c7861a21b206d9dd", - "sha256:9de47f375baf1ea07cdb3436ff39d7a9c76042c10a769c52353ec46e4e8fc3b9" - ], - "version": "==1.1.0" + "version": "==2.0.1" }, "stdeb": { "hashes": [ @@ -684,7 +659,8 @@ "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", + "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" }, @@ -698,31 +674,32 @@ "towncrier": { "editable": true, "git": "https://github.com/hawkowl/towncrier.git", - "ref": "47754a607a9b03f06affaf167d65b990786aae25" + "ref": "ecd438c9c0ef132a92aba2eecc4dc672ccf9ec63", + "version": "==19.2.0" }, "tqdm": { "hashes": [ - "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", - "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + "sha256:d385c95361699e5cf7622485d9b9eae2d4864b21cd5a2374a9c381ffed701021", + "sha256:e22977e3ebe961f72362f6ddfb9197cc531c9737aaf5f607ef09740c849ecd05" ], - "version": "==4.28.1" + "version": "==4.31.1" }, "twine": { "hashes": [ - "sha256:7d89bc6acafb31d124e6e5b295ef26ac77030bf098960c2a4c4e058335827c5c", - "sha256:fad6f1251195f7ddd1460cb76d6ea106c93adb4e56c41e0da79658e56e547d2c" + "sha256:0fb0bfa3df4f62076cab5def36b1a71a2e4acb4d1fa5c97475b048117b1a6446", + "sha256:d6c29c933ecfc74e9b1d9fa13aa1f87c5d5770e119f5a4ce032092f0ff5b14dc" ], "index": "pypi", - "version": "==1.12.1" + "version": "==1.13.0" }, "typing": { "hashes": [ - "sha256:3a887b021a77b292e151afb75323dea88a7bc1b3dfa92176cff8e44c8b68bddf", - "sha256:b2c689d54e1144bbcfd191b0832980a21c2dbcf7b5ff7a66248a60c90e951eb8", - "sha256:d400a9344254803a2368533e4533a4200d21eb7b6b729c173bc38201a74db3f2" + "sha256:4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d", + "sha256:57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4", + "sha256:a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" ], "markers": "python_version < '3.5'", - "version": "==3.6.4" + "version": "==3.6.6" }, "urllib3": { "hashes": [ @@ -733,26 +710,28 @@ }, "virtualenv": { "hashes": [ - "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", - "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" + "sha256:8b9abfc51c38b70f61634bf265e5beacf6fae11fc25d355d1871f49b8e45f0db", + "sha256:cceab52aa7d4df1e1871a70236eb2b89fcfe29b6b43510d9738689787c513261" ], - "version": "==16.1.0" + "version": "==16.4.0" }, "virtualenv-clone": { "hashes": [ - "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" + "sha256:217bd3f0880c9f85672c0bcc9ad9e0354ab7dfa89c2f117e63aa878b4279f5bf", + "sha256:316c8a05432a7adb5e461709759aca18c51433ffc2c33e2e80c9e51c452d339f", + "sha256:f2a07ed255f3abaceef8c8442512d8cdb2ba9f867e212d8a51680c7790a85033" ], - "version": "==0.4.0" + "version": "==0.5.1" }, "vistir": { "extras": [ "spinner" ], "hashes": [ - "sha256:3a1020fb7be000b268af96641ced9ead844b1f75840c41e20e473647688fc630", - "sha256:6d2005ad670f77bd9c9b5415c4e2a4a20dce5b0cf0e0d11598eb463b2e0ebe44" + "sha256:510408ec63a4b423967fd630bf0885c8d6a1d5d126f8bb1be6aba86a0da5e815", + "sha256:fc5cca7a14e92feaa6f85dd91da74d834904280a96a21190aecb4cd1d1048e0e" ], - "version": "==0.2.5" + "version": "==0.3.0" }, "webencodings": { "hashes": [ @@ -770,17 +749,17 @@ }, "wheel": { "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + "sha256:12363e6df5678ecf9daf8429f06f97e7106e701405898f24318ce7f0b79c611a", + "sha256:b79ffea026bc0dbd940868347ae9eee36789b6496b6623bd2dec7c7c540a8f99" ], - "version": "==0.32.3" + "version": "==0.33.0" }, "yaspin": { "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + "sha256:441f8a6761e347652d04614899fd0a9cfda7439e2d5682e664bd31230c656176", + "sha256:d3ebcf8162e0ef8bb5484b8751d5b6d2fbf0720112c81f64614c308576a03b1d" ], - "version": "==0.14.0" + "version": "==0.14.1" } } } diff --git a/news/2722.bugfix.rst b/news/2722.bugfix.rst new file mode 100644 index 00000000..8c26df8d --- /dev/null +++ b/news/2722.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which caused editable package resolution to sometimes fail with an unhelpful setuptools-related error message. diff --git a/news/3053.bugfix.rst b/news/3053.bugfix.rst new file mode 100644 index 00000000..21134f59 --- /dev/null +++ b/news/3053.bugfix.rst @@ -0,0 +1 @@ +Dependency resolution now writes hashes for local and remote files to the lockfile. diff --git a/news/3071.bugfix.rst b/news/3071.bugfix.rst new file mode 100644 index 00000000..dd4145ea --- /dev/null +++ b/news/3071.bugfix.rst @@ -0,0 +1 @@ +Fixed a bug which prevented ``pipenv graph`` from correctly showing all dependencies when running from within ``pipenv shell``. diff --git a/news/3148.bugfix.rst b/news/3148.bugfix.rst new file mode 100644 index 00000000..1f0f4a62 --- /dev/null +++ b/news/3148.bugfix.rst @@ -0,0 +1 @@ +Fixed resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3148.feature.rst b/news/3148.feature.rst new file mode 100644 index 00000000..e33434db --- /dev/null +++ b/news/3148.feature.rst @@ -0,0 +1 @@ +Added support for resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/news/3298.bugfix.rst b/news/3298.bugfix.rst new file mode 100644 index 00000000..aa378723 --- /dev/null +++ b/news/3298.bugfix.rst @@ -0,0 +1,3 @@ +Fixed a bug which caused failures in warning reporting when running pipenv inside a virtualenv under some circumstances. + +- Fixed a bug with package discovery when running ``pipenv clean``. diff --git a/news/3298.feature.rst b/news/3298.feature.rst new file mode 100644 index 00000000..65a49424 --- /dev/null +++ b/news/3298.feature.rst @@ -0,0 +1,5 @@ +Added full support for resolution of all dependency types including direct URLs, zip archives, tarballs, etc. + +- Improved error handling and formatting. + +- Introduced improved cross platform stream wrappers for better ``stdout`` and ``stderr`` consistency. diff --git a/news/3298.vendor.rst b/news/3298.vendor.rst new file mode 100644 index 00000000..40f9b510 --- /dev/null +++ b/news/3298.vendor.rst @@ -0,0 +1,28 @@ +Updated vendored dependencies: + + - **attrs**: ``18.2.0`` => ``19.1.0`` + - **certifi**: ``2018.10.15`` => ``2018.11.29`` + - **cached_property**: ``1.4.3`` => ``1.5.1`` + - **colorama**: ``0.3.9`` => ``0.4.1`` + - **idna**: ``2.7`` => ``2.8`` + - **markupsafe**: ``1.0`` => ``1.1.1`` + - **orderedmultidict**: ``(new)`` => ``1.0`` + - **packaging**: ``18.0`` => ``19.0`` + - **parse**: ``1.9.0`` => ``1.11.1`` + - **pathlib2**: ``2.3.2`` => ``2.3.3`` + - **pep517**: ``(new)`` => ``0.5.0`` + - **pipdeptree**: ``0.13.0`` => ``0.13.2`` + - **pyparsing**: ``2.2.2`` => ``2.3.1`` + - **python-dotenv**: ``0.9.1`` => ``0.10.1`` + - **pythonfinder**: ``1.1.10`` => ``1.2.0`` + - **pytoml**: ``(new)`` => ``0.1.20`` + - **requests**: ``2.20.1`` => ``2.21.0`` + - **requirementslib**: ``1.3.3`` => ``1.4.2`` + - **shellingham**: ``1.2.7`` => ``1.2.8`` + - **six**: ``1.11.0`` => ``1.12.0`` + - **tomlkit**: ``0.5.2`` => ``0.5.3`` + - **urllib3**: ``1.24`` => ``1.24.1`` + - **vistir**: ``0.3.0`` => ``0.3.1`` + - **yaspin**: ``0.14.0`` => ``0.14.1`` + +- Removed vendored dependency **cursor**. diff --git a/news/3328.feature.rst b/news/3328.feature.rst new file mode 100644 index 00000000..7e92d39f --- /dev/null +++ b/news/3328.feature.rst @@ -0,0 +1 @@ +Pipenv will now successfully recursively lock VCS sub-dependencies. diff --git a/news/3368.feature.rst b/news/3368.feature.rst new file mode 100644 index 00000000..a998fce1 --- /dev/null +++ b/news/3368.feature.rst @@ -0,0 +1 @@ +Pipenv will now discover and resolve the intrinsic dependencies of **all** VCS dependencies, whether they are editable or not, to prevent resolution conflicts. diff --git a/news/3404.bugfix.rst b/news/3404.bugfix.rst new file mode 100644 index 00000000..fa678d6a --- /dev/null +++ b/news/3404.bugfix.rst @@ -0,0 +1 @@ +Fixed a keyerror which could occur when locking VCS dependencies in some cases. diff --git a/news/3427.bugfix.rst b/news/3427.bugfix.rst index 42f659f0..76aeb489 100644 --- a/news/3427.bugfix.rst +++ b/news/3427.bugfix.rst @@ -1 +1 @@ -Fix a bug that ``ValidationError`` is thrown when some fields are missing in source section. +Fixed a bug that ``ValidationError`` is thrown when some fields are missing in source section. diff --git a/news/3446.trivial.rst b/news/3446.trivial.rst index 1eb98bbb..c3f6a000 100644 --- a/news/3446.trivial.rst +++ b/news/3446.trivial.rst @@ -1 +1 @@ -Fix the wrong order of old and new hashes in message. +Fixed the wrong order of old and new hashes in message. diff --git a/news/3449.bugfix.rst b/news/3449.bugfix.rst index 72e8d228..4ed07046 100644 --- a/news/3449.bugfix.rst +++ b/news/3449.bugfix.rst @@ -1 +1 @@ -Update the index names in lock file when source name in Pipfile is changed. +Updated the index names in lock file when source name in Pipfile is changed. diff --git a/news/3584.bugfix.rst b/news/3584.bugfix.rst new file mode 100644 index 00000000..09684d1d --- /dev/null +++ b/news/3584.bugfix.rst @@ -0,0 +1 @@ +Fix the issue that lock file can't be created when ``PIPENV_PIPFILE`` is not under working directory. diff --git a/pipenv/_compat.py b/pipenv/_compat.py index ca3bbd0f..caba80fe 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -5,9 +5,6 @@ Exposes a standard API that enables compatibility across python versions, operating systems, etc. """ -import functools -import importlib -import io import os import sys import warnings @@ -122,6 +119,12 @@ UNICODE_TO_ASCII_TRANSLATION_MAP = { } +def decode_for_output(output, target=sys.stdout): + return vistir.misc.decode_for_output( + output, sys.stdout, translation_map=UNICODE_TO_ASCII_TRANSLATION_MAP + ) + + def decode_output(output): if not isinstance(output, six.string_types): return output diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index ec1bef61..f1fac2b2 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -12,8 +12,6 @@ import click_completion import crayons import delegator -from click_didyoumean import DYMCommandCollection - from ..__version__ import __version__ from .options import ( CONTEXT_SETTINGS, PipenvGroup, code_option, common_options, deploy_option, @@ -300,6 +298,7 @@ def uninstall( if retcode: sys.exit(retcode) + @cli.command(short_help="Generates Pipfile.lock.", context_settings=CONTEXT_SETTINGS) @lock_options @pass_state @@ -400,7 +399,6 @@ def shell( def run(state, command, args): """Spawns a command installed into the virtualenv.""" from ..core import do_run - do_run( command=command, args=args, three=state.three, python=state.python, pypi_mirror=state.pypi_mirror ) @@ -629,11 +627,9 @@ def sync( def clean(ctx, state, dry_run=False, bare=False, user=False): """Uninstalls all packages not specified in Pipfile.lock.""" from ..core import do_clean - do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run) + do_clean(ctx=ctx, three=state.three, python=state.python, dry_run=dry_run, + system=state.system) -# Only invoke the "did you mean" when an argument wasn't passed (it breaks those). -if "-" not in "".join(sys.argv) and len(sys.argv) > 1: - cli = DYMCommandCollection(sources=[cli]) if __name__ == "__main__": cli() diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index d6be0bac..d0a70c00 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -8,6 +8,7 @@ import click.types from click import ( BadParameter, Group, Option, argument, echo, make_pass_decorator, option ) +from click_didyoumean import DYMMixin from .. import environments from ..utils import is_valid_url @@ -19,7 +20,7 @@ CONTEXT_SETTINGS = { } -class PipenvGroup(Group): +class PipenvGroup(DYMMixin, Group): """Custom Group class provides formatted main help""" def get_help_option(self, ctx): diff --git a/pipenv/core.py b/pipenv/core.py index 934f13b6..7daff2be 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- - +from __future__ import absolute_import, print_function import json as simplejson import logging import os @@ -11,7 +11,7 @@ import warnings import click import six import urllib3.util as urllib3_util -import vistir +from pipenv.vendor import vistir import click_completion import crayons @@ -20,7 +20,7 @@ import dotenv import pipfile from . import environments, exceptions, pep508checker, progress -from ._compat import fix_utf8 +from ._compat import fix_utf8, decode_for_output from .cmdparse import Script from .environments import ( PIPENV_CACHE_DIR, PIPENV_COLORBLIND, PIPENV_DEFAULT_PYTHON_VERSION, @@ -675,7 +675,7 @@ def batch_install(deps_list, procs, failed_deps_queue, requirements_dir, no_deps=False, ignore_hashes=False, allow_global=False, blocking=False, pypi_mirror=None, nprocs=PIPENV_MAX_SUBPROCESS, retry=True): - + from .vendor.requirementslib.models.utils import strip_extras_markers_from_requirement failed = (not retry) if not failed: label = INSTALL_LABEL if os.name != "nt" else "" @@ -690,6 +690,10 @@ def batch_install(deps_list, procs, failed_deps_queue, trusted_hosts = [] # Install these because for dep in deps_list_bar: + if dep.req.req: + dep.req.req = strip_extras_markers_from_requirement(dep.req.req) + if dep.markers: + dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers())) index = None if dep.index: index = project.find_source(dep.index) @@ -698,10 +702,18 @@ def batch_install(deps_list, procs, failed_deps_queue, trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) # Install the module. is_artifact = False + if no_deps: + link = getattr(dep.req, "link", None) + is_wheel = False + if link: + is_wheel = link.is_wheel if dep.is_file_or_url and (dep.is_direct_url or any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] )): is_artifact = True + elif dep.is_vcs: + is_artifact = True + needs_deps = not no_deps if no_deps is True else is_artifact extra_indexes = [] if not index and indexes: @@ -714,24 +726,24 @@ def batch_install(deps_list, procs, failed_deps_queue, os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] - if no_deps: + if not needs_deps: link = getattr(dep.req, "link", None) is_wheel = False if link: is_wheel = link.is_wheel - is_non_editable_vcs = (dep.is_vcs and not dep.editable) - no_deps = not (dep.is_file_or_url and not (is_wheel or dep.editable)) + needs_deps = dep.is_file_or_url and not (is_wheel or dep.editable) c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, + no_deps=not needs_deps, block=any([dep.editable, dep.is_vcs, blocking]), index=index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes + extra_indexes=extra_indexes, + use_pep517=not retry, ) if dep.is_vcs or dep.editable: c.block() @@ -814,6 +826,7 @@ def do_install_dependencies( else: install_kwargs["nprocs"] = 1 + # with project.environment.activated(): batch_install( deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs ) @@ -1016,9 +1029,12 @@ def do_lock( dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. for is_dev in [True, False]: - pipfile_section = "dev_packages" if is_dev else "packages" + pipfile_section = "dev-packages" if is_dev else "packages" lockfile_section = "develop" if is_dev else "default" - packages = getattr(project, pipfile_section) + if project.pipfile_exists: + packages = project.parsed_pipfile.get(pipfile_section, {}) + else: + packages = getattr(project, pipfile_section.replace("-", "_")) if write: # Alert the user of progress. @@ -1031,12 +1047,8 @@ def do_lock( err=True, ) - deps = convert_deps_to_pip( - packages, project, r=False, include_index=True - ) - # Mutates the lockfile venv_resolve_deps( - deps, + packages, which=which, project=project, dev=is_dev, @@ -1257,12 +1269,12 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, - trusted_hosts=None + trusted_hosts=None, + use_pep517=True ): from pipenv.patched.notpip._internal import logger as piplogger - from .utils import Mapping + from .vendor.vistir.compat import Mapping from .vendor.urllib3.util import parse_url - src = [] write_to_tmpfile = False if requirement: @@ -1280,6 +1292,10 @@ def pip_install( crayons.normal("Installing {0!r}".format(requirement.name), bold=True), err=True, ) + + if requirement: + ignore_hashes = True if not requirement.hashes else ignore_hashes + # Create files for hash mode. if write_to_tmpfile: if not requirements_dir: @@ -1289,12 +1305,13 @@ def pip_install( prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False ) - f.write(vistir.misc.to_bytes(requirement.as_line())) + line = requirement.as_line(include_hashes=not ignore_hashes) + f.write(vistir.misc.to_bytes(line)) r = f.name f.close() - # Install dependencies when a package is a VCS dependency. + if requirement and requirement.vcs: - no_deps = False + # Install dependencies when a package is a non-editable VCS dependency. # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) @@ -1340,25 +1357,84 @@ def pip_install( create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source for source in sources ] + + line_kwargs = {"as_list": True, "include_hashes": not ignore_hashes} + + # Install dependencies when a package is a VCS dependency. + if requirement and requirement.vcs: + ignore_hashes = True + # Don't specify a source directory when using --system. + src_dir = None + if "PIP_SRC" in os.environ: + src_dir = os.environ["PIP_SRC"] + src = ["--src", os.environ["PIP_SRC"]] + if not requirement.editable: + no_deps = False + + if src_dir is not None: + repo = requirement.req.get_vcs_repo(src_dir=src_dir) + else: + repo = requirement.req.get_vcs_repo() + write_to_tmpfile = True + line_kwargs["include_markers"] = False + line_kwargs["include_hashes"] = False + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir(prefix="pipenv", + suffix="requirements") + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + line = "-e" if requirement.editable else "" + if requirement.editable or requirement.name is not None: + name = requirement.name + if requirement.extras: + name = "{0}{1}".format(name, requirement.extras_as_pip) + line = "-e {0}#egg={1}".format(vistir.path.path_to_url(repo.checkout_directory), requirement.name) + if repo.subdirectory: + line = "{0}&subdirectory={1}".format(line, repo.subdirectory) + else: + line = requirement.as_line(**line_kwargs) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + + # Create files for hash mode. + if write_to_tmpfile and not r: + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + ignore_hashes = True if not requirement.hashes else ignore_hashes + line = requirement.as_line(include_hashes=not ignore_hashes) + line = "{0} {1}".format(line, " ".join(src)) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + if (requirement and requirement.editable) and not r: - line_kwargs = {"as_list": True} - if requirement.markers: - line_kwargs["include_markers"] = False + line_kwargs["include_markers"] = False + line_kwargs["include_hashes"] = False install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] + possible_hashes = install_reqs[:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not all(item.startswith("--hash") for item in install_reqs): - ignore_hashes = True + + # hashes must be passed via a file + ignore_hashes = True elif r: install_reqs = ["-r", r] with open(r) as f: if "--hash" not in f.read(): ignore_hashes = True else: - ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + ignore_hashes = True + install_reqs = requirement.as_line(as_list=True, include_hashes=not ignore_hashes) if not requirement.markers: install_reqs = [escape_cmd(r) for r in install_reqs] elif len(install_reqs) > 1: @@ -1379,7 +1455,11 @@ def pip_install( pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: pip_command.append("--require-hashes") - + if not use_pep517: + from .vendor.packaging.version import parse as parse_version + pip_command.append("--no-build-isolation") + if project.environment.pip_version >= parse_version("19.0"): + pip_command.append("--no-use-pep517") if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) @@ -1398,6 +1478,8 @@ def pip_install( ) cmd = Script.parse(pip_command) pip_command = cmd.cmdify() + c = None + # with project.environment.activated(): c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1856,7 +1938,7 @@ def do_install( # This is for if the user passed in dependencies, then we want to make sure we else: - from .vendor.requirementslib import Requirement + from .vendor.requirementslib.models.requirements import Requirement # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] @@ -1907,6 +1989,15 @@ def do_install( pypi_mirror=pypi_mirror, ) if not c.ok: + sp.write_err(vistir.compat.fs_str( + "{0}: {1}".format( + crayons.red("WARNING"), + "Failed installing package {0}".format(pkg_line) + ), + )) + sp.write_err(vistir.compat.fs_str( + "Error text: {0}".format(c.out) + )) raise RuntimeError(c.err) except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str( @@ -1954,12 +2045,20 @@ def do_install( crayons.normal(fix_utf8("…"), bold=True), ) )) + # Add the package to the Pipfile. + try: + project.add_package_to_pipfile(pkg_requirement, dev) + except ValueError as e: + import traceback + sp.write_err( + "{0} {1}".format( + crayons.red("Error:", bold=True), traceback.format_exc() + ) + ) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed adding package to Pipfile" + )) sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) - # Add the package to the Pipfile. - try: - project.add_package_to_pipfile(pkg_requirement, dev) - except ValueError as e: - raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -2158,7 +2257,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror # Only set PIPENV_ACTIVE after finishing reading virtualenv_location # otherwise its value will be changed os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) if fancy: @@ -2297,6 +2396,8 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): load_dot_env() + previous_pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() @@ -2304,10 +2405,9 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): # Only set PIPENV_ACTIVE after finishing reading virtualenv_location # such as in inline_activate_virtual_environment # otherwise its value will be changed + previous_pipenv_active_value = os.environ.get("PIPENV_ACTIVE") os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) - try: script = project.build_script(command, args) cmd_string = ' '.join([script.command] + script.args) @@ -2315,10 +2415,21 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): click.echo(crayons.normal("$ {0}".format(cmd_string)), err=True) except ScriptEmptyError: click.echo("Can't run script {0!r}-it's empty?", err=True) + run_args = [script] + run_kwargs = {} if os.name == "nt": - do_run_nt(script) + run_fn = do_run_nt else: - do_run_posix(script, command=command) + run_fn = do_run_posix + run_kwargs = {"command": command} + try: + run_fn(*run_args, **run_kwargs) + finally: + os.environ.pop("PIPENV_ACTIVE", None) + if previous_pipenv_active_value is not None: + os.environ["PIPENV_ACTIVE"] = previous_pipenv_active_value + if previous_pip_shims_module is not None: + os.environ["PIP_SHIMS_BASE_MODULE"] = previous_pip_shims_module def do_check( @@ -2607,34 +2718,32 @@ def do_sync( click.echo(crayons.green("All dependencies are now up-to-date!")) -def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None): +def do_clean( + ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirror=None, + system=False +): # Ensure that virtualenv is available. from packaging.utils import canonicalize_name ensure_project(three=three, python=python, validate=False, pypi_mirror=pypi_mirror) ensure_lockfile(pypi_mirror=pypi_mirror) # Make sure that the virtualenv's site packages are configured correctly # otherwise we may end up removing from the global site packages directory - installed_package_names = [ - canonicalize_name(pkg.project_name) for pkg - in project.environment.get_installed_packages() - ] + installed_package_names = project.installed_package_names.copy() # Remove known "bad packages" from the list. for bad_package in BAD_PACKAGES: if canonicalize_name(bad_package) in installed_package_names: if environments.is_verbose(): click.echo("Ignoring {0}.".format(bad_package), err=True) - del installed_package_names[installed_package_names.index( - canonicalize_name(bad_package) - )] + installed_package_names.remove(canonicalize_name(bad_package)) # Intelligently detect if --dev should be used or not. - develop = [canonicalize_name(k) for k in project.lockfile_content["develop"].keys()] - default = [canonicalize_name(k) for k in project.lockfile_content["default"].keys()] - for used_package in set(develop + default): + locked_packages = { + canonicalize_name(pkg) for pkg in project.lockfile_package_names["combined"] + } + for used_package in locked_packages: if used_package in installed_package_names: - del installed_package_names[installed_package_names.index( - canonicalize_name(used_package) - )] + installed_package_names.remove(used_package) failure = False + cmd = [which_pip(allow_global=system), "uninstall", "-y", "-qq"] for apparent_bad_package in installed_package_names: if dry_run and not bare: click.echo(apparent_bad_package) @@ -2646,9 +2755,8 @@ def do_clean(ctx, three=None, python=None, dry_run=False, bare=False, pypi_mirro ) ) # Uninstall the package. - c = delegator.run( - "{0} uninstall {1} -y".format(which_pip(), apparent_bad_package) - ) + cmd_str = Script.parse(cmd + [apparent_bad_package]).cmdify() + c = delegator.run(cmd_str, block=True) if c.return_code != 0: failure = True sys.exit(int(failure)) diff --git a/pipenv/environment.py b/pipenv/environment.py index 06757938..602859e2 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -11,15 +11,16 @@ import sys from distutils.sysconfig import get_python_lib from sysconfig import get_paths +import itertools import pkg_resources import six -import vistir import pipenv -from cached_property import cached_property +from .vendor.cached_property import cached_property +import vistir -from .utils import normalize_path +from .utils import normalize_path, make_posix BASE_WORKING_SET = pkg_resources.WorkingSet(sys.path) @@ -92,12 +93,16 @@ class Environment(object): deps |= cls.resolve_dist(dist, working_set) return deps - def add_dist(self, dist_name): - dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + def extend_dists(self, dist): extras = self.resolve_dist(dist, self.base_working_set) + self.extra_dists.append(dist) if extras: self.extra_dists.extend(extras) + def add_dist(self, dist_name): + dist = pkg_resources.get_distribution(pkg_resources.Requirement(dist_name)) + self.extend_dists(dist) + @cached_property def python_version(self): with self.activated(): @@ -145,7 +150,7 @@ class Environment(object): 'stdlib': '/home/hawk/.pyenv/versions/3.7.1/lib/python3.7'} """ - prefix = self.prefix.as_posix() + prefix = make_posix(self.prefix.as_posix()) install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' paths = get_paths(install_scheme, vars={ 'base': prefix, @@ -154,8 +159,8 @@ class Environment(object): paths["PATH"] = paths["scripts"] + os.pathsep + os.defpath if "prefix" not in paths: paths["prefix"] = prefix - purelib = get_python_lib(plat_specific=0, prefix=prefix) - platlib = get_python_lib(plat_specific=1, prefix=prefix) + purelib = make_posix(get_python_lib(plat_specific=0, prefix=prefix)) + platlib = make_posix(get_python_lib(plat_specific=1, prefix=prefix)) if purelib == platlib: lib_dirs = purelib else: @@ -163,7 +168,7 @@ class Environment(object): paths["libdir"] = purelib paths["purelib"] = purelib paths["platlib"] = platlib - paths['PYTHONPATH'] = lib_dirs + paths['PYTHONPATH'] = os.pathsep.join(["", ".", lib_dirs]) paths["libdirs"] = lib_dirs return paths @@ -176,19 +181,21 @@ class Environment(object): @property def python(self): """Path to the environment python""" - py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").as_posix() + py = vistir.compat.Path(self.base_paths["scripts"]).joinpath("python").absolute().as_posix() if not py: return vistir.compat.Path(sys.executable).as_posix() return py @cached_property def sys_path(self): - """The system path inside the environment + """ + The system path inside the environment :return: The :data:`sys.path` from the environment :rtype: list """ + from .vendor.vistir.compat import JSONDecodeError current_executable = vistir.compat.Path(sys.executable).as_posix() if not self.python or self.python == current_executable: return sys.path @@ -196,12 +203,16 @@ class Environment(object): return sys.path cmd_args = [self.python, "-c", "import json, sys; print(json.dumps(sys.path))"] path, _ = vistir.misc.run(cmd_args, return_object=False, nospin=True, block=True, combine_stderr=False, write_to_stdout=False) - path = json.loads(path.strip()) + try: + path = json.loads(path.strip()) + except JSONDecodeError: + path = sys.path return path @cached_property def sys_prefix(self): - """The prefix run inside the context of the environment + """ + The prefix run inside the context of the environment :return: The python prefix inside the environment :rtype: :data:`sys.prefix` @@ -236,15 +247,33 @@ class Environment(object): return "purelib", purelib return "platlib", self.paths["platlib"] + @property + def pip_version(self): + """ + Get the pip version in the environment. Useful for knowing which args we can use + when installing. + """ + from .vendor.packaging.version import parse as parse_version + pip = next(iter( + pkg for pkg in self.get_installed_packages() if pkg.key == "pip" + ), None) + if pip is not None: + pip_version = parse_version(pip.version) + return parse_version("18.0") + def get_distributions(self): - """Retrives the distributions installed on the library path of the environment + """ + Retrives the distributions installed on the library path of the environment :return: A set of distributions found on the library path :rtype: iterator """ pkg_resources = self.safe_import("pkg_resources") - return pkg_resources.find_distributions(self.paths["PYTHONPATH"]) + libdirs = self.base_paths["libdirs"].split(os.pathsep) + dists = (pkg_resources.find_distributions(libdir) for libdir in libdirs) + for dist in itertools.chain.from_iterable(dists): + yield dist def find_egg(self, egg_dist): """Find an egg by name in the given environment""" @@ -271,21 +300,28 @@ class Environment(object): def dist_is_in_project(self, dist): """Determine whether the supplied distribution is in the environment.""" from .project import _normalized - prefix = _normalized(self.base_paths["prefix"]) + prefixes = [ + _normalized(prefix) for prefix in self.base_paths["libdirs"].split(os.pathsep) + if _normalized(prefix).startswith(_normalized(self.prefix.as_posix())) + ] location = self.locate_dist(dist) if not location: return False - return _normalized(location).startswith(prefix) + location = _normalized(make_posix(location)) + return any(location.startswith(prefix) for prefix in prefixes) def get_installed_packages(self): """Returns all of the installed packages in a given environment""" workingset = self.get_working_set() - packages = [pkg for pkg in workingset if self.dist_is_in_project(pkg)] + packages = [ + pkg for pkg in workingset + if self.dist_is_in_project(pkg) and pkg.key != "python" + ] return packages @contextlib.contextmanager def get_finder(self, pre=False): - from .vendor.pip_shims import Command, cmdoptions, index_group, PackageFinder + from .vendor.pip_shims.shims import Command, cmdoptions, index_group, PackageFinder from .environments import PIPENV_CACHE_DIR index_urls = [source.get("url") for source in self.sources] @@ -475,6 +511,7 @@ class Environment(object): vendor_dir = parent_path.joinpath("vendor").as_posix() patched_dir = parent_path.joinpath("patched").as_posix() parent_path = parent_path.as_posix() + self.add_dist("pip") prefix = self.prefix.as_posix() with vistir.contextmanagers.temp_environ(), vistir.contextmanagers.temp_path(): os.environ["PATH"] = os.pathsep.join([ @@ -484,12 +521,24 @@ class Environment(object): ]) os.environ["PYTHONIOENCODING"] = vistir.compat.fs_str("utf-8") os.environ["PYTHONDONTWRITEBYTECODE"] = vistir.compat.fs_str("1") - os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + from .environments import PIPENV_USE_SYSTEM if self.is_venv: + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] os.environ["VIRTUAL_ENV"] = vistir.compat.fs_str(prefix) + else: + if not PIPENV_USE_SYSTEM and not os.environ.get("VIRTUAL_ENV"): + os.environ["PYTHONPATH"] = self.base_paths["PYTHONPATH"] + os.environ.pop("PYTHONHOME", None) sys.path = self.sys_path sys.prefix = self.sys_prefix site.addsitedir(self.base_paths["purelib"]) + pip = self.safe_import("pip") + pip_vendor = self.safe_import("pip._vendor") + pep517_dir = os.path.join(os.path.dirname(pip_vendor.__file__), "pep517") + site.addsitedir(pep517_dir) + os.environ["PYTHONPATH"] = os.pathsep.join([ + os.environ.get("PYTHONPATH", self.base_paths["PYTHONPATH"]), pep517_dir + ]) if include_extras: site.addsitedir(parent_path) sys.path.extend([parent_path, patched_dir, vendor_dir]) @@ -593,10 +642,7 @@ class Environment(object): monkey_patch.activate() pip_shims = self.safe_import("pip_shims") pathset_base = pip_shims.UninstallPathSet - import recursive_monkey_patch - recursive_monkey_patch.monkey_patch( - PatchedUninstaller, pathset_base - ) + pathset_base._permitted = PatchedUninstaller._permitted dist = next( iter(filter(lambda d: d.project_name == pkgname, self.get_working_set())), None diff --git a/pipenv/environments.py b/pipenv/environments.py index 445e3b63..f4510621 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -280,13 +280,37 @@ def is_quiet(threshold=-1): def is_in_virtualenv(): - pipenv_active = os.environ.get("PIPENV_ACTIVE") - virtual_env = os.environ.get("VIRTUAL_ENV") - return (PIPENV_USE_SYSTEM or virtual_env) and not ( - pipenv_active or PIPENV_IGNORE_VIRTUALENVS + """ + Check virtualenv membership dynamically + + :return: True or false depending on whether we are in a regular virtualenv or not + :rtype: bool + """ + + pipenv_active = os.environ.get("PIPENV_ACTIVE", False) + virtual_env = None + use_system = False + ignore_virtualenvs = bool(os.environ.get("PIPENV_IGNORE_VIRTUALENVS", False)) + + if not pipenv_active and not ignore_virtualenvs: + virtual_env = os.environ.get("VIRTUAL_ENV") + use_system = bool(virtual_env) + return (use_system or virtual_env) and not ( + pipenv_active or ignore_virtualenvs ) PIPENV_SPINNER_FAIL_TEXT = fix_utf8(u"✘ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") PIPENV_SPINNER_OK_TEXT = fix_utf8(u"✔ {0}") if not PIPENV_HIDE_EMOJIS else ("{0}") + + +def is_type_checking(): + try: + from typing import TYPE_CHECKING + except ImportError: + return False + return TYPE_CHECKING + + +MYPY_RUNNING = is_type_checking() diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 0da42fe7..47dfc718 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -4,7 +4,7 @@ import itertools import sys from pprint import pformat -from traceback import format_exception +from traceback import format_exception, format_tb import six @@ -25,8 +25,8 @@ def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): hook(exc_type, exception, traceback) else: exc = format_exception(exc_type, exception, traceback) - lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) - lines = list(lines)[-11:-1] + tb = format_tb(traceback, limit=-6) + lines = itertools.chain.from_iterable([frame.splitlines() for frame in tb]) for line in lines: line = line.strip("'").strip('"').strip("\n").strip() if not line.startswith("File"): @@ -179,7 +179,8 @@ class SystemUsageError(PipenvOptionsError): crayons.red("Warning", bold=True) ), ] - message = crayons.blue("See also: {0}".format(crayons.white("-deploy flag."))) + if message is None: + message = crayons.blue("See also: {0}".format(crayons.white("--deploy flag."))) super(SystemUsageError, self).__init__(option_name, message=message, ctx=ctx, extra=extra, **kwargs) @@ -235,11 +236,11 @@ class UninstallError(PipenvException): crayons.yellow("$ {0}".format(command), bold=True) )),] extra.extend([crayons.blue(line.strip()) for line in return_values.splitlines()]) - if isinstance(package, (tuple, list)): + if isinstance(package, (tuple, list, set)): package = " ".join(package) - message = "{0} {1}...".format( + message = "{0!s} {1!s}...".format( crayons.normal("Failed to uninstall package(s)"), - crayons.yellow(package, bold=True) + crayons.yellow(str(package), bold=True) ) self.exit_code = return_code PipenvException.__init__(self, message=fix_utf8(message), extra=extra) @@ -248,8 +249,14 @@ class UninstallError(PipenvException): class InstallError(PipenvException): def __init__(self, package, **kwargs): - message = "{0} {1}".format( + package_message = "" + if package is not None: + package_message = crayons.normal("Couldn't install package {0}\n".format( + crayons.white(package, bold=True) + )) + message = "{0} {1} {2}".format( crayons.red("ERROR:", bold=True), + package_message, crayons.yellow("Package installation failed...") ) extra = kwargs.pop("extra", []) diff --git a/pipenv/patched/safety/__init__.py b/pipenv/patched/safety/__init__.py index 56b497d2..69563274 100644 --- a/pipenv/patched/safety/__init__.py +++ b/pipenv/patched/safety/__init__.py @@ -2,4 +2,4 @@ __author__ = """pyup.io""" __email__ = 'support@pyup.io' -__version__ = '1.8.4' +__version__ = '1.8.5' diff --git a/pipenv/patched/safety/cli.py b/pipenv/patched/safety/cli.py index 9ebd9e8a..2e8f88df 100644 --- a/pipenv/patched/safety/cli.py +++ b/pipenv/patched/safety/cli.py @@ -6,17 +6,13 @@ from safety import __version__ from safety import safety from safety.formatter import report import itertools -from safety.util import read_requirements +from safety.util import read_requirements, read_vulnerabilities from safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError - try: - # pip 9 - from pipenv.patched.notpip import get_installed_distributions + from json.decoder import JSONDecodeError except ImportError: - # pip 10 - from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - + JSONDecodeError = ValueError @click.group() @click.version_option(version=__version__) @@ -46,10 +42,17 @@ def cli(): help="Read input from one (or multiple) requirement files. Default: empty") @click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[], help="Ignore one (or multiple) vulnerabilities by ID. Default: empty") -def check(key, db, json, full_report, bare, stdin, files, cache, ignore): - +@click.option("--output", "-o", default="", + help="Path to where output file will be placed. Default: empty") +@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None, + help="Proxy host IP or DNS --proxy-host") +@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80, + help="Proxy port number --proxy-port") +@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http', + help="Proxy protocol (https or http) --proxy-protocol") +def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport): if files and stdin: - click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red") + click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr) sys.exit(-1) if files: @@ -57,33 +60,72 @@ def check(key, db, json, full_report, bare, stdin, files, cache, ignore): elif stdin: packages = list(read_requirements(sys.stdin)) else: - packages = get_installed_distributions() - + import pkg_resources + packages = [ + d for d in pkg_resources.working_set + if d.key not in {"python", "wsgiref", "argparse"} + ] + proxy_dictionary = {} + if proxyhost is not None: + if proxyprotocol in ["http", "https"]: + proxy_dictionary = {proxyprotocol: "{0}://{1}:{2}".format(proxyprotocol, proxyhost, str(proxyport))} + else: + click.secho("Proxy Protocol should be http or https only.", fg="red") + sys.exit(-1) try: - vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore) - click.secho(report( - vulns=vulns, - full=full_report, - json_report=json, - bare_report=bare, - checked_packages=len(packages), - db=db, - key=key - ) - ) + vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary) + output_report = report(vulns=vulns, + full=full_report, + json_report=json, + bare_report=bare, + checked_packages=len(packages), + db=db, + key=key) + + if output: + with open(output, 'w+') as output_file: + output_file.write(output_report) + else: + click.secho(output_report, nl=False if bare and not vulns else True) sys.exit(-1 if vulns else 0) except InvalidKeyError: click.secho("Your API Key '{key}' is invalid. See {link}".format( key=key, link='https://goo.gl/O7Y1rS'), - fg="red") + fg="red", + file=sys.stderr) sys.exit(-1) except DatabaseFileNotFoundError: - click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red") + click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red", file=sys.stderr) sys.exit(-1) except DatabaseFetchError: - click.secho("Unable to load vulnerability database", fg="red") + click.secho("Unable to load vulnerability database", fg="red", file=sys.stderr) sys.exit(-1) +@cli.command() +@click.option("--full-report/--short-report", default=False, + help='Full reports include a security advisory (if available). Default: ' + '--short-report') +@click.option("--bare/--not-bare", default=False, + help='Output vulnerable packages only. Useful in combination with other tools.' + 'Default: --not-bare') +@click.option("file", "--file", "-f", type=click.File(), required=True, + help="Read input from an insecure report file. Default: empty") +def review(full_report, bare, file): + if full_report and bare: + click.secho("Can't choose both --bare and --full-report/--short-report", fg="red") + sys.exit(-1) + + try: + input_vulns = read_vulnerabilities(file) + except JSONDecodeError: + click.secho("Not a valid JSON file", fg="red") + sys.exit(-1) + + vulns = safety.review(input_vulns) + output_report = report(vulns=vulns, full=full_report, bare_report=bare) + click.secho(output_report, nl=False if bare and not vulns else True) + + if __name__ == "__main__": cli() diff --git a/pipenv/patched/safety/formatter.py b/pipenv/patched/safety/formatter.py index 8bc57ec1..c19bff1b 100644 --- a/pipenv/patched/safety/formatter.py +++ b/pipenv/patched/safety/formatter.py @@ -3,6 +3,7 @@ import platform import sys import json import os +import textwrap # python 2.7 compat try: @@ -110,9 +111,10 @@ class SheetReport(object): descr = get_advisory(vuln) - for chunk in [descr[i:i + 76] for i in range(0, len(descr), 76)]: - - for line in chunk.splitlines(): + for pn, paragraph in enumerate(descr.replace('\r', '').split('\n\n')): + if pn: + table.append("│ {:76} │".format('')) + for line in textwrap.wrap(paragraph, width=76): try: table.append("│ {:76} │".format(line.encode('utf-8'))) except TypeError: diff --git a/pipenv/patched/safety/safety.py b/pipenv/patched/safety/safety.py index 99b99125..871bd775 100644 --- a/pipenv/patched/safety/safety.py +++ b/pipenv/patched/safety/safety.py @@ -9,6 +9,7 @@ import json import time import errno + class Vulnerability(namedtuple("Vulnerability", ["name", "spec", "version", "advisory", "vuln_id"])): pass @@ -64,7 +65,7 @@ def write_to_cache(db_name, data): f.write(json.dumps(cache)) -def fetch_database_url(mirror, db_name, key, cached): +def fetch_database_url(mirror, db_name, key, cached, proxy): headers = {} if key: @@ -74,9 +75,8 @@ def fetch_database_url(mirror, db_name, key, cached): cached_data = get_from_cache(db_name=db_name) if cached_data: return cached_data - url = mirror + db_name - r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers) + r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy) if r.status_code == 200: data = r.json() if cached: @@ -94,7 +94,7 @@ def fetch_database_file(path, db_name): return json.loads(f.read()) -def fetch_database(full=False, key=False, db=False, cached=False): +def fetch_database(full=False, key=False, db=False, cached=False, proxy={}): if db: mirrors = [db] @@ -105,7 +105,7 @@ def fetch_database(full=False, key=False, db=False, cached=False): for mirror in mirrors: # mirror can either be a local path or a URL if mirror.startswith("http://") or mirror.startswith("https://"): - data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached) + data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy) else: data = fetch_database_file(mirror, db_name=db_name) if data: @@ -120,10 +120,9 @@ def get_vulnerabilities(pkg, spec, db): yield entry -def check(packages, key, db_mirror, cached, ignore_ids): - +def check(packages, key, db_mirror, cached, ignore_ids, proxy): key = key if key else os.environ.get("SAFETY_API_KEY", False) - db = fetch_database(key=key, db=db_mirror, cached=cached) + db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy) db_full = None vulnerable_packages = frozenset(db.keys()) vulnerable = [] @@ -152,3 +151,19 @@ def check(packages, key, db_mirror, cached, ignore_ids): ) ) return vulnerable + + +def review(vulnerabilities): + vulnerable = [] + for vuln in vulnerabilities: + current_vuln = { + "name": vuln[0], + "spec": vuln[1], + "version": vuln[2], + "advisory": vuln[3], + "vuln_id": vuln[4], + } + vulnerable.append( + Vulnerability(**current_vuln) + ) + return vulnerable diff --git a/pipenv/patched/safety/util.py b/pipenv/patched/safety/util.py index 2e6efaec..16062f41 100644 --- a/pipenv/patched/safety/util.py +++ b/pipenv/patched/safety/util.py @@ -1,11 +1,17 @@ from dparse.parser import setuptools_parse_requirements_backport as _parse_requirements from collections import namedtuple import click +import sys +import json import os Package = namedtuple("Package", ["key", "version"]) RequirementFile = namedtuple("RequirementFile", ["path"]) +def read_vulnerabilities(fh): + return json.load(fh) + + def iter_lines(fh, lineno=0): for line in fh.readlines()[lineno:]: yield line @@ -85,7 +91,8 @@ def read_requirements(fh, resolve=False): "Warning: unpinned requirement '{req}' found in {fname}, " "unable to check.".format(req=req.name, fname=fname), - fg="yellow" + fg="yellow", + file=sys.stderr ) except ValueError: continue diff --git a/pipenv/project.py b/pipenv/project.py index edbcff87..8c813170 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -20,7 +20,7 @@ from first import first import pipfile import pipfile.api -from cached_property import cached_property +from .vendor.cached_property import cached_property from .cmdparse import Script from .environment import Environment @@ -29,12 +29,13 @@ from .environments import ( PIPENV_PIPFILE, PIPENV_PYTHON, PIPENV_TEST_INDEX, PIPENV_VENV_IN_PROJECT, is_in_virtualenv ) +from .vendor.requirementslib.models.utils import get_default_pyproject_backend from .utils import ( cleanup_toml, convert_toml_outline_tables, find_requirements, get_canonical_names, get_url_name, get_workon_home, is_editable, is_installable_file, is_star, is_valid_url, is_virtual_environment, looks_like_dir, normalize_drive, pep423_name, proper_case, python_version, - safe_expandvars + safe_expandvars, get_pipenv_dist ) @@ -98,6 +99,9 @@ if PIPENV_PIPFILE: else: PIPENV_PIPFILE = _normalized(PIPENV_PIPFILE) + # Overwrite environment variable so that subprocesses can get the correct path. + # See https://github.com/pypa/pipenv/issues/3584 + os.environ['PIPENV_PIPFILE'] = PIPENV_PIPFILE # (path, file contents) => TOMLFile # keeps track of pipfiles that we've seen so we do not need to re-parse 'em _pipfile_cache = {} @@ -340,7 +344,11 @@ class Project(object): prefix=prefix, is_venv=is_venv, sources=sources, pipfile=self.parsed_pipfile, project=self ) - self._environment.add_dist("pipenv") + pipenv_dist = get_pipenv_dist(pkg="pipenv") + if pipenv_dist: + self._environment.extend_dists(pipenv_dist) + else: + self._environment.add_dist("pipenv") return self._environment def get_outdated_packages(self): @@ -525,18 +533,19 @@ class Project(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8.0", "wheel"], + "build-backend": get_default_pyproject_backend(), } self._build_system = build_system @property def build_requires(self): - return self._build_system.get("requires", []) + return self._build_system.get("requires", ["setuptools>=40.8.0", "wheel"]) + @property def build_backend(self): - return self._build_system.get("build-backend", None) + return self._build_system.get("build-backend", get_default_pyproject_backend()) @property def settings(self): @@ -603,10 +612,8 @@ class Project(object): def _get_editable_packages(self, dev=False): section = "dev-packages" if dev else "packages" - # section = "{0}-editable".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_editable(k) or is_editable(v) } @@ -615,10 +622,8 @@ class Project(object): def _get_vcs_packages(self, dev=False): from pipenv.vendor.requirementslib.utils import is_vcs section = "dev-packages" if dev else "packages" - # section = "{0}-vcs".format(section) packages = { k: v - # for k, v in self._pipfile[section].items() for k, v in self.parsed_pipfile.get(section, {}).items() if is_vcs(v) or is_vcs(k) } @@ -681,7 +686,7 @@ class Project(object): ) name = self.name if self.name is not None else "Pipfile" - config_parser = ConfigOptionParser(name=self.name) + config_parser = ConfigOptionParser(name=name) config_parser.add_option_group(make_option_group(index_group, config_parser)) install = config_parser.option_groups[0] indexes = ( @@ -856,7 +861,8 @@ class Project(object): return self.pipfile_sources def find_source(self, source): - """given a source, find it. + """ + Given a source, find it. source can be a url or an index name. """ @@ -869,23 +875,34 @@ class Project(object): source = self.get_source(url=source) return source - def get_source(self, name=None, url=None): + def get_source(self, name=None, url=None, refresh=False): + from .utils import is_url_equal + def find_source(sources, name=None, url=None): source = None if name: - source = [s for s in sources if s.get("name") == name] + source = next(iter( + s for s in sources if "name" in s and s["name"] == name + ), None) elif url: - source = [s for s in sources if url.startswith(s.get("url"))] - if source: - return first(source) + source = next(iter( + s for s in sources + if "url" in s and is_url_equal(url, s.get("url", "")) + ), None) + if source is not None: + return source - found_source = find_source(self.sources, name=name, url=url) - if found_source: - return found_source - found_source = find_source(self.pipfile_sources, name=name, url=url) - if found_source: - return found_source - raise SourceNotFound(name or url) + sources = (self.sources, self.pipfile_sources) + if refresh: + self.clear_pipfile_cache() + sources = reversed(sources) + found = next( + iter(find_source(source, name=name, url=url) for source in sources), None + ) + target = next(iter(t for t in (name, url) if t is not None)) + if found is None: + raise SourceNotFound(target) + return found def get_package_name_in_pipfile(self, package_name, dev=False): """Get the equivalent package name in pipfile""" @@ -930,17 +947,17 @@ class Project(object): # Don't re-capitalize file URLs or VCSs. if not isinstance(package, Requirement): package = Requirement.from_line(package.strip()) - _, converted = package.pipfile_entry + req_name, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - name = self.get_package_name_in_pipfile(package.name, dev) + name = self.get_package_name_in_pipfile(req_name, dev) if name and is_star(converted): # Skip for wildcard version return # Add the package to the group. - p[key][name or pep423_name(package.name)] = converted + p[key][name or pep423_name(req_name)] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/resolver.py b/pipenv/resolver.py index ed474fb4..693c85ad 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -7,12 +7,48 @@ import sys os.environ["PIP_PYTHON_PATH"] = str(sys.executable) -def _patch_path(): +def find_site_path(pkg, site_dir=None): + import pkg_resources + if site_dir is not None: + site_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + working_set = pkg_resources.WorkingSet([site_dir] + sys.path[:]) + for dist in working_set: + root = dist.location + base_name = dist.project_name if dist.project_name else dist.key + name = None + if "top_level.txt" in dist.metadata_listdir(""): + name = next(iter([l.strip() for l in dist.get_metadata_lines("top_level.txt") if l is not None]), None) + if name is None: + name = pkg_resources.safe_name(base_name).replace("-", "_") + if not any(pkg == _ for _ in [base_name, name]): + continue + path_options = [name, "{0}.py".format(name)] + path_options = [os.path.join(root, p) for p in path_options if p is not None] + path = next(iter(p for p in path_options if os.path.exists(p)), None) + if path is not None: + return (dist, path) + return (None, None) + + +def _patch_path(pipenv_site=None): import site pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) pipenv_site_dir = os.path.dirname(pipenv_libdir) - site.addsitedir(pipenv_site_dir) - for _dir in ("vendor", "patched"): + pipenv_dist = None + if pipenv_site is not None: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site) + else: + pipenv_dist, pipenv_path = find_site_path("pipenv", site_dir=pipenv_site_dir) + if pipenv_dist is not None: + pipenv_dist.activate() + else: + site.addsitedir(next(iter( + sitedir for sitedir in (pipenv_site, pipenv_site_dir) + if sitedir is not None + ), None)) + if pipenv_path is not None: + pipenv_libdir = pipenv_path + for _dir in ("vendor", "patched", pipenv_libdir): sys.path.insert(0, os.path.join(pipenv_libdir, _dir)) @@ -24,8 +60,11 @@ def get_parser(): parser.add_argument("--verbose", "-v", action="count", default=False) parser.add_argument("--debug", action="store_true", default=False) parser.add_argument("--system", action="store_true", default=False) + parser.add_argument("--parse-only", action="store_true", default=False) + parser.add_argument("--pipenv-site", metavar="pipenv_site_dir", action="store", + default=os.environ.get("PIPENV_SITE_DIR")) parser.add_argument("--requirements-dir", metavar="requirements_dir", action="store", - default=os.environ.get("PIPENV_REQ_DIR")) + default=os.environ.get("PIPENV_REQ_DIR")) parser.add_argument("packages", nargs="*") return parser @@ -41,22 +80,53 @@ def handle_parsed_args(parsed): logging.getLogger("notpip").setLevel(logging.DEBUG) elif parsed.verbose > 0: logging.getLogger("notpip").setLevel(logging.INFO) + os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose) if "PIPENV_PACKAGES" in os.environ: parsed.packages += os.environ.get("PIPENV_PACKAGES", "").strip().split("\n") return parsed -def _main(pre, clear, verbose, system, requirements_dir, packages): - os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) - os.environ["PIP_PYTHON_PATH"] = str(sys.executable) +def parse_packages(packages, pre, clear, system, requirements_dir=None): + from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.vendor.vistir.contextmanagers import cd, temp_path + from pipenv.utils import parse_indexes + parsed_packages = [] + for package in packages: + indexes, trusted_hosts, line = parse_indexes(package) + line = " ".join(line) + pf = dict() + req = Requirement.from_line(line) + if not req.name: + with temp_path(), cd(req.req.setup_info.base_dir): + sys.path.insert(0, req.req.setup_info.base_dir) + req.req._setup_info.get_info() + req.update_name_from_path(req.req.setup_info.base_dir) + print(os.listdir(req.req.setup_info.base_dir)) + try: + name, entry = req.pipfile_entry + except Exception: + continue + else: + if name is not None and entry is not None: + pf[name] = entry + parsed_packages.append(pf) + print("RESULTS:") + if parsed_packages: + print(json.dumps(parsed_packages)) + else: + print(json.dumps([])) + +def resolve_packages(pre, clear, verbose, system, requirements_dir, packages): from pipenv.utils import create_mirror_source, resolve_deps, replace_pypi_sources - pypi_mirror_source = ( create_mirror_source(os.environ["PIPENV_PYPI_MIRROR"]) if "PIPENV_PYPI_MIRROR" in os.environ else None ) + # os.environ["PIP_NO_BUILD_ISOLATION"] = "1" + # os.environ["PIP_NO_USE_PEP517"] = "1" + # os.environ["PIP_NO_DEPS"] = "1" def resolve(packages, pre, project, sources, clear, system, requirements_dir=None): return resolve_deps( @@ -76,15 +146,8 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): if pypi_mirror_source else project.pipfile_sources ) - results = resolve( - packages, - pre=pre, - project=project, - sources=sources, - clear=clear, - system=system, - requirements_dir=requirements_dir, - ) + results = resolve(packages, pre=pre, project=project, sources=sources, clear=clear, + system=system, requirements_dir=requirements_dir) print("RESULTS:") if results: print(json.dumps(results)) @@ -92,36 +155,46 @@ def _main(pre, clear, verbose, system, requirements_dir, packages): print(json.dumps([])) -def main(): - _patch_path() - import warnings - from pipenv.vendor.vistir.compat import ResourceWarning - warnings.simplefilter("ignore", category=ResourceWarning) - import io - import six - if six.PY3: - import atexit - stdout_wrapper = io.TextIOWrapper(sys.stdout.buffer, encoding='utf8') - atexit.register(stdout_wrapper.close) - stderr_wrapper = io.TextIOWrapper(sys.stderr.buffer, encoding='utf8') - atexit.register(stderr_wrapper.close) - sys.stdout = stdout_wrapper - sys.stderr = stderr_wrapper +def _main(pre, clear, verbose, system, requirements_dir, packages, parse_only=False): + os.environ["PIP_PYTHON_VERSION"] = ".".join([str(s) for s in sys.version_info[:3]]) + os.environ["PIP_PYTHON_PATH"] = str(sys.executable) + if parse_only: + parse_packages( + packages, + pre=pre, + clear=clear, + system=system, + requirements_dir=requirements_dir, + ) else: - from pipenv._compat import force_encoding - force_encoding() - os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") - os.environ["PYTHONIOENCODING"] = str("utf-8") + resolve_packages(pre, clear, verbose, system, requirements_dir, packages) + + +def main(): parser = get_parser() parsed, remaining = parser.parse_known_args() - # sys.argv = remaining + _patch_path(pipenv_site=parsed.pipenv_site) + import warnings + from pipenv.vendor.vistir.compat import ResourceWarning + from pipenv.vendor.vistir.misc import get_wrapped_stream + warnings.simplefilter("ignore", category=ResourceWarning) + import six + if six.PY3: + stdout = sys.stdout.buffer + stderr = sys.stderr.buffer + else: + stdout = sys.stdout + stderr = sys.stderr + sys.stderr = get_wrapped_stream(stderr) + sys.stdout = get_wrapped_stream(stdout) + from pipenv.vendor import colorama + colorama.init() + os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = str("1") + os.environ["PYTHONIOENCODING"] = str("utf-8") parsed = handle_parsed_args(parsed) _main(parsed.pre, parsed.clear, parsed.verbose, parsed.system, - parsed.requirements_dir, parsed.packages) + parsed.requirements_dir, parsed.packages, parse_only=parsed.parse_only) if __name__ == "__main__": - _patch_path() - from pipenv.vendor import colorama - colorama.init() main() diff --git a/pipenv/utils.py b/pipenv/utils.py index 1fc4fd2d..3771f61f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -3,6 +3,7 @@ import contextlib import errno import logging import os +import posixpath import re import shutil import stat @@ -17,15 +18,13 @@ import toml import tomlkit from click import echo as click_echo -from first import first six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # noqa six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # noqa six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # noqa from six.moves import Mapping, Sequence, Set from six.moves.urllib.parse import urlparse -from urllib3 import util as urllib3_util -from vistir.compat import ResourceWarning -from vistir.misc import fs_str +from .vendor.vistir.compat import ResourceWarning, lru_cache +from .vendor.vistir.misc import fs_str import crayons import parse @@ -33,6 +32,13 @@ import parse from . import environments from .exceptions import PipenvUsageError from .pep508checker import lookup +from .vendor.urllib3 import util as urllib3_util + + +if environments.MYPY_RUNNING: + from typing import Tuple, Dict, Any, List, Union, Optional, Text + from .vendor.requirementslib.models.requirements import Requirement, Line + from .project import Project logging.basicConfig(level=logging.ERROR) @@ -41,7 +47,7 @@ specifiers = [k for k in lookup.keys()] # List of version control systems we support. VCS_LIST = ("git", "svn", "hg", "bzr") SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") -requests_session = None +requests_session = None # type: ignore def _get_requests_session(): @@ -228,30 +234,16 @@ def prepare_pip_source_args(sources, pip_args=None): return pip_args -def get_resolver_metadata(deps, index_lookup, markers_lookup, project, sources): - from .vendor.requirementslib.models.requirements import Requirement - constraints = [] - for dep in deps: - if not dep: - continue - url = None - indexes, trusted_hosts, remainder = parse_indexes(dep) - if indexes: - url = indexes[0] - dep = " ".join(remainder) - req = Requirement.from_line(dep) - constraints.append(req.constraint_line) - if url: - source = first( - s for s in sources if s.get("url") and url.startswith(s["url"])) - if source: - index_lookup[req.name] = source.get("name") - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.name] = req.markers.replace('"', "'") - return constraints +@lru_cache() +def get_pipenv_sitedir(): + # type: () -> Optional[str] + import pkg_resources + site_dir = next( + iter(d for d in pkg_resources.working_set if d.key.lower() == "pipenv"), None + ) + if site_dir is not None: + return site_dir.location + return None class Resolver(object): @@ -286,8 +278,10 @@ class Resolver(object): "sources={self.sources})>".format(self=self) ) - def _get_pip_command(self): - from pip_shims.shims import Command + @staticmethod + @lru_cache() + def _get_pip_command(): + from .vendor.pip_shims.shims import Command class PipCommand(Command): """Needed for pip-tools.""" @@ -297,6 +291,140 @@ class Resolver(object): from pipenv.patched.piptools.scripts.compile import get_pip_command return get_pip_command() + @classmethod + def get_metadata( + cls, + deps, # type: List[str] + index_lookup, # type: Dict[str, str] + markers_lookup, # type: Dict[str, str] + project, # type: Project + sources # type: Dict[str, str] + ): + # type: (...) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]], Dict[str, str], Dict[str, str]] + constraints = set() # type: Set[str] + skipped = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + for dep in deps: + if not dep: + continue + req, req_idx, markers_idx = cls.parse_line( + dep, index_lookup=index_lookup, markers_lookup=markers_lookup, project=project + ) + index_lookup.update(req_idx) + markers_lookup.update(markers_idx) + constraint_update, lockfile_update = cls.get_deps_from_req(req) + constraints |= constraint_update + skipped.update(lockfile_update) + return constraints, skipped, index_lookup, markers_lookup + + @classmethod + def parse_line( + cls, + line, # type: str + index_lookup=None, # type: Dict[str, str] + markers_lookup=None, # type: Dict[str, str] + project=None # type: Optional[Project] + ): + # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]] + from .vendor.requirementslib.models.requirements import Requirement + from .exceptions import ResolutionFailure + if index_lookup is None: + index_lookup = {} + if markers_lookup is None: + markers_lookup = {} + if project is None: + from .project import Project + project = Project() + url = None + indexes, trusted_hosts, remainder = parse_indexes(line) + if indexes: + url = indexes[0] + line = " ".join(remainder) + try: + req = Requirement.from_line(line) + except ValueError: + raise ResolutionFailure("Failed to resolve requirement from line: {0!s}".format(line)) + if url: + index_lookup[req.normalized_name] = project.get_source( + url=url, refresh=True).get("name") + # strip the marker and re-add it later after resolution + # but we will need a fallback in case resolution fails + # eg pypiwin32 + if req.markers: + markers_lookup[req.normalized_name] = req.markers.replace('"', "'") + return req, index_lookup, markers_lookup + + @classmethod + def get_deps_from_line(cls, line): + # type: (str) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + req, _, _ = cls.parse_line(line) + return cls.get_deps_from_req(req) + + @classmethod + def get_deps_from_req(cls, req): + # type: (Requirement) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]] + from requirementslib.models.utils import _requirement_to_str_lowercase_name + constraints = set() # type: Set[str] + locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]] + if req.is_file_or_url or req.is_vcs and not req.is_wheel: + # for local packages with setup.py files and potential direct url deps: + if req.is_vcs: + req_list, lockfile = get_vcs_deps(reqs=[req]) + req = next(iter(req for req in req_list if req is not None), req_list) + entry = lockfile[pep423_name(req.normalized_name)] + else: + _, entry = req.pipfile_entry + parsed_line = req.req.parsed_line # type: Line + setup_info = None # type: Any + name = req.normalized_name + setup_info = req.req.setup_info + locked_deps[pep423_name(name)] = entry + requirements = [v for v in getattr(setup_info, "requires", {}).values()] + for r in requirements: + if getattr(r, "url", None) and not getattr(r, "editable", False): + if r is not None: + if not r.url: + continue + line = _requirement_to_str_lowercase_name(r) + new_req, _, _ = cls.parse_line(line) + if r.marker and not r.marker.evaluate(): + new_constraints = {} + _, new_entry = req.pipfile_entry + new_lock = { + pep_423_name(new_req.normalized_name): new_entry + } + else: + new_constraints, new_lock = cls.get_deps_from_req(new_req) + locked_deps.update(new_lock) + constraints |= new_constraints + else: + if r is not None: + line = _requirement_to_str_lowercase_name(r) + constraints.add(line) + # ensure the top level entry remains as provided + # note that we shouldn't pin versions for editable vcs deps + if (not req.is_vcs or (req.is_vcs and not req.editable)): + if req.specifiers: + locked_deps[name]["version"] = req.specifiers + elif parsed_line.setup_info and parsed_line.setup_info.version: + locked_deps[name]["version"] = "=={}".format( + parsed_line.setup_info.version + ) + # if not req.is_vcs: + locked_deps.update({name: entry}) + if req.is_vcs and req.editable: + constraints.add(req.constraint_line) + if req.is_file_or_url and req.req.is_local and req.editable and ( + req.req.setup_path is not None and os.path.exists(req.req.setup_path)): + constraints.add(req.constraint_line) + else: + constraints.add(req.constraint_line) + return constraints, locked_deps + return constraints, locked_deps + @property def pip_command(self): if self._pip_command is None: @@ -418,6 +546,59 @@ class Resolver(object): self.resolved_tree.update(results) return self.resolved_tree + @classmethod + def prepend_hash_types(cls, checksums): + cleaned_checksums = [] + for checksum in checksums: + if not checksum: + continue + if not checksum.startswith("sha256:"): + checksum = "sha256:{0}".format(checksum) + cleaned_checksums.append(checksum) + return cleaned_checksums + + def collect_hashes(self, ireq): + from .vendor.requests import ConnectionError + collected_hashes = [] + if ireq in self.hashes: + collected_hashes += list(self.hashes.get(ireq, [])) + if self._should_include_hash(ireq): + try: + hash_map = self.get_hash(ireq) + collected_hashes += list(hash_map) + except (ValueError, KeyError, IndexError, ConnectionError): + pass + elif any( + "python.org" in source["url"] or "pypi.org" in source["url"] + for source in self.sources + ): + pkg_url = "https://pypi.org/pypi/{0}/json".format(ireq.name) + session = _get_requests_session() + try: + # Grab the hashes from the new warehouse API. + r = session.get(pkg_url, timeout=10) + api_releases = r.json()["releases"] + cleaned_releases = {} + for api_version, api_info in api_releases.items(): + api_version = clean_pkg_version(api_version) + cleaned_releases[api_version] = api_info + version = "" + if ireq.specifier: + spec = next(iter(s for s in list(ireq.specifier._specs)), None) + if spec: + version = spec.version + for release in cleaned_releases[version]: + collected_hashes.append(release["digests"]["sha256"]) + collected_hashes = self.prepend_hash_types(collected_hashes) + except (ValueError, KeyError, ConnectionError): + if environments.is_verbose(): + click_echo( + "{0}: Error generating hash for {1}".format( + crayons.red("Warning", bold=True), ireq.name + ), err=True + ) + return collected_hashes + @staticmethod def _should_include_hash(ireq): from pipenv.vendor.vistir.compat import Path, to_native_string @@ -442,31 +623,43 @@ class Resolver(object): return False return True + def get_hash(self, ireq, ireq_hashes=None): + """ + Retrieve hashes for a specific ``InstallRequirement`` instance. + + :param ireq: An ``InstallRequirement`` to retrieve hashes for + :type ireq: :class:`~pip_shims.InstallRequirement` + :return: A set of hashes. + :rtype: Set + """ + + # We _ALWAYS MUST PRIORITIZE_ the inclusion of hashes from local sources + # PLEASE *DO NOT MODIFY THIS* TO CHECK WHETHER AN IREQ ALREADY HAS A HASH + # RESOLVED. The resolver will pull hashes from PyPI and only from PyPI. + # The entire purpose of this approach is to include missing hashes. + # This fixes a race condition in resolution for missing dependency caches + # see pypa/pipenv#3289 + if self._should_include_hash(ireq) and ( + not ireq_hashes or ireq.link.scheme == "file" + ): + if not ireq_hashes: + ireq_hashes = set() + new_hashes = self.resolver.repository._hash_cache.get_hash(ireq.link) + ireq_hashes = add_to_set(ireq_hashes, new_hashes) + else: + ireq_hashes = set(ireq_hashes) + # The _ONLY CASE_ where we flat out set the value is if it isn't present + # It's a set, so otherwise we *always* need to do a union update + if ireq not in self.hashes: + return ireq_hashes + else: + return self.hashes[ireq] | ireq_hashes + def resolve_hashes(self): if self.results is not None: resolved_hashes = self.resolver.resolve_hashes(self.results) for ireq, ireq_hashes in resolved_hashes.items(): - # We _ALWAYS MUST PRIORITIZE_ the inclusion of hashes from local sources - # PLEASE *DO NOT MODIFY THIS* TO CHECK WHETHER AN IREQ ALREADY HAS A HASH - # RESOLVED. The resolver will pull hashes from PyPI and only from PyPI. - # The entire purpose of this approach is to include missing hashes. - # This fixes a race condition in resolution for missing dependency caches - # see pypa/pipenv#3289 - if self._should_include_hash(ireq) and ( - not ireq_hashes or ireq.link.scheme == "file" - ): - if not ireq_hashes: - ireq_hashes = set() - new_hashes = self.resolver.repository._hash_cache.get_hash(ireq.link) - add_to_set(ireq_hashes, new_hashes) - else: - ireq_hashes = set(ireq_hashes) - # The _ONLY CASE_ where we flat out set the value is if it isn't present - # It's a set, so otherwise we *always* need to do a union update - if ireq not in self.hashes: - self.hashes[ireq] = ireq_hashes - else: - self.hashes[ireq] |= ireq_hashes + self.hashes[ireq] = self.get_hash(ireq, ireq_hashes=ireq_hashes) return self.hashes @@ -487,42 +680,106 @@ def actually_resolve_deps( req_dir=None, ): from pipenv.vendor.vistir.path import create_tracked_tempdir + from pipenv.vendor.requirementslib.models.requirements import Requirement if not req_dir: req_dir = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") warning_list = [] with warnings.catch_warnings(record=True) as warning_list: - constraints = get_resolver_metadata( + constraints, skipped, index_lookup, markers_lookup = Resolver.get_metadata( deps, index_lookup, markers_lookup, project, sources, ) resolver = Resolver(constraints, req_dir, project, sources, clear=clear, pre=pre) resolved_tree = resolver.resolve() hashes = resolver.resolve_hashes() - + reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in resolved_tree] + results = {} + for req, ireq in reqs: + if (req.vcs and req.editable and not req.is_direct_url): + continue + collected_hashes = resolver.collect_hashes(ireq) + if collected_hashes: + req = req.add_hashes(collected_hashes) + elif resolver._should_include_hash(ireq): + existing_hashes = hashes.get(ireq, set()) + discovered_hashes = existing_hashes | resolver.get_hash(ireq) + if discovered_hashes: + req = req.add_hashes(discovered_hashes) + resolver.hashes[ireq] = discovered_hashes + if req.specifiers: + version = str(req.get_version()) + else: + version = None + index = index_lookup.get(req.normalized_name) + markers = markers_lookup.get(req.normalized_name) + req.index = index + name, pf_entry = req.pipfile_entry + name = pep423_name(req.name) + entry = {} + if isinstance(pf_entry, six.string_types): + entry["version"] = pf_entry.lstrip("=") + else: + entry.update(pf_entry) + if version is not None: + entry["version"] = version + if req.line_instance.is_direct_url: + entry["file"] = req.req.uri + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + entry["name"] = name + if index: # and index != next(iter(project.sources), {}).get("name"): + entry.update({"index": index}) + if markers: + entry.update({"markers": markers}) + entry = translate_markers(entry) + if name in results: + results[name].update(entry) + else: + results[name] = entry + for k in list(skipped.keys()): + req = Requirement.from_pipfile(k, skipped[k]) + ref = None + if req.is_vcs: + ref = req.commit_hash + ireq = req.as_ireq() + entry = skipped[k].copy() + entry["name"] = req.name + ref = ref if ref is not None else entry.get("ref") + if ref: + entry["ref"] = ref + if resolver._should_include_hash(ireq): + collected_hashes = resolver.collect_hashes(ireq) + if collected_hashes: + entry["hashes"] = sorted(set(collected_hashes)) + if k in results: + results[k].update(entry) + else: + results[k] = entry + results = list(results.values()) for warning in warning_list: _show_warning(warning.message, warning.category, warning.filename, warning.lineno, warning.line) - return (resolved_tree, hashes, markers_lookup, resolver) + return (results, hashes, markers_lookup, resolver, skipped) @contextlib.contextmanager def create_spinner(text, nospin=None, spinner_name=None): - import vistir.spin + from .vendor.vistir import spin + from .vendor.vistir.misc import fs_str if not spinner_name: spinner_name = environments.PIPENV_SPINNER if nospin is None: nospin = environments.PIPENV_NOSPIN - with vistir.spin.create_spinner( + with spin.create_spinner( spinner_name=spinner_name, - start_text=vistir.compat.fs_str(text), + start_text=fs_str(text), nospin=nospin, write_to_stdout=False ) as sp: yield sp - def resolve(cmd, sp): - from .vendor import delegator + import delegator from .cmdparse import Script from .vendor.pexpect.exceptions import EOF, TIMEOUT from .vendor.vistir.compat import to_native_string @@ -560,7 +817,7 @@ def resolve(cmd, sp): return c -def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): +def get_locked_dep(dep, pipfile_section, prefer_pipfile=True): # the prefer pipfile flag is not used yet, but we are introducing # it now for development purposes # TODO: Is this implementation clear? How can it be improved? @@ -570,8 +827,11 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): "pipfile_entry": None } if isinstance(dep, Mapping) and dep.get("name", ""): - name_options = [dep["name"], pep423_name(dep["name"])] - name = next(iter(k for k in name_options if k in pipfile_section), None) + dep_name = pep423_name(dep["name"]) + name = next(iter( + k for k in pipfile_section.keys() + if pep423_name(k) == dep_name + ), None) entry = pipfile_section[name] if name else None if entry: @@ -581,24 +841,33 @@ def get_locked_dep(dep, pipfile_section, prefer_pipfile=False): version = entry.get("version", "") if entry else "" else: version = entry if entry else "" - lockfile_version = lockfile_entry.get("version", "") + lockfile_name, lockfile_dict = lockfile_entry.copy().popitem() + lockfile_version = lockfile_dict.get("version", "") # Keep pins from the lockfile if prefer_pipfile and lockfile_version != version and version.startswith("=="): - lockfile_version = version + lockfile_dict["version"] = version + lockfile_entry[lockfile_name] = lockfile_dict return lockfile_entry def prepare_lockfile(results, pipfile, lockfile): - from .vendor.requirementslib.utils import is_vcs + # from .vendor.requirementslib.utils import is_vcs for dep in results: + if not dep: + continue # Merge in any relevant information from the pipfile entry, including # markers, normalized names, URL info, etc that we may have dropped during lock - if not is_vcs(dep): - lockfile_entry = get_locked_dep(dep, pipfile) - name = next(iter(k for k in lockfile_entry.keys())) - current_entry = lockfile.get(name) - if not current_entry or not is_vcs(current_entry): - lockfile.update(lockfile_entry) + # if not is_vcs(dep): + lockfile_entry = get_locked_dep(dep, pipfile) + name = next(iter(k for k in lockfile_entry.keys())) + current_entry = lockfile.get(name) + if current_entry: + if not isinstance(current_entry, Mapping): + lockfile[name] = lockfile_entry[name] + else: + lockfile[name].update(lockfile_entry[name]) + else: + lockfile[name] = lockfile_entry[name] return lockfile @@ -614,39 +883,52 @@ def venv_resolve_deps( pipfile=None, lockfile=None ): + """ + Resolve dependencies for a pipenv project, acts as a portal to the target environment. + + Regardless of whether a virtual environment is present or not, this will spawn + a subproces which is isolated to the target environment and which will perform + dependency resolution. This function reads the output of that call and mutates + the provided lockfile accordingly, returning nothing. + + :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve. + :param Callable which: [description] + :param project: The pipenv Project instance to use during resolution + :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False + :param Optional[bool] clear: Whether to clear the cache during resolution, defaults to False + :param Optional[bool] allow_global: Whether to use *sys.executable* as the python binary, defaults to False + :param Optional[str] pypi_mirror: A URL to substitute any time *pypi.org* is encountered, defaults to None + :param Optional[bool] dev: Whether to target *dev-packages* or not, defaults to False + :param pipfile: A Pipfile section to operate on, defaults to None + :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]] + :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None + :raises RuntimeError: Raised on resolution failure + :return: Nothing + :rtype: None + """ + from .vendor.vistir.misc import fs_str - from .vendor.vistir.compat import Path, to_native_string, JSONDecodeError + from .vendor.vistir.compat import Path, JSONDecodeError from .vendor.vistir.path import create_tracked_tempdir from . import resolver + from ._compat import decode_for_output import json - vcs_deps = [] - vcs_lockfile = {} results = [] - pipfile_section = "dev_packages" if dev else "packages" + pipfile_section = "dev-packages" if dev else "packages" lockfile_section = "develop" if dev else "default" - vcs_section = "vcs_{0}".format(pipfile_section) - vcs_deps = getattr(project, vcs_section, {}) - if not deps and not vcs_deps: - return {} + if not deps: + if not project.pipfile_exists: + return None + deps = project.parsed_pipfile.get(pipfile_section, {}) + if not deps: + return None if not pipfile: - pipfile = getattr(project, pipfile_section, None) + pipfile = getattr(project, pipfile_section, {}) if not lockfile: lockfile = project._lockfile req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - if vcs_deps: - with create_spinner(text=fs_str("Pinning VCS Packages...")) as sp: - vcs_reqs, vcs_lockfile = get_vcs_deps( - project, - which=which, - clear=clear, - pre=pre, - allow_global=allow_global, - dev=dev, - ) - vcs_deps = [req.as_line() for req in vcs_reqs if req.editable] - lockfile[lockfile_section].update(vcs_lockfile) cmd = [ which("python", allow_global=allow_global), Path(resolver.__file__.rstrip("co")).as_posix() @@ -657,49 +939,43 @@ def venv_resolve_deps( cmd.append("--clear") if allow_global: cmd.append("--system") + if dev: + cmd.append("--dev") with temp_environ(): - os.environ = {fs_str(k): fs_str(val) for k, val in os.environ.items()} - os.environ["PIPENV_PACKAGES"] = str("\n".join(deps)) + os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) if pypi_mirror: os.environ["PIPENV_PYPI_MIRROR"] = str(pypi_mirror) os.environ["PIPENV_VERBOSITY"] = str(environments.PIPENV_VERBOSITY) os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") - with create_spinner(text=fs_str("Locking...")) as sp: + os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + with create_spinner(text=decode_for_output("Locking...")) as sp: + # This conversion is somewhat slow on local and file-type requirements since + # we now download those requirements / make temporary folders to perform + # dependency resolution on them, so we are including this step inside the + # spinner context manager for the UX improvement + sp.write(decode_for_output("Building requirements...")) + deps = convert_deps_to_pip( + deps, project, r=False, include_index=True + ) + constraints = set(deps) + os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) + sp.write(decode_for_output("Resolving dependencies...")) c = resolve(cmd, sp) - results = c.out - if vcs_deps: - with temp_environ(): - os.environ["PIPENV_PACKAGES"] = str("\n".join(vcs_deps)) - sp.text = to_native_string("Locking VCS Dependencies...") - vcs_c = resolve(cmd, sp) - vcs_results, vcs_err = vcs_c.out, vcs_c.err - else: - vcs_results, vcs_err = "", "" + results = c.out.strip() sp.green.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Success!")) - outputs = [results, vcs_results] if environments.is_verbose(): - for output in outputs: - click_echo(output.split("RESULTS:")[0], err=True) + click_echo(results.split("RESULTS:")[1], err=True) try: results = json.loads(results.split("RESULTS:")[1].strip()) - if vcs_results: - # For vcs dependencies, treat the initial pass at locking (i.e. checkout) - # as the pipfile entry because it gets us an actual ref to use - vcs_results = json.loads(vcs_results.split("RESULTS:")[1].strip()) - vcs_lockfile = prepare_lockfile(vcs_results, vcs_lockfile.copy(), vcs_lockfile) - else: - vcs_results = [] except (IndexError, JSONDecodeError): - for out, err in [(c.out, c.err), (vcs_results, vcs_err)]: - click_echo(out.strip(), err=True) - click_echo(err.strip(), err=True) + click_echo(c.out.strip(), err=True) + click_echo(c.err.strip(), err=True) raise RuntimeError("There was a problem with locking.") - lockfile[lockfile_section] = prepare_lockfile(results, pipfile, lockfile[lockfile_section]) - for k, v in vcs_lockfile.items(): - if k in getattr(project, vcs_section, {}) or k not in lockfile[lockfile_section]: - lockfile[lockfile_section][k].update(v) + if lockfile_section not in lockfile: + lockfile[lockfile_section] = {} + prepare_lockfile(results, pipfile, lockfile[lockfile_section]) def resolve_deps( @@ -716,9 +992,6 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - from .vendor.requests.exceptions import ConnectionError - from .vendor.requirementslib.models.requirements import Requirement - index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) @@ -735,7 +1008,7 @@ def resolve_deps( req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_version=python, python_path=python_path): try: - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -757,7 +1030,7 @@ def resolve_deps( try: # Attempt to resolve again, with different Python version information, # particularly for particularly particular packages. - resolved_tree, hashes, markers_lookup, resolver = actually_resolve_deps( + resolved_tree, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -769,64 +1042,7 @@ def resolve_deps( ) except RuntimeError: sys.exit(1) - for result in resolved_tree: - if not result.editable: - req = Requirement.from_ireq(result) - name = pep423_name(req.name) - version = str(req.get_version()) - index = index_lookup.get(result.name) - req.index = index - collected_hashes = [] - if result in hashes: - collected_hashes = list(hashes.get(result)) - elif any( - "python.org" in source["url"] or "pypi.org" in source["url"] - for source in sources - ): - pkg_url = "https://pypi.org/pypi/{0}/json".format(name) - session = _get_requests_session() - try: - # Grab the hashes from the new warehouse API. - r = session.get(pkg_url, timeout=10) - api_releases = r.json()["releases"] - cleaned_releases = {} - for api_version, api_info in api_releases.items(): - api_version = clean_pkg_version(api_version) - cleaned_releases[api_version] = api_info - for release in cleaned_releases[version]: - collected_hashes.append(release["digests"]["sha256"]) - collected_hashes = ["sha256:" + s for s in collected_hashes] - except (ValueError, KeyError, ConnectionError): - if environments.is_verbose(): - click_echo( - "{0}: Error generating hash for {1}".format( - crayons.red("Warning", bold=True), name - ), err=True - ) - # # Collect un-collectable hashes (should work with devpi). - # try: - # collected_hashes = collected_hashes + list( - # list(resolver.resolve_hashes([result]).items())[0][1] - # ) - # except (ValueError, KeyError, ConnectionError, IndexError): - # if verbose: - # print('Error generating hash for {}'.format(name)) - req.hashes = sorted(set(collected_hashes)) - name, _entry = req.pipfile_entry - entry = {} - if isinstance(_entry, six.string_types): - entry["version"] = _entry.lstrip("=") - else: - entry.update(_entry) - entry["version"] = version - entry["name"] = name - # if index: - # d.update({"index": index}) - if markers_lookup.get(result.name): - entry.update({"markers": markers_lookup.get(result.name)}) - entry = translate_markers(entry) - results.append(entry) - return results + return resolved_tree def is_star(val): @@ -845,7 +1061,9 @@ def convert_deps_to_pip(deps, project=None, r=True, include_index=True): dependencies = [] for dep_name, dep in deps.items(): - indexes = project.pipfile_sources if hasattr(project, "pipfile_sources") else [] + if project: + project.clear_pipfile_cache() + indexes = getattr(project, "pipfile_sources", []) if project is not None else [] new_dep = Requirement.from_pipfile(dep_name, dep) if new_dep.index: include_index = True @@ -1287,7 +1505,7 @@ def handle_remove_readonly(func, path, exc): warnings.warn(default_warning_message.format(path), ResourceWarning) return - raise + raise exc def escape_cmd(cmd): @@ -1305,37 +1523,53 @@ def safe_expandvars(value): def get_vcs_deps( - project, - which=None, - clear=False, - pre=False, - allow_global=False, + project=None, dev=False, pypi_mirror=None, + packages=None, + reqs=None ): from .vendor.requirementslib.models.requirements import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" - reqs = [] + if reqs is None: + reqs = [] lockfile = {} - try: - packages = getattr(project, section) - except AttributeError: - return [], [] - for pkg_name, pkg_pipfile in packages.items(): - requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) + if not reqs: + if not project and not packages: + raise ValueError( + "Must supply either a project or a pipfile section to lock vcs dependencies." + ) + if not packages: + try: + packages = getattr(project, section) + except AttributeError: + return [], [] + reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()] + result = [] + for requirement in reqs: name = requirement.normalized_name commit_hash = None if requirement.is_vcs: try: - with locked_repository(requirement) as repo: + with temp_path(), locked_repository(requirement) as repo: + from pipenv.vendor.requirementslib.models.requirements import Requirement + # from distutils.sysconfig import get_python_lib + # sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] commit_hash = repo.get_commit_hash() + name = requirement.normalized_name + version = requirement._specifiers = "=={0}".format(requirement.req.setup_info.version) lockfile[name] = requirement.pipfile_entry[1] lockfile[name]['ref'] = commit_hash - reqs.append(requirement) + result.append(requirement) + version = requirement.specifiers + if not version and requirement.specifiers: + version = requirement.specifiers + if version: + lockfile[name]['version'] = version except OSError: continue - return reqs, lockfile + return result, lockfile def translate_markers(pipfile_entry): @@ -1378,25 +1612,38 @@ def translate_markers(pipfile_entry): def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): + from .vendor.requirementslib.utils import is_vcs name = pep423_name(dep["name"]) + lockfile = {} # We use this to determine if there are any markers on top level packages # So we can make sure those win out during resolution if the packages reoccur - lockfile = {"version": "=={0}".format(dep["version"])} - for key in ["hashes", "index", "extras"]: + if "version" in dep: + version = "{0}".format(dep["version"]) + if not version.startswith("=="): + version = "=={0}".format(version) + lockfile["version"] = version + if is_vcs(dep): + ref = dep.get("ref", None) + if ref is not None: + lockfile["ref"] = ref + vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None) + if vcs_type: + lockfile[vcs_type] = dep[vcs_type] + if "subdirectory" in dep: + lockfile["subdirectory"] = dep["subdirectory"] + for key in ["hashes", "index", "extras", "editable"]: if key in dep: lockfile[key] = dep[key] # In case we lock a uri or a file when the user supplied a path # remove the uri or file keys from the entry and keep the path - if pipfile_entry and any(k in pipfile_entry for k in ["file", "path"]): - fs_key = next((k for k in ["path", "file"] if k in pipfile_entry), None) - lockfile_key = next((k for k in ["uri", "file", "path"] if k in lockfile), None) - if fs_key != lockfile_key: - try: - del lockfile[lockfile_key] - except KeyError: - # pass when there is no lock file, usually because it's the first time - pass - lockfile[fs_key] = pipfile_entry[fs_key] + fs_key = next(iter(k for k in ["path", "file"] if k in dep), None) + pipfile_fs_key = None + if pipfile_entry: + pipfile_fs_key = next(iter(k for k in ["path", "file"] if k in pipfile_entry), None) + if fs_key and pipfile_fs_key and fs_key != pipfile_fs_key: + lockfile[pipfile_fs_key] = pipfile_entry[pipfile_fs_key] + elif fs_key is not None: + lockfile[fs_key] = dep[fs_key] # If a package is **PRESENT** in the pipfile but has no markers, make sure we # **NEVER** include markers in the lockfile @@ -1536,3 +1783,70 @@ def add_to_set(original_set, element): original_set |= set(element) else: original_set.add(element) + return original_set + + +def is_url_equal(url, other_url): + # type: (str, str) -> bool + """ + Compare two urls by scheme, host, and path, ignoring auth + + :param str url: The initial URL to compare + :param str url: Second url to compare to the first + :return: Whether the URLs are equal without **auth**, **query**, and **fragment** + :rtype: bool + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://user2:pass2@mydomain.com/some/path") + True + + >>> is_url_equal("https://user:pass@mydomain.com/some/path?some_query", + "https://mydomain.com/some?some_query") + False + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(url)) + if not isinstance(other_url, six.string_types): + raise TypeError("Expected string for url, received {0!r}".format(other_url)) + parsed_url = urllib3_util.parse_url(url) + parsed_other_url = urllib3_util.parse_url(other_url) + unparsed = parsed_url._replace(auth=None, query=None, fragment=None).url + unparsed_other = parsed_other_url._replace(auth=None, query=None, fragment=None).url + return unparsed == unparsed_other + + +@lru_cache() +def make_posix(path): + # type: (str) -> str + """ + Convert a path with possible windows-style separators to a posix-style path + (with **/** separators instead of **\\** separators). + + :param Text path: A path to convert. + :return: A converted posix-style path + :rtype: Text + + >>> make_posix("c:/users/user/venvs/some_venv\\Lib\\site-packages") + "c:/users/user/venvs/some_venv/Lib/site-packages" + + >>> make_posix("c:\\users\\user\\venvs\\some_venv") + "c:/users/user/venvs/some_venv" + """ + if not isinstance(path, six.string_types): + raise TypeError("Expected a string for path, received {0!r}...".format(path)) + starts_with_sep = path.startswith(os.path.sep) + separated = normalize_path(path).split(os.path.sep) + if isinstance(separated, (list, tuple)): + path = posixpath.join(*separated) + if starts_with_sep: + path = "/{0}".format(path) + return path + + +def get_pipenv_dist(pkg="pipenv", pipenv_site=None): + from .resolver import find_site_path + pipenv_libdir = os.path.dirname(os.path.abspath(__file__)) + if pipenv_site is None: + pipenv_site = os.path.dirname(pipenv_libdir) + pipenv_dist, _ = find_site_path(pkg, site_dir=pipenv_site) + return pipenv_dist diff --git a/pipenv/vendor/attr/__init__.py b/pipenv/vendor/attr/__init__.py index debfd57b..0ebe5197 100644 --- a/pipenv/vendor/attr/__init__.py +++ b/pipenv/vendor/attr/__init__.py @@ -18,7 +18,7 @@ from ._make import ( ) -__version__ = "18.2.0" +__version__ = "19.1.0" __title__ = "attrs" __description__ = "Classes Without Boilerplate" diff --git a/pipenv/vendor/attr/__init__.pyi b/pipenv/vendor/attr/__init__.pyi index 492fb85e..fcb93b18 100644 --- a/pipenv/vendor/attr/__init__.pyi +++ b/pipenv/vendor/attr/__init__.pyi @@ -23,9 +23,9 @@ from . import validators as validators _T = TypeVar("_T") _C = TypeVar("_C", bound=type) -_ValidatorType = Callable[[Any, Attribute, _T], Any] +_ValidatorType = Callable[[Any, Attribute[_T], _T], Any] _ConverterType = Callable[[Any], _T] -_FilterType = Callable[[Attribute, Any], bool] +_FilterType = Callable[[Attribute[_T], _T], bool] # FIXME: in reality, if multiple validators are passed they must be in a list or tuple, # but those are invariant and so would prevent subtypes of _ValidatorType from working # when passed in a list or tuple. @@ -57,10 +57,10 @@ class Attribute(Generic[_T]): metadata: Dict[Any, Any] type: Optional[Type[_T]] kw_only: bool - def __lt__(self, x: Attribute) -> bool: ... - def __le__(self, x: Attribute) -> bool: ... - def __gt__(self, x: Attribute) -> bool: ... - def __ge__(self, x: Attribute) -> bool: ... + def __lt__(self, x: Attribute[_T]) -> bool: ... + def __le__(self, x: Attribute[_T]) -> bool: ... + def __gt__(self, x: Attribute[_T]) -> bool: ... + def __ge__(self, x: Attribute[_T]) -> bool: ... # NOTE: We had several choices for the annotation to use for type arg: # 1) Type[_T] @@ -167,6 +167,7 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> _C: ... @overload def attrs( @@ -184,14 +185,15 @@ def attrs( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> Callable[[_C], _C]: ... # TODO: add support for returning NamedTuple from the mypy plugin -class _Fields(Tuple[Attribute, ...]): - def __getattr__(self, name: str) -> Attribute: ... +class _Fields(Tuple[Attribute[Any], ...]): + def __getattr__(self, name: str) -> Attribute[Any]: ... def fields(cls: type) -> _Fields: ... -def fields_dict(cls: type) -> Dict[str, Attribute]: ... +def fields_dict(cls: type) -> Dict[str, Attribute[Any]]: ... def validate(inst: Any) -> None: ... # TODO: add support for returning a proper attrs class from the mypy plugin @@ -212,6 +214,7 @@ def make_class( auto_attribs: bool = ..., kw_only: bool = ..., cache_hash: bool = ..., + auto_exc: bool = ..., ) -> type: ... # _funcs -- @@ -223,7 +226,7 @@ def make_class( def asdict( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., + filter: Optional[_FilterType[Any]] = ..., dict_factory: Type[Mapping[Any, Any]] = ..., retain_collection_types: bool = ..., ) -> Dict[str, Any]: ... @@ -232,8 +235,8 @@ def asdict( def astuple( inst: Any, recurse: bool = ..., - filter: Optional[_FilterType] = ..., - tuple_factory: Type[Sequence] = ..., + filter: Optional[_FilterType[Any]] = ..., + tuple_factory: Type[Sequence[Any]] = ..., retain_collection_types: bool = ..., ) -> Tuple[Any, ...]: ... def has(cls: type) -> bool: ... diff --git a/pipenv/vendor/attr/_compat.py b/pipenv/vendor/attr/_compat.py index 5bb06593..9a99dcd9 100644 --- a/pipenv/vendor/attr/_compat.py +++ b/pipenv/vendor/attr/_compat.py @@ -20,6 +20,7 @@ else: if PY2: from UserDict import IterableUserDict + from collections import Mapping, Sequence # noqa # We 'bundle' isclass instead of using inspect as importing inspect is # fairly expensive (order of 10-15 ms for a modern machine in 2016) @@ -89,8 +90,27 @@ if PY2: res.data.update(d) # We blocked update, so we have to do it like this. return res + def just_warn(*args, **kw): # pragma: nocover + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ -else: + +else: # Python 3 and later. + from collections.abc import Mapping, Sequence # noqa + + def just_warn(*args, **kw): + """ + We only warn on Python 3 because we are not aware of any concrete + consequences of not setting the cell on Python 2. + """ + warnings.warn( + "Missing ctypes. Some features like bare super() or accessing " + "__class__ will not work with slotted classes.", + RuntimeWarning, + stacklevel=2, + ) def isclass(klass): return isinstance(klass, type) @@ -113,30 +133,6 @@ def import_ctypes(): return ctypes -if not PY2: - - def just_warn(*args, **kw): - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - warnings.warn( - "Missing ctypes. Some features like bare super() or accessing " - "__class__ will not work with slots classes.", - RuntimeWarning, - stacklevel=2, - ) - - -else: - - def just_warn(*args, **kw): # pragma: nocover - """ - We only warn on Python 3 because we are not aware of any concrete - consequences of not setting the cell on Python 2. - """ - - def make_set_closure_cell(): """ Moved into a function for testability. diff --git a/pipenv/vendor/attr/_make.py b/pipenv/vendor/attr/_make.py index f7fd05e7..827175a4 100644 --- a/pipenv/vendor/attr/_make.py +++ b/pipenv/vendor/attr/_make.py @@ -409,12 +409,11 @@ def _transform_attrs(cls, these, auto_attribs, kw_only): a.kw_only is False ): had_default = True - if was_kw_only is True and a.kw_only is False: + if was_kw_only is True and a.kw_only is False and a.init is True: raise ValueError( "Non keyword-only attributes are not allowed after a " - "keyword-only attribute. Attribute in question: {a!r}".format( - a=a - ) + "keyword-only attribute (unless they are init=False). " + "Attribute in question: {a!r}".format(a=a) ) if was_kw_only is False and a.init is True and a.kw_only is True: was_kw_only = True @@ -454,6 +453,7 @@ class _ClassBuilder(object): "_has_post_init", "_delete_attribs", "_base_attr_map", + "_is_exc", ) def __init__( @@ -466,6 +466,7 @@ class _ClassBuilder(object): auto_attribs, kw_only, cache_hash, + is_exc, ): attrs, base_attrs, base_map = _transform_attrs( cls, these, auto_attribs, kw_only @@ -483,6 +484,7 @@ class _ClassBuilder(object): self._cache_hash = cache_hash self._has_post_init = bool(getattr(cls, "__attrs_post_init__", False)) self._delete_attribs = not bool(these) + self._is_exc = is_exc self._cls_dict["__attrs_attrs__"] = self._attrs @@ -530,6 +532,26 @@ class _ClassBuilder(object): for name, value in self._cls_dict.items(): setattr(cls, name, value) + # Attach __setstate__. This is necessary to clear the hash code + # cache on deserialization. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for dict classes. + # For slotted classes, see similar code in _create_slots_class . + if self._cache_hash: + existing_set_state_method = getattr(cls, "__setstate__", None) + if existing_set_state_method: + raise NotImplementedError( + "Currently you cannot use hash caching if " + "you specify your own __setstate__ method." + "See https://github.com/python-attrs/attrs/issues/494 ." + ) + + def cache_hash_set_state(chss_self, _): + # clear hash code cache + setattr(chss_self, _hash_cache_field, None) + + setattr(cls, "__setstate__", cache_hash_set_state) + return cls def _create_slots_class(self): @@ -582,6 +604,8 @@ class _ClassBuilder(object): """ return tuple(getattr(self, name) for name in state_attr_names) + hash_caching_enabled = self._cache_hash + def slots_setstate(self, state): """ Automatically created by attrs. @@ -589,6 +613,13 @@ class _ClassBuilder(object): __bound_setattr = _obj_setattr.__get__(self, Attribute) for name, value in zip(state_attr_names, state): __bound_setattr(name, value) + # Clearing the hash code cache on deserialization is needed + # because hash codes can change from run to run. See issue + # https://github.com/python-attrs/attrs/issues/482 . + # Note that this code only handles setstate for slotted classes. + # For dict classes, see similar code in _patch_original_class . + if hash_caching_enabled: + __bound_setattr(_hash_cache_field, None) # slots and frozen require __getstate__/__setstate__ to work cd["__getstate__"] = slots_getstate @@ -660,6 +691,7 @@ class _ClassBuilder(object): self._slots, self._cache_hash, self._base_attr_map, + self._is_exc, ) ) @@ -710,6 +742,7 @@ def attrs( auto_attribs=False, kw_only=False, cache_hash=False, + auto_exc=False, ): r""" A class decorator that adds `dunder @@ -815,10 +848,23 @@ def attrs( :param bool cache_hash: Ensure that the object's hash code is computed only once and stored on the object. If this is set to ``True``, hashing must be either explicitly or implicitly enabled for this - class. If the hash code is cached, then no attributes of this - class which participate in hash code computation may be mutated - after object creation. + class. If the hash code is cached, avoid any reassignments of + fields involved in hash code computation or mutations of the objects + those fields point to after object creation. If such changes occur, + the behavior of the object's hash code is undefined. + :param bool auto_exc: If the class subclasses :class:`BaseException` + (which implicitly includes any subclass of any exception), the + following happens to behave like a well-behaved Python exceptions + class: + - the values for *cmp* and *hash* are ignored and the instances compare + and hash by the instance's ids (N.B. ``attrs`` will *not* remove + existing implementations of ``__hash__`` or the equality methods. It + just won't add own ones.), + - all attributes that are either passed into ``__init__`` or have a + default value are additionally available as a tuple in the ``args`` + attribute, + - the value of *str* is ignored leaving ``__str__`` to base classes. .. versionadded:: 16.0.0 *slots* .. versionadded:: 16.1.0 *frozen* @@ -838,12 +884,16 @@ def attrs( to each other. .. versionadded:: 18.2.0 *kw_only* .. versionadded:: 18.2.0 *cache_hash* + .. versionadded:: 19.1.0 *auto_exc* """ def wrap(cls): + if getattr(cls, "__class__", None) is None: raise TypeError("attrs only works with new-style classes.") + is_exc = auto_exc is True and issubclass(cls, BaseException) + builder = _ClassBuilder( cls, these, @@ -853,13 +903,14 @@ def attrs( auto_attribs, kw_only, cache_hash, + is_exc, ) if repr is True: builder.add_repr(repr_ns) if str is True: builder.add_str() - if cmp is True: + if cmp is True and not is_exc: builder.add_cmp() if hash is not True and hash is not False and hash is not None: @@ -874,7 +925,11 @@ def attrs( " hashing must be either explicitly or implicitly " "enabled." ) - elif hash is True or (hash is None and cmp is True and frozen is True): + elif ( + hash is True + or (hash is None and cmp is True and frozen is True) + and is_exc is False + ): builder.add_hash() else: if cache_hash: @@ -1213,7 +1268,9 @@ def _add_repr(cls, ns=None, attrs=None): return cls -def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): +def _make_init( + attrs, post_init, frozen, slots, cache_hash, base_attr_map, is_exc +): attrs = [a for a in attrs if a.init or a.default is not NOTHING] # We cache the generated init methods for the same kinds of attributes. @@ -1222,16 +1279,18 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): unique_filename = "".format(sha1.hexdigest()) script, globs, annotations = _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ) locs = {} bytecode = compile(script, unique_filename, "exec") attr_dict = dict((a.name, a) for a in attrs) globs.update({"NOTHING": NOTHING, "attr_dict": attr_dict}) + if frozen is True: # Save the lookup overhead in __init__ if we need to circumvent # immutability. globs["_cached_setattr"] = _obj_setattr + eval(bytecode, globs, locs) # In order of debuggers like PDB being able to step through the code, @@ -1245,24 +1304,10 @@ def _make_init(attrs, post_init, frozen, slots, cache_hash, base_attr_map): __init__ = locs["__init__"] __init__.__annotations__ = annotations + return __init__ -def _add_init(cls, frozen): - """ - Add a __init__ method to *cls*. If *frozen* is True, make it immutable. - """ - cls.__init__ = _make_init( - cls.__attrs_attrs__, - getattr(cls, "__attrs_post_init__", False), - frozen, - _is_slot_cls(cls), - cache_hash=False, - base_attr_map={}, - ) - return cls - - def fields(cls): """ Return the tuple of ``attrs`` attributes for a class. @@ -1348,7 +1393,7 @@ def _is_slot_attr(a_name, base_attr_map): def _attrs_to_init_script( - attrs, frozen, slots, post_init, cache_hash, base_attr_map + attrs, frozen, slots, post_init, cache_hash, base_attr_map, is_exc ): """ Return a script of an initializer for *attrs* and a dict of globals. @@ -1597,6 +1642,13 @@ def _attrs_to_init_script( init_hash_cache = "self.%s = %s" lines.append(init_hash_cache % (_hash_cache_field, "None")) + # For exceptions we rely on BaseException.__init__ for proper + # initialization. + if is_exc: + vals = ",".join("self." + a.name for a in attrs if a.init) + + lines.append("BaseException.__init__(self, %s)" % (vals,)) + args = ", ".join(args) if kw_only_args: if PY2: diff --git a/pipenv/vendor/attr/filters.pyi b/pipenv/vendor/attr/filters.pyi index a618140c..68368fe2 100644 --- a/pipenv/vendor/attr/filters.pyi +++ b/pipenv/vendor/attr/filters.pyi @@ -1,5 +1,5 @@ -from typing import Union +from typing import Union, Any from . import Attribute, _FilterType -def include(*what: Union[type, Attribute]) -> _FilterType: ... -def exclude(*what: Union[type, Attribute]) -> _FilterType: ... +def include(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... +def exclude(*what: Union[type, Attribute[Any]]) -> _FilterType[Any]: ... diff --git a/pipenv/vendor/attr/validators.py b/pipenv/vendor/attr/validators.py index f12d0aa5..7fc4446b 100644 --- a/pipenv/vendor/attr/validators.py +++ b/pipenv/vendor/attr/validators.py @@ -136,7 +136,7 @@ class _InValidator(object): def __call__(self, inst, attr, value): try: in_options = value in self.options - except TypeError as e: # e.g. `1 in "abc"` + except TypeError: # e.g. `1 in "abc"` in_options = False if not in_options: @@ -168,3 +168,115 @@ def in_(options): .. versionadded:: 17.1.0 """ return _InValidator(options) + + +@attrs(repr=False, slots=False, hash=True) +class _IsCallableValidator(object): + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if not callable(value): + raise TypeError("'{name}' must be callable".format(name=attr.name)) + + def __repr__(self): + return "" + + +def is_callable(): + """ + A validator that raises a :class:`TypeError` if the initializer is called + with a value for this particular attribute that is not callable. + + .. versionadded:: 19.1.0 + + :raises TypeError: With a human readable error message containing the + attribute (of type :class:`attr.Attribute`) name. + """ + return _IsCallableValidator() + + +@attrs(repr=False, slots=True, hash=True) +class _DeepIterable(object): + member_validator = attrib(validator=is_callable()) + iterable_validator = attrib( + default=None, validator=optional(is_callable()) + ) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.iterable_validator is not None: + self.iterable_validator(inst, attr, value) + + for member in value: + self.member_validator(inst, attr, member) + + def __repr__(self): + iterable_identifier = ( + "" + if self.iterable_validator is None + else " {iterable!r}".format(iterable=self.iterable_validator) + ) + return ( + "" + ).format( + iterable_identifier=iterable_identifier, + member=self.member_validator, + ) + + +def deep_iterable(member_validator, iterable_validator=None): + """ + A validator that performs deep validation of an iterable. + + :param member_validator: Validator to apply to iterable members + :param iterable_validator: Validator to apply to iterable itself + (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepIterable(member_validator, iterable_validator) + + +@attrs(repr=False, slots=True, hash=True) +class _DeepMapping(object): + key_validator = attrib(validator=is_callable()) + value_validator = attrib(validator=is_callable()) + mapping_validator = attrib(default=None, validator=optional(is_callable())) + + def __call__(self, inst, attr, value): + """ + We use a callable class to be able to change the ``__repr__``. + """ + if self.mapping_validator is not None: + self.mapping_validator(inst, attr, value) + + for key in value: + self.key_validator(inst, attr, key) + self.value_validator(inst, attr, value[key]) + + def __repr__(self): + return ( + "" + ).format(key=self.key_validator, value=self.value_validator) + + +def deep_mapping(key_validator, value_validator, mapping_validator=None): + """ + A validator that performs deep validation of a dictionary. + + :param key_validator: Validator to apply to dictionary keys + :param value_validator: Validator to apply to dictionary values + :param mapping_validator: Validator to apply to top-level mapping + attribute (optional) + + .. versionadded:: 19.1.0 + + :raises TypeError: if any sub-validators fail + """ + return _DeepMapping(key_validator, value_validator, mapping_validator) diff --git a/pipenv/vendor/attr/validators.pyi b/pipenv/vendor/attr/validators.pyi index abbaedf1..01af0684 100644 --- a/pipenv/vendor/attr/validators.pyi +++ b/pipenv/vendor/attr/validators.pyi @@ -12,3 +12,13 @@ def optional( ) -> _ValidatorType[Optional[_T]]: ... def in_(options: Container[_T]) -> _ValidatorType[_T]: ... def and_(*validators: _ValidatorType[_T]) -> _ValidatorType[_T]: ... +def deep_iterable( + member_validator: _ValidatorType[_T], + iterable_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def deep_mapping( + key_validator: _ValidatorType[_T], + value_validator: _ValidatorType[_T], + mapping_validator: Optional[_ValidatorType[_T]], +) -> _ValidatorType[_T]: ... +def is_callable() -> _ValidatorType[_T]: ... diff --git a/pipenv/vendor/cached_property.py b/pipenv/vendor/cached_property.py index a06be97a..125f6195 100644 --- a/pipenv/vendor/cached_property.py +++ b/pipenv/vendor/cached_property.py @@ -2,7 +2,7 @@ __author__ = "Daniel Greenfeld" __email__ = "pydanny@gmail.com" -__version__ = "1.4.3" +__version__ = "1.5.1" __license__ = "BSD" from time import time diff --git a/pipenv/vendor/certifi/__init__.py b/pipenv/vendor/certifi/__init__.py index 50f2e130..ef71f3af 100644 --- a/pipenv/vendor/certifi/__init__.py +++ b/pipenv/vendor/certifi/__init__.py @@ -1,3 +1,3 @@ -from .core import where, old_where +from .core import where -__version__ = "2018.10.15" +__version__ = "2018.11.29" diff --git a/pipenv/vendor/certifi/cacert.pem b/pipenv/vendor/certifi/cacert.pem index e75d85b3..db68797e 100644 --- a/pipenv/vendor/certifi/cacert.pem +++ b/pipenv/vendor/certifi/cacert.pem @@ -4268,3 +4268,245 @@ rYy0UGYwEAYJKwYBBAGCNxUBBAMCAQAwCgYIKoZIzj0EAwMDaAAwZQIwJsdpW9zV 57LnyAyMjMPdeYwbY9XJUpROTYJKcx6ygISpJcBMWm1JKWB4E+J+SOtkAjEA2zQg Mgj/mkkCtojeFK9dbJlxjRo/i9fgojaGHAeCOnZT/cKi7e97sIBPWA9LUzm9 -----END CERTIFICATE----- + +# Issuer: CN=GTS Root R1 O=Google Trust Services LLC +# Subject: CN=GTS Root R1 O=Google Trust Services LLC +# Label: "GTS Root R1" +# Serial: 146587175971765017618439757810265552097 +# MD5 Fingerprint: 82:1a:ef:d4:d2:4a:f2:9f:e2:3d:97:06:14:70:72:85 +# SHA1 Fingerprint: e1:c9:50:e6:ef:22:f8:4c:56:45:72:8b:92:20:60:d7:d5:a7:a3:e8 +# SHA256 Fingerprint: 2a:57:54:71:e3:13:40:bc:21:58:1c:bd:2c:f1:3e:15:84:63:20:3e:ce:94:bc:f9:d3:cc:19:6b:f0:9a:54:72 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxUtHDA3sM9CJuRz04TANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjEwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQC2EQKLHuOhd5s73L+UPreVp0A8of2C+X0yBoJx9vaM +f/vo27xqLpeXo4xL+Sv2sfnOhB2x+cWX3u+58qPpvBKJXqeqUqv4IyfLpLGcY9vX +mX7wCl7raKb0xlpHDU0QM+NOsROjyBhsS+z8CZDfnWQpJSMHobTSPS5g4M/SCYe7 +zUjwTcLCeoiKu7rPWRnWr4+wB7CeMfGCwcDfLqZtbBkOtdh+JhpFAz2weaSUKK0P +fyblqAj+lug8aJRT7oM6iCsVlgmy4HqMLnXWnOunVmSPlk9orj2XwoSPwLxAwAtc +vfaHszVsrBhQf4TgTM2S0yDpM7xSma8ytSmzJSq0SPly4cpk9+aCEI3oncKKiPo4 +Zor8Y/kB+Xj9e1x3+naH+uzfsQ55lVe0vSbv1gHR6xYKu44LtcXFilWr06zqkUsp +zBmkMiVOKvFlRNACzqrOSbTqn3yDsEB750Orp2yjj32JgfpMpf/VjsPOS+C12LOO +Rc92wO1AK/1TD7Cn1TsNsYqiA94xrcx36m97PtbfkSIS5r762DL8EGMUUXLeXdYW +k70paDPvOmbsB4om3xPXV2V4J95eSRQAogB/mqghtqmxlbCluQ0WEdrHbEg8QOB+ +DVrNVjzRlwW5y0vtOUucxD/SVRNuJLDWcfr0wbrM7Rv1/oFB2ACYPTrIrnqYNxgF +lQIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQU5K8rJnEaK0gnhS9SZizv8IkTcT4wDQYJKoZIhvcNAQEMBQADggIBADiW +Cu49tJYeX++dnAsznyvgyv3SjgofQXSlfKqE1OXyHuY3UjKcC9FhHb8owbZEKTV1 +d5iyfNm9dKyKaOOpMQkpAWBz40d8U6iQSifvS9efk+eCNs6aaAyC58/UEBZvXw6Z +XPYfcX3v73svfuo21pdwCxXu11xWajOl40k4DLh9+42FpLFZXvRq4d2h9mREruZR +gyFmxhE+885H7pwoHyXa/6xmld01D1zvICxi/ZG6qcz8WpyTgYMpl0p8WnK0OdC3 +d8t5/Wk6kjftbjhlRn7pYL15iJdfOBL07q9bgsiG1eGZbYwE8na6SfZu6W0eX6Dv +J4J2QPim01hcDyxC2kLGe4g0x8HYRZvBPsVhHdljUEn2NIVq4BjFbkerQUIpm/Zg +DdIx02OYI5NaAIFItO/Nis3Jz5nu2Z6qNuFoS3FJFDYoOj0dzpqPJeaAcWErtXvM ++SUWgeExX6GjfhaknBZqlxi9dnKlC54dNuYvoS++cJEPqOba+MSSQGwlfnuzCdyy +F62ARPBopY+Udf90WuioAnwMCeKpSwughQtiue+hMZL77/ZRBIls6Kl0obsXs7X9 +SQ98POyDGCBDTtWTurQ0sR8WNh8M5mQ5Fkzc4P4dyKliPUDqysU0ArSuiYgzNdws +E3PYJ/HQcu51OyLemGhmW/HGY0dVHLqlCFF1pkgl +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R2 O=Google Trust Services LLC +# Subject: CN=GTS Root R2 O=Google Trust Services LLC +# Label: "GTS Root R2" +# Serial: 146587176055767053814479386953112547951 +# MD5 Fingerprint: 44:ed:9a:0e:a4:09:3b:00:f2:ae:4c:a3:c6:61:b0:8b +# SHA1 Fingerprint: d2:73:96:2a:2a:5e:39:9f:73:3f:e1:c7:1e:64:3f:03:38:34:fc:4d +# SHA256 Fingerprint: c4:5d:7b:b0:8e:6d:67:e6:2e:42:35:11:0b:56:4e:5f:78:fd:92:ef:05:8c:84:0a:ea:4e:64:55:d7:58:5c:60 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQbkepxlqz5yDFMJo/aFLybzANBgkqhkiG9w0BAQwFADBH +MQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExM +QzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIy +MDAwMDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNl +cnZpY2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjIwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQDO3v2m++zsFDQ8BwZabFn3GTXd98GdVarTzTukk3Lv +CvptnfbwhYBboUhSnznFt+4orO/LdmgUud+tAWyZH8QiHZ/+cnfgLFuv5AS/T3Kg +GjSY6Dlo7JUle3ah5mm5hRm9iYz+re026nO8/4Piy33B0s5Ks40FnotJk9/BW9Bu +XvAuMC6C/Pq8tBcKSOWIm8Wba96wyrQD8Nr0kLhlZPdcTK3ofmZemde4wj7I0BOd +re7kRXuJVfeKH2JShBKzwkCX44ofR5GmdFrS+LFjKBC4swm4VndAoiaYecb+3yXu +PuWgf9RhD1FLPD+M2uFwdNjCaKH5wQzpoeJ/u1U8dgbuak7MkogwTZq9TwtImoS1 +mKPV+3PBV2HdKFZ1E66HjucMUQkQdYhMvI35ezzUIkgfKtzra7tEscszcTJGr61K +8YzodDqs5xoic4DSMPclQsciOzsSrZYuxsN2B6ogtzVJV+mSSeh2FnIxZyuWfoqj +x5RWIr9qS34BIbIjMt/kmkRtWVtd9QCgHJvGeJeNkP+byKq0rxFROV7Z+2et1VsR +nTKaG73VululycslaVNVJ1zgyjbLiGH7HrfQy+4W+9OmTN6SpdTi3/UGVN4unUu0 +kzCqgc7dGtxRcw1PcOnlthYhGXmy5okLdWTK1au8CcEYof/UVKGFPP0UJAOyh9Ok +twIDAQABo0IwQDAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAdBgNV +HQ4EFgQUu//KjiOfT5nK2+JopqUVJxce2Q4wDQYJKoZIhvcNAQEMBQADggIBALZp +8KZ3/p7uC4Gt4cCpx/k1HUCCq+YEtN/L9x0Pg/B+E02NjO7jMyLDOfxA325BS0JT +vhaI8dI4XsRomRyYUpOM52jtG2pzegVATX9lO9ZY8c6DR2Dj/5epnGB3GFW1fgiT +z9D2PGcDFWEJ+YF59exTpJ/JjwGLc8R3dtyDovUMSRqodt6Sm2T4syzFJ9MHwAiA +pJiS4wGWAqoC7o87xdFtCjMwc3i5T1QWvwsHoaRc5svJXISPD+AVdyx+Jn7axEvb +pxZ3B7DNdehyQtaVhJ2Gg/LkkM0JR9SLA3DaWsYDQvTtN6LwG1BUSw7YhN4ZKJmB +R64JGz9I0cNv4rBgF/XuIwKl2gBbbZCr7qLpGzvpx0QnRY5rn/WkhLx3+WuXrD5R +RaIRpsyF7gpo8j5QOHokYh4XIDdtak23CZvJ/KRY9bb7nE4Yu5UC56GtmwfuNmsk +0jmGwZODUNKBRqhfYlcsu2xkiAhu7xNUX90txGdj08+JN7+dIPT7eoOboB6BAFDC +5AwiWVIQ7UNWhwD4FFKnHYuTjKJNRn8nxnGbJN7k2oaLDX5rIMHAnuFl2GqjpuiF +izoHCBy69Y9Vmhh1fuXsgWbRIXOhNUQLgD1bnF5vKheW0YMjiGZt5obicDIvUiLn +yOd/xCxgXS/Dr55FBcOEArf9LAhST4Ldo/DUhgkC +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R3 O=Google Trust Services LLC +# Subject: CN=GTS Root R3 O=Google Trust Services LLC +# Label: "GTS Root R3" +# Serial: 146587176140553309517047991083707763997 +# MD5 Fingerprint: 1a:79:5b:6b:04:52:9c:5d:c7:74:33:1b:25:9a:f9:25 +# SHA1 Fingerprint: 30:d4:24:6f:07:ff:db:91:89:8a:0b:e9:49:66:11:eb:8c:5e:46:e5 +# SHA256 Fingerprint: 15:d5:b8:77:46:19:ea:7d:54:ce:1c:a6:d0:b0:c4:03:e0:37:a9:17:f1:31:e8:a0:4e:1e:6b:7a:71:ba:bc:e5 +-----BEGIN CERTIFICATE----- +MIICDDCCAZGgAwIBAgIQbkepx2ypcyRAiQ8DVd2NHTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjMwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjMwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAAQfTzOHMymKoYTey8chWEGJ6ladK0uFxh1MJ7x/JlFyb+Kf1qPKzEUURout +736GjOyxfi//qXGdGIRFBEFVbivqJn+7kAHjSxm65FSWRQmx1WyRRK2EE46ajA2A +DDL24CejQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBTB8Sa6oC2uhYHP0/EqEr24Cmf9vDAKBggqhkjOPQQDAwNpADBmAjEAgFuk +fCPAlaUs3L6JbyO5o91lAFJekazInXJ0glMLfalAvWhgxeG4VDvBNhcl2MG9AjEA +njWSdIUlUfUk7GRSJFClH9voy8l27OyCbvWFGFPouOOaKaqW04MjyaR7YbPMAuhd +-----END CERTIFICATE----- + +# Issuer: CN=GTS Root R4 O=Google Trust Services LLC +# Subject: CN=GTS Root R4 O=Google Trust Services LLC +# Label: "GTS Root R4" +# Serial: 146587176229350439916519468929765261721 +# MD5 Fingerprint: 5d:b6:6a:c4:60:17:24:6a:1a:99:a8:4b:ee:5e:b4:26 +# SHA1 Fingerprint: 2a:1d:60:27:d9:4a:b1:0a:1c:4d:91:5c:cd:33:a0:cb:3e:2d:54:cb +# SHA256 Fingerprint: 71:cc:a5:39:1f:9e:79:4b:04:80:25:30:b3:63:e1:21:da:8a:30:43:bb:26:66:2f:ea:4d:ca:7f:c9:51:a4:bd +-----BEGIN CERTIFICATE----- +MIICCjCCAZGgAwIBAgIQbkepyIuUtui7OyrYorLBmTAKBggqhkjOPQQDAzBHMQsw +CQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZpY2VzIExMQzEU +MBIGA1UEAxMLR1RTIFJvb3QgUjQwHhcNMTYwNjIyMDAwMDAwWhcNMzYwNjIyMDAw +MDAwWjBHMQswCQYDVQQGEwJVUzEiMCAGA1UEChMZR29vZ2xlIFRydXN0IFNlcnZp +Y2VzIExMQzEUMBIGA1UEAxMLR1RTIFJvb3QgUjQwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATzdHOnaItgrkO4NcWBMHtLSZ37wWHO5t5GvWvVYRg1rkDdc/eJkTBa6zzu +hXyiQHY7qca4R9gq55KRanPpsXI5nymfopjTX15YhmUPoYRlBtHci8nHc8iMai/l +xKvRHYqjQjBAMA4GA1UdDwEB/wQEAwIBBjAPBgNVHRMBAf8EBTADAQH/MB0GA1Ud +DgQWBBSATNbrdP9JNqPV2Py1PsVq8JQdjDAKBggqhkjOPQQDAwNnADBkAjBqUFJ0 +CMRw3J5QdCHojXohw0+WbhXRIjVhLfoIN+4Zba3bssx9BzT1YBkstTTZbyACMANx +sbqjYAuG7ZoIapVon+Kz4ZNkfF6Tpt95LY2F45TPI11xzPKwTdb+mciUqXWi4w== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Global G2 Root O=UniTrust +# Subject: CN=UCA Global G2 Root O=UniTrust +# Label: "UCA Global G2 Root" +# Serial: 124779693093741543919145257850076631279 +# MD5 Fingerprint: 80:fe:f0:c4:4a:f0:5c:62:32:9f:1c:ba:78:a9:50:f8 +# SHA1 Fingerprint: 28:f9:78:16:19:7a:ff:18:25:18:aa:44:fe:c1:a0:ce:5c:b6:4c:8a +# SHA256 Fingerprint: 9b:ea:11:c9:76:fe:01:47:64:c1:be:56:a6:f9:14:b5:a5:60:31:7a:bd:99:88:39:33:82:e5:16:1a:a0:49:3c +-----BEGIN CERTIFICATE----- +MIIFRjCCAy6gAwIBAgIQXd+x2lqj7V2+WmUgZQOQ7zANBgkqhkiG9w0BAQsFADA9 +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxGzAZBgNVBAMMElVDQSBH +bG9iYWwgRzIgUm9vdDAeFw0xNjAzMTEwMDAwMDBaFw00MDEyMzEwMDAwMDBaMD0x +CzAJBgNVBAYTAkNOMREwDwYDVQQKDAhVbmlUcnVzdDEbMBkGA1UEAwwSVUNBIEds +b2JhbCBHMiBSb290MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxeYr +b3zvJgUno4Ek2m/LAfmZmqkywiKHYUGRO8vDaBsGxUypK8FnFyIdK+35KYmToni9 +kmugow2ifsqTs6bRjDXVdfkX9s9FxeV67HeToI8jrg4aA3++1NDtLnurRiNb/yzm +VHqUwCoV8MmNsHo7JOHXaOIxPAYzRrZUEaalLyJUKlgNAQLx+hVRZ2zA+te2G3/R +VogvGjqNO7uCEeBHANBSh6v7hn4PJGtAnTRnvI3HLYZveT6OqTwXS3+wmeOwcWDc +C/Vkw85DvG1xudLeJ1uK6NjGruFZfc8oLTW4lVYa8bJYS7cSN8h8s+1LgOGN+jIj +tm+3SJUIsUROhYw6AlQgL9+/V087OpAh18EmNVQg7Mc/R+zvWr9LesGtOxdQXGLY +D0tK3Cv6brxzks3sx1DoQZbXqX5t2Okdj4q1uViSukqSKwxW/YDrCPBeKW4bHAyv +j5OJrdu9o54hyokZ7N+1wxrrFv54NkzWbtA+FxyQF2smuvt6L78RHBgOLXMDj6Dl +NaBa4kx1HXHhOThTeEDMg5PXCp6dW4+K5OXgSORIskfNTip1KnvyIvbJvgmRlld6 +iIis7nCs+dwp4wwcOxJORNanTrAmyPPZGpeRaOrvjUYG0lZFWJo8DA+DuAUlwznP +O6Q0ibd5Ei9Hxeepl2n8pndntd978XplFeRhVmUCAwEAAaNCMEAwDgYDVR0PAQH/ +BAQDAgEGMA8GA1UdEwEB/wQFMAMBAf8wHQYDVR0OBBYEFIHEjMz15DD/pQwIX4wV +ZyF0Ad/fMA0GCSqGSIb3DQEBCwUAA4ICAQATZSL1jiutROTL/7lo5sOASD0Ee/oj +L3rtNtqyzm325p7lX1iPyzcyochltq44PTUbPrw7tgTQvPlJ9Zv3hcU2tsu8+Mg5 +1eRfB70VVJd0ysrtT7q6ZHafgbiERUlMjW+i67HM0cOU2kTC5uLqGOiiHycFutfl +1qnN3e92mI0ADs0b+gO3joBYDic/UvuUospeZcnWhNq5NXHzJsBPd+aBJ9J3O5oU +b3n09tDh05S60FdRvScFDcH9yBIw7m+NESsIndTUv4BFFJqIRNow6rSn4+7vW4LV +PtateJLbXDzz2K36uGt/xDYotgIVilQsnLAXc47QN6MUPJiVAAwpBVueSUmxX8fj +y88nZY41F7dXyDDZQVu5FLbowg+UMaeUmMxq67XhJ/UQqAHojhJi6IjMtX9Gl8Cb +EGY4GjZGXyJoPd/JxhMnq1MGrKI8hgZlb7F+sSlEmqO6SWkoaY/X5V+tBIZkbxqg +DMUIYs6Ao9Dz7GjevjPHF1t/gMRMTLGmhIrDO7gJzRSBuhjjVFc2/tsvfEehOjPI ++Vg7RE+xygKJBJYoaMVLuCaJu9YzL1DV/pqJuhgyklTGW+Cd+V7lDSKb9triyCGy +YiGqhkCyLmTTX8jjfhFnRR8F/uOi77Oos/N9j/gMHyIfLXC0uAE0djAA5SN4p1bX +UB+K+wb1whnw0A== +-----END CERTIFICATE----- + +# Issuer: CN=UCA Extended Validation Root O=UniTrust +# Subject: CN=UCA Extended Validation Root O=UniTrust +# Label: "UCA Extended Validation Root" +# Serial: 106100277556486529736699587978573607008 +# MD5 Fingerprint: a1:f3:5f:43:c6:34:9b:da:bf:8c:7e:05:53:ad:96:e2 +# SHA1 Fingerprint: a3:a1:b0:6f:24:61:23:4a:e3:36:a5:c2:37:fc:a6:ff:dd:f0:d7:3a +# SHA256 Fingerprint: d4:3a:f9:b3:54:73:75:5c:96:84:fc:06:d7:d8:cb:70:ee:5c:28:e7:73:fb:29:4e:b4:1e:e7:17:22:92:4d:24 +-----BEGIN CERTIFICATE----- +MIIFWjCCA0KgAwIBAgIQT9Irj/VkyDOeTzRYZiNwYDANBgkqhkiG9w0BAQsFADBH +MQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNVBAMMHFVDQSBF +eHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwHhcNMTUwMzEzMDAwMDAwWhcNMzgxMjMx +MDAwMDAwWjBHMQswCQYDVQQGEwJDTjERMA8GA1UECgwIVW5pVHJ1c3QxJTAjBgNV +BAMMHFVDQSBFeHRlbmRlZCBWYWxpZGF0aW9uIFJvb3QwggIiMA0GCSqGSIb3DQEB +AQUAA4ICDwAwggIKAoICAQCpCQcoEwKwmeBkqh5DFnpzsZGgdT6o+uM4AHrsiWog +D4vFsJszA1qGxliG1cGFu0/GnEBNyr7uaZa4rYEwmnySBesFK5pI0Lh2PpbIILvS +sPGP2KxFRv+qZ2C0d35qHzwaUnoEPQc8hQ2E0B92CvdqFN9y4zR8V05WAT558aop +O2z6+I9tTcg1367r3CTueUWnhbYFiN6IXSV8l2RnCdm/WhUFhvMJHuxYMjMR83dk +sHYf5BA1FxvyDrFspCqjc/wJHx4yGVMR59mzLC52LqGj3n5qiAno8geK+LLNEOfi +c0CTuwjRP+H8C5SzJe98ptfRr5//lpr1kXuYC3fUfugH0mK1lTnj8/FtDw5lhIpj +VMWAtuCeS31HJqcBCF3RiJ7XwzJE+oJKCmhUfzhTA8ykADNkUVkLo4KRel7sFsLz +KuZi2irbWWIQJUoqgQtHB0MGcIfS+pMRKXpITeuUx3BNr2fVUbGAIAEBtHoIppB/ +TuDvB0GHr2qlXov7z1CymlSvw4m6WC31MJixNnI5fkkE/SmnTHnkBVfblLkWU41G +sx2VYVdWf6/wFlthWG82UBEL2KwrlRYaDh8IzTY0ZRBiZtWAXxQgXy0MoHgKaNYs +1+lvK9JKBZP8nm9rZ/+I8U6laUpSNwXqxhaN0sSZ0YIrO7o1dfdRUVjzyAfd5LQD +fwIDAQABo0IwQDAdBgNVHQ4EFgQU2XQ65DA9DfcS3H5aBZ8eNJr34RQwDwYDVR0T +AQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQADggIBADaN +l8xCFWQpN5smLNb7rhVpLGsaGvdftvkHTFnq88nIua7Mui563MD1sC3AO6+fcAUR +ap8lTwEpcOPlDOHqWnzcSbvBHiqB9RZLcpHIojG5qtr8nR/zXUACE/xOHAbKsxSQ +VBcZEhrxH9cMaVr2cXj0lH2RC47skFSOvG+hTKv8dGT9cZr4QQehzZHkPJrgmzI5 +c6sq1WnIeJEmMX3ixzDx/BR4dxIOE/TdFpS/S2d7cFOFyrC78zhNLJA5wA3CXWvp +4uXViI3WLL+rG761KIcSF3Ru/H38j9CHJrAb+7lsq+KePRXBOy5nAliRn+/4Qh8s +t2j1da3Ptfb/EX3C8CSlrdP6oDyp+l3cpaDvRKS+1ujl5BOWF3sGPjLtx7dCvHaj +2GU4Kzg1USEODm8uNBNA4StnDG1KQTAYI1oyVZnJF+A83vbsea0rWBmirSwiGpWO +vpaQXUJXxPkUAzUrHC1RVwinOt4/5Mi0A3PCwSaAuwtCH60NryZy2sy+s6ODWA2C +xR9GUeOcGMyNm43sSet1UNWMKFnKdDTajAshqx7qG+XH/RU+wBeq+yNuJkbL+vmx +cmtpzyKEC2IPrNkZAJSidjzULZrtBJ4tBmIQN1IchXIbJ+XMxjHsN+xjWZsLHXbM +fjKaiJUINlK73nZfdklJrX+9ZSCyycErdhh2n1ax +-----END CERTIFICATE----- + +# Issuer: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Subject: CN=Certigna Root CA O=Dhimyotis OU=0002 48146308100036 +# Label: "Certigna Root CA" +# Serial: 269714418870597844693661054334862075617 +# MD5 Fingerprint: 0e:5c:30:62:27:eb:5b:bc:d7:ae:62:ba:e9:d5:df:77 +# SHA1 Fingerprint: 2d:0d:52:14:ff:9e:ad:99:24:01:74:20:47:6e:6c:85:27:27:f5:43 +# SHA256 Fingerprint: d4:8d:3d:23:ee:db:50:a4:59:e5:51:97:60:1c:27:77:4b:9d:7b:18:c9:4d:5a:05:95:11:a1:02:50:b9:31:68 +-----BEGIN CERTIFICATE----- +MIIGWzCCBEOgAwIBAgIRAMrpG4nxVQMNo+ZBbcTjpuEwDQYJKoZIhvcNAQELBQAw +WjELMAkGA1UEBhMCRlIxEjAQBgNVBAoMCURoaW15b3RpczEcMBoGA1UECwwTMDAw +MiA0ODE0NjMwODEwMDAzNjEZMBcGA1UEAwwQQ2VydGlnbmEgUm9vdCBDQTAeFw0x +MzEwMDEwODMyMjdaFw0zMzEwMDEwODMyMjdaMFoxCzAJBgNVBAYTAkZSMRIwEAYD +VQQKDAlEaGlteW90aXMxHDAaBgNVBAsMEzAwMDIgNDgxNDYzMDgxMDAwMzYxGTAX +BgNVBAMMEENlcnRpZ25hIFJvb3QgQ0EwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAw +ggIKAoICAQDNGDllGlmx6mQWDoyUJJV8g9PFOSbcDO8WV43X2KyjQn+Cyu3NW9sO +ty3tRQgXstmzy9YXUnIo245Onoq2C/mehJpNdt4iKVzSs9IGPjA5qXSjklYcoW9M +CiBtnyN6tMbaLOQdLNyzKNAT8kxOAkmhVECe5uUFoC2EyP+YbNDrihqECB63aCPu +I9Vwzm1RaRDuoXrC0SIxwoKF0vJVdlB8JXrJhFwLrN1CTivngqIkicuQstDuI7pm +TLtipPlTWmR7fJj6o0ieD5Wupxj0auwuA0Wv8HT4Ks16XdG+RCYyKfHx9WzMfgIh +C59vpD++nVPiz32pLHxYGpfhPTc3GGYo0kDFUYqMwy3OU4gkWGQwFsWq4NYKpkDf +ePb1BHxpE4S80dGnBs8B92jAqFe7OmGtBIyT46388NtEbVncSVmurJqZNjBBe3Yz +IoejwpKGbvlw7q6Hh5UbxHq9MfPU0uWZ/75I7HX1eBYdpnDBfzwboZL7z8g81sWT +Co/1VTp2lc5ZmIoJlXcymoO6LAQ6l73UL77XbJuiyn1tJslV1c/DeVIICZkHJC1k +JWumIWmbat10TWuXekG9qxf5kBdIjzb5LdXF2+6qhUVB+s06RbFo5jZMm5BX7CO5 +hwjCxAnxl4YqKE3idMDaxIzb3+KhF1nOJFl0Mdp//TBt2dzhauH8XwIDAQABo4IB +GjCCARYwDwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAQYwHQYDVR0OBBYE +FBiHVuBud+4kNTxOc5of1uHieX4rMB8GA1UdIwQYMBaAFBiHVuBud+4kNTxOc5of +1uHieX4rMEQGA1UdIAQ9MDswOQYEVR0gADAxMC8GCCsGAQUFBwIBFiNodHRwczov +L3d3d3cuY2VydGlnbmEuZnIvYXV0b3JpdGVzLzBtBgNVHR8EZjBkMC+gLaArhilo +dHRwOi8vY3JsLmNlcnRpZ25hLmZyL2NlcnRpZ25hcm9vdGNhLmNybDAxoC+gLYYr +aHR0cDovL2NybC5kaGlteW90aXMuY29tL2NlcnRpZ25hcm9vdGNhLmNybDANBgkq +hkiG9w0BAQsFAAOCAgEAlLieT/DjlQgi581oQfccVdV8AOItOoldaDgvUSILSo3L +6btdPrtcPbEo/uRTVRPPoZAbAh1fZkYJMyjhDSSXcNMQH+pkV5a7XdrnxIxPTGRG +HVyH41neQtGbqH6mid2PHMkwgu07nM3A6RngatgCdTer9zQoKJHyBApPNeNgJgH6 +0BGM+RFq7q89w1DTj18zeTyGqHNFkIwgtnJzFyO+B2XleJINugHA64wcZr+shncB +lA2c5uk5jR+mUYyZDDl34bSb+hxnV29qao6pK0xXeXpXIs/NX2NGjVxZOob4Mkdi +o2cNGJHc+6Zr9UhhcyNZjgKnvETq9Emd8VRY+WCv2hikLyhF3HqgiIZd8zvn/yk1 +gPxkQ5Tm4xxvvq0OKmOZK8l+hfZx6AYDlf7ej0gcWtSS6Cvu5zHbugRqh5jnxV/v +faci9wHYTfmJ0A6aBVmknpjZbyvKcL5kwlWj9Omvw5Ip3IgWJJk8jSaYtlu3zM63 +Nwf9JtmYhST/WSMDmu2dnajkXjjO11INb9I/bbEFa0nOipFGc/T2L/Coc3cOZayh +jWZSaX5LaAzHHjcng6WMxwLkFM1JAbBzs/3GkDpv0mztO+7skb6iQ12LAEpmJURw +3kAP+HwV96LOPNdeE4yBFxgX0b3xdxA61GU5wSesVywlVP+i2k+KYTlerj1KjL0= +-----END CERTIFICATE----- diff --git a/pipenv/vendor/certifi/core.py b/pipenv/vendor/certifi/core.py index eab9d1d1..2d02ea44 100644 --- a/pipenv/vendor/certifi/core.py +++ b/pipenv/vendor/certifi/core.py @@ -8,14 +8,6 @@ certifi.py This module returns the installation location of cacert.pem. """ import os -import warnings - - -class DeprecatedBundleWarning(DeprecationWarning): - """ - The weak security bundle is being deprecated. Please bother your service - provider to get them to stop using cross-signed roots. - """ def where(): @@ -24,14 +16,5 @@ def where(): return os.path.join(f, 'cacert.pem') -def old_where(): - warnings.warn( - "The weak security bundle has been removed. certifi.old_where() is now an alias " - "of certifi.where(). Please update your code to use certifi.where() instead. " - "certifi.old_where() will be removed in 2018.", - DeprecatedBundleWarning - ) - return where() - if __name__ == '__main__': print(where()) diff --git a/pipenv/vendor/colorama/LICENSE.txt b/pipenv/vendor/colorama/LICENSE.txt index 5f567799..3105888e 100644 --- a/pipenv/vendor/colorama/LICENSE.txt +++ b/pipenv/vendor/colorama/LICENSE.txt @@ -25,4 +25,3 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - diff --git a/pipenv/vendor/colorama/__init__.py b/pipenv/vendor/colorama/__init__.py index f4d9ce21..2a3bf471 100644 --- a/pipenv/vendor/colorama/__init__.py +++ b/pipenv/vendor/colorama/__init__.py @@ -3,5 +3,4 @@ from .initialise import init, deinit, reinit, colorama_text from .ansi import Fore, Back, Style, Cursor from .ansitowin32 import AnsiToWin32 -__version__ = '0.3.9' - +__version__ = '0.4.1' diff --git a/pipenv/vendor/colorama/ansitowin32.py b/pipenv/vendor/colorama/ansitowin32.py index 1d6e6059..359c92be 100644 --- a/pipenv/vendor/colorama/ansitowin32.py +++ b/pipenv/vendor/colorama/ansitowin32.py @@ -13,14 +13,6 @@ if windll is not None: winterm = WinTerm() -def is_stream_closed(stream): - return not hasattr(stream, 'closed') or stream.closed - - -def is_a_tty(stream): - return hasattr(stream, 'isatty') and stream.isatty() - - class StreamWrapper(object): ''' Wraps a stream (such as stdout), acting as a transparent proxy for all @@ -36,9 +28,38 @@ class StreamWrapper(object): def __getattr__(self, name): return getattr(self.__wrapped, name) + def __enter__(self, *args, **kwargs): + # special method lookup bypasses __getattr__/__getattribute__, see + # https://stackoverflow.com/questions/12632894/why-doesnt-getattr-work-with-exit + # thus, contextlib magic methods are not proxied via __getattr__ + return self.__wrapped.__enter__(*args, **kwargs) + + def __exit__(self, *args, **kwargs): + return self.__wrapped.__exit__(*args, **kwargs) + def write(self, text): self.__convertor.write(text) + def isatty(self): + stream = self.__wrapped + if 'PYCHARM_HOSTED' in os.environ: + if stream is not None and (stream is sys.__stdout__ or stream is sys.__stderr__): + return True + try: + stream_isatty = stream.isatty + except AttributeError: + return False + else: + return stream_isatty() + + @property + def closed(self): + stream = self.__wrapped + try: + return stream.closed + except AttributeError: + return True + class AnsiToWin32(object): ''' @@ -68,12 +89,12 @@ class AnsiToWin32(object): # should we strip ANSI sequences from our output? if strip is None: - strip = conversion_supported or (not is_stream_closed(wrapped) and not is_a_tty(wrapped)) + strip = conversion_supported or (not self.stream.closed and not self.stream.isatty()) self.strip = strip # should we should convert ANSI sequences into win32 calls? if convert is None: - convert = conversion_supported and not is_stream_closed(wrapped) and is_a_tty(wrapped) + convert = conversion_supported and not self.stream.closed and self.stream.isatty() self.convert = convert # dict of ansi codes to win32 functions and parameters @@ -149,7 +170,7 @@ class AnsiToWin32(object): def reset_all(self): if self.convert: self.call_win32('m', (0,)) - elif not self.strip and not is_stream_closed(self.wrapped): + elif not self.strip and not self.stream.closed: self.wrapped.write(Style.RESET_ALL) diff --git a/pipenv/vendor/colorama/initialise.py b/pipenv/vendor/colorama/initialise.py index 834962a3..430d0668 100644 --- a/pipenv/vendor/colorama/initialise.py +++ b/pipenv/vendor/colorama/initialise.py @@ -78,5 +78,3 @@ def wrap_stream(stream, convert, strip, autoreset, wrap): if wrapper.should_wrap(): stream = wrapper.stream return stream - - diff --git a/pipenv/vendor/colorama/win32.py b/pipenv/vendor/colorama/win32.py index 8262e350..c2d83603 100644 --- a/pipenv/vendor/colorama/win32.py +++ b/pipenv/vendor/colorama/win32.py @@ -89,11 +89,6 @@ else: ] _SetConsoleTitleW.restype = wintypes.BOOL - handles = { - STDOUT: _GetStdHandle(STDOUT), - STDERR: _GetStdHandle(STDERR), - } - def _winapi_test(handle): csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( @@ -101,17 +96,18 @@ else: return bool(success) def winapi_test(): - return any(_winapi_test(h) for h in handles.values()) + return any(_winapi_test(h) for h in + (_GetStdHandle(STDOUT), _GetStdHandle(STDERR))) def GetConsoleScreenBufferInfo(stream_id=STDOUT): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) csbi = CONSOLE_SCREEN_BUFFER_INFO() success = _GetConsoleScreenBufferInfo( handle, byref(csbi)) return csbi def SetConsoleTextAttribute(stream_id, attrs): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleTextAttribute(handle, attrs) def SetConsoleCursorPosition(stream_id, position, adjust=True): @@ -129,11 +125,11 @@ else: adjusted_position.Y += sr.Top adjusted_position.X += sr.Left # Resume normal processing - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) return _SetConsoleCursorPosition(handle, adjusted_position) def FillConsoleOutputCharacter(stream_id, char, length, start): - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) char = c_char(char.encode()) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) @@ -144,7 +140,7 @@ else: def FillConsoleOutputAttribute(stream_id, attr, length, start): ''' FillConsoleOutputAttribute( hConsole, csbi.wAttributes, dwConSize, coordScreen, &cCharsWritten )''' - handle = handles[stream_id] + handle = _GetStdHandle(stream_id) attribute = wintypes.WORD(attr) length = wintypes.DWORD(length) num_written = wintypes.DWORD(0) diff --git a/pipenv/vendor/colorama/winterm.py b/pipenv/vendor/colorama/winterm.py index 60309d3c..0fdb4ec4 100644 --- a/pipenv/vendor/colorama/winterm.py +++ b/pipenv/vendor/colorama/winterm.py @@ -44,6 +44,7 @@ class WinTerm(object): def reset_all(self, on_stderr=None): self.set_attrs(self._default) self.set_console(attrs=self._default) + self._light = 0 def fore(self, fore=None, light=False, on_stderr=False): if fore is None: @@ -122,12 +123,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = cells_in_screen - cells_before_cursor - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, 0) cells_to_erase = cells_before_cursor elif mode == 2: from_coord = win32.COORD(0, 0) cells_to_erase = cells_in_screen + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly @@ -147,12 +151,15 @@ class WinTerm(object): if mode == 0: from_coord = csbi.dwCursorPosition cells_to_erase = csbi.dwSize.X - csbi.dwCursorPosition.X - if mode == 1: + elif mode == 1: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwCursorPosition.X elif mode == 2: from_coord = win32.COORD(0, csbi.dwCursorPosition.Y) cells_to_erase = csbi.dwSize.X + else: + # invalid mode + return # fill the entire screen with blanks win32.FillConsoleOutputCharacter(handle, ' ', cells_to_erase, from_coord) # now set the buffer's attributes accordingly diff --git a/pipenv/vendor/cursor/LICENSE b/pipenv/vendor/cursor/LICENSE deleted file mode 100644 index 00023c80..00000000 --- a/pipenv/vendor/cursor/LICENSE +++ /dev/null @@ -1,5 +0,0 @@ -This work is licensed under the Creative Commons -Attribution-ShareAlike 2.5 International License. To view a copy of -this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -send a letter to Creative Commons, PO Box 1866, Mountain View, -CA 94042, USA. diff --git a/pipenv/vendor/cursor/__init__.py b/pipenv/vendor/cursor/__init__.py deleted file mode 100644 index 76a4f671..00000000 --- a/pipenv/vendor/cursor/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .cursor import hide, show, HiddenCursor - -__all__ = ["hide", "show", "HiddenCursor"] - diff --git a/pipenv/vendor/cursor/cursor.py b/pipenv/vendor/cursor/cursor.py deleted file mode 100644 index e4407c02..00000000 --- a/pipenv/vendor/cursor/cursor.py +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env python2 -# -*- coding: utf-8 -*- - -## Author: James Spencer: http://stackoverflow.com/users/1375885/james-spencer -## Packager: Gijs TImmers: https://github.com/GijsTimmers - -## Based on James Spencer's answer on StackOverflow: -## http://stackoverflow.com/questions/5174810/how-to-turn-off-blinking-cursor-in-command-window - -## Licence: CC-BY-SA-2.5 -## http://creativecommons.org/licenses/by-sa/2.5/ - -## This work is licensed under the Creative Commons -## Attribution-ShareAlike 2.5 International License. To view a copy of -## this license, visit http://creativecommons.org/licenses/by-sa/2.5/ or -## send a letter to Creative Commons, PO Box 1866, Mountain View, -## CA 94042, USA. - -import sys -import os - -if os.name == 'nt': - import ctypes - - class _CursorInfo(ctypes.Structure): - _fields_ = [("size", ctypes.c_int), - ("visible", ctypes.c_byte)] - -def hide(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = False - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25l") - stream.flush() - -def show(stream=sys.stdout): - if os.name == 'nt': - ci = _CursorInfo() - handle = ctypes.windll.kernel32.GetStdHandle(-11) - ctypes.windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(ci)) - ci.visible = True - ctypes.windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(ci)) - elif os.name == 'posix': - stream.write("\033[?25h") - stream.flush() - -class HiddenCursor(object): - def __init__(self, stream=sys.stdout): - self._stream = stream - def __enter__(self): - hide(stream=self._stream) - def __exit__(self, type, value, traceback): - show(stream=self._stream) \ No newline at end of file diff --git a/pipenv/vendor/dotenv/compat.py b/pipenv/vendor/dotenv/compat.py index c4a481e6..f6baa361 100644 --- a/pipenv/vendor/dotenv/compat.py +++ b/pipenv/vendor/dotenv/compat.py @@ -1,4 +1,9 @@ +import sys try: from StringIO import StringIO # noqa except ImportError: from io import StringIO # noqa + +PY2 = sys.version_info[0] == 2 +WIN = sys.platform.startswith('win') +text_type = unicode if PY2 else str # noqa diff --git a/pipenv/vendor/dotenv/environ.py b/pipenv/vendor/dotenv/environ.py new file mode 100644 index 00000000..ad357165 --- /dev/null +++ b/pipenv/vendor/dotenv/environ.py @@ -0,0 +1,54 @@ +import os + + +class UndefinedValueError(Exception): + pass + + +class Undefined(object): + """Class to represent undefined type. """ + pass + + +# Reference instance to represent undefined values +undefined = Undefined() + + +def _cast_boolean(value): + """ + Helper to convert config values to boolean as ConfigParser do. + """ + _BOOLEANS = {'1': True, 'yes': True, 'true': True, 'on': True, + '0': False, 'no': False, 'false': False, 'off': False, '': False} + value = str(value) + if value.lower() not in _BOOLEANS: + raise ValueError('Not a boolean: %s' % value) + + return _BOOLEANS[value.lower()] + + +def getenv(option, default=undefined, cast=undefined): + """ + Return the value for option or default if defined. + """ + + # We can't avoid __contains__ because value may be empty. + if option in os.environ: + value = os.environ[option] + else: + if isinstance(default, Undefined): + raise UndefinedValueError('{} not found. Declare it as envvar or define a default value.'.format(option)) + + value = default + + if isinstance(cast, Undefined): + return value + + if cast is bool: + value = _cast_boolean(value) + elif cast is list: + value = [x for x in value.split(',') if x] + else: + value = cast(value) + + return value diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py index 6ba28bbb..98b22ec0 100644 --- a/pipenv/vendor/dotenv/main.py +++ b/pipenv/vendor/dotenv/main.py @@ -2,47 +2,90 @@ from __future__ import absolute_import, print_function, unicode_literals import codecs -import fileinput import io import os import re +import shutil import sys -from subprocess import Popen, PIPE, STDOUT +from subprocess import Popen +import tempfile import warnings -from collections import OrderedDict +from collections import OrderedDict, namedtuple +from contextlib import contextmanager -from .compat import StringIO +from .compat import StringIO, PY2, WIN, text_type -__escape_decoder = codecs.getdecoder('unicode_escape') -__posix_variable = re.compile('\$\{[^\}]*\}') +__posix_variable = re.compile(r'\$\{[^\}]*\}') + +_binding = re.compile( + r""" + ( + \s* # leading whitespace + (?:export{0}+)? # export + + ( '[^']+' # single-quoted key + | [^=\#\s]+ # or unquoted key + )? + + (?: + (?:{0}*={0}*) # equal sign + + ( '(?:\\'|[^'])*' # single-quoted value + | "(?:\\"|[^"])*" # or double-quoted value + | [^\#\r\n]* # or unquoted value + ) + )? + + \s* # trailing whitespace + (?:\#[^\r\n]*)? # comment + (?:\r|\n|\r\n)? # newline + ) + """.format(r'[^\S\r\n]'), + re.MULTILINE | re.VERBOSE, +) + +_escape_sequence = re.compile(r"\\[\\'\"abfnrtv]") -def decode_escaped(escaped): - return __escape_decoder(escaped)[0] +Binding = namedtuple('Binding', 'key value original') -def parse_line(line): - line = line.strip() +def decode_escapes(string): + def decode_match(match): + return codecs.decode(match.group(0), 'unicode-escape') - # Ignore lines with `#` or which doesn't have `=` in it. - if not line or line.startswith('#') or '=' not in line: - return None, None + return _escape_sequence.sub(decode_match, string) - k, v = line.split('=', 1) - if k.startswith('export '): - (_, _, k) = k.partition('export ') +def is_surrounded_by(string, char): + return ( + len(string) > 1 + and string[0] == string[-1] == char + ) - # Remove any leading and trailing spaces in key, value - k, v = k.strip(), v.strip() - if v: - v = v.encode('unicode-escape').decode('ascii') - quoted = v[0] == v[-1] in ['"', "'"] - if quoted: - v = decode_escaped(v[1:-1]) +def parse_binding(string, position): + match = _binding.match(string, position) + (matched, key, value) = match.groups() + if key is None or value is None: + key = None + value = None + else: + value_quoted = is_surrounded_by(value, "'") or is_surrounded_by(value, '"') + if value_quoted: + value = decode_escapes(value[1:-1]) + else: + value = value.strip() + return (Binding(key=key, value=value, original=matched), match.end()) - return k, v + +def parse_stream(stream): + string = stream.read() + position = 0 + length = len(string) + while position < length: + (binding, position) = parse_binding(string, position) + yield binding class DotEnv(): @@ -52,19 +95,17 @@ class DotEnv(): self._dict = None self.verbose = verbose + @contextmanager def _get_stream(self): - self._is_file = False if isinstance(self.dotenv_path, StringIO): - return self.dotenv_path - - if os.path.exists(self.dotenv_path): - self._is_file = True - return io.open(self.dotenv_path) - - if self.verbose: - warnings.warn("File doesn't exist {}".format(self.dotenv_path)) - - return StringIO('') + yield self.dotenv_path + elif os.path.isfile(self.dotenv_path): + with io.open(self.dotenv_path) as stream: + yield stream + else: + if self.verbose: + warnings.warn("File doesn't exist {}".format(self.dotenv_path)) + yield StringIO('') def dict(self): """Return dotenv as dict""" @@ -76,17 +117,10 @@ class DotEnv(): return self._dict def parse(self): - f = self._get_stream() - - for line in f: - key, value = parse_line(line) - if not key: - continue - - yield key, value - - if self._is_file: - f.close() + with self._get_stream() as stream: + for mapping in parse_stream(stream): + if mapping.key is not None and mapping.value is not None: + yield mapping.key, mapping.value def set_as_environment_variables(self, override=False): """ @@ -95,13 +129,12 @@ class DotEnv(): for k, v in self.dict().items(): if k in os.environ and not override: continue - # With Python 2 on Windows, ensuree environment variables are - # system strings to avoid "TypeError: environment can only contain - # strings" in Python's subprocess module. - if sys.version_info.major < 3 and sys.platform == 'win32': - from pipenv.utils import fs_str - k = fs_str(k) - v = fs_str(v) + # With Python2 on Windows, force environment variables to str to avoid + # "TypeError: environment can only contain strings" in Python's subprocess.py. + if PY2 and WIN: + if isinstance(k, text_type) or isinstance(v, text_type): + k = k.encode('ascii') + v = v.encode('ascii') os.environ[k] = v return True @@ -127,6 +160,20 @@ def get_key(dotenv_path, key_to_get): return DotEnv(dotenv_path, verbose=True).get(key_to_get) +@contextmanager +def rewrite(path): + try: + with tempfile.NamedTemporaryFile(mode="w+", delete=False) as dest: + with io.open(path) as source: + yield (source, dest) + except BaseException: + if os.path.isfile(dest.name): + os.unlink(dest.name) + raise + else: + shutil.move(dest.name, path) + + def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): """ Adds or Updates a key/value to the given .env @@ -142,20 +189,19 @@ def set_key(dotenv_path, key_to_set, value_to_set, quote_mode="always"): if " " in value_to_set: quote_mode = "always" - line_template = '{}="{}"' if quote_mode == "always" else '{}={}' + line_template = '{}="{}"\n' if quote_mode == "always" else '{}={}\n' line_out = line_template.format(key_to_set, value_to_set) - replaced = False - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_set: - replaced = True - line = line_out - print(line, end='') - - if not replaced: - with io.open(dotenv_path, "a") as f: - f.write("{}\n".format(line_out)) + with rewrite(dotenv_path) as (source, dest): + replaced = False + for mapping in parse_stream(source): + if mapping.key == key_to_set: + dest.write(line_out) + replaced = True + else: + dest.write(mapping.original) + if not replaced: + dest.write(line_out) return True, key_to_set, value_to_set @@ -167,18 +213,17 @@ def unset_key(dotenv_path, key_to_unset, quote_mode="always"): If the .env path given doesn't exist, fails If the given key doesn't exist in the .env, fails """ - removed = False - if not os.path.exists(dotenv_path): warnings.warn("can't delete from %s - it doesn't exist." % dotenv_path) return None, key_to_unset - for line in fileinput.input(dotenv_path, inplace=True): - k, v = parse_line(line) - if k == key_to_unset: - removed = True - line = '' - print(line, end='') + removed = False + with rewrite(dotenv_path) as (source, dest): + for mapping in parse_stream(source): + if mapping.key == key_to_unset: + removed = True + else: + dest.write(mapping.original) if not removed: warnings.warn("key %s not removed from %s - key doesn't exist." % (key_to_unset, dotenv_path)) @@ -194,7 +239,7 @@ def resolve_nested_variables(values): first search in environ, if not found, then look into the dotenv variables """ - ret = os.getenv(name, values.get(name, "")) + ret = os.getenv(name, new_values.get(name, "")) return ret def _re_sub_callback(match_object): @@ -204,10 +249,12 @@ def resolve_nested_variables(values): """ return _replacement(match_object.group()[2:-1]) - for k, v in values.items(): - values[k] = __posix_variable.sub(_re_sub_callback, v) + new_values = {} - return values + for k, v in values.items(): + new_values[k] = __posix_variable.sub(_re_sub_callback, v) + + return new_values def _walk_to_root(path): @@ -248,7 +295,7 @@ def find_dotenv(filename='.env', raise_error_if_not_found=False, usecwd=False): for dirname in _walk_to_root(path): check_path = os.path.join(dirname, filename) - if os.path.exists(check_path): + if os.path.isfile(check_path): return check_path if raise_error_if_not_found: @@ -292,19 +339,10 @@ def run_command(command, env): cmd_env.update(env) p = Popen(command, - stdin=PIPE, - stdout=PIPE, - stderr=STDOUT, universal_newlines=True, bufsize=0, shell=False, env=cmd_env) - try: - out, _ = p.communicate() - print(out) - except Exception: - warnings.warn('An error occured, running the command:') - out, _ = p.communicate() - warnings.warn(out) + _, _ = p.communicate() return p.returncode diff --git a/pipenv/vendor/dotenv/version.py b/pipenv/vendor/dotenv/version.py index d69d16e9..1f4c4d43 100644 --- a/pipenv/vendor/dotenv/version.py +++ b/pipenv/vendor/dotenv/version.py @@ -1 +1 @@ -__version__ = "0.9.1" +__version__ = "0.10.1" diff --git a/pipenv/vendor/idna/__init__.py b/pipenv/vendor/idna/__init__.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/codec.py b/pipenv/vendor/idna/codec.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/compat.py b/pipenv/vendor/idna/compat.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/core.py b/pipenv/vendor/idna/core.py old mode 100755 new mode 100644 index 090c2c18..104624ad --- a/pipenv/vendor/idna/core.py +++ b/pipenv/vendor/idna/core.py @@ -267,10 +267,7 @@ def alabel(label): try: label = label.encode('ascii') - try: - ulabel(label) - except IDNAError: - raise IDNAError('The label {0} is not a valid A-label'.format(label)) + ulabel(label) if not valid_label_length(label): raise IDNAError('Label too long') return label diff --git a/pipenv/vendor/idna/idnadata.py b/pipenv/vendor/idna/idnadata.py old mode 100755 new mode 100644 index 17974e23..a80c959d --- a/pipenv/vendor/idna/idnadata.py +++ b/pipenv/vendor/idna/idnadata.py @@ -1,6 +1,6 @@ # This file is automatically generated by tools/idna-data -__version__ = "10.0.0" +__version__ = "11.0.0" scripts = { 'Greek': ( 0x37000000374, @@ -49,7 +49,7 @@ scripts = { 0x30210000302a, 0x30380000303c, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xf9000000fa6e, 0xfa700000fada, 0x200000002a6d7, @@ -62,7 +62,7 @@ scripts = { 'Hebrew': ( 0x591000005c8, 0x5d0000005eb, - 0x5f0000005f5, + 0x5ef000005f5, 0xfb1d0000fb37, 0xfb380000fb3d, 0xfb3e0000fb3f, @@ -248,6 +248,7 @@ joining_types = { 0x6fb: 68, 0x6fc: 68, 0x6ff: 68, + 0x70f: 84, 0x710: 82, 0x712: 68, 0x713: 68, @@ -522,6 +523,7 @@ joining_types = { 0x1875: 68, 0x1876: 68, 0x1877: 68, + 0x1878: 68, 0x1880: 85, 0x1881: 85, 0x1882: 85, @@ -690,6 +692,70 @@ joining_types = { 0x10bad: 68, 0x10bae: 68, 0x10baf: 85, + 0x10d00: 76, + 0x10d01: 68, + 0x10d02: 68, + 0x10d03: 68, + 0x10d04: 68, + 0x10d05: 68, + 0x10d06: 68, + 0x10d07: 68, + 0x10d08: 68, + 0x10d09: 68, + 0x10d0a: 68, + 0x10d0b: 68, + 0x10d0c: 68, + 0x10d0d: 68, + 0x10d0e: 68, + 0x10d0f: 68, + 0x10d10: 68, + 0x10d11: 68, + 0x10d12: 68, + 0x10d13: 68, + 0x10d14: 68, + 0x10d15: 68, + 0x10d16: 68, + 0x10d17: 68, + 0x10d18: 68, + 0x10d19: 68, + 0x10d1a: 68, + 0x10d1b: 68, + 0x10d1c: 68, + 0x10d1d: 68, + 0x10d1e: 68, + 0x10d1f: 68, + 0x10d20: 68, + 0x10d21: 68, + 0x10d22: 82, + 0x10d23: 68, + 0x10f30: 68, + 0x10f31: 68, + 0x10f32: 68, + 0x10f33: 82, + 0x10f34: 68, + 0x10f35: 68, + 0x10f36: 68, + 0x10f37: 68, + 0x10f38: 68, + 0x10f39: 68, + 0x10f3a: 68, + 0x10f3b: 68, + 0x10f3c: 68, + 0x10f3d: 68, + 0x10f3e: 68, + 0x10f3f: 68, + 0x10f40: 68, + 0x10f41: 68, + 0x10f42: 68, + 0x10f43: 68, + 0x10f44: 68, + 0x10f45: 85, + 0x10f51: 68, + 0x10f52: 68, + 0x10f53: 68, + 0x10f54: 82, + 0x110bd: 85, + 0x110cd: 85, 0x1e900: 68, 0x1e901: 68, 0x1e902: 68, @@ -1034,14 +1100,15 @@ codepoint_classes = { 0x52d0000052e, 0x52f00000530, 0x5590000055a, - 0x56100000587, + 0x56000000587, + 0x58800000589, 0x591000005be, 0x5bf000005c0, 0x5c1000005c3, 0x5c4000005c6, 0x5c7000005c8, 0x5d0000005eb, - 0x5f0000005f3, + 0x5ef000005f3, 0x6100000061b, 0x62000000640, 0x64100000660, @@ -1054,12 +1121,13 @@ codepoint_classes = { 0x7100000074b, 0x74d000007b2, 0x7c0000007f6, + 0x7fd000007fe, 0x8000000082e, 0x8400000085c, 0x8600000086b, 0x8a0000008b5, 0x8b6000008be, - 0x8d4000008e2, + 0x8d3000008e2, 0x8e300000958, 0x96000000964, 0x96600000970, @@ -1077,6 +1145,7 @@ codepoint_classes = { 0x9e0000009e4, 0x9e6000009f2, 0x9fc000009fd, + 0x9fe000009ff, 0xa0100000a04, 0xa0500000a0b, 0xa0f00000a11, @@ -1136,8 +1205,7 @@ codepoint_classes = { 0xbd000000bd1, 0xbd700000bd8, 0xbe600000bf0, - 0xc0000000c04, - 0xc0500000c0d, + 0xc0000000c0d, 0xc0e00000c11, 0xc1200000c29, 0xc2a00000c3a, @@ -1276,7 +1344,7 @@ codepoint_classes = { 0x17dc000017de, 0x17e0000017ea, 0x18100000181a, - 0x182000001878, + 0x182000001879, 0x1880000018ab, 0x18b0000018f6, 0x19000000191f, @@ -1544,11 +1612,11 @@ codepoint_classes = { 0x309d0000309f, 0x30a1000030fb, 0x30fc000030ff, - 0x31050000312f, + 0x310500003130, 0x31a0000031bb, 0x31f000003200, 0x340000004db6, - 0x4e0000009feb, + 0x4e0000009ff0, 0xa0000000a48d, 0xa4d00000a4fe, 0xa5000000a60d, @@ -1655,8 +1723,10 @@ codepoint_classes = { 0xa7a50000a7a6, 0xa7a70000a7a8, 0xa7a90000a7aa, + 0xa7af0000a7b0, 0xa7b50000a7b6, 0xa7b70000a7b8, + 0xa7b90000a7ba, 0xa7f70000a7f8, 0xa7fa0000a828, 0xa8400000a874, @@ -1664,8 +1734,7 @@ codepoint_classes = { 0xa8d00000a8da, 0xa8e00000a8f8, 0xa8fb0000a8fc, - 0xa8fd0000a8fe, - 0xa9000000a92e, + 0xa8fd0000a92e, 0xa9300000a954, 0xa9800000a9c1, 0xa9cf0000a9da, @@ -1743,7 +1812,7 @@ codepoint_classes = { 0x10a0500010a07, 0x10a0c00010a14, 0x10a1500010a18, - 0x10a1900010a34, + 0x10a1900010a36, 0x10a3800010a3b, 0x10a3f00010a40, 0x10a6000010a7d, @@ -1756,6 +1825,11 @@ codepoint_classes = { 0x10b8000010b92, 0x10c0000010c49, 0x10cc000010cf3, + 0x10d0000010d28, + 0x10d3000010d3a, + 0x10f0000010f1d, + 0x10f2700010f28, + 0x10f3000010f51, 0x1100000011047, 0x1106600011070, 0x1107f000110bb, @@ -1763,10 +1837,11 @@ codepoint_classes = { 0x110f0000110fa, 0x1110000011135, 0x1113600011140, + 0x1114400011147, 0x1115000011174, 0x1117600011177, 0x11180000111c5, - 0x111ca000111cd, + 0x111c9000111cd, 0x111d0000111db, 0x111dc000111dd, 0x1120000011212, @@ -1786,7 +1861,7 @@ codepoint_classes = { 0x1132a00011331, 0x1133200011334, 0x113350001133a, - 0x1133c00011345, + 0x1133b00011345, 0x1134700011349, 0x1134b0001134e, 0x1135000011351, @@ -1796,6 +1871,7 @@ codepoint_classes = { 0x1137000011375, 0x114000001144b, 0x114500001145a, + 0x1145e0001145f, 0x11480000114c6, 0x114c7000114c8, 0x114d0000114da, @@ -1807,15 +1883,17 @@ codepoint_classes = { 0x116500001165a, 0x11680000116b8, 0x116c0000116ca, - 0x117000001171a, + 0x117000001171b, 0x1171d0001172c, 0x117300001173a, + 0x118000001183b, 0x118c0000118ea, 0x118ff00011900, 0x11a0000011a3f, 0x11a4700011a48, 0x11a5000011a84, 0x11a8600011a9a, + 0x11a9d00011a9e, 0x11ac000011af9, 0x11c0000011c09, 0x11c0a00011c37, @@ -1831,6 +1909,13 @@ codepoint_classes = { 0x11d3c00011d3e, 0x11d3f00011d48, 0x11d5000011d5a, + 0x11d6000011d66, + 0x11d6700011d69, + 0x11d6a00011d8f, + 0x11d9000011d92, + 0x11d9300011d99, + 0x11da000011daa, + 0x11ee000011ef7, 0x120000001239a, 0x1248000012544, 0x130000001342f, @@ -1845,11 +1930,12 @@ codepoint_classes = { 0x16b5000016b5a, 0x16b6300016b78, 0x16b7d00016b90, + 0x16e6000016e80, 0x16f0000016f45, 0x16f5000016f7f, 0x16f8f00016fa0, 0x16fe000016fe2, - 0x17000000187ed, + 0x17000000187f2, 0x1880000018af3, 0x1b0000001b11f, 0x1b1700001b2fc, diff --git a/pipenv/vendor/idna/intranges.py b/pipenv/vendor/idna/intranges.py old mode 100755 new mode 100644 diff --git a/pipenv/vendor/idna/package_data.py b/pipenv/vendor/idna/package_data.py old mode 100755 new mode 100644 index 39c192ba..257e8989 --- a/pipenv/vendor/idna/package_data.py +++ b/pipenv/vendor/idna/package_data.py @@ -1,2 +1,2 @@ -__version__ = '2.7' +__version__ = '2.8' diff --git a/pipenv/vendor/idna/uts46data.py b/pipenv/vendor/idna/uts46data.py old mode 100755 new mode 100644 index 79731cb9..a68ed4c0 --- a/pipenv/vendor/idna/uts46data.py +++ b/pipenv/vendor/idna/uts46data.py @@ -4,7 +4,7 @@ """IDNA Mapping Table from UTS46.""" -__version__ = "10.0.0" +__version__ = "11.0.0" def _seg_0(): return [ (0x0, '3'), @@ -1029,11 +1029,8 @@ def _seg_9(): (0x556, 'M', u'ֆ'), (0x557, 'X'), (0x559, 'V'), - (0x560, 'X'), - (0x561, 'V'), (0x587, 'M', u'եւ'), - (0x588, 'X'), - (0x589, 'V'), + (0x588, 'V'), (0x58B, 'X'), (0x58D, 'V'), (0x590, 'X'), @@ -1041,15 +1038,15 @@ def _seg_9(): (0x5C8, 'X'), (0x5D0, 'V'), (0x5EB, 'X'), - (0x5F0, 'V'), + (0x5EF, 'V'), (0x5F5, 'X'), + (0x606, 'V'), + (0x61C, 'X'), + (0x61E, 'V'), ] def _seg_10(): return [ - (0x606, 'V'), - (0x61C, 'X'), - (0x61E, 'V'), (0x675, 'M', u'اٴ'), (0x676, 'M', u'وٴ'), (0x677, 'M', u'ۇٴ'), @@ -1064,7 +1061,7 @@ def _seg_10(): (0x7B2, 'X'), (0x7C0, 'V'), (0x7FB, 'X'), - (0x800, 'V'), + (0x7FD, 'V'), (0x82E, 'X'), (0x830, 'V'), (0x83F, 'X'), @@ -1078,7 +1075,7 @@ def _seg_10(): (0x8B5, 'X'), (0x8B6, 'V'), (0x8BE, 'X'), - (0x8D4, 'V'), + (0x8D3, 'V'), (0x8E2, 'X'), (0x8E3, 'V'), (0x958, 'M', u'क़'), @@ -1118,7 +1115,7 @@ def _seg_10(): (0x9E0, 'V'), (0x9E4, 'X'), (0x9E6, 'V'), - (0x9FE, 'X'), + (0x9FF, 'X'), (0xA01, 'V'), (0xA04, 'X'), (0xA05, 'V'), @@ -1147,19 +1144,19 @@ def _seg_10(): (0xA4E, 'X'), (0xA51, 'V'), (0xA52, 'X'), + (0xA59, 'M', u'ਖ਼'), + (0xA5A, 'M', u'ਗ਼'), + (0xA5B, 'M', u'ਜ਼'), ] def _seg_11(): return [ - (0xA59, 'M', u'ਖ਼'), - (0xA5A, 'M', u'ਗ਼'), - (0xA5B, 'M', u'ਜ਼'), (0xA5C, 'V'), (0xA5D, 'X'), (0xA5E, 'M', u'ਫ਼'), (0xA5F, 'X'), (0xA66, 'V'), - (0xA76, 'X'), + (0xA77, 'X'), (0xA81, 'V'), (0xA84, 'X'), (0xA85, 'V'), @@ -1250,16 +1247,14 @@ def _seg_11(): (0xBE6, 'V'), (0xBFB, 'X'), (0xC00, 'V'), - (0xC04, 'X'), - ] - -def _seg_12(): - return [ - (0xC05, 'V'), (0xC0D, 'X'), (0xC0E, 'V'), (0xC11, 'X'), (0xC12, 'V'), + ] + +def _seg_12(): + return [ (0xC29, 'X'), (0xC2A, 'V'), (0xC3A, 'X'), @@ -1278,8 +1273,6 @@ def _seg_12(): (0xC66, 'V'), (0xC70, 'X'), (0xC78, 'V'), - (0xC84, 'X'), - (0xC85, 'V'), (0xC8D, 'X'), (0xC8E, 'V'), (0xC91, 'X'), @@ -1355,10 +1348,6 @@ def _seg_12(): (0xE83, 'X'), (0xE84, 'V'), (0xE85, 'X'), - ] - -def _seg_13(): - return [ (0xE87, 'V'), (0xE89, 'X'), (0xE8A, 'V'), @@ -1366,6 +1355,10 @@ def _seg_13(): (0xE8D, 'V'), (0xE8E, 'X'), (0xE94, 'V'), + ] + +def _seg_13(): + return [ (0xE98, 'X'), (0xE99, 'V'), (0xEA0, 'X'), @@ -1459,10 +1452,6 @@ def _seg_13(): (0x124E, 'X'), (0x1250, 'V'), (0x1257, 'X'), - ] - -def _seg_14(): - return [ (0x1258, 'V'), (0x1259, 'X'), (0x125A, 'V'), @@ -1470,6 +1459,10 @@ def _seg_14(): (0x1260, 'V'), (0x1289, 'X'), (0x128A, 'V'), + ] + +def _seg_14(): + return [ (0x128E, 'X'), (0x1290, 'V'), (0x12B1, 'X'), @@ -1538,7 +1531,7 @@ def _seg_14(): (0x1810, 'V'), (0x181A, 'X'), (0x1820, 'V'), - (0x1878, 'X'), + (0x1879, 'X'), (0x1880, 'V'), (0x18AB, 'X'), (0x18B0, 'V'), @@ -1563,10 +1556,6 @@ def _seg_14(): (0x19DB, 'X'), (0x19DE, 'V'), (0x1A1C, 'X'), - ] - -def _seg_15(): - return [ (0x1A1E, 'V'), (0x1A5F, 'X'), (0x1A60, 'V'), @@ -1574,6 +1563,10 @@ def _seg_15(): (0x1A7F, 'V'), (0x1A8A, 'X'), (0x1A90, 'V'), + ] + +def _seg_15(): + return [ (0x1A9A, 'X'), (0x1AA0, 'V'), (0x1AAE, 'X'), @@ -1667,10 +1660,6 @@ def _seg_15(): (0x1D68, 'M', u'ρ'), (0x1D69, 'M', u'φ'), (0x1D6A, 'M', u'χ'), - ] - -def _seg_16(): - return [ (0x1D6B, 'V'), (0x1D78, 'M', u'н'), (0x1D79, 'V'), @@ -1678,6 +1667,10 @@ def _seg_16(): (0x1D9C, 'M', u'c'), (0x1D9D, 'M', u'ɕ'), (0x1D9E, 'M', u'ð'), + ] + +def _seg_16(): + return [ (0x1D9F, 'M', u'ɜ'), (0x1DA0, 'M', u'f'), (0x1DA1, 'M', u'ɟ'), @@ -1771,10 +1764,6 @@ def _seg_16(): (0x1E36, 'M', u'ḷ'), (0x1E37, 'V'), (0x1E38, 'M', u'ḹ'), - ] - -def _seg_17(): - return [ (0x1E39, 'V'), (0x1E3A, 'M', u'ḻ'), (0x1E3B, 'V'), @@ -1782,6 +1771,10 @@ def _seg_17(): (0x1E3D, 'V'), (0x1E3E, 'M', u'ḿ'), (0x1E3F, 'V'), + ] + +def _seg_17(): + return [ (0x1E40, 'M', u'ṁ'), (0x1E41, 'V'), (0x1E42, 'M', u'ṃ'), @@ -1875,10 +1868,6 @@ def _seg_17(): (0x1E9F, 'V'), (0x1EA0, 'M', u'ạ'), (0x1EA1, 'V'), - ] - -def _seg_18(): - return [ (0x1EA2, 'M', u'ả'), (0x1EA3, 'V'), (0x1EA4, 'M', u'ấ'), @@ -1886,6 +1875,10 @@ def _seg_18(): (0x1EA6, 'M', u'ầ'), (0x1EA7, 'V'), (0x1EA8, 'M', u'ẩ'), + ] + +def _seg_18(): + return [ (0x1EA9, 'V'), (0x1EAA, 'M', u'ẫ'), (0x1EAB, 'V'), @@ -1979,10 +1972,6 @@ def _seg_18(): (0x1F0B, 'M', u'ἃ'), (0x1F0C, 'M', u'ἄ'), (0x1F0D, 'M', u'ἅ'), - ] - -def _seg_19(): - return [ (0x1F0E, 'M', u'ἆ'), (0x1F0F, 'M', u'ἇ'), (0x1F10, 'V'), @@ -1990,6 +1979,10 @@ def _seg_19(): (0x1F18, 'M', u'ἐ'), (0x1F19, 'M', u'ἑ'), (0x1F1A, 'M', u'ἒ'), + ] + +def _seg_19(): + return [ (0x1F1B, 'M', u'ἓ'), (0x1F1C, 'M', u'ἔ'), (0x1F1D, 'M', u'ἕ'), @@ -2083,10 +2076,6 @@ def _seg_19(): (0x1F9A, 'M', u'ἢι'), (0x1F9B, 'M', u'ἣι'), (0x1F9C, 'M', u'ἤι'), - ] - -def _seg_20(): - return [ (0x1F9D, 'M', u'ἥι'), (0x1F9E, 'M', u'ἦι'), (0x1F9F, 'M', u'ἧι'), @@ -2094,6 +2083,10 @@ def _seg_20(): (0x1FA1, 'M', u'ὡι'), (0x1FA2, 'M', u'ὢι'), (0x1FA3, 'M', u'ὣι'), + ] + +def _seg_20(): + return [ (0x1FA4, 'M', u'ὤι'), (0x1FA5, 'M', u'ὥι'), (0x1FA6, 'M', u'ὦι'), @@ -2187,10 +2180,6 @@ def _seg_20(): (0x2024, 'X'), (0x2027, 'V'), (0x2028, 'X'), - ] - -def _seg_21(): - return [ (0x202F, '3', u' '), (0x2030, 'V'), (0x2033, 'M', u'′′'), @@ -2198,6 +2187,10 @@ def _seg_21(): (0x2035, 'V'), (0x2036, 'M', u'‵‵'), (0x2037, 'M', u'‵‵‵'), + ] + +def _seg_21(): + return [ (0x2038, 'V'), (0x203C, '3', u'!!'), (0x203D, 'V'), @@ -2291,10 +2284,6 @@ def _seg_21(): (0x2120, 'M', u'sm'), (0x2121, 'M', u'tel'), (0x2122, 'M', u'tm'), - ] - -def _seg_22(): - return [ (0x2123, 'V'), (0x2124, 'M', u'z'), (0x2125, 'V'), @@ -2302,6 +2291,10 @@ def _seg_22(): (0x2127, 'V'), (0x2128, 'M', u'z'), (0x2129, 'V'), + ] + +def _seg_22(): + return [ (0x212A, 'M', u'k'), (0x212B, 'M', u'å'), (0x212C, 'M', u'b'), @@ -2395,10 +2388,6 @@ def _seg_22(): (0x226E, '3'), (0x2270, 'V'), (0x2329, 'M', u'〈'), - ] - -def _seg_23(): - return [ (0x232A, 'M', u'〉'), (0x232B, 'V'), (0x2427, 'X'), @@ -2406,6 +2395,10 @@ def _seg_23(): (0x244B, 'X'), (0x2460, 'M', u'1'), (0x2461, 'M', u'2'), + ] + +def _seg_23(): + return [ (0x2462, 'M', u'3'), (0x2463, 'M', u'4'), (0x2464, 'M', u'5'), @@ -2499,10 +2492,6 @@ def _seg_23(): (0x24CF, 'M', u'z'), (0x24D0, 'M', u'a'), (0x24D1, 'M', u'b'), - ] - -def _seg_24(): - return [ (0x24D2, 'M', u'c'), (0x24D3, 'M', u'd'), (0x24D4, 'M', u'e'), @@ -2510,6 +2499,10 @@ def _seg_24(): (0x24D6, 'M', u'g'), (0x24D7, 'M', u'h'), (0x24D8, 'M', u'i'), + ] + +def _seg_24(): + return [ (0x24D9, 'M', u'j'), (0x24DA, 'M', u'k'), (0x24DB, 'M', u'l'), @@ -2541,13 +2534,9 @@ def _seg_24(): (0x2B76, 'V'), (0x2B96, 'X'), (0x2B98, 'V'), - (0x2BBA, 'X'), - (0x2BBD, 'V'), (0x2BC9, 'X'), (0x2BCA, 'V'), - (0x2BD3, 'X'), - (0x2BEC, 'V'), - (0x2BF0, 'X'), + (0x2BFF, 'X'), (0x2C00, 'M', u'ⰰ'), (0x2C01, 'M', u'ⰱ'), (0x2C02, 'M', u'ⰲ'), @@ -2603,10 +2592,6 @@ def _seg_24(): (0x2C62, 'M', u'ɫ'), (0x2C63, 'M', u'ᵽ'), (0x2C64, 'M', u'ɽ'), - ] - -def _seg_25(): - return [ (0x2C65, 'V'), (0x2C67, 'M', u'ⱨ'), (0x2C68, 'V'), @@ -2618,6 +2603,10 @@ def _seg_25(): (0x2C6E, 'M', u'ɱ'), (0x2C6F, 'M', u'ɐ'), (0x2C70, 'M', u'ɒ'), + ] + +def _seg_25(): + return [ (0x2C71, 'V'), (0x2C72, 'M', u'ⱳ'), (0x2C73, 'V'), @@ -2707,10 +2696,6 @@ def _seg_25(): (0x2CCD, 'V'), (0x2CCE, 'M', u'ⳏ'), (0x2CCF, 'V'), - ] - -def _seg_26(): - return [ (0x2CD0, 'M', u'ⳑ'), (0x2CD1, 'V'), (0x2CD2, 'M', u'ⳓ'), @@ -2722,6 +2707,10 @@ def _seg_26(): (0x2CD8, 'M', u'ⳙ'), (0x2CD9, 'V'), (0x2CDA, 'M', u'ⳛ'), + ] + +def _seg_26(): + return [ (0x2CDB, 'V'), (0x2CDC, 'M', u'ⳝ'), (0x2CDD, 'V'), @@ -2768,7 +2757,7 @@ def _seg_26(): (0x2DD8, 'V'), (0x2DDF, 'X'), (0x2DE0, 'V'), - (0x2E4A, 'X'), + (0x2E4F, 'X'), (0x2E80, 'V'), (0x2E9A, 'X'), (0x2E9B, 'V'), @@ -2811,10 +2800,6 @@ def _seg_26(): (0x2F20, 'M', u'士'), (0x2F21, 'M', u'夂'), (0x2F22, 'M', u'夊'), - ] - -def _seg_27(): - return [ (0x2F23, 'M', u'夕'), (0x2F24, 'M', u'大'), (0x2F25, 'M', u'女'), @@ -2826,6 +2811,10 @@ def _seg_27(): (0x2F2B, 'M', u'尸'), (0x2F2C, 'M', u'屮'), (0x2F2D, 'M', u'山'), + ] + +def _seg_27(): + return [ (0x2F2E, 'M', u'巛'), (0x2F2F, 'M', u'工'), (0x2F30, 'M', u'己'), @@ -2915,10 +2904,6 @@ def _seg_27(): (0x2F84, 'M', u'至'), (0x2F85, 'M', u'臼'), (0x2F86, 'M', u'舌'), - ] - -def _seg_28(): - return [ (0x2F87, 'M', u'舛'), (0x2F88, 'M', u'舟'), (0x2F89, 'M', u'艮'), @@ -2930,6 +2915,10 @@ def _seg_28(): (0x2F8F, 'M', u'行'), (0x2F90, 'M', u'衣'), (0x2F91, 'M', u'襾'), + ] + +def _seg_28(): + return [ (0x2F92, 'M', u'見'), (0x2F93, 'M', u'角'), (0x2F94, 'M', u'言'), @@ -3019,13 +3008,9 @@ def _seg_28(): (0x309F, 'M', u'より'), (0x30A0, 'V'), (0x30FF, 'M', u'コト'), - ] - -def _seg_29(): - return [ (0x3100, 'X'), (0x3105, 'V'), - (0x312F, 'X'), + (0x3130, 'X'), (0x3131, 'M', u'ᄀ'), (0x3132, 'M', u'ᄁ'), (0x3133, 'M', u'ᆪ'), @@ -3034,6 +3019,10 @@ def _seg_29(): (0x3136, 'M', u'ᆭ'), (0x3137, 'M', u'ᄃ'), (0x3138, 'M', u'ᄄ'), + ] + +def _seg_29(): + return [ (0x3139, 'M', u'ᄅ'), (0x313A, 'M', u'ᆰ'), (0x313B, 'M', u'ᆱ'), @@ -3123,10 +3112,6 @@ def _seg_29(): (0x318F, 'X'), (0x3190, 'V'), (0x3192, 'M', u'一'), - ] - -def _seg_30(): - return [ (0x3193, 'M', u'二'), (0x3194, 'M', u'三'), (0x3195, 'M', u'四'), @@ -3138,6 +3123,10 @@ def _seg_30(): (0x319B, 'M', u'丙'), (0x319C, 'M', u'丁'), (0x319D, 'M', u'天'), + ] + +def _seg_30(): + return [ (0x319E, 'M', u'地'), (0x319F, 'M', u'人'), (0x31A0, 'V'), @@ -3227,10 +3216,6 @@ def _seg_30(): (0x3256, 'M', u'26'), (0x3257, 'M', u'27'), (0x3258, 'M', u'28'), - ] - -def _seg_31(): - return [ (0x3259, 'M', u'29'), (0x325A, 'M', u'30'), (0x325B, 'M', u'31'), @@ -3242,6 +3227,10 @@ def _seg_31(): (0x3261, 'M', u'ᄂ'), (0x3262, 'M', u'ᄃ'), (0x3263, 'M', u'ᄅ'), + ] + +def _seg_31(): + return [ (0x3264, 'M', u'ᄆ'), (0x3265, 'M', u'ᄇ'), (0x3266, 'M', u'ᄉ'), @@ -3331,10 +3320,6 @@ def _seg_31(): (0x32BA, 'M', u'45'), (0x32BB, 'M', u'46'), (0x32BC, 'M', u'47'), - ] - -def _seg_32(): - return [ (0x32BD, 'M', u'48'), (0x32BE, 'M', u'49'), (0x32BF, 'M', u'50'), @@ -3346,6 +3331,10 @@ def _seg_32(): (0x32C5, 'M', u'6月'), (0x32C6, 'M', u'7月'), (0x32C7, 'M', u'8月'), + ] + +def _seg_32(): + return [ (0x32C8, 'M', u'9月'), (0x32C9, 'M', u'10月'), (0x32CA, 'M', u'11月'), @@ -3435,10 +3424,6 @@ def _seg_32(): (0x331E, 'M', u'コーポ'), (0x331F, 'M', u'サイクル'), (0x3320, 'M', u'サンチーム'), - ] - -def _seg_33(): - return [ (0x3321, 'M', u'シリング'), (0x3322, 'M', u'センチ'), (0x3323, 'M', u'セント'), @@ -3450,6 +3435,10 @@ def _seg_33(): (0x3329, 'M', u'ノット'), (0x332A, 'M', u'ハイツ'), (0x332B, 'M', u'パーセント'), + ] + +def _seg_33(): + return [ (0x332C, 'M', u'パーツ'), (0x332D, 'M', u'バーレル'), (0x332E, 'M', u'ピアストル'), @@ -3539,10 +3528,6 @@ def _seg_33(): (0x3382, 'M', u'μa'), (0x3383, 'M', u'ma'), (0x3384, 'M', u'ka'), - ] - -def _seg_34(): - return [ (0x3385, 'M', u'kb'), (0x3386, 'M', u'mb'), (0x3387, 'M', u'gb'), @@ -3554,6 +3539,10 @@ def _seg_34(): (0x338D, 'M', u'μg'), (0x338E, 'M', u'mg'), (0x338F, 'M', u'kg'), + ] + +def _seg_34(): + return [ (0x3390, 'M', u'hz'), (0x3391, 'M', u'khz'), (0x3392, 'M', u'mhz'), @@ -3643,10 +3632,6 @@ def _seg_34(): (0x33E6, 'M', u'7日'), (0x33E7, 'M', u'8日'), (0x33E8, 'M', u'9日'), - ] - -def _seg_35(): - return [ (0x33E9, 'M', u'10日'), (0x33EA, 'M', u'11日'), (0x33EB, 'M', u'12日'), @@ -3658,6 +3643,10 @@ def _seg_35(): (0x33F1, 'M', u'18日'), (0x33F2, 'M', u'19日'), (0x33F3, 'M', u'20日'), + ] + +def _seg_35(): + return [ (0x33F4, 'M', u'21日'), (0x33F5, 'M', u'22日'), (0x33F6, 'M', u'23日'), @@ -3673,7 +3662,7 @@ def _seg_35(): (0x3400, 'V'), (0x4DB6, 'X'), (0x4DC0, 'V'), - (0x9FEB, 'X'), + (0x9FF0, 'X'), (0xA000, 'V'), (0xA48D, 'X'), (0xA490, 'V'), @@ -3747,10 +3736,6 @@ def _seg_35(): (0xA692, 'M', u'ꚓ'), (0xA693, 'V'), (0xA694, 'M', u'ꚕ'), - ] - -def _seg_36(): - return [ (0xA695, 'V'), (0xA696, 'M', u'ꚗ'), (0xA697, 'V'), @@ -3762,6 +3747,10 @@ def _seg_36(): (0xA69D, 'M', u'ь'), (0xA69E, 'V'), (0xA6F8, 'X'), + ] + +def _seg_36(): + return [ (0xA700, 'V'), (0xA722, 'M', u'ꜣ'), (0xA723, 'V'), @@ -3851,10 +3840,6 @@ def _seg_36(): (0xA780, 'M', u'ꞁ'), (0xA781, 'V'), (0xA782, 'M', u'ꞃ'), - ] - -def _seg_37(): - return [ (0xA783, 'V'), (0xA784, 'M', u'ꞅ'), (0xA785, 'V'), @@ -3866,6 +3851,10 @@ def _seg_37(): (0xA78E, 'V'), (0xA790, 'M', u'ꞑ'), (0xA791, 'V'), + ] + +def _seg_37(): + return [ (0xA792, 'M', u'ꞓ'), (0xA793, 'V'), (0xA796, 'M', u'ꞗ'), @@ -3893,7 +3882,7 @@ def _seg_37(): (0xA7AC, 'M', u'ɡ'), (0xA7AD, 'M', u'ɬ'), (0xA7AE, 'M', u'ɪ'), - (0xA7AF, 'X'), + (0xA7AF, 'V'), (0xA7B0, 'M', u'ʞ'), (0xA7B1, 'M', u'ʇ'), (0xA7B2, 'M', u'ʝ'), @@ -3903,6 +3892,8 @@ def _seg_37(): (0xA7B6, 'M', u'ꞷ'), (0xA7B7, 'V'), (0xA7B8, 'X'), + (0xA7B9, 'V'), + (0xA7BA, 'X'), (0xA7F7, 'V'), (0xA7F8, 'M', u'ħ'), (0xA7F9, 'M', u'œ'), @@ -3917,8 +3908,6 @@ def _seg_37(): (0xA8CE, 'V'), (0xA8DA, 'X'), (0xA8E0, 'V'), - (0xA8FE, 'X'), - (0xA900, 'V'), (0xA954, 'X'), (0xA95F, 'V'), (0xA97D, 'X'), @@ -3955,10 +3944,6 @@ def _seg_37(): (0xAB5F, 'M', u'ꭒ'), (0xAB60, 'V'), (0xAB66, 'X'), - ] - -def _seg_38(): - return [ (0xAB70, 'M', u'Ꭰ'), (0xAB71, 'M', u'Ꭱ'), (0xAB72, 'M', u'Ꭲ'), @@ -3970,6 +3955,10 @@ def _seg_38(): (0xAB78, 'M', u'Ꭸ'), (0xAB79, 'M', u'Ꭹ'), (0xAB7A, 'M', u'Ꭺ'), + ] + +def _seg_38(): + return [ (0xAB7B, 'M', u'Ꭻ'), (0xAB7C, 'M', u'Ꭼ'), (0xAB7D, 'M', u'Ꭽ'), @@ -4059,10 +4048,6 @@ def _seg_38(): (0xF907, 'M', u'龜'), (0xF909, 'M', u'契'), (0xF90A, 'M', u'金'), - ] - -def _seg_39(): - return [ (0xF90B, 'M', u'喇'), (0xF90C, 'M', u'奈'), (0xF90D, 'M', u'懶'), @@ -4074,6 +4059,10 @@ def _seg_39(): (0xF913, 'M', u'邏'), (0xF914, 'M', u'樂'), (0xF915, 'M', u'洛'), + ] + +def _seg_39(): + return [ (0xF916, 'M', u'烙'), (0xF917, 'M', u'珞'), (0xF918, 'M', u'落'), @@ -4163,10 +4152,6 @@ def _seg_39(): (0xF96C, 'M', u'塞'), (0xF96D, 'M', u'省'), (0xF96E, 'M', u'葉'), - ] - -def _seg_40(): - return [ (0xF96F, 'M', u'說'), (0xF970, 'M', u'殺'), (0xF971, 'M', u'辰'), @@ -4178,6 +4163,10 @@ def _seg_40(): (0xF977, 'M', u'亮'), (0xF978, 'M', u'兩'), (0xF979, 'M', u'凉'), + ] + +def _seg_40(): + return [ (0xF97A, 'M', u'梁'), (0xF97B, 'M', u'糧'), (0xF97C, 'M', u'良'), @@ -4267,10 +4256,6 @@ def _seg_40(): (0xF9D0, 'M', u'類'), (0xF9D1, 'M', u'六'), (0xF9D2, 'M', u'戮'), - ] - -def _seg_41(): - return [ (0xF9D3, 'M', u'陸'), (0xF9D4, 'M', u'倫'), (0xF9D5, 'M', u'崙'), @@ -4282,6 +4267,10 @@ def _seg_41(): (0xF9DB, 'M', u'率'), (0xF9DC, 'M', u'隆'), (0xF9DD, 'M', u'利'), + ] + +def _seg_41(): + return [ (0xF9DE, 'M', u'吏'), (0xF9DF, 'M', u'履'), (0xF9E0, 'M', u'易'), @@ -4371,10 +4360,6 @@ def _seg_41(): (0xFA39, 'M', u'塀'), (0xFA3A, 'M', u'墨'), (0xFA3B, 'M', u'層'), - ] - -def _seg_42(): - return [ (0xFA3C, 'M', u'屮'), (0xFA3D, 'M', u'悔'), (0xFA3E, 'M', u'慨'), @@ -4386,6 +4371,10 @@ def _seg_42(): (0xFA44, 'M', u'梅'), (0xFA45, 'M', u'海'), (0xFA46, 'M', u'渚'), + ] + +def _seg_42(): + return [ (0xFA47, 'M', u'漢'), (0xFA48, 'M', u'煮'), (0xFA49, 'M', u'爫'), @@ -4475,10 +4464,6 @@ def _seg_42(): (0xFA9F, 'M', u'犯'), (0xFAA0, 'M', u'猪'), (0xFAA1, 'M', u'瑱'), - ] - -def _seg_43(): - return [ (0xFAA2, 'M', u'甆'), (0xFAA3, 'M', u'画'), (0xFAA4, 'M', u'瘝'), @@ -4490,6 +4475,10 @@ def _seg_43(): (0xFAAA, 'M', u'着'), (0xFAAB, 'M', u'磌'), (0xFAAC, 'M', u'窱'), + ] + +def _seg_43(): + return [ (0xFAAD, 'M', u'節'), (0xFAAE, 'M', u'类'), (0xFAAF, 'M', u'絛'), @@ -4579,10 +4568,6 @@ def _seg_43(): (0xFB38, 'M', u'טּ'), (0xFB39, 'M', u'יּ'), (0xFB3A, 'M', u'ךּ'), - ] - -def _seg_44(): - return [ (0xFB3B, 'M', u'כּ'), (0xFB3C, 'M', u'לּ'), (0xFB3D, 'X'), @@ -4594,6 +4579,10 @@ def _seg_44(): (0xFB43, 'M', u'ףּ'), (0xFB44, 'M', u'פּ'), (0xFB45, 'X'), + ] + +def _seg_44(): + return [ (0xFB46, 'M', u'צּ'), (0xFB47, 'M', u'קּ'), (0xFB48, 'M', u'רּ'), @@ -4683,10 +4672,6 @@ def _seg_44(): (0xFC19, 'M', u'خج'), (0xFC1A, 'M', u'خح'), (0xFC1B, 'M', u'خم'), - ] - -def _seg_45(): - return [ (0xFC1C, 'M', u'سج'), (0xFC1D, 'M', u'سح'), (0xFC1E, 'M', u'سخ'), @@ -4698,6 +4683,10 @@ def _seg_45(): (0xFC24, 'M', u'ضخ'), (0xFC25, 'M', u'ضم'), (0xFC26, 'M', u'طح'), + ] + +def _seg_45(): + return [ (0xFC27, 'M', u'طم'), (0xFC28, 'M', u'ظم'), (0xFC29, 'M', u'عج'), @@ -4787,10 +4776,6 @@ def _seg_45(): (0xFC7D, 'M', u'في'), (0xFC7E, 'M', u'قى'), (0xFC7F, 'M', u'قي'), - ] - -def _seg_46(): - return [ (0xFC80, 'M', u'كا'), (0xFC81, 'M', u'كل'), (0xFC82, 'M', u'كم'), @@ -4802,6 +4787,10 @@ def _seg_46(): (0xFC88, 'M', u'ما'), (0xFC89, 'M', u'مم'), (0xFC8A, 'M', u'نر'), + ] + +def _seg_46(): + return [ (0xFC8B, 'M', u'نز'), (0xFC8C, 'M', u'نم'), (0xFC8D, 'M', u'نن'), @@ -4891,10 +4880,6 @@ def _seg_46(): (0xFCE1, 'M', u'بم'), (0xFCE2, 'M', u'به'), (0xFCE3, 'M', u'تم'), - ] - -def _seg_47(): - return [ (0xFCE4, 'M', u'ته'), (0xFCE5, 'M', u'ثم'), (0xFCE6, 'M', u'ثه'), @@ -4906,6 +4891,10 @@ def _seg_47(): (0xFCEC, 'M', u'كم'), (0xFCED, 'M', u'لم'), (0xFCEE, 'M', u'نم'), + ] + +def _seg_47(): + return [ (0xFCEF, 'M', u'نه'), (0xFCF0, 'M', u'يم'), (0xFCF1, 'M', u'يه'), @@ -4995,10 +4984,6 @@ def _seg_47(): (0xFD57, 'M', u'تمخ'), (0xFD58, 'M', u'جمح'), (0xFD5A, 'M', u'حمي'), - ] - -def _seg_48(): - return [ (0xFD5B, 'M', u'حمى'), (0xFD5C, 'M', u'سحج'), (0xFD5D, 'M', u'سجح'), @@ -5010,6 +4995,10 @@ def _seg_48(): (0xFD66, 'M', u'صمم'), (0xFD67, 'M', u'شحم'), (0xFD69, 'M', u'شجي'), + ] + +def _seg_48(): + return [ (0xFD6A, 'M', u'شمخ'), (0xFD6C, 'M', u'شمم'), (0xFD6E, 'M', u'ضحى'), @@ -5099,10 +5088,6 @@ def _seg_48(): (0xFDF3, 'M', u'اكبر'), (0xFDF4, 'M', u'محمد'), (0xFDF5, 'M', u'صلعم'), - ] - -def _seg_49(): - return [ (0xFDF6, 'M', u'رسول'), (0xFDF7, 'M', u'عليه'), (0xFDF8, 'M', u'وسلم'), @@ -5114,6 +5099,10 @@ def _seg_49(): (0xFDFE, 'X'), (0xFE00, 'I'), (0xFE10, '3', u','), + ] + +def _seg_49(): + return [ (0xFE11, 'M', u'、'), (0xFE12, 'X'), (0xFE13, '3', u':'), @@ -5203,10 +5192,6 @@ def _seg_49(): (0xFE8F, 'M', u'ب'), (0xFE93, 'M', u'ة'), (0xFE95, 'M', u'ت'), - ] - -def _seg_50(): - return [ (0xFE99, 'M', u'ث'), (0xFE9D, 'M', u'ج'), (0xFEA1, 'M', u'ح'), @@ -5218,6 +5203,10 @@ def _seg_50(): (0xFEB1, 'M', u'س'), (0xFEB5, 'M', u'ش'), (0xFEB9, 'M', u'ص'), + ] + +def _seg_50(): + return [ (0xFEBD, 'M', u'ض'), (0xFEC1, 'M', u'ط'), (0xFEC5, 'M', u'ظ'), @@ -5307,10 +5296,6 @@ def _seg_50(): (0xFF41, 'M', u'a'), (0xFF42, 'M', u'b'), (0xFF43, 'M', u'c'), - ] - -def _seg_51(): - return [ (0xFF44, 'M', u'd'), (0xFF45, 'M', u'e'), (0xFF46, 'M', u'f'), @@ -5322,6 +5307,10 @@ def _seg_51(): (0xFF4C, 'M', u'l'), (0xFF4D, 'M', u'm'), (0xFF4E, 'M', u'n'), + ] + +def _seg_51(): + return [ (0xFF4F, 'M', u'o'), (0xFF50, 'M', u'p'), (0xFF51, 'M', u'q'), @@ -5411,10 +5400,6 @@ def _seg_51(): (0xFFA5, 'M', u'ᆬ'), (0xFFA6, 'M', u'ᆭ'), (0xFFA7, 'M', u'ᄃ'), - ] - -def _seg_52(): - return [ (0xFFA8, 'M', u'ᄄ'), (0xFFA9, 'M', u'ᄅ'), (0xFFAA, 'M', u'ᆰ'), @@ -5426,6 +5411,10 @@ def _seg_52(): (0xFFB0, 'M', u'ᄚ'), (0xFFB1, 'M', u'ᄆ'), (0xFFB2, 'M', u'ᄇ'), + ] + +def _seg_52(): + return [ (0xFFB3, 'M', u'ᄈ'), (0xFFB4, 'M', u'ᄡ'), (0xFFB5, 'M', u'ᄉ'), @@ -5515,10 +5504,6 @@ def _seg_52(): (0x10300, 'V'), (0x10324, 'X'), (0x1032D, 'V'), - ] - -def _seg_53(): - return [ (0x1034B, 'X'), (0x10350, 'V'), (0x1037B, 'X'), @@ -5530,6 +5515,10 @@ def _seg_53(): (0x103D6, 'X'), (0x10400, 'M', u'𐐨'), (0x10401, 'M', u'𐐩'), + ] + +def _seg_53(): + return [ (0x10402, 'M', u'𐐪'), (0x10403, 'M', u'𐐫'), (0x10404, 'M', u'𐐬'), @@ -5619,10 +5608,6 @@ def _seg_53(): (0x10570, 'X'), (0x10600, 'V'), (0x10737, 'X'), - ] - -def _seg_54(): - return [ (0x10740, 'V'), (0x10756, 'X'), (0x10760, 'V'), @@ -5634,6 +5619,10 @@ def _seg_54(): (0x1080A, 'V'), (0x10836, 'X'), (0x10837, 'V'), + ] + +def _seg_54(): + return [ (0x10839, 'X'), (0x1083C, 'V'), (0x1083D, 'X'), @@ -5666,11 +5655,11 @@ def _seg_54(): (0x10A15, 'V'), (0x10A18, 'X'), (0x10A19, 'V'), - (0x10A34, 'X'), + (0x10A36, 'X'), (0x10A38, 'V'), (0x10A3B, 'X'), (0x10A3F, 'V'), - (0x10A48, 'X'), + (0x10A49, 'X'), (0x10A50, 'V'), (0x10A59, 'X'), (0x10A60, 'V'), @@ -5723,10 +5712,6 @@ def _seg_54(): (0x10C9B, 'M', u'𐳛'), (0x10C9C, 'M', u'𐳜'), (0x10C9D, 'M', u'𐳝'), - ] - -def _seg_55(): - return [ (0x10C9E, 'M', u'𐳞'), (0x10C9F, 'M', u'𐳟'), (0x10CA0, 'M', u'𐳠'), @@ -5738,6 +5723,10 @@ def _seg_55(): (0x10CA6, 'M', u'𐳦'), (0x10CA7, 'M', u'𐳧'), (0x10CA8, 'M', u'𐳨'), + ] + +def _seg_55(): + return [ (0x10CA9, 'M', u'𐳩'), (0x10CAA, 'M', u'𐳪'), (0x10CAB, 'M', u'𐳫'), @@ -5752,9 +5741,15 @@ def _seg_55(): (0x10CC0, 'V'), (0x10CF3, 'X'), (0x10CFA, 'V'), - (0x10D00, 'X'), + (0x10D28, 'X'), + (0x10D30, 'V'), + (0x10D3A, 'X'), (0x10E60, 'V'), (0x10E7F, 'X'), + (0x10F00, 'V'), + (0x10F28, 'X'), + (0x10F30, 'V'), + (0x10F5A, 'X'), (0x11000, 'V'), (0x1104E, 'X'), (0x11052, 'V'), @@ -5770,7 +5765,7 @@ def _seg_55(): (0x11100, 'V'), (0x11135, 'X'), (0x11136, 'V'), - (0x11144, 'X'), + (0x11147, 'X'), (0x11150, 'V'), (0x11177, 'X'), (0x11180, 'V'), @@ -5811,7 +5806,7 @@ def _seg_55(): (0x11334, 'X'), (0x11335, 'V'), (0x1133A, 'X'), - (0x1133C, 'V'), + (0x1133B, 'V'), (0x11345, 'X'), (0x11347, 'V'), (0x11349, 'X'), @@ -5827,16 +5822,16 @@ def _seg_55(): (0x1136D, 'X'), (0x11370, 'V'), (0x11375, 'X'), - ] - -def _seg_56(): - return [ (0x11400, 'V'), (0x1145A, 'X'), (0x1145B, 'V'), (0x1145C, 'X'), (0x1145D, 'V'), - (0x1145E, 'X'), + ] + +def _seg_56(): + return [ + (0x1145F, 'X'), (0x11480, 'V'), (0x114C8, 'X'), (0x114D0, 'V'), @@ -5856,11 +5851,13 @@ def _seg_56(): (0x116C0, 'V'), (0x116CA, 'X'), (0x11700, 'V'), - (0x1171A, 'X'), + (0x1171B, 'X'), (0x1171D, 'V'), (0x1172C, 'X'), (0x11730, 'V'), (0x11740, 'X'), + (0x11800, 'V'), + (0x1183C, 'X'), (0x118A0, 'M', u'𑣀'), (0x118A1, 'M', u'𑣁'), (0x118A2, 'M', u'𑣂'), @@ -5902,8 +5899,6 @@ def _seg_56(): (0x11A50, 'V'), (0x11A84, 'X'), (0x11A86, 'V'), - (0x11A9D, 'X'), - (0x11A9E, 'V'), (0x11AA3, 'X'), (0x11AC0, 'V'), (0x11AF9, 'X'), @@ -5931,14 +5926,28 @@ def _seg_56(): (0x11D3B, 'X'), (0x11D3C, 'V'), (0x11D3E, 'X'), - ] - -def _seg_57(): - return [ (0x11D3F, 'V'), (0x11D48, 'X'), (0x11D50, 'V'), (0x11D5A, 'X'), + (0x11D60, 'V'), + ] + +def _seg_57(): + return [ + (0x11D66, 'X'), + (0x11D67, 'V'), + (0x11D69, 'X'), + (0x11D6A, 'V'), + (0x11D8F, 'X'), + (0x11D90, 'V'), + (0x11D92, 'X'), + (0x11D93, 'V'), + (0x11D99, 'X'), + (0x11DA0, 'V'), + (0x11DAA, 'X'), + (0x11EE0, 'V'), + (0x11EF9, 'X'), (0x12000, 'V'), (0x1239A, 'X'), (0x12400, 'V'), @@ -5973,6 +5982,8 @@ def _seg_57(): (0x16B78, 'X'), (0x16B7D, 'V'), (0x16B90, 'X'), + (0x16E60, 'V'), + (0x16E9B, 'X'), (0x16F00, 'V'), (0x16F45, 'X'), (0x16F50, 'V'), @@ -5982,7 +5993,7 @@ def _seg_57(): (0x16FE0, 'V'), (0x16FE2, 'X'), (0x17000, 'V'), - (0x187ED, 'X'), + (0x187F2, 'X'), (0x18800, 'V'), (0x18AF3, 'X'), (0x1B000, 'V'), @@ -6024,21 +6035,23 @@ def _seg_57(): (0x1D1C1, 'V'), (0x1D1E9, 'X'), (0x1D200, 'V'), + ] + +def _seg_58(): + return [ (0x1D246, 'X'), + (0x1D2E0, 'V'), + (0x1D2F4, 'X'), (0x1D300, 'V'), (0x1D357, 'X'), (0x1D360, 'V'), - (0x1D372, 'X'), + (0x1D379, 'X'), (0x1D400, 'M', u'a'), (0x1D401, 'M', u'b'), (0x1D402, 'M', u'c'), (0x1D403, 'M', u'd'), (0x1D404, 'M', u'e'), (0x1D405, 'M', u'f'), - ] - -def _seg_58(): - return [ (0x1D406, 'M', u'g'), (0x1D407, 'M', u'h'), (0x1D408, 'M', u'i'), @@ -6126,6 +6139,10 @@ def _seg_58(): (0x1D45A, 'M', u'm'), (0x1D45B, 'M', u'n'), (0x1D45C, 'M', u'o'), + ] + +def _seg_59(): + return [ (0x1D45D, 'M', u'p'), (0x1D45E, 'M', u'q'), (0x1D45F, 'M', u'r'), @@ -6139,10 +6156,6 @@ def _seg_58(): (0x1D467, 'M', u'z'), (0x1D468, 'M', u'a'), (0x1D469, 'M', u'b'), - ] - -def _seg_59(): - return [ (0x1D46A, 'M', u'c'), (0x1D46B, 'M', u'd'), (0x1D46C, 'M', u'e'), @@ -6230,6 +6243,10 @@ def _seg_59(): (0x1D4C1, 'M', u'l'), (0x1D4C2, 'M', u'm'), (0x1D4C3, 'M', u'n'), + ] + +def _seg_60(): + return [ (0x1D4C4, 'X'), (0x1D4C5, 'M', u'p'), (0x1D4C6, 'M', u'q'), @@ -6243,10 +6260,6 @@ def _seg_59(): (0x1D4CE, 'M', u'y'), (0x1D4CF, 'M', u'z'), (0x1D4D0, 'M', u'a'), - ] - -def _seg_60(): - return [ (0x1D4D1, 'M', u'b'), (0x1D4D2, 'M', u'c'), (0x1D4D3, 'M', u'd'), @@ -6334,6 +6347,10 @@ def _seg_60(): (0x1D526, 'M', u'i'), (0x1D527, 'M', u'j'), (0x1D528, 'M', u'k'), + ] + +def _seg_61(): + return [ (0x1D529, 'M', u'l'), (0x1D52A, 'M', u'm'), (0x1D52B, 'M', u'n'), @@ -6347,10 +6364,6 @@ def _seg_60(): (0x1D533, 'M', u'v'), (0x1D534, 'M', u'w'), (0x1D535, 'M', u'x'), - ] - -def _seg_61(): - return [ (0x1D536, 'M', u'y'), (0x1D537, 'M', u'z'), (0x1D538, 'M', u'a'), @@ -6438,6 +6451,10 @@ def _seg_61(): (0x1D58C, 'M', u'g'), (0x1D58D, 'M', u'h'), (0x1D58E, 'M', u'i'), + ] + +def _seg_62(): + return [ (0x1D58F, 'M', u'j'), (0x1D590, 'M', u'k'), (0x1D591, 'M', u'l'), @@ -6451,10 +6468,6 @@ def _seg_61(): (0x1D599, 'M', u't'), (0x1D59A, 'M', u'u'), (0x1D59B, 'M', u'v'), - ] - -def _seg_62(): - return [ (0x1D59C, 'M', u'w'), (0x1D59D, 'M', u'x'), (0x1D59E, 'M', u'y'), @@ -6542,6 +6555,10 @@ def _seg_62(): (0x1D5F0, 'M', u'c'), (0x1D5F1, 'M', u'd'), (0x1D5F2, 'M', u'e'), + ] + +def _seg_63(): + return [ (0x1D5F3, 'M', u'f'), (0x1D5F4, 'M', u'g'), (0x1D5F5, 'M', u'h'), @@ -6555,10 +6572,6 @@ def _seg_62(): (0x1D5FD, 'M', u'p'), (0x1D5FE, 'M', u'q'), (0x1D5FF, 'M', u'r'), - ] - -def _seg_63(): - return [ (0x1D600, 'M', u's'), (0x1D601, 'M', u't'), (0x1D602, 'M', u'u'), @@ -6646,6 +6659,10 @@ def _seg_63(): (0x1D654, 'M', u'y'), (0x1D655, 'M', u'z'), (0x1D656, 'M', u'a'), + ] + +def _seg_64(): + return [ (0x1D657, 'M', u'b'), (0x1D658, 'M', u'c'), (0x1D659, 'M', u'd'), @@ -6659,10 +6676,6 @@ def _seg_63(): (0x1D661, 'M', u'l'), (0x1D662, 'M', u'm'), (0x1D663, 'M', u'n'), - ] - -def _seg_64(): - return [ (0x1D664, 'M', u'o'), (0x1D665, 'M', u'p'), (0x1D666, 'M', u'q'), @@ -6750,6 +6763,10 @@ def _seg_64(): (0x1D6B9, 'M', u'θ'), (0x1D6BA, 'M', u'σ'), (0x1D6BB, 'M', u'τ'), + ] + +def _seg_65(): + return [ (0x1D6BC, 'M', u'υ'), (0x1D6BD, 'M', u'φ'), (0x1D6BE, 'M', u'χ'), @@ -6763,10 +6780,6 @@ def _seg_64(): (0x1D6C6, 'M', u'ε'), (0x1D6C7, 'M', u'ζ'), (0x1D6C8, 'M', u'η'), - ] - -def _seg_65(): - return [ (0x1D6C9, 'M', u'θ'), (0x1D6CA, 'M', u'ι'), (0x1D6CB, 'M', u'κ'), @@ -6854,6 +6867,10 @@ def _seg_65(): (0x1D71F, 'M', u'δ'), (0x1D720, 'M', u'ε'), (0x1D721, 'M', u'ζ'), + ] + +def _seg_66(): + return [ (0x1D722, 'M', u'η'), (0x1D723, 'M', u'θ'), (0x1D724, 'M', u'ι'), @@ -6867,10 +6884,6 @@ def _seg_65(): (0x1D72C, 'M', u'ρ'), (0x1D72D, 'M', u'θ'), (0x1D72E, 'M', u'σ'), - ] - -def _seg_66(): - return [ (0x1D72F, 'M', u'τ'), (0x1D730, 'M', u'υ'), (0x1D731, 'M', u'φ'), @@ -6958,6 +6971,10 @@ def _seg_66(): (0x1D785, 'M', u'φ'), (0x1D786, 'M', u'χ'), (0x1D787, 'M', u'ψ'), + ] + +def _seg_67(): + return [ (0x1D788, 'M', u'ω'), (0x1D789, 'M', u'∂'), (0x1D78A, 'M', u'ε'), @@ -6971,10 +6988,6 @@ def _seg_66(): (0x1D792, 'M', u'γ'), (0x1D793, 'M', u'δ'), (0x1D794, 'M', u'ε'), - ] - -def _seg_67(): - return [ (0x1D795, 'M', u'ζ'), (0x1D796, 'M', u'η'), (0x1D797, 'M', u'θ'), @@ -7062,6 +7075,10 @@ def _seg_67(): (0x1D7EC, 'M', u'0'), (0x1D7ED, 'M', u'1'), (0x1D7EE, 'M', u'2'), + ] + +def _seg_68(): + return [ (0x1D7EF, 'M', u'3'), (0x1D7F0, 'M', u'4'), (0x1D7F1, 'M', u'5'), @@ -7075,10 +7092,6 @@ def _seg_67(): (0x1D7F9, 'M', u'3'), (0x1D7FA, 'M', u'4'), (0x1D7FB, 'M', u'5'), - ] - -def _seg_68(): - return [ (0x1D7FC, 'M', u'6'), (0x1D7FD, 'M', u'7'), (0x1D7FE, 'M', u'8'), @@ -7143,6 +7156,8 @@ def _seg_68(): (0x1E95A, 'X'), (0x1E95E, 'V'), (0x1E960, 'X'), + (0x1EC71, 'V'), + (0x1ECB5, 'X'), (0x1EE00, 'M', u'ا'), (0x1EE01, 'M', u'ب'), (0x1EE02, 'M', u'ج'), @@ -7164,6 +7179,10 @@ def _seg_68(): (0x1EE12, 'M', u'ق'), (0x1EE13, 'M', u'ر'), (0x1EE14, 'M', u'ش'), + ] + +def _seg_69(): + return [ (0x1EE15, 'M', u'ت'), (0x1EE16, 'M', u'ث'), (0x1EE17, 'M', u'خ'), @@ -7179,10 +7198,6 @@ def _seg_68(): (0x1EE21, 'M', u'ب'), (0x1EE22, 'M', u'ج'), (0x1EE23, 'X'), - ] - -def _seg_69(): - return [ (0x1EE24, 'M', u'ه'), (0x1EE25, 'X'), (0x1EE27, 'M', u'ح'), @@ -7268,6 +7283,10 @@ def _seg_69(): (0x1EE81, 'M', u'ب'), (0x1EE82, 'M', u'ج'), (0x1EE83, 'M', u'د'), + ] + +def _seg_70(): + return [ (0x1EE84, 'M', u'ه'), (0x1EE85, 'M', u'و'), (0x1EE86, 'M', u'ز'), @@ -7283,10 +7302,6 @@ def _seg_69(): (0x1EE90, 'M', u'ف'), (0x1EE91, 'M', u'ص'), (0x1EE92, 'M', u'ق'), - ] - -def _seg_70(): - return [ (0x1EE93, 'M', u'ر'), (0x1EE94, 'M', u'ش'), (0x1EE95, 'M', u'ت'), @@ -7372,6 +7387,10 @@ def _seg_70(): (0x1F122, '3', u'(s)'), (0x1F123, '3', u'(t)'), (0x1F124, '3', u'(u)'), + ] + +def _seg_71(): + return [ (0x1F125, '3', u'(v)'), (0x1F126, '3', u'(w)'), (0x1F127, '3', u'(x)'), @@ -7382,15 +7401,11 @@ def _seg_70(): (0x1F12C, 'M', u'r'), (0x1F12D, 'M', u'cd'), (0x1F12E, 'M', u'wz'), - (0x1F12F, 'X'), + (0x1F12F, 'V'), (0x1F130, 'M', u'a'), (0x1F131, 'M', u'b'), (0x1F132, 'M', u'c'), (0x1F133, 'M', u'd'), - ] - -def _seg_71(): - return [ (0x1F134, 'M', u'e'), (0x1F135, 'M', u'f'), (0x1F136, 'M', u'g'), @@ -7476,6 +7491,10 @@ def _seg_71(): (0x1F239, 'M', u'割'), (0x1F23A, 'M', u'営'), (0x1F23B, 'M', u'配'), + ] + +def _seg_72(): + return [ (0x1F23C, 'X'), (0x1F240, 'M', u'〔本〕'), (0x1F241, 'M', u'〔三〕'), @@ -7491,21 +7510,17 @@ def _seg_71(): (0x1F251, 'M', u'可'), (0x1F252, 'X'), (0x1F260, 'V'), - ] - -def _seg_72(): - return [ (0x1F266, 'X'), (0x1F300, 'V'), (0x1F6D5, 'X'), (0x1F6E0, 'V'), (0x1F6ED, 'X'), (0x1F6F0, 'V'), - (0x1F6F9, 'X'), + (0x1F6FA, 'X'), (0x1F700, 'V'), (0x1F774, 'X'), (0x1F780, 'V'), - (0x1F7D5, 'X'), + (0x1F7D9, 'X'), (0x1F800, 'V'), (0x1F80C, 'X'), (0x1F810, 'V'), @@ -7521,15 +7536,21 @@ def _seg_72(): (0x1F910, 'V'), (0x1F93F, 'X'), (0x1F940, 'V'), - (0x1F94D, 'X'), - (0x1F950, 'V'), - (0x1F96C, 'X'), - (0x1F980, 'V'), - (0x1F998, 'X'), + (0x1F971, 'X'), + (0x1F973, 'V'), + (0x1F977, 'X'), + (0x1F97A, 'V'), + (0x1F97B, 'X'), + (0x1F97C, 'V'), + (0x1F9A3, 'X'), + (0x1F9B0, 'V'), + (0x1F9BA, 'X'), (0x1F9C0, 'V'), - (0x1F9C1, 'X'), + (0x1F9C3, 'X'), (0x1F9D0, 'V'), - (0x1F9E7, 'X'), + (0x1FA00, 'X'), + (0x1FA60, 'V'), + (0x1FA6E, 'X'), (0x20000, 'V'), (0x2A6D7, 'X'), (0x2A700, 'V'), @@ -7574,6 +7595,10 @@ def _seg_72(): (0x2F81F, 'M', u'㓟'), (0x2F820, 'M', u'刻'), (0x2F821, 'M', u'剆'), + ] + +def _seg_73(): + return [ (0x2F822, 'M', u'割'), (0x2F823, 'M', u'剷'), (0x2F824, 'M', u'㔕'), @@ -7595,10 +7620,6 @@ def _seg_72(): (0x2F836, 'M', u'及'), (0x2F837, 'M', u'叟'), (0x2F838, 'M', u'𠭣'), - ] - -def _seg_73(): - return [ (0x2F839, 'M', u'叫'), (0x2F83A, 'M', u'叱'), (0x2F83B, 'M', u'吆'), @@ -7678,6 +7699,10 @@ def _seg_73(): (0x2F887, 'M', u'幩'), (0x2F888, 'M', u'㡢'), (0x2F889, 'M', u'𢆃'), + ] + +def _seg_74(): + return [ (0x2F88A, 'M', u'㡼'), (0x2F88B, 'M', u'庰'), (0x2F88C, 'M', u'庳'), @@ -7699,10 +7724,6 @@ def _seg_73(): (0x2F89E, 'M', u'志'), (0x2F89F, 'M', u'忹'), (0x2F8A0, 'M', u'悁'), - ] - -def _seg_74(): - return [ (0x2F8A1, 'M', u'㤺'), (0x2F8A2, 'M', u'㤜'), (0x2F8A3, 'M', u'悔'), @@ -7782,6 +7803,10 @@ def _seg_74(): (0x2F8ED, 'M', u'櫛'), (0x2F8EE, 'M', u'㰘'), (0x2F8EF, 'M', u'次'), + ] + +def _seg_75(): + return [ (0x2F8F0, 'M', u'𣢧'), (0x2F8F1, 'M', u'歔'), (0x2F8F2, 'M', u'㱎'), @@ -7803,10 +7828,6 @@ def _seg_74(): (0x2F902, 'M', u'流'), (0x2F903, 'M', u'浩'), (0x2F904, 'M', u'浸'), - ] - -def _seg_75(): - return [ (0x2F905, 'M', u'涅'), (0x2F906, 'M', u'𣴞'), (0x2F907, 'M', u'洴'), @@ -7886,6 +7907,10 @@ def _seg_75(): (0x2F953, 'M', u'祖'), (0x2F954, 'M', u'𥚚'), (0x2F955, 'M', u'𥛅'), + ] + +def _seg_76(): + return [ (0x2F956, 'M', u'福'), (0x2F957, 'M', u'秫'), (0x2F958, 'M', u'䄯'), @@ -7907,10 +7932,6 @@ def _seg_75(): (0x2F969, 'M', u'糣'), (0x2F96A, 'M', u'紀'), (0x2F96B, 'M', u'𥾆'), - ] - -def _seg_76(): - return [ (0x2F96C, 'M', u'絣'), (0x2F96D, 'M', u'䌁'), (0x2F96E, 'M', u'緇'), @@ -7990,6 +8011,10 @@ def _seg_76(): (0x2F9B8, 'M', u'蚈'), (0x2F9B9, 'M', u'蜎'), (0x2F9BA, 'M', u'蛢'), + ] + +def _seg_77(): + return [ (0x2F9BB, 'M', u'蝹'), (0x2F9BC, 'M', u'蜨'), (0x2F9BD, 'M', u'蝫'), @@ -8011,10 +8036,6 @@ def _seg_76(): (0x2F9CD, 'M', u'䚾'), (0x2F9CE, 'M', u'䛇'), (0x2F9CF, 'M', u'誠'), - ] - -def _seg_77(): - return [ (0x2F9D0, 'M', u'諭'), (0x2F9D1, 'M', u'變'), (0x2F9D2, 'M', u'豕'), @@ -8094,6 +8115,10 @@ def _seg_77(): (0x2FA1D, 'M', u'𪘀'), (0x2FA1E, 'X'), (0xE0100, 'I'), + ] + +def _seg_78(): + return [ (0xE01F0, 'X'), ] @@ -8176,4 +8201,5 @@ uts46data = tuple( + _seg_75() + _seg_76() + _seg_77() + + _seg_78() ) diff --git a/pipenv/vendor/markupsafe/LICENSE b/pipenv/vendor/markupsafe/LICENSE deleted file mode 100644 index 5d269389..00000000 --- a/pipenv/vendor/markupsafe/LICENSE +++ /dev/null @@ -1,33 +0,0 @@ -Copyright (c) 2010 by Armin Ronacher and contributors. See AUTHORS -for more details. - -Some rights reserved. - -Redistribution and use in source and binary forms of the software as well -as documentation, with or without modification, are permitted provided -that the following conditions are met: - -* Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above - copyright notice, this list of conditions and the following - disclaimer in the documentation and/or other materials provided - with the distribution. - -* The names of the contributors may not be used to endorse or - promote products derived from this software without specific - prior written permission. - -THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER -OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE AND DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH -DAMAGE. diff --git a/pipenv/vendor/markupsafe/LICENSE.rst b/pipenv/vendor/markupsafe/LICENSE.rst new file mode 100644 index 00000000..9d227a0c --- /dev/null +++ b/pipenv/vendor/markupsafe/LICENSE.rst @@ -0,0 +1,28 @@ +Copyright 2010 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/markupsafe/__init__.py b/pipenv/vendor/markupsafe/__init__.py index 68dc85f6..da05ed32 100644 --- a/pipenv/vendor/markupsafe/__init__.py +++ b/pipenv/vendor/markupsafe/__init__.py @@ -1,75 +1,74 @@ # -*- coding: utf-8 -*- """ - markupsafe - ~~~~~~~~~~ +markupsafe +~~~~~~~~~~ - Implements a Markup string. +Implements an escape function and a Markup string to replace HTML +special characters with safe representations. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import re import string -from collections import Mapping -from markupsafe._compat import text_type, string_types, int_types, \ - unichr, iteritems, PY2 -__version__ = "1.0" +from ._compat import int_types +from ._compat import iteritems +from ._compat import Mapping +from ._compat import PY2 +from ._compat import string_types +from ._compat import text_type +from ._compat import unichr -__all__ = ['Markup', 'soft_unicode', 'escape', 'escape_silent'] +__version__ = "1.1.1" +__all__ = ["Markup", "soft_unicode", "escape", "escape_silent"] -_striptags_re = re.compile(r'(|<[^>]*>)') -_entity_re = re.compile(r'&([^& ;]+);') +_striptags_re = re.compile(r"(|<[^>]*>)") +_entity_re = re.compile(r"&([^& ;]+);") class Markup(text_type): - r"""Marks a string as being safe for inclusion in HTML/XML output without - needing to be escaped. This implements the `__html__` interface a couple - of frameworks and web applications use. :class:`Markup` is a direct - subclass of `unicode` and provides all the methods of `unicode` just that - it escapes arguments passed and always returns `Markup`. + """A string that is ready to be safely inserted into an HTML or XML + document, either because it was escaped or because it was marked + safe. - The `escape` function returns markup objects so that double escaping can't - happen. + Passing an object to the constructor converts it to text and wraps + it to mark it safe without escaping. To escape the text, use the + :meth:`escape` class method instead. - The constructor of the :class:`Markup` class can be used for three - different things: When passed an unicode object it's assumed to be safe, - when passed an object with an HTML representation (has an `__html__` - method) that representation is used, otherwise the object passed is - converted into a unicode string and then assumed to be safe: + >>> Markup('Hello, World!') + Markup('Hello, World!') + >>> Markup(42) + Markup('42') + >>> Markup.escape('Hello, World!') + Markup('Hello <em>World</em>!') - >>> Markup("Hello World!") - Markup(u'Hello World!') - >>> class Foo(object): - ... def __html__(self): - ... return 'foo' + This implements the ``__html__()`` interface that some frameworks + use. Passing an object that implements ``__html__()`` will wrap the + output of that method, marking it safe. + + >>> class Foo: + ... def __html__(self): + ... return 'foo' ... >>> Markup(Foo()) - Markup(u'foo') + Markup('foo') - If you want object passed being always treated as unsafe you can use the - :meth:`escape` classmethod to create a :class:`Markup` object: + This is a subclass of the text type (``str`` in Python 3, + ``unicode`` in Python 2). It has the same methods as that type, but + all methods escape their arguments and return a ``Markup`` instance. - >>> Markup.escape("Hello World!") - Markup(u'Hello <em>World</em>!') - - Operations on a markup string are markup aware which means that all - arguments are passed through the :func:`escape` function: - - >>> em = Markup("%s") - >>> em % "foo & bar" - Markup(u'foo & bar') - >>> strong = Markup("%(text)s") - >>> strong % {'text': 'hacker here'} - Markup(u'<blink>hacker here</blink>') - >>> Markup("Hello ") + "" - Markup(u'Hello <foo>') + >>> Markup('%s') % 'foo & bar' + Markup('foo & bar') + >>> Markup('Hello ') + '' + Markup('Hello <foo>') """ + __slots__ = () - def __new__(cls, base=u'', encoding=None, errors='strict'): - if hasattr(base, '__html__'): + def __new__(cls, base=u"", encoding=None, errors="strict"): + if hasattr(base, "__html__"): base = base.__html__() if encoding is None: return text_type.__new__(cls, base) @@ -79,12 +78,12 @@ class Markup(text_type): return self def __add__(self, other): - if isinstance(other, string_types) or hasattr(other, '__html__'): + if isinstance(other, string_types) or hasattr(other, "__html__"): return self.__class__(super(Markup, self).__add__(self.escape(other))) return NotImplemented def __radd__(self, other): - if hasattr(other, '__html__') or isinstance(other, string_types): + if hasattr(other, "__html__") or isinstance(other, string_types): return self.escape(other).__add__(self) return NotImplemented @@ -92,6 +91,7 @@ class Markup(text_type): if isinstance(num, int_types): return self.__class__(text_type.__mul__(self, num)) return NotImplemented + __rmul__ = __mul__ def __mod__(self, arg): @@ -102,115 +102,124 @@ class Markup(text_type): return self.__class__(text_type.__mod__(self, arg)) def __repr__(self): - return '%s(%s)' % ( - self.__class__.__name__, - text_type.__repr__(self) - ) + return "%s(%s)" % (self.__class__.__name__, text_type.__repr__(self)) def join(self, seq): return self.__class__(text_type.join(self, map(self.escape, seq))) + join.__doc__ = text_type.join.__doc__ def split(self, *args, **kwargs): return list(map(self.__class__, text_type.split(self, *args, **kwargs))) + split.__doc__ = text_type.split.__doc__ def rsplit(self, *args, **kwargs): return list(map(self.__class__, text_type.rsplit(self, *args, **kwargs))) + rsplit.__doc__ = text_type.rsplit.__doc__ def splitlines(self, *args, **kwargs): - return list(map(self.__class__, text_type.splitlines( - self, *args, **kwargs))) + return list(map(self.__class__, text_type.splitlines(self, *args, **kwargs))) + splitlines.__doc__ = text_type.splitlines.__doc__ def unescape(self): - r"""Unescape markup again into an text_type string. This also resolves - known HTML4 and XHTML entities: + """Convert escaped markup back into a text string. This replaces + HTML entities with the characters they represent. - >>> Markup("Main » About").unescape() - u'Main \xbb About' + >>> Markup('Main » About').unescape() + 'Main » About' """ - from markupsafe._constants import HTML_ENTITIES + from ._constants import HTML_ENTITIES + def handle_match(m): name = m.group(1) if name in HTML_ENTITIES: return unichr(HTML_ENTITIES[name]) try: - if name[:2] in ('#x', '#X'): + if name[:2] in ("#x", "#X"): return unichr(int(name[2:], 16)) - elif name.startswith('#'): + elif name.startswith("#"): return unichr(int(name[1:])) except ValueError: pass # Don't modify unexpected input. return m.group() + return _entity_re.sub(handle_match, text_type(self)) def striptags(self): - r"""Unescape markup into an text_type string and strip all tags. This - also resolves known HTML4 and XHTML entities. Whitespace is - normalized to one: + """:meth:`unescape` the markup, remove tags, and normalize + whitespace to single spaces. - >>> Markup("Main » About").striptags() - u'Main \xbb About' + >>> Markup('Main »\tAbout').striptags() + 'Main » About' """ - stripped = u' '.join(_striptags_re.sub('', self).split()) + stripped = u" ".join(_striptags_re.sub("", self).split()) return Markup(stripped).unescape() @classmethod def escape(cls, s): - """Escape the string. Works like :func:`escape` with the difference - that for subclasses of :class:`Markup` this function would return the - correct subclass. + """Escape a string. Calls :func:`escape` and ensures that for + subclasses the correct type is returned. """ rv = escape(s) if rv.__class__ is not cls: return cls(rv) return rv - def make_simple_escaping_wrapper(name): + def make_simple_escaping_wrapper(name): # noqa: B902 orig = getattr(text_type, name) + def func(self, *args, **kwargs): args = _escape_argspec(list(args), enumerate(args), self.escape) _escape_argspec(kwargs, iteritems(kwargs), self.escape) return self.__class__(orig(self, *args, **kwargs)) + func.__name__ = orig.__name__ func.__doc__ = orig.__doc__ return func - for method in '__getitem__', 'capitalize', \ - 'title', 'lower', 'upper', 'replace', 'ljust', \ - 'rjust', 'lstrip', 'rstrip', 'center', 'strip', \ - 'translate', 'expandtabs', 'swapcase', 'zfill': + for method in ( + "__getitem__", + "capitalize", + "title", + "lower", + "upper", + "replace", + "ljust", + "rjust", + "lstrip", + "rstrip", + "center", + "strip", + "translate", + "expandtabs", + "swapcase", + "zfill", + ): locals()[method] = make_simple_escaping_wrapper(method) - # new in python 2.5 - if hasattr(text_type, 'partition'): - def partition(self, sep): - return tuple(map(self.__class__, - text_type.partition(self, self.escape(sep)))) - def rpartition(self, sep): - return tuple(map(self.__class__, - text_type.rpartition(self, self.escape(sep)))) + def partition(self, sep): + return tuple(map(self.__class__, text_type.partition(self, self.escape(sep)))) - # new in python 2.6 - if hasattr(text_type, 'format'): - def format(*args, **kwargs): - self, args = args[0], args[1:] - formatter = EscapeFormatter(self.escape) - kwargs = _MagicFormatMapping(args, kwargs) - return self.__class__(formatter.vformat(self, args, kwargs)) + def rpartition(self, sep): + return tuple(map(self.__class__, text_type.rpartition(self, self.escape(sep)))) - def __html_format__(self, format_spec): - if format_spec: - raise ValueError('Unsupported format specification ' - 'for Markup.') - return self + def format(self, *args, **kwargs): + formatter = EscapeFormatter(self.escape) + kwargs = _MagicFormatMapping(args, kwargs) + return self.__class__(formatter.vformat(self, args, kwargs)) + + def __html_format__(self, format_spec): + if format_spec: + raise ValueError("Unsupported format specification " "for Markup.") + return self # not in python 3 - if hasattr(text_type, '__getslice__'): - __getslice__ = make_simple_escaping_wrapper('__getslice__') + if hasattr(text_type, "__getslice__"): + __getslice__ = make_simple_escaping_wrapper("__getslice__") del method, make_simple_escaping_wrapper @@ -229,7 +238,7 @@ class _MagicFormatMapping(Mapping): self._last_index = 0 def __getitem__(self, key): - if key == '': + if key == "": idx = self._last_index self._last_index += 1 try: @@ -246,35 +255,37 @@ class _MagicFormatMapping(Mapping): return len(self._kwargs) -if hasattr(text_type, 'format'): - class EscapeFormatter(string.Formatter): +if hasattr(text_type, "format"): + class EscapeFormatter(string.Formatter): def __init__(self, escape): self.escape = escape def format_field(self, value, format_spec): - if hasattr(value, '__html_format__'): + if hasattr(value, "__html_format__"): rv = value.__html_format__(format_spec) - elif hasattr(value, '__html__'): + elif hasattr(value, "__html__"): if format_spec: - raise ValueError('No format specification allowed ' - 'when formatting an object with ' - 'its __html__ method.') + raise ValueError( + "Format specifier {0} given, but {1} does not" + " define __html_format__. A class that defines" + " __html__ must define __html_format__ to work" + " with format specifiers.".format(format_spec, type(value)) + ) rv = value.__html__() else: # We need to make sure the format spec is unicode here as # otherwise the wrong callback methods are invoked. For # instance a byte string there would invoke __str__ and # not __unicode__. - rv = string.Formatter.format_field( - self, value, text_type(format_spec)) + rv = string.Formatter.format_field(self, value, text_type(format_spec)) return text_type(self.escape(rv)) def _escape_argspec(obj, iterable, escape): """Helper for various string-wrapped functions.""" for key, value in iterable: - if hasattr(value, '__html__') or isinstance(value, string_types): + if hasattr(value, "__html__") or isinstance(value, string_types): obj[key] = escape(value) return obj @@ -286,20 +297,31 @@ class _MarkupEscapeHelper(object): self.obj = obj self.escape = escape - __getitem__ = lambda s, x: _MarkupEscapeHelper(s.obj[x], s.escape) - __unicode__ = __str__ = lambda s: text_type(s.escape(s.obj)) - __repr__ = lambda s: str(s.escape(repr(s.obj))) - __int__ = lambda s: int(s.obj) - __float__ = lambda s: float(s.obj) + def __getitem__(self, item): + return _MarkupEscapeHelper(self.obj[item], self.escape) + + def __str__(self): + return text_type(self.escape(self.obj)) + + __unicode__ = __str__ + + def __repr__(self): + return str(self.escape(repr(self.obj))) + + def __int__(self): + return int(self.obj) + + def __float__(self): + return float(self.obj) # we have to import it down here as the speedups and native # modules imports the markup type which is define above. try: - from markupsafe._speedups import escape, escape_silent, soft_unicode + from ._speedups import escape, escape_silent, soft_unicode except ImportError: - from markupsafe._native import escape, escape_silent, soft_unicode + from ._native import escape, escape_silent, soft_unicode if not PY2: soft_str = soft_unicode - __all__.append('soft_str') + __all__.append("soft_str") diff --git a/pipenv/vendor/markupsafe/_compat.py b/pipenv/vendor/markupsafe/_compat.py index 62e5632a..bc05090f 100644 --- a/pipenv/vendor/markupsafe/_compat.py +++ b/pipenv/vendor/markupsafe/_compat.py @@ -1,12 +1,10 @@ # -*- coding: utf-8 -*- """ - markupsafe._compat - ~~~~~~~~~~~~~~~~~~ +markupsafe._compat +~~~~~~~~~~~~~~~~~~ - Compatibility module for different Python versions. - - :copyright: (c) 2013 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ import sys @@ -17,10 +15,19 @@ if not PY2: string_types = (str,) unichr = chr int_types = (int,) - iteritems = lambda x: iter(x.items()) + + def iteritems(x): + return iter(x.items()) + + from collections.abc import Mapping + else: text_type = unicode string_types = (str, unicode) unichr = unichr int_types = (int, long) - iteritems = lambda x: x.iteritems() + + def iteritems(x): + return x.iteritems() + + from collections import Mapping diff --git a/pipenv/vendor/markupsafe/_constants.py b/pipenv/vendor/markupsafe/_constants.py index 919bf03c..7c57c2d2 100644 --- a/pipenv/vendor/markupsafe/_constants.py +++ b/pipenv/vendor/markupsafe/_constants.py @@ -1,267 +1,264 @@ # -*- coding: utf-8 -*- """ - markupsafe._constants - ~~~~~~~~~~~~~~~~~~~~~ +markupsafe._constants +~~~~~~~~~~~~~~~~~~~~~ - Highlevel implementation of the Markup string. - - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ - HTML_ENTITIES = { - 'AElig': 198, - 'Aacute': 193, - 'Acirc': 194, - 'Agrave': 192, - 'Alpha': 913, - 'Aring': 197, - 'Atilde': 195, - 'Auml': 196, - 'Beta': 914, - 'Ccedil': 199, - 'Chi': 935, - 'Dagger': 8225, - 'Delta': 916, - 'ETH': 208, - 'Eacute': 201, - 'Ecirc': 202, - 'Egrave': 200, - 'Epsilon': 917, - 'Eta': 919, - 'Euml': 203, - 'Gamma': 915, - 'Iacute': 205, - 'Icirc': 206, - 'Igrave': 204, - 'Iota': 921, - 'Iuml': 207, - 'Kappa': 922, - 'Lambda': 923, - 'Mu': 924, - 'Ntilde': 209, - 'Nu': 925, - 'OElig': 338, - 'Oacute': 211, - 'Ocirc': 212, - 'Ograve': 210, - 'Omega': 937, - 'Omicron': 927, - 'Oslash': 216, - 'Otilde': 213, - 'Ouml': 214, - 'Phi': 934, - 'Pi': 928, - 'Prime': 8243, - 'Psi': 936, - 'Rho': 929, - 'Scaron': 352, - 'Sigma': 931, - 'THORN': 222, - 'Tau': 932, - 'Theta': 920, - 'Uacute': 218, - 'Ucirc': 219, - 'Ugrave': 217, - 'Upsilon': 933, - 'Uuml': 220, - 'Xi': 926, - 'Yacute': 221, - 'Yuml': 376, - 'Zeta': 918, - 'aacute': 225, - 'acirc': 226, - 'acute': 180, - 'aelig': 230, - 'agrave': 224, - 'alefsym': 8501, - 'alpha': 945, - 'amp': 38, - 'and': 8743, - 'ang': 8736, - 'apos': 39, - 'aring': 229, - 'asymp': 8776, - 'atilde': 227, - 'auml': 228, - 'bdquo': 8222, - 'beta': 946, - 'brvbar': 166, - 'bull': 8226, - 'cap': 8745, - 'ccedil': 231, - 'cedil': 184, - 'cent': 162, - 'chi': 967, - 'circ': 710, - 'clubs': 9827, - 'cong': 8773, - 'copy': 169, - 'crarr': 8629, - 'cup': 8746, - 'curren': 164, - 'dArr': 8659, - 'dagger': 8224, - 'darr': 8595, - 'deg': 176, - 'delta': 948, - 'diams': 9830, - 'divide': 247, - 'eacute': 233, - 'ecirc': 234, - 'egrave': 232, - 'empty': 8709, - 'emsp': 8195, - 'ensp': 8194, - 'epsilon': 949, - 'equiv': 8801, - 'eta': 951, - 'eth': 240, - 'euml': 235, - 'euro': 8364, - 'exist': 8707, - 'fnof': 402, - 'forall': 8704, - 'frac12': 189, - 'frac14': 188, - 'frac34': 190, - 'frasl': 8260, - 'gamma': 947, - 'ge': 8805, - 'gt': 62, - 'hArr': 8660, - 'harr': 8596, - 'hearts': 9829, - 'hellip': 8230, - 'iacute': 237, - 'icirc': 238, - 'iexcl': 161, - 'igrave': 236, - 'image': 8465, - 'infin': 8734, - 'int': 8747, - 'iota': 953, - 'iquest': 191, - 'isin': 8712, - 'iuml': 239, - 'kappa': 954, - 'lArr': 8656, - 'lambda': 955, - 'lang': 9001, - 'laquo': 171, - 'larr': 8592, - 'lceil': 8968, - 'ldquo': 8220, - 'le': 8804, - 'lfloor': 8970, - 'lowast': 8727, - 'loz': 9674, - 'lrm': 8206, - 'lsaquo': 8249, - 'lsquo': 8216, - 'lt': 60, - 'macr': 175, - 'mdash': 8212, - 'micro': 181, - 'middot': 183, - 'minus': 8722, - 'mu': 956, - 'nabla': 8711, - 'nbsp': 160, - 'ndash': 8211, - 'ne': 8800, - 'ni': 8715, - 'not': 172, - 'notin': 8713, - 'nsub': 8836, - 'ntilde': 241, - 'nu': 957, - 'oacute': 243, - 'ocirc': 244, - 'oelig': 339, - 'ograve': 242, - 'oline': 8254, - 'omega': 969, - 'omicron': 959, - 'oplus': 8853, - 'or': 8744, - 'ordf': 170, - 'ordm': 186, - 'oslash': 248, - 'otilde': 245, - 'otimes': 8855, - 'ouml': 246, - 'para': 182, - 'part': 8706, - 'permil': 8240, - 'perp': 8869, - 'phi': 966, - 'pi': 960, - 'piv': 982, - 'plusmn': 177, - 'pound': 163, - 'prime': 8242, - 'prod': 8719, - 'prop': 8733, - 'psi': 968, - 'quot': 34, - 'rArr': 8658, - 'radic': 8730, - 'rang': 9002, - 'raquo': 187, - 'rarr': 8594, - 'rceil': 8969, - 'rdquo': 8221, - 'real': 8476, - 'reg': 174, - 'rfloor': 8971, - 'rho': 961, - 'rlm': 8207, - 'rsaquo': 8250, - 'rsquo': 8217, - 'sbquo': 8218, - 'scaron': 353, - 'sdot': 8901, - 'sect': 167, - 'shy': 173, - 'sigma': 963, - 'sigmaf': 962, - 'sim': 8764, - 'spades': 9824, - 'sub': 8834, - 'sube': 8838, - 'sum': 8721, - 'sup': 8835, - 'sup1': 185, - 'sup2': 178, - 'sup3': 179, - 'supe': 8839, - 'szlig': 223, - 'tau': 964, - 'there4': 8756, - 'theta': 952, - 'thetasym': 977, - 'thinsp': 8201, - 'thorn': 254, - 'tilde': 732, - 'times': 215, - 'trade': 8482, - 'uArr': 8657, - 'uacute': 250, - 'uarr': 8593, - 'ucirc': 251, - 'ugrave': 249, - 'uml': 168, - 'upsih': 978, - 'upsilon': 965, - 'uuml': 252, - 'weierp': 8472, - 'xi': 958, - 'yacute': 253, - 'yen': 165, - 'yuml': 255, - 'zeta': 950, - 'zwj': 8205, - 'zwnj': 8204 + "AElig": 198, + "Aacute": 193, + "Acirc": 194, + "Agrave": 192, + "Alpha": 913, + "Aring": 197, + "Atilde": 195, + "Auml": 196, + "Beta": 914, + "Ccedil": 199, + "Chi": 935, + "Dagger": 8225, + "Delta": 916, + "ETH": 208, + "Eacute": 201, + "Ecirc": 202, + "Egrave": 200, + "Epsilon": 917, + "Eta": 919, + "Euml": 203, + "Gamma": 915, + "Iacute": 205, + "Icirc": 206, + "Igrave": 204, + "Iota": 921, + "Iuml": 207, + "Kappa": 922, + "Lambda": 923, + "Mu": 924, + "Ntilde": 209, + "Nu": 925, + "OElig": 338, + "Oacute": 211, + "Ocirc": 212, + "Ograve": 210, + "Omega": 937, + "Omicron": 927, + "Oslash": 216, + "Otilde": 213, + "Ouml": 214, + "Phi": 934, + "Pi": 928, + "Prime": 8243, + "Psi": 936, + "Rho": 929, + "Scaron": 352, + "Sigma": 931, + "THORN": 222, + "Tau": 932, + "Theta": 920, + "Uacute": 218, + "Ucirc": 219, + "Ugrave": 217, + "Upsilon": 933, + "Uuml": 220, + "Xi": 926, + "Yacute": 221, + "Yuml": 376, + "Zeta": 918, + "aacute": 225, + "acirc": 226, + "acute": 180, + "aelig": 230, + "agrave": 224, + "alefsym": 8501, + "alpha": 945, + "amp": 38, + "and": 8743, + "ang": 8736, + "apos": 39, + "aring": 229, + "asymp": 8776, + "atilde": 227, + "auml": 228, + "bdquo": 8222, + "beta": 946, + "brvbar": 166, + "bull": 8226, + "cap": 8745, + "ccedil": 231, + "cedil": 184, + "cent": 162, + "chi": 967, + "circ": 710, + "clubs": 9827, + "cong": 8773, + "copy": 169, + "crarr": 8629, + "cup": 8746, + "curren": 164, + "dArr": 8659, + "dagger": 8224, + "darr": 8595, + "deg": 176, + "delta": 948, + "diams": 9830, + "divide": 247, + "eacute": 233, + "ecirc": 234, + "egrave": 232, + "empty": 8709, + "emsp": 8195, + "ensp": 8194, + "epsilon": 949, + "equiv": 8801, + "eta": 951, + "eth": 240, + "euml": 235, + "euro": 8364, + "exist": 8707, + "fnof": 402, + "forall": 8704, + "frac12": 189, + "frac14": 188, + "frac34": 190, + "frasl": 8260, + "gamma": 947, + "ge": 8805, + "gt": 62, + "hArr": 8660, + "harr": 8596, + "hearts": 9829, + "hellip": 8230, + "iacute": 237, + "icirc": 238, + "iexcl": 161, + "igrave": 236, + "image": 8465, + "infin": 8734, + "int": 8747, + "iota": 953, + "iquest": 191, + "isin": 8712, + "iuml": 239, + "kappa": 954, + "lArr": 8656, + "lambda": 955, + "lang": 9001, + "laquo": 171, + "larr": 8592, + "lceil": 8968, + "ldquo": 8220, + "le": 8804, + "lfloor": 8970, + "lowast": 8727, + "loz": 9674, + "lrm": 8206, + "lsaquo": 8249, + "lsquo": 8216, + "lt": 60, + "macr": 175, + "mdash": 8212, + "micro": 181, + "middot": 183, + "minus": 8722, + "mu": 956, + "nabla": 8711, + "nbsp": 160, + "ndash": 8211, + "ne": 8800, + "ni": 8715, + "not": 172, + "notin": 8713, + "nsub": 8836, + "ntilde": 241, + "nu": 957, + "oacute": 243, + "ocirc": 244, + "oelig": 339, + "ograve": 242, + "oline": 8254, + "omega": 969, + "omicron": 959, + "oplus": 8853, + "or": 8744, + "ordf": 170, + "ordm": 186, + "oslash": 248, + "otilde": 245, + "otimes": 8855, + "ouml": 246, + "para": 182, + "part": 8706, + "permil": 8240, + "perp": 8869, + "phi": 966, + "pi": 960, + "piv": 982, + "plusmn": 177, + "pound": 163, + "prime": 8242, + "prod": 8719, + "prop": 8733, + "psi": 968, + "quot": 34, + "rArr": 8658, + "radic": 8730, + "rang": 9002, + "raquo": 187, + "rarr": 8594, + "rceil": 8969, + "rdquo": 8221, + "real": 8476, + "reg": 174, + "rfloor": 8971, + "rho": 961, + "rlm": 8207, + "rsaquo": 8250, + "rsquo": 8217, + "sbquo": 8218, + "scaron": 353, + "sdot": 8901, + "sect": 167, + "shy": 173, + "sigma": 963, + "sigmaf": 962, + "sim": 8764, + "spades": 9824, + "sub": 8834, + "sube": 8838, + "sum": 8721, + "sup": 8835, + "sup1": 185, + "sup2": 178, + "sup3": 179, + "supe": 8839, + "szlig": 223, + "tau": 964, + "there4": 8756, + "theta": 952, + "thetasym": 977, + "thinsp": 8201, + "thorn": 254, + "tilde": 732, + "times": 215, + "trade": 8482, + "uArr": 8657, + "uacute": 250, + "uarr": 8593, + "ucirc": 251, + "ugrave": 249, + "uml": 168, + "upsih": 978, + "upsilon": 965, + "uuml": 252, + "weierp": 8472, + "xi": 958, + "yacute": 253, + "yen": 165, + "yuml": 255, + "zeta": 950, + "zwj": 8205, + "zwnj": 8204, } diff --git a/pipenv/vendor/markupsafe/_native.py b/pipenv/vendor/markupsafe/_native.py index 5e83f10a..cd08752c 100644 --- a/pipenv/vendor/markupsafe/_native.py +++ b/pipenv/vendor/markupsafe/_native.py @@ -1,36 +1,49 @@ # -*- coding: utf-8 -*- """ - markupsafe._native - ~~~~~~~~~~~~~~~~~~ +markupsafe._native +~~~~~~~~~~~~~~~~~~ - Native Python implementation the C module is not compiled. +Native Python implementation used when the C module is not compiled. - :copyright: (c) 2010 by Armin Ronacher. - :license: BSD, see LICENSE for more details. +:copyright: 2010 Pallets +:license: BSD-3-Clause """ -from markupsafe import Markup -from markupsafe._compat import text_type +from . import Markup +from ._compat import text_type def escape(s): - """Convert the characters &, <, >, ' and " in string s to HTML-safe - sequences. Use this if you need to display text that might contain - such characters in HTML. Marks return value as markup string. + """Replace the characters ``&``, ``<``, ``>``, ``'``, and ``"`` in + the string with HTML-safe sequences. Use this if you need to display + text that might contain such characters in HTML. + + If the object has an ``__html__`` method, it is called and the + return value is assumed to already be safe for HTML. + + :param s: An object to be converted to a string and escaped. + :return: A :class:`Markup` string with the escaped text. """ - if hasattr(s, '__html__'): - return s.__html__() - return Markup(text_type(s) - .replace('&', '&') - .replace('>', '>') - .replace('<', '<') - .replace("'", ''') - .replace('"', '"') + if hasattr(s, "__html__"): + return Markup(s.__html__()) + return Markup( + text_type(s) + .replace("&", "&") + .replace(">", ">") + .replace("<", "<") + .replace("'", "'") + .replace('"', """) ) def escape_silent(s): - """Like :func:`escape` but converts `None` into an empty - markup string. + """Like :func:`escape` but treats ``None`` as the empty string. + Useful with optional values, as otherwise you get the string + ``'None'`` when the value is ``None``. + + >>> escape(None) + Markup('None') + >>> escape_silent(None) + Markup('') """ if s is None: return Markup() @@ -38,8 +51,18 @@ def escape_silent(s): def soft_unicode(s): - """Make a string unicode if it isn't already. That way a markup - string is not converted back to unicode. + """Convert an object to a string if it isn't already. This preserves + a :class:`Markup` string rather than converting it back to a basic + string, so it will still be marked as safe and won't be escaped + again. + + >>> value = escape('') + >>> value + Markup('<User 1>') + >>> escape(str(value)) + Markup('&lt;User 1&gt;') + >>> escape(soft_unicode(value)) + Markup('<User 1>') """ if not isinstance(s, text_type): s = text_type(s) diff --git a/pipenv/vendor/markupsafe/_speedups.c b/pipenv/vendor/markupsafe/_speedups.c index d779a68c..12d2c4a7 100644 --- a/pipenv/vendor/markupsafe/_speedups.c +++ b/pipenv/vendor/markupsafe/_speedups.c @@ -2,33 +2,30 @@ * markupsafe._speedups * ~~~~~~~~~~~~~~~~~~~~ * - * This module implements functions for automatic escaping in C for better - * performance. + * C implementation of escaping for better performance. Used instead of + * the native Python implementation when compiled. * - * :copyright: (c) 2010 by Armin Ronacher. - * :license: BSD. + * :copyright: 2010 Pallets + * :license: BSD-3-Clause */ - #include +#if PY_MAJOR_VERSION < 3 #define ESCAPED_CHARS_TABLE_SIZE 63 #define UNICHR(x) (PyUnicode_AS_UNICODE((PyUnicodeObject*)PyUnicode_DecodeASCII(x, strlen(x), NULL))); -#if PY_VERSION_HEX < 0x02050000 && !defined(PY_SSIZE_T_MIN) -typedef int Py_ssize_t; -#define PY_SSIZE_T_MAX INT_MAX -#define PY_SSIZE_T_MIN INT_MIN -#endif - - -static PyObject* markup; static Py_ssize_t escaped_chars_delta_len[ESCAPED_CHARS_TABLE_SIZE]; static Py_UNICODE *escaped_chars_repl[ESCAPED_CHARS_TABLE_SIZE]; +#endif + +static PyObject* markup; static int init_constants(void) { PyObject *module; + +#if PY_MAJOR_VERSION < 3 /* mapping of characters to replace */ escaped_chars_repl['"'] = UNICHR("""); escaped_chars_repl['\''] = UNICHR("'"); @@ -41,6 +38,7 @@ init_constants(void) escaped_chars_delta_len['"'] = escaped_chars_delta_len['\''] = \ escaped_chars_delta_len['&'] = 4; escaped_chars_delta_len['<'] = escaped_chars_delta_len['>'] = 3; +#endif /* import markup type so that we can mark the return value */ module = PyImport_ImportModule("markupsafe"); @@ -52,6 +50,7 @@ init_constants(void) return 1; } +#if PY_MAJOR_VERSION < 3 static PyObject* escape_unicode(PyUnicodeObject *in) { @@ -112,13 +111,192 @@ escape_unicode(PyUnicodeObject *in) return (PyObject*)out; } +#else /* PY_MAJOR_VERSION < 3 */ +#define GET_DELTA(inp, inp_end, delta) \ + while (inp < inp_end) { \ + switch (*inp++) { \ + case '"': \ + case '\'': \ + case '&': \ + delta += 4; \ + break; \ + case '<': \ + case '>': \ + delta += 3; \ + break; \ + } \ + } + +#define DO_ESCAPE(inp, inp_end, outp) \ + { \ + Py_ssize_t ncopy = 0; \ + while (inp < inp_end) { \ + switch (*inp) { \ + case '"': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '4'; \ + *outp++ = ';'; \ + break; \ + case '\'': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = '#'; \ + *outp++ = '3'; \ + *outp++ = '9'; \ + *outp++ = ';'; \ + break; \ + case '&': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'a'; \ + *outp++ = 'm'; \ + *outp++ = 'p'; \ + *outp++ = ';'; \ + break; \ + case '<': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'l'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + case '>': \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + outp += ncopy; ncopy = 0; \ + *outp++ = '&'; \ + *outp++ = 'g'; \ + *outp++ = 't'; \ + *outp++ = ';'; \ + break; \ + default: \ + ncopy++; \ + } \ + inp++; \ + } \ + memcpy(outp, inp-ncopy, sizeof(*outp)*ncopy); \ + } + +static PyObject* +escape_unicode_kind1(PyUnicodeObject *in) +{ + Py_UCS1 *inp = PyUnicode_1BYTE_DATA(in); + Py_UCS1 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS1 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, + PyUnicode_IS_ASCII(in) ? 127 : 255); + if (!out) + return NULL; + + inp = PyUnicode_1BYTE_DATA(in); + outp = PyUnicode_1BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode_kind2(PyUnicodeObject *in) +{ + Py_UCS2 *inp = PyUnicode_2BYTE_DATA(in); + Py_UCS2 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS2 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 65535); + if (!out) + return NULL; + + inp = PyUnicode_2BYTE_DATA(in); + outp = PyUnicode_2BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + + +static PyObject* +escape_unicode_kind4(PyUnicodeObject *in) +{ + Py_UCS4 *inp = PyUnicode_4BYTE_DATA(in); + Py_UCS4 *inp_end = inp + PyUnicode_GET_LENGTH(in); + Py_UCS4 *outp; + PyObject *out; + Py_ssize_t delta = 0; + + GET_DELTA(inp, inp_end, delta); + if (!delta) { + Py_INCREF(in); + return (PyObject*)in; + } + + out = PyUnicode_New(PyUnicode_GET_LENGTH(in) + delta, 1114111); + if (!out) + return NULL; + + inp = PyUnicode_4BYTE_DATA(in); + outp = PyUnicode_4BYTE_DATA(out); + DO_ESCAPE(inp, inp_end, outp); + return out; +} + +static PyObject* +escape_unicode(PyUnicodeObject *in) +{ + if (PyUnicode_READY(in)) + return NULL; + + switch (PyUnicode_KIND(in)) { + case PyUnicode_1BYTE_KIND: + return escape_unicode_kind1(in); + case PyUnicode_2BYTE_KIND: + return escape_unicode_kind2(in); + case PyUnicode_4BYTE_KIND: + return escape_unicode_kind4(in); + } + assert(0); /* shouldn't happen */ + return NULL; +} +#endif /* PY_MAJOR_VERSION < 3 */ static PyObject* escape(PyObject *self, PyObject *text) { + static PyObject *id_html; PyObject *s = NULL, *rv = NULL, *html; + if (id_html == NULL) { +#if PY_MAJOR_VERSION < 3 + id_html = PyString_InternFromString("__html__"); +#else + id_html = PyUnicode_InternFromString("__html__"); +#endif + if (id_html == NULL) { + return NULL; + } + } + /* we don't have to escape integers, bools or floats */ if (PyLong_CheckExact(text) || #if PY_MAJOR_VERSION < 3 @@ -129,10 +307,16 @@ escape(PyObject *self, PyObject *text) return PyObject_CallFunctionObjArgs(markup, text, NULL); /* if the object has an __html__ method that performs the escaping */ - html = PyObject_GetAttrString(text, "__html__"); + html = PyObject_GetAttr(text ,id_html); if (html) { - rv = PyObject_CallObject(html, NULL); + s = PyObject_CallObject(html, NULL); Py_DECREF(html); + if (s == NULL) { + return NULL; + } + /* Convert to Markup object */ + rv = PyObject_CallFunctionObjArgs(markup, (PyObject*)s, NULL); + Py_DECREF(s); return rv; } diff --git a/pipenv/vendor/orderedmultidict/LICENSE.md b/pipenv/vendor/orderedmultidict/LICENSE.md new file mode 100644 index 00000000..210e8658 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/LICENSE.md @@ -0,0 +1,31 @@ +Build Amazing Things. + +*** + +### Unlicense + +This is free and unencumbered software released into the public\ +domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or distribute\ +this software, either in source code form or as a compiled binary, for any\ +purpose, commercial or non-commercial, and by any means. + +In jurisdictions that recognize copyright laws, the author or authors of\ +this software dedicate any and all copyright interest in the software to the\ +public domain. We make this dedication for the benefit of the public at\ +large and to the detriment of our heirs and successors. We intend this\ +dedication to be an overt act of relinquishment in perpetuity of all\ +present and future rights to this software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF\ +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED\ +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A\ +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT\ +SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR\ +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT\ +OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION\ +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\ +SOFTWARE. + +For more information, please refer to diff --git a/pipenv/vendor/orderedmultidict/__init__.py b/pipenv/vendor/orderedmultidict/__init__.py new file mode 100755 index 00000000..4f0ce2f7 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/__init__.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from .orderedmultidict import * # noqa + +__title__ = 'orderedmultidict' +__version__ = '1.0' +__author__ = 'Ansgar Grunseid' +__contact__ = 'grunseid@gmail.com' +__license__ = 'Unlicense' +__url__ = 'https://github.com/gruns/orderedmultidict' diff --git a/pipenv/vendor/orderedmultidict/itemlist.py b/pipenv/vendor/orderedmultidict/itemlist.py new file mode 100755 index 00000000..e9e96c72 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/itemlist.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from six.moves import zip_longest + +_absent = object() # Marker that means no parameter was provided. + + +class itemnode(object): + + """ + Dictionary key:value items wrapped in a node to be members of itemlist, the + doubly linked list defined below. + """ + + def __init__(self, prev=None, next=None, key=_absent, value=_absent): + self.prev = prev + self.next = next + self.key = key + self.value = value + + +class itemlist(object): + + """ + Doubly linked list of itemnodes. + + This class is used as the key:value item storage of orderedmultidict. + Methods below were only added as needed for use with orderedmultidict, so + some otherwise common list methods may be missing. + """ + + def __init__(self, items=[]): + self.root = itemnode() + self.root.next = self.root.prev = self.root + self.size = 0 + + for key, value in items: + self.append(key, value) + + def append(self, key, value): + tail = self.root.prev if self.root.prev is not self.root else self.root + node = itemnode(tail, self.root, key=key, value=value) + tail.next = node + self.root.prev = node + self.size += 1 + return node + + def removenode(self, node): + node.prev.next = node.next + node.next.prev = node.prev + self.size -= 1 + return self + + def clear(self): + for node, key, value in self: + self.removenode(node) + return self + + def items(self): + return list(self.iteritems()) + + def keys(self): + return list(self.iterkeys()) + + def values(self): + return list(self.itervalues()) + + def iteritems(self): + for node, key, value in self: + yield key, value + + def iterkeys(self): + for node, key, value in self: + yield key + + def itervalues(self): + for node, key, value in self: + yield value + + def reverse(self): + for node, key, value in self: + node.prev, node.next = node.next, node.prev + self.root.prev, self.root.next = self.root.next, self.root.prev + return self + + def __len__(self): + return self.size + + def __iter__(self): + current = self.root.next + while current and current is not self.root: + # Record current.next here in case current.next changes after the + # yield and before we return for the next iteration. For example, + # methods like reverse() will change current.next() before yield + # gets executed again. + nextnode = current.next + yield current, current.key, current.value + current = nextnode + + def __contains__(self, item): + """ + Params: + item: Can either be a (key,value) tuple or an itemnode reference. + """ + node = key = value = _absent + if hasattr(item, '__len__') and callable(item.__len__): + if len(item) == 2: + key, value = item + elif len(item) == 3: + node, key, value = item + else: + node = item + + if node is not _absent or _absent not in [key, value]: + for selfnode, selfkey, selfvalue in self: + if ((node is _absent and key == selfkey and value == selfvalue) + or (node is not _absent and node == selfnode)): + return True + return False + + def __getitem__(self, index): + # Only support direct access to the first or last element, as this is + # all orderedmultidict needs for now. + if index == 0 and self.root.next is not self.root: + return self.root.next + elif index == -1 and self.root.prev is not self.root: + return self.root.prev + raise IndexError(index) + + def __delitem__(self, index): + self.removenode(self[index]) + + def __eq__(self, other): + for (n1, key1, value1), (n2, key2, value2) in zip_longest(self, other): + if key1 != key2 or value1 != value2: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __nonzero__(self): + return self.size > 0 + + def __str__(self): + return '[%s]' % self.items() diff --git a/pipenv/vendor/orderedmultidict/orderedmultidict.py b/pipenv/vendor/orderedmultidict/orderedmultidict.py new file mode 100755 index 00000000..924dd8d2 --- /dev/null +++ b/pipenv/vendor/orderedmultidict/orderedmultidict.py @@ -0,0 +1,811 @@ +# -*- coding: utf-8 -*- + +# +# omdict - Ordered Multivalue Dictionary. +# +# Ansgar Grunseid +# grunseid.com +# grunseid@gmail.com +# +# License: Build Amazing Things (Unlicense) + +from __future__ import absolute_import + +from itertools import chain +from collections import MutableMapping + +import six +from six.moves import map, zip_longest + +from .itemlist import itemlist + +try: + from collections import OrderedDict as odict # Python 2.7 and later. +except ImportError: + from ordereddict import OrderedDict as odict # Python 2.6 and earlier. + +import sys +items_attr = 'items' if sys.version_info[0] >= 3 else 'iteritems' + +_absent = object() # Marker that means no parameter was provided. + + +def callable_attr(obj, attr): + return hasattr(obj, attr) and callable(getattr(obj, attr)) + + +# +# TODO(grun): Create a subclass of list that values(), getlist(), allitems(), +# etc return that the user can manipulate directly to control the omdict() +# object. +# +# For example, users should be able to do things like +# +# omd = omdict([(1,1), (1,11)]) +# omd.values(1).append('sup') +# omd.allitems() == [(1,1), (1,11), (1,'sup')] +# omd.values(1).remove(11) +# omd.allitems() == [(1,1), (1,'sup')] +# omd.values(1).extend(['two', 'more']) +# omd.allitems() == [(1,1), (1,'sup'), (1,'two'), (1,'more')] +# +# or +# +# omd = omdict([(1,1), (1,11)]) +# omd.allitems().extend([(2,2), (2,22)]) +# omd.allitems() == [(1,1), (1,11), (2,2), (2,22)]) +# +# or +# +# omd = omdict() +# omd.values(1) = [1, 11] +# omd.allitems() == [(1,1), (1,11)] +# omd.values(1) = list(map(lambda i: i * -10, omd.values(1))) +# omd.allitems() == [(1,-10), (1,-110)] +# omd.allitems() = filter(lambda (k,v): v > -100, omd.allitems()) +# omd.allitems() == [(1,-10)] +# +# etc. +# +# To accomplish this, subclass list in such a manner that each list element is +# really a two tuple, where the first tuple value is the actual value and the +# second tuple value is a reference to the itemlist node for that value. Users +# only interact with the first tuple values, the actual values, but behind the +# scenes when an element is modified, deleted, inserted, etc, the according +# itemlist nodes are modified, deleted, inserted, etc accordingly. In this +# manner, users can manipulate omdict objects directly through direct list +# manipulation. +# +# Once accomplished, some methods become redundant and should be removed in +# favor of the more intuitive direct value list manipulation. Such redundant +# methods include getlist() (removed in favor of values()?), addlist(), and +# setlist(). +# +# With the removal of many of the 'list' methods, think about renaming all +# remaining 'list' methods to 'values' methods, like poplist() -> popvalues(), +# poplistitem() -> popvaluesitem(), etc. This would be an easy switch for most +# methods, but wouldn't fit others so well. For example, iterlists() would +# become itervalues(), a name extremely similar to iterallvalues() but quite +# different in function. +# + + +class omdict(MutableMapping): + + """ + Ordered Multivalue Dictionary. + + A multivalue dictionary is a dictionary that can store multiple values per + key. An ordered multivalue dictionary is a multivalue dictionary that + retains the order of insertions and deletions. + + Internally, items are stored in a doubly linked list, self._items. A + dictionary, self._map, is also maintained and stores an ordered list of + linked list node references, one for each value associated with that key. + + Standard dict methods interact with the first value associated with a given + key. This means that omdict retains method parity with dict, and a dict + object can be replaced with an omdict object and all interaction will + behave identically. All dict methods that retain parity with omdict are: + + get(), setdefault(), pop(), popitem(), + clear(), copy(), update(), fromkeys(), len() + __getitem__(), __setitem__(), __delitem__(), __contains__(), + items(), keys(), values(), iteritems(), iterkeys(), itervalues(), + + Optional parameters have been added to some dict methods, but because the + added parameters are optional, existing use remains unaffected. An optional + parameter has been added to these methods: + + items(), values(), iteritems(), itervalues() + + New methods have also been added to omdict. Methods with 'list' in their + name interact with lists of values, and methods with 'all' in their name + interact with all items in the dictionary, including multiple items with + the same key. + + The new omdict methods are: + + load(), size(), reverse(), + getlist(), add(), addlist(), set(), setlist(), setdefaultlist(), + poplist(), popvalue(), popvalues(), popitem(), poplistitem(), + allitems(), allkeys(), allvalues(), lists(), listitems(), + iterallitems(), iterallkeys(), iterallvalues(), iterlists(), + iterlistitems() + + Explanations and examples of the new methods above can be found in the + function comments below and online at + + https://github.com/gruns/orderedmultidict + + Additional omdict information and documentation can also be found at the + above url. + """ + + def __init__(self, *args, **kwargs): + # Doubly linked list of itemnodes. Each itemnode stores a key:value + # item. + self._items = itemlist() + + # Ordered dictionary of keys and itemnode references. Each itemnode + # reference points to one of that keys values. + self._map = odict() + + self.load(*args, **kwargs) + + def load(self, *args, **kwargs): + """ + Clear all existing key:value items and import all key:value items from + . If multiple values exist for the same key in , they + are all be imported. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.load([(4,4), (4,44), (5,5)]) + omd.allitems() == [(4,4), (4,44), (5,5)] + + Returns: . + """ + self.clear() + self.updateall(*args, **kwargs) + return self + + def copy(self): + return self.__class__(self.allitems()) + + def clear(self): + self._map.clear() + self._items.clear() + + def size(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.size() == 5 + + Returns: Total number of items, including multiple items with the same + key. + """ + return len(self._items) + + @classmethod + def fromkeys(cls, iterable, value=None): + return cls([(key, value) for key in iterable]) + + def has_key(self, key): + return key in self + + def update(self, *args, **kwargs): + self._update_updateall(True, *args, **kwargs) + + def updateall(self, *args, **kwargs): + """ + Update this dictionary with the items from , replacing + existing key:value items with shared keys before adding new key:value + items. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.updateall([(2,'two'), (1,'one'), (2,222), (1,111)]) + omd.allitems() == [(1, 'one'), (2, 'two'), (2, 222), (1, 111)] + + Returns: . + """ + self._update_updateall(False, *args, **kwargs) + return self + + def _update_updateall(self, replace_at_most_one, *args, **kwargs): + # Bin the items in and into or + # . Items in are new values to replace old + # values for a given key, and items in are new items to be + # added. + replacements, leftovers = dict(), [] + for mapping in chain(args, [kwargs]): + self._bin_update_items( + self._items_iterator(mapping), replace_at_most_one, + replacements, leftovers) + + # First, replace existing values for each key. + for key, values in six.iteritems(replacements): + self.setlist(key, values) + # Then, add the leftover items to the end of the list of all items. + for key, value in leftovers: + self.add(key, value) + + def _bin_update_items(self, items, replace_at_most_one, + replacements, leftovers): + """ + are modified directly, ala pass by + reference. + """ + for key, value in items: + # If there are existing items with key that have yet to be + # marked for replacement, mark that item's value to be replaced by + # by appending it to . + if key in self and key not in replacements: + replacements[key] = [value] + elif (key in self and not replace_at_most_one and + len(replacements[key]) < len(self.values(key))): + replacements[key].append(value) + else: + if replace_at_most_one: + replacements[key] = [value] + else: + leftovers.append((key, value)) + + def _items_iterator(self, container): + cont = container + iterator = iter(cont) + if callable_attr(cont, 'iterallitems'): + iterator = cont.iterallitems() + elif callable_attr(cont, 'allitems'): + iterator = iter(cont.allitems()) + elif callable_attr(cont, 'iteritems'): + iterator = cont.iteritems() + elif callable_attr(cont, 'items'): + iterator = iter(cont.items()) + return iterator + + def get(self, key, default=None): + if key in self: + return self._map[key][0].value + return default + + def getlist(self, key, default=[]): + """ + Returns: The list of values for if is in the dictionary, + else . If is not provided, an empty list is + returned. + """ + if key in self: + return [node.value for node in self._map[key]] + return default + + def setdefault(self, key, default=None): + if key in self: + return self[key] + self.add(key, default) + return default + + def setdefaultlist(self, key, defaultlist=[None]): + """ + Similar to setdefault() except is a list of values to set + for . If already exists, its existing list of values is + returned. + + If isn't a key and is an empty list, [], no values + are added for and will not be added as a key. + + Returns: List of 's values if exists in the dictionary, + otherwise . + """ + if key in self: + return self.getlist(key) + self.addlist(key, defaultlist) + return defaultlist + + def add(self, key, value=None): + """ + Add to the list of values for . If is not in the + dictionary, then is added as the sole value for . + + Example: + omd = omdict() + omd.add(1, 1) # omd.allitems() == [(1,1)] + omd.add(1, 11) # omd.allitems() == [(1,1), (1,11)] + omd.add(2, 2) # omd.allitems() == [(1,1), (1,11), (2,2)] + + Returns: . + """ + self._map.setdefault(key, []) + node = self._items.append(key, value) + self._map[key].append(node) + return self + + def addlist(self, key, valuelist=[]): + """ + Add the values in to the list of values for . If + is not in the dictionary, the values in become the values + for . + + Example: + omd = omdict([(1,1)]) + omd.addlist(1, [11, 111]) + omd.allitems() == [(1, 1), (1, 11), (1, 111)] + omd.addlist(2, [2]) + omd.allitems() == [(1, 1), (1, 11), (1, 111), (2, 2)] + + Returns: . + """ + for value in valuelist: + self.add(key, value) + return self + + def set(self, key, value=None): + """ + Sets 's value to . Identical in function to __setitem__(). + + Returns: . + """ + self[key] = value + return self + + def setlist(self, key, values): + """ + Sets 's list of values to . Existing items with key + are first replaced with new values from . Any remaining old + items that haven't been replaced with new values are deleted, and any + new values from that don't have corresponding items with + to replace are appended to the end of the list of all items. + + If values is an empty list, [], is deleted, equivalent in action + to del self[]. + + Example: + omd = omdict([(1,1), (2,2)]) + omd.setlist(1, [11, 111]) + omd.allitems() == [(1,11), (2,2), (1,111)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, [None]) + omd.allitems() == [(1,None), (2,2)] + + omd = omdict([(1,1), (1,11), (2,2), (1,111)]) + omd.setlist(1, []) + omd.allitems() == [(2,2)] + + Returns: . + """ + if not values and key in self: + self.pop(key) + else: + it = zip_longest( + list(self._map.get(key, [])), values, fillvalue=_absent) + for node, value in it: + if node is not _absent and value is not _absent: + node.value = value + elif node is _absent: + self.add(key, value) + elif value is _absent: + self._map[key].remove(node) + self._items.removenode(node) + return self + + def removevalues(self, key, values): + """ + Removes all from the values of . If has no + remaining values after removevalues(), the key is popped. + + Example: + omd = omdict([(1, 1), (1, 11), (1, 1), (1, 111)]) + omd.removevalues(1, [1, 111]) + omd.allitems() == [(1, 11)] + + Returns: . + """ + self.setlist(key, [v for v in self.getlist(key) if v not in values]) + return self + + def pop(self, key, default=_absent): + if key in self: + return self.poplist(key)[0] + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def poplist(self, key, default=_absent): + """ + If is in the dictionary, pop it and return its list of values. If + is not in the dictionary, return . KeyError is raised if + is not provided and is not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplist(1) == [1, 11, 111] + omd.allitems() == [(2,2), (3,3)] + omd.poplist(2) == [2] + omd.allitems() == [(3,3)] + + Raises: KeyError if isn't in the dictionary and isn't + provided. + Returns: List of 's values. + """ + if key in self: + values = self.getlist(key) + del self._map[key] + for node, nodekey, nodevalue in self._items: + if nodekey == key: + self._items.removenode(node) + return values + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popvalue(self, key, value=_absent, default=_absent, last=True): + """ + If is provided, pops the first or last (key,value) item in the + dictionary if is in the dictionary. + + If is not provided, pops the first or last value for if + is in the dictionary. + + If no longer has any values after a popvalue() call, is + removed from the dictionary. If isn't in the dictionary and + was provided, return default. KeyError is raised if + is not provided and is not in the dictionary. ValueError is + raised if is provided but isn't a value for . + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3), (2,22)]) + omd.popvalue(1) == 111 + omd.allitems() == [(1,11), (1,111), (2,2), (3,3), (2,22)] + omd.popvalue(1, last=False) == 1 + omd.allitems() == [(1,11), (2,2), (3,3), (2,22)] + omd.popvalue(2, 2) == 2 + omd.allitems() == [(1,11), (3,3), (2,22)] + omd.popvalue(1, 11) == 11 + omd.allitems() == [(3,3), (2,22)] + omd.popvalue('not a key', default='sup') == 'sup' + + Params: + last: Boolean whether to return 's first value ( is False) + or last value ( is True). + Raises: + KeyError if isn't in the dictionary and isn't + provided. + ValueError if isn't a value for . + Returns: The first or last of 's values. + """ + def pop_node_with_index(key, index): + node = self._map[key].pop(index) + if not self._map[key]: + del self._map[key] + self._items.removenode(node) + return node + + if key in self: + if value is not _absent: + if last: + pos = self.values(key)[::-1].index(value) + else: + pos = self.values(key).index(value) + if pos == -1: + raise ValueError(value) + else: + index = (len(self.values(key)) - 1 - pos) if last else pos + return pop_node_with_index(key, index).value + else: + return pop_node_with_index(key, -1 if last else 0).value + elif key not in self._map and default is not _absent: + return default + raise KeyError(key) + + def popitem(self, fromall=False, last=True): + """ + Pop and return a key:value item. + + If is False, items()[0] is popped if is False or + items()[-1] is popped if is True. All remaining items with the + same key are removed. + + If is True, allitems()[0] is popped if is False or + allitems()[-1] is popped if is True. Any remaining items with + the same key remain. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem() == (3,3) + omd.popitem(fromall=False, last=False) == (1,1) + omd.popitem(fromall=False, last=False) == (2,2) + + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.popitem(fromall=True, last=False) == (1,1) + omd.popitem(fromall=True, last=False) == (1,11) + omd.popitem(fromall=True, last=True) == (3,3) + omd.popitem(fromall=True, last=False) == (1,111) + + Params: + fromall: Whether to pop an item from items() ( is True) or + allitems() ( is False). + last: Boolean whether to pop the first item or last item of items() + or allitems(). + Raises: KeyError if the dictionary is empty. + Returns: The first or last item from item() or allitem(). + """ + if not self._items: + raise KeyError('popitem(): %s is empty' % self.__class__.__name__) + + if fromall: + node = self._items[-1 if last else 0] + key = node.key + return key, self.popvalue(key, last=last) + else: + key = list(self._map.keys())[-1 if last else 0] + return key, self.pop(key) + + def poplistitem(self, last=True): + """ + Pop and return a key:valuelist item comprised of a key and that key's + list of values. If is False, a key:valuelist item comprised of + keys()[0] and its list of values is popped and returned. If is + True, a key:valuelist item comprised of keys()[-1] and its list of + values is popped and returned. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.poplistitem(last=True) == (3,[3]) + omd.poplistitem(last=False) == (1,[1,11,111]) + + Params: + last: Boolean whether to pop the first or last key and its associated + list of values. + Raises: KeyError if the dictionary is empty. + Returns: A two-tuple comprised of the first or last key and its + associated list of values. + """ + if not self._items: + s = 'poplistitem(): %s is empty' % self.__class__.__name__ + raise KeyError(s) + + key = self.keys()[-1 if last else 0] + return key, self.poplist(key) + + def items(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iteritems(). Only items with key + are returned if is provided and is a dictionary key. + """ + return list(self.iteritems(key)) + + def keys(self): + return list(self.iterkeys()) + + def values(self, key=_absent): + """ + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from itervalues().If is provided and + is a dictionary key, only values of items with key are + returned. + """ + if key is not _absent and key in self._map: + return self.getlist(key) + return list(self.itervalues()) + + def lists(self): + """ + Returns: List created from iterlists(). + """ + return list(self.iterlists()) + + def listitems(self): + """ + Returns: List created from iterlistitems(). + """ + return list(self.iterlistitems()) + + def iteritems(self, key=_absent): + """ + Parity with dict.iteritems() except the optional parameter has + been added. If is provided, only items with the provided key are + iterated over. KeyError is raised if is provided and not in the + dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iteritems(1) -> (1,1) -> (1,11) -> (1,111) + omd.iteritems() -> (1,1) -> (2,2) -> (3,3) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over the items() of the dictionary, or only items + with the key if is provided. + """ + if key is not _absent: + if key in self: + items = [(node.key, node.value) for node in self._map[key]] + return iter(items) + raise KeyError(key) + items = six.iteritems(self._map) + return iter((key, nodes[0].value) for (key, nodes) in items) + + def iterkeys(self): + return six.iterkeys(self._map) + + def itervalues(self, key=_absent): + """ + Parity with dict.itervalues() except the optional parameter has + been added. If is provided, only values from items with the + provided key are iterated over. KeyError is raised if is provided + and not in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.itervalues(1) -> 1 -> 11 -> 111 + omd.itervalues() -> 1 -> 11 -> 111 -> 2 -> 3 + + Raises: KeyError if is provided and isn't in the dictionary. + Returns: An iterator over the values() of the dictionary, or only the + values of key if is provided. + """ + if key is not _absent: + if key in self: + return iter([node.value for node in self._map[key]]) + raise KeyError(key) + return iter([nodes[0].value for nodes in six.itervalues(self._map)]) + + def allitems(self, key=_absent): + ''' + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallitems(). + ''' + return list(self.iterallitems(key)) + + def allkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allkeys() == [1,1,1,2,3] + + Returns: List created from iterallkeys(). + ''' + return list(self.iterallkeys()) + + def allvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.allvalues() == [1,11,111,2,3] + omd.allvalues(1) == [1,11,111] + + Raises: KeyError if is provided and not in the dictionary. + Returns: List created from iterallvalues(). + ''' + return list(self.iterallvalues(key)) + + def iterallitems(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallitems() == (1,1) -> (1,11) -> (1,111) -> (2,2) -> (3,3) + omd.iterallitems(1) == (1,1) -> (1,11) -> (1,111) + + Raises: KeyError if is provided and not in the dictionary. + Returns: An iterator over every item in the diciontary. If is + provided, only items with the key are iterated over. + ''' + if key is not _absent: + # Raises KeyError if is not in self._map. + return self.iteritems(key) + return self._items.iteritems() + + def iterallkeys(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallkeys() == 1 -> 1 -> 1 -> 2 -> 3 + + Returns: An iterator over the keys of every item in the dictionary. + ''' + return self._items.iterkeys() + + def iterallvalues(self, key=_absent): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterallvalues() == 1 -> 11 -> 111 -> 2 -> 3 + + Returns: An iterator over the values of every item in the dictionary. + ''' + if key is not _absent: + if key in self: + return iter(self.getlist(key)) + raise KeyError(key) + return self._items.itervalues() + + def iterlists(self): + ''' + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlists() -> [1,11,111] -> [2] -> [3] + + Returns: An iterator over the list comprised of the lists of values for + each key. + ''' + return map(lambda key: self.getlist(key), self) + + def iterlistitems(self): + """ + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.iterlistitems() -> (1,[1,11,111]) -> (2,[2]) -> (3,[3]) + + Returns: An iterator over the list of key:valuelist items. + """ + return map(lambda key: (key, self.getlist(key)), self) + + def reverse(self): + """ + Reverse the order of all items in the dictionary. + + Example: + omd = omdict([(1,1), (1,11), (1,111), (2,2), (3,3)]) + omd.reverse() + omd.allitems() == [(3,3), (2,2), (1,111), (1,11), (1,1)] + + Returns: . + """ + for key in six.iterkeys(self._map): + self._map[key].reverse() + self._items.reverse() + return self + + def __eq__(self, other): + if callable_attr(other, 'iterallitems'): + myiter, otheriter = self.iterallitems(), other.iterallitems() + for i1, i2 in zip_longest(myiter, otheriter, fillvalue=_absent): + if i1 != i2 or i1 is _absent or i2 is _absent: + return False + elif not hasattr(other, '__len__') or not hasattr(other, items_attr): + return False + # Ignore order so we can compare ordered omdicts with unordered dicts. + else: + if len(self) != len(other): + return False + for key, value in six.iteritems(other): + if self.get(key, _absent) != value: + return False + return True + + def __ne__(self, other): + return not self.__eq__(other) + + def __len__(self): + return len(self._map) + + def __iter__(self): + for key in self.iterkeys(): + yield key + + def __contains__(self, key): + return key in self._map + + def __getitem__(self, key): + if key in self: + return self.get(key) + raise KeyError(key) + + def __setitem__(self, key, value): + self.setlist(key, [value]) + + def __delitem__(self, key): + return self.pop(key) + + def __nonzero__(self): + return bool(self._map) + + def __str__(self): + return '{%s}' % ', '.join( + map(lambda p: '%r: %r' % (p[0], p[1]), self.iterallitems())) + + def __repr__(self): + return '%s(%s)' % (self.__class__.__name__, self.allitems()) diff --git a/pipenv/vendor/packaging/__about__.py b/pipenv/vendor/packaging/__about__.py index 21fc6ce3..7481c9e2 100644 --- a/pipenv/vendor/packaging/__about__.py +++ b/pipenv/vendor/packaging/__about__.py @@ -4,18 +4,24 @@ from __future__ import absolute_import, division, print_function __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] __title__ = "packaging" __summary__ = "Core utilities for Python packages" __uri__ = "https://github.com/pypa/packaging" -__version__ = "18.0" +__version__ = "19.0" __author__ = "Donald Stufft and individual contributors" __email__ = "donald@stufft.io" __license__ = "BSD or Apache License, Version 2.0" -__copyright__ = "Copyright 2014-2018 %s" % __author__ +__copyright__ = "Copyright 2014-2019 %s" % __author__ diff --git a/pipenv/vendor/packaging/__init__.py b/pipenv/vendor/packaging/__init__.py index 5ee62202..a0cf67df 100644 --- a/pipenv/vendor/packaging/__init__.py +++ b/pipenv/vendor/packaging/__init__.py @@ -4,11 +4,23 @@ from __future__ import absolute_import, division, print_function from .__about__ import ( - __author__, __copyright__, __email__, __license__, __summary__, __title__, - __uri__, __version__ + __author__, + __copyright__, + __email__, + __license__, + __summary__, + __title__, + __uri__, + __version__, ) __all__ = [ - "__title__", "__summary__", "__uri__", "__version__", "__author__", - "__email__", "__license__", "__copyright__", + "__title__", + "__summary__", + "__uri__", + "__version__", + "__author__", + "__email__", + "__license__", + "__copyright__", ] diff --git a/pipenv/vendor/packaging/_compat.py b/pipenv/vendor/packaging/_compat.py index 210bb80b..25da473c 100644 --- a/pipenv/vendor/packaging/_compat.py +++ b/pipenv/vendor/packaging/_compat.py @@ -12,9 +12,9 @@ PY3 = sys.version_info[0] == 3 # flake8: noqa if PY3: - string_types = str, + string_types = (str,) else: - string_types = basestring, + string_types = (basestring,) def with_metaclass(meta, *bases): @@ -27,4 +27,5 @@ def with_metaclass(meta, *bases): class metaclass(meta): def __new__(cls, name, this_bases, d): return meta(name, bases, d) - return type.__new__(metaclass, 'temporary_class', (), {}) + + return type.__new__(metaclass, "temporary_class", (), {}) diff --git a/pipenv/vendor/packaging/_structures.py b/pipenv/vendor/packaging/_structures.py index e9fc4a04..68dcca63 100644 --- a/pipenv/vendor/packaging/_structures.py +++ b/pipenv/vendor/packaging/_structures.py @@ -5,7 +5,6 @@ from __future__ import absolute_import, division, print_function class Infinity(object): - def __repr__(self): return "Infinity" @@ -38,7 +37,6 @@ Infinity = Infinity() class NegativeInfinity(object): - def __repr__(self): return "-Infinity" diff --git a/pipenv/vendor/packaging/markers.py b/pipenv/vendor/packaging/markers.py index 5fdf510c..eff5abbb 100644 --- a/pipenv/vendor/packaging/markers.py +++ b/pipenv/vendor/packaging/markers.py @@ -17,8 +17,11 @@ from .specifiers import Specifier, InvalidSpecifier __all__ = [ - "InvalidMarker", "UndefinedComparison", "UndefinedEnvironmentName", - "Marker", "default_environment", + "InvalidMarker", + "UndefinedComparison", + "UndefinedEnvironmentName", + "Marker", + "default_environment", ] @@ -42,7 +45,6 @@ class UndefinedEnvironmentName(ValueError): class Node(object): - def __init__(self, value): self.value = value @@ -57,62 +59,52 @@ class Node(object): class Variable(Node): - def serialize(self): return str(self) class Value(Node): - def serialize(self): return '"{0}"'.format(self) class Op(Node): - def serialize(self): return str(self) VARIABLE = ( - L("implementation_version") | - L("platform_python_implementation") | - L("implementation_name") | - L("python_full_version") | - L("platform_release") | - L("platform_version") | - L("platform_machine") | - L("platform_system") | - L("python_version") | - L("sys_platform") | - L("os_name") | - L("os.name") | # PEP-345 - L("sys.platform") | # PEP-345 - L("platform.version") | # PEP-345 - L("platform.machine") | # PEP-345 - L("platform.python_implementation") | # PEP-345 - L("python_implementation") | # undocumented setuptools legacy - L("extra") + L("implementation_version") + | L("platform_python_implementation") + | L("implementation_name") + | L("python_full_version") + | L("platform_release") + | L("platform_version") + | L("platform_machine") + | L("platform_system") + | L("python_version") + | L("sys_platform") + | L("os_name") + | L("os.name") + | L("sys.platform") # PEP-345 + | L("platform.version") # PEP-345 + | L("platform.machine") # PEP-345 + | L("platform.python_implementation") # PEP-345 + | L("python_implementation") # PEP-345 + | L("extra") # undocumented setuptools legacy ) ALIASES = { - 'os.name': 'os_name', - 'sys.platform': 'sys_platform', - 'platform.version': 'platform_version', - 'platform.machine': 'platform_machine', - 'platform.python_implementation': 'platform_python_implementation', - 'python_implementation': 'platform_python_implementation' + "os.name": "os_name", + "sys.platform": "sys_platform", + "platform.version": "platform_version", + "platform.machine": "platform_machine", + "platform.python_implementation": "platform_python_implementation", + "python_implementation": "platform_python_implementation", } VARIABLE.setParseAction(lambda s, l, t: Variable(ALIASES.get(t[0], t[0]))) VERSION_CMP = ( - L("===") | - L("==") | - L(">=") | - L("<=") | - L("!=") | - L("~=") | - L(">") | - L("<") + L("===") | L("==") | L(">=") | L("<=") | L("!=") | L("~=") | L(">") | L("<") ) MARKER_OP = VERSION_CMP | L("not in") | L("in") @@ -152,8 +144,11 @@ def _format_marker(marker, first=True): # where the single item is itself it's own list. In that case we want skip # the rest of this function so that we don't get extraneous () on the # outside. - if (isinstance(marker, list) and len(marker) == 1 and - isinstance(marker[0], (list, tuple))): + if ( + isinstance(marker, list) + and len(marker) == 1 + and isinstance(marker[0], (list, tuple)) + ): return _format_marker(marker[0]) if isinstance(marker, list): @@ -239,20 +234,20 @@ def _evaluate_markers(markers, environment): def format_full_version(info): - version = '{0.major}.{0.minor}.{0.micro}'.format(info) + version = "{0.major}.{0.minor}.{0.micro}".format(info) kind = info.releaselevel - if kind != 'final': + if kind != "final": version += kind[0] + str(info.serial) return version def default_environment(): - if hasattr(sys, 'implementation'): + if hasattr(sys, "implementation"): iver = format_full_version(sys.implementation.version) implementation_name = sys.implementation.name else: - iver = '0' - implementation_name = '' + iver = "0" + implementation_name = "" return { "implementation_name": implementation_name, @@ -270,13 +265,13 @@ def default_environment(): class Marker(object): - def __init__(self, marker): try: self._markers = _coerce_parse_result(MARKER.parseString(marker)) except ParseException as e: err_str = "Invalid marker: {0!r}, parse error at {1!r}".format( - marker, marker[e.loc:e.loc + 8]) + marker, marker[e.loc : e.loc + 8] + ) raise InvalidMarker(err_str) def __str__(self): diff --git a/pipenv/vendor/packaging/requirements.py b/pipenv/vendor/packaging/requirements.py index e8008a6d..4d9688b9 100644 --- a/pipenv/vendor/packaging/requirements.py +++ b/pipenv/vendor/packaging/requirements.py @@ -38,8 +38,8 @@ IDENTIFIER = Combine(ALPHANUM + ZeroOrMore(IDENTIFIER_END)) NAME = IDENTIFIER("name") EXTRA = IDENTIFIER -URI = Regex(r'[^ ]+')("url") -URL = (AT + URI) +URI = Regex(r"[^ ]+")("url") +URL = AT + URI EXTRAS_LIST = EXTRA + ZeroOrMore(COMMA + EXTRA) EXTRAS = (LBRACKET + Optional(EXTRAS_LIST) + RBRACKET)("extras") @@ -48,17 +48,18 @@ VERSION_PEP440 = Regex(Specifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_LEGACY = Regex(LegacySpecifier._regex_str, re.VERBOSE | re.IGNORECASE) VERSION_ONE = VERSION_PEP440 ^ VERSION_LEGACY -VERSION_MANY = Combine(VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), - joinString=",", adjacent=False)("_raw_spec") +VERSION_MANY = Combine( + VERSION_ONE + ZeroOrMore(COMMA + VERSION_ONE), joinString=",", adjacent=False +)("_raw_spec") _VERSION_SPEC = Optional(((LPAREN + VERSION_MANY + RPAREN) | VERSION_MANY)) -_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or '') +_VERSION_SPEC.setParseAction(lambda s, l, t: t._raw_spec or "") VERSION_SPEC = originalTextFor(_VERSION_SPEC)("specifier") VERSION_SPEC.setParseAction(lambda s, l, t: t[1]) MARKER_EXPR = originalTextFor(MARKER_EXPR())("marker") MARKER_EXPR.setParseAction( - lambda s, l, t: Marker(s[t._original_start:t._original_end]) + lambda s, l, t: Marker(s[t._original_start : t._original_end]) ) MARKER_SEPARATOR = SEMICOLON MARKER = MARKER_SEPARATOR + MARKER_EXPR @@ -66,8 +67,7 @@ MARKER = MARKER_SEPARATOR + MARKER_EXPR VERSION_AND_MARKER = VERSION_SPEC + Optional(MARKER) URL_AND_MARKER = URL + Optional(MARKER) -NAMED_REQUIREMENT = \ - NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) +NAMED_REQUIREMENT = NAME + Optional(EXTRAS) + (URL_AND_MARKER | VERSION_AND_MARKER) REQUIREMENT = stringStart + NAMED_REQUIREMENT + stringEnd # pyparsing isn't thread safe during initialization, so we do it eagerly, see @@ -92,15 +92,21 @@ class Requirement(object): try: req = REQUIREMENT.parseString(requirement_string) except ParseException as e: - raise InvalidRequirement("Parse error at \"{0!r}\": {1}".format( - requirement_string[e.loc:e.loc + 8], e.msg - )) + raise InvalidRequirement( + 'Parse error at "{0!r}": {1}'.format( + requirement_string[e.loc : e.loc + 8], e.msg + ) + ) self.name = req.name if req.url: parsed_url = urlparse.urlparse(req.url) - if not (parsed_url.scheme and parsed_url.netloc) or ( - not parsed_url.scheme and not parsed_url.netloc): + if parsed_url.scheme == "file": + if urlparse.urlunparse(parsed_url) != req.url: + raise InvalidRequirement("Invalid URL given") + elif not (parsed_url.scheme and parsed_url.netloc) or ( + not parsed_url.scheme and not parsed_url.netloc + ): raise InvalidRequirement("Invalid URL: {0}".format(req.url)) self.url = req.url else: @@ -120,6 +126,8 @@ class Requirement(object): if self.url: parts.append("@ {0}".format(self.url)) + if self.marker: + parts.append(" ") if self.marker: parts.append("; {0}".format(self.marker)) diff --git a/pipenv/vendor/packaging/specifiers.py b/pipenv/vendor/packaging/specifiers.py index 4c798999..743576a0 100644 --- a/pipenv/vendor/packaging/specifiers.py +++ b/pipenv/vendor/packaging/specifiers.py @@ -19,7 +19,6 @@ class InvalidSpecifier(ValueError): class BaseSpecifier(with_metaclass(abc.ABCMeta, object)): - @abc.abstractmethod def __str__(self): """ @@ -84,10 +83,7 @@ class _IndividualSpecifier(BaseSpecifier): if not match: raise InvalidSpecifier("Invalid specifier: '{0}'".format(spec)) - self._spec = ( - match.group("operator").strip(), - match.group("version").strip(), - ) + self._spec = (match.group("operator").strip(), match.group("version").strip()) # Store whether or not this Specifier should accept prereleases self._prereleases = prereleases @@ -99,11 +95,7 @@ class _IndividualSpecifier(BaseSpecifier): else "" ) - return "<{0}({1!r}{2})>".format( - self.__class__.__name__, - str(self), - pre, - ) + return "<{0}({1!r}{2})>".format(self.__class__.__name__, str(self), pre) def __str__(self): return "{0}{1}".format(*self._spec) @@ -194,8 +186,9 @@ class _IndividualSpecifier(BaseSpecifier): # If our version is a prerelease, and we were not set to allow # prereleases, then we'll store it for later incase nothing # else matches this specifier. - if (parsed_version.is_prerelease and not - (prereleases or self.prereleases)): + if parsed_version.is_prerelease and not ( + prereleases or self.prereleases + ): found_prereleases.append(version) # Either this is not a prerelease, or we should have been # accepting prereleases from the beginning. @@ -213,8 +206,7 @@ class _IndividualSpecifier(BaseSpecifier): class LegacySpecifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(==|!=|<=|>=|<|>)) \s* (?P @@ -225,10 +217,8 @@ class LegacySpecifier(_IndividualSpecifier): # them, and a comma since it's a version separator. ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "==": "equal", @@ -269,13 +259,13 @@ def _require_version_compare(fn): if not isinstance(prospective, Version): return False return fn(self, prospective, spec) + return wrapped class Specifier(_IndividualSpecifier): - _regex_str = ( - r""" + _regex_str = r""" (?P(~=|==|!=|<=|>=|<|>|===)) (?P (?: @@ -367,10 +357,8 @@ class Specifier(_IndividualSpecifier): ) ) """ - ) - _regex = re.compile( - r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) + _regex = re.compile(r"^\s*" + _regex_str + r"\s*$", re.VERBOSE | re.IGNORECASE) _operators = { "~=": "compatible", @@ -397,8 +385,7 @@ class Specifier(_IndividualSpecifier): prefix = ".".join( list( itertools.takewhile( - lambda x: (not x.startswith("post") and not - x.startswith("dev")), + lambda x: (not x.startswith("post") and not x.startswith("dev")), _version_split(spec), ) )[:-1] @@ -407,8 +394,9 @@ class Specifier(_IndividualSpecifier): # Add the prefix notation to the end of our string prefix += ".*" - return (self._get_operator(">=")(prospective, spec) and - self._get_operator("==")(prospective, prefix)) + return self._get_operator(">=")(prospective, spec) and self._get_operator("==")( + prospective, prefix + ) @_require_version_compare def _compare_equal(self, prospective, spec): @@ -428,7 +416,7 @@ class Specifier(_IndividualSpecifier): # Shorten the prospective version to be the same length as the spec # so that we can determine if the specifier is a prefix of the # prospective version or not. - prospective = prospective[:len(spec)] + prospective = prospective[: len(spec)] # Pad out our two sides with zeros so that they both equal the same # length. @@ -567,27 +555,17 @@ def _pad_version(left, right): right_split.append(list(itertools.takewhile(lambda x: x.isdigit(), right))) # Get the rest of our versions - left_split.append(left[len(left_split[0]):]) - right_split.append(right[len(right_split[0]):]) + left_split.append(left[len(left_split[0]) :]) + right_split.append(right[len(right_split[0]) :]) # Insert our padding - left_split.insert( - 1, - ["0"] * max(0, len(right_split[0]) - len(left_split[0])), - ) - right_split.insert( - 1, - ["0"] * max(0, len(left_split[0]) - len(right_split[0])), - ) + left_split.insert(1, ["0"] * max(0, len(right_split[0]) - len(left_split[0]))) + right_split.insert(1, ["0"] * max(0, len(left_split[0]) - len(right_split[0]))) - return ( - list(itertools.chain(*left_split)), - list(itertools.chain(*right_split)), - ) + return (list(itertools.chain(*left_split)), list(itertools.chain(*right_split))) class SpecifierSet(BaseSpecifier): - def __init__(self, specifiers="", prereleases=None): # Split on , to break each indidivual specifier into it's own item, and # strip each item to remove leading/trailing whitespace. @@ -721,10 +699,7 @@ class SpecifierSet(BaseSpecifier): # given version is contained within all of them. # Note: This use of all() here means that an empty set of specifiers # will always return True, this is an explicit design decision. - return all( - s.contains(item, prereleases=prereleases) - for s in self._specs - ) + return all(s.contains(item, prereleases=prereleases) for s in self._specs) def filter(self, iterable, prereleases=None): # Determine if we're forcing a prerelease or not, if we're not forcing diff --git a/pipenv/vendor/packaging/utils.py b/pipenv/vendor/packaging/utils.py index 4b94a82f..88418786 100644 --- a/pipenv/vendor/packaging/utils.py +++ b/pipenv/vendor/packaging/utils.py @@ -36,13 +36,7 @@ def canonicalize_version(version): # Release segment # NB: This strips trailing '.0's to normalize - parts.append( - re.sub( - r'(\.0)+$', - '', - ".".join(str(x) for x in version.release) - ) - ) + parts.append(re.sub(r"(\.0)+$", "", ".".join(str(x) for x in version.release))) # Pre-release if version.pre is not None: diff --git a/pipenv/vendor/packaging/version.py b/pipenv/vendor/packaging/version.py index 6ed5cbbd..95157a1f 100644 --- a/pipenv/vendor/packaging/version.py +++ b/pipenv/vendor/packaging/version.py @@ -10,14 +10,11 @@ import re from ._structures import Infinity -__all__ = [ - "parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN" -] +__all__ = ["parse", "Version", "LegacyVersion", "InvalidVersion", "VERSION_PATTERN"] _Version = collections.namedtuple( - "_Version", - ["epoch", "release", "dev", "pre", "post", "local"], + "_Version", ["epoch", "release", "dev", "pre", "post", "local"] ) @@ -40,7 +37,6 @@ class InvalidVersion(ValueError): class _BaseVersion(object): - def __hash__(self): return hash(self._key) @@ -70,7 +66,6 @@ class _BaseVersion(object): class LegacyVersion(_BaseVersion): - def __init__(self, version): self._version = str(version) self._key = _legacy_cmpkey(self._version) @@ -126,12 +121,14 @@ class LegacyVersion(_BaseVersion): return False -_legacy_version_component_re = re.compile( - r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE, -) +_legacy_version_component_re = re.compile(r"(\d+ | [a-z]+ | \.| -)", re.VERBOSE) _legacy_version_replacement_map = { - "pre": "c", "preview": "c", "-": "final-", "rc": "c", "dev": "@", + "pre": "c", + "preview": "c", + "-": "final-", + "rc": "c", + "dev": "@", } @@ -215,10 +212,7 @@ VERSION_PATTERN = r""" class Version(_BaseVersion): - _regex = re.compile( - r"^\s*" + VERSION_PATTERN + r"\s*$", - re.VERBOSE | re.IGNORECASE, - ) + _regex = re.compile(r"^\s*" + VERSION_PATTERN + r"\s*$", re.VERBOSE | re.IGNORECASE) def __init__(self, version): # Validate the version and parse it into pieces @@ -230,18 +224,11 @@ class Version(_BaseVersion): self._version = _Version( epoch=int(match.group("epoch")) if match.group("epoch") else 0, release=tuple(int(i) for i in match.group("release").split(".")), - pre=_parse_letter_version( - match.group("pre_l"), - match.group("pre_n"), - ), + pre=_parse_letter_version(match.group("pre_l"), match.group("pre_n")), post=_parse_letter_version( - match.group("post_l"), - match.group("post_n1") or match.group("post_n2"), - ), - dev=_parse_letter_version( - match.group("dev_l"), - match.group("dev_n"), + match.group("post_l"), match.group("post_n1") or match.group("post_n2") ), + dev=_parse_letter_version(match.group("dev_l"), match.group("dev_n")), local=_parse_local_version(match.group("local")), ) @@ -395,12 +382,7 @@ def _cmpkey(epoch, release, pre, post, dev, local): # re-reverse it back into the correct order and make it a tuple and use # that for our sorting key. release = tuple( - reversed(list( - itertools.dropwhile( - lambda x: x == 0, - reversed(release), - ) - )) + reversed(list(itertools.dropwhile(lambda x: x == 0, reversed(release)))) ) # We need to "trick" the sorting algorithm to put 1.0.dev0 before 1.0a0. @@ -433,9 +415,6 @@ def _cmpkey(epoch, release, pre, post, dev, local): # - Numeric segments sort numerically # - Shorter versions sort before longer versions when the prefixes # match exactly - local = tuple( - (i, "") if isinstance(i, int) else (-Infinity, i) - for i in local - ) + local = tuple((i, "") if isinstance(i, int) else (-Infinity, i) for i in local) return epoch, release, pre, post, dev, local diff --git a/pipenv/vendor/parse.py b/pipenv/vendor/parse.py index 7f9f0786..b5d543f9 100644 --- a/pipenv/vendor/parse.py +++ b/pipenv/vendor/parse.py @@ -3,7 +3,7 @@ r'''Parse strings using a specification based on the Python format() syntax. ``parse()`` is the opposite of ``format()`` The module is set up to only export ``parse()``, ``search()``, ``findall()``, -and ``with_pattern()`` when ``import *`` is used: +and ``with_pattern()`` when ``import \*`` is used: >>> from parse import * @@ -78,9 +78,11 @@ Some simple parse() format string examples: {'item': 'hand grenade'} >>> print(r['item']) hand grenade +>>> 'item' in r +True -Dotted names and indexes are possible though the application must make -additional sense of the result: +Note that `in` only works if you have named fields. Dotted names and indexes +are possible though the application must make additional sense of the result: >>> r = parse("Mmm, {food.type}, I love it!", "Mmm, spam, I love it!") >>> print(r) @@ -132,38 +134,39 @@ The differences between `parse()` and `format()` are: ===== =========================================== ======== Type Characters Matched Output ===== =========================================== ======== - w Letters and underscore str - W Non-letter and underscore str - s Whitespace str - S Non-whitespace str - d Digits (effectively integer numbers) int - D Non-digit str - n Numbers with thousands separators (, or .) int - % Percentage (converted to value/100.0) float - f Fixed-point numbers float - F Decimal numbers Decimal - e Floating-point numbers with exponent float +l Letters (ASCII) str +w Letters, numbers and underscore str +W Not letters, numbers and underscore str +s Whitespace str +S Non-whitespace str +d Digits (effectively integer numbers) int +D Non-digit str +n Numbers with thousands separators (, or .) int +% Percentage (converted to value/100.0) float +f Fixed-point numbers float +F Decimal numbers Decimal +e Floating-point numbers with exponent float e.g. 1.1e-10, NAN (all case insensitive) - g General number format (either d, f or e) float - b Binary numbers int - o Octal numbers int - x Hexadecimal numbers (lower and upper case) int - ti ISO 8601 format date/time datetime +g General number format (either d, f or e) float +b Binary numbers int +o Octal numbers int +x Hexadecimal numbers (lower and upper case) int +ti ISO 8601 format date/time datetime e.g. 1972-01-20T10:21:36Z ("T" and "Z" optional) - te RFC2822 e-mail format date/time datetime +te RFC2822 e-mail format date/time datetime e.g. Mon, 20 Jan 1972 10:21:36 +1000 - tg Global (day/month) format date/time datetime +tg Global (day/month) format date/time datetime e.g. 20/1/1972 10:21:36 AM +1:00 - ta US (month/day) format date/time datetime +ta US (month/day) format date/time datetime e.g. 1/20/1972 10:21:36 PM +10:30 - tc ctime() format date/time datetime +tc ctime() format date/time datetime e.g. Sun Sep 16 01:03:52 1973 - th HTTP log format date/time datetime +th HTTP log format date/time datetime e.g. 21/Nov/2011:00:07:11 +0000 - ts Linux system log format date/time datetime +ts Linux system log format date/time datetime e.g. Nov 9 03:37:44 - tt Time time +tt Time time e.g. 10:21:36 PM -5:30 ===== =========================================== ======== @@ -342,6 +345,13 @@ the pattern, the actual match represents the shortest successful match for **Version history (in brief)**: +- 1.11.1 Revert having unicode char in docstring, it breaks Bamboo builds(?!) +- 1.11.0 Implement `__contains__` for Result instances. +- 1.10.0 Introduce a "letters" matcher, since "w" matches numbers + also. +- 1.9.1 Fix deprecation warnings around backslashes in regex strings + (thanks Mickael Schoentgen). Also fix some documentation formatting + issues. - 1.9.0 We now honor precision and width specifiers when parsing numbers and strings, allowing parsing of concatenated elements of fixed width (thanks Julia Signell) @@ -400,12 +410,12 @@ the pattern, the actual match represents the shortest successful match for and removed the restriction on mixing fixed-position and named fields - 1.0.0 initial release -This code is copyright 2012-2017 Richard Jones +This code is copyright 2012-2019 Richard Jones See the end of the source file for the license of use. ''' from __future__ import absolute_import -__version__ = '1.9.0' +__version__ = '1.11.1' # yes, I now have two problems import re @@ -530,9 +540,9 @@ MONTHS_MAP = dict( Nov=11, November=11, Dec=12, December=12 ) -DAYS_PAT = '(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' -MONTHS_PAT = '(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' -ALL_MONTHS_PAT = '(%s)' % '|'.join(MONTHS_MAP) +DAYS_PAT = r'(Mon|Tue|Wed|Thu|Fri|Sat|Sun)' +MONTHS_PAT = r'(Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)' +ALL_MONTHS_PAT = r'(%s)' % '|'.join(MONTHS_MAP) TIME_PAT = r'(\d{1,2}:\d{1,2}(:\d{1,2}(\.\d+)?)?)' AM_PAT = r'(\s+[AP]M)' TZ_PAT = r'(\s+[-+]\d\d?:?\d\d)' @@ -550,11 +560,11 @@ def date_convert(string, match, ymd=None, mdy=None, dmy=None, m=groups[mm] d=groups[dd] elif ymd is not None: - y, m, d = re.split('[-/\s]', groups[ymd]) + y, m, d = re.split(r'[-/\s]', groups[ymd]) elif mdy is not None: - m, d, y = re.split('[-/\s]', groups[mdy]) + m, d, y = re.split(r'[-/\s]', groups[mdy]) elif dmy is not None: - d, m, y = re.split('[-/\s]', groups[dmy]) + d, m, y = re.split(r'[-/\s]', groups[dmy]) elif d_m_y is not None: d, m, y = d_m_y d = groups[d] @@ -636,10 +646,10 @@ class RepeatedNameError(ValueError): # note: {} are handled separately # note: I don't use r'' here because Sublime Text 2 syntax highlight has a fit -REGEX_SAFETY = re.compile('([?\\\\.[\]()*+\^$!\|])') +REGEX_SAFETY = re.compile(r'([?\\\\.[\]()*+\^$!\|])') # allowed field types -ALLOWED_TYPES = set(list('nbox%fFegwWdDsS') + +ALLOWED_TYPES = set(list('nbox%fFegwWdDsSl') + ['t' + c for c in 'ieahgcts']) @@ -745,7 +755,7 @@ class Parser(object): @property def _match_re(self): if self.__match_re is None: - expression = '^%s$' % self._expression + expression = r'^%s$' % self._expression try: self.__match_re = re.compile(expression, self._re_flags) except AssertionError: @@ -923,16 +933,16 @@ class Parser(object): name, self._name_types[name])) group = self._name_to_group_map[name] # match previously-seen value - return '(?P=%s)' % group + return r'(?P=%s)' % group else: group = self._to_group_name(name) self._name_types[name] = format self._named_fields.append(group) # this will become a group, which must not contain dots - wrap = '(?P<%s>%%s)' % group + wrap = r'(?P<%s>%%s)' % group else: self._fixed_fields.append(self._group_index) - wrap = '(%s)' + wrap = r'(%s)' if ':' in field: format = field[1:] group = self._group_index @@ -940,7 +950,7 @@ class Parser(object): # simplest case: no type specifier ({} or {name}) if not format: self._group_index += 1 - return wrap % '.+?' + return wrap % r'.+?' # decode the format specification format = extract_format(format, self._extra_types) @@ -960,19 +970,19 @@ class Parser(object): return type_converter(string) self._type_conversions[group] = f elif type == 'n': - s = '\d{1,3}([,.]\d{3})*' + s = r'\d{1,3}([,.]\d{3})*' self._group_index += 1 self._type_conversions[group] = int_convert(10) elif type == 'b': - s = '(0[bB])?[01]+' + s = r'(0[bB])?[01]+' self._type_conversions[group] = int_convert(2) self._group_index += 1 elif type == 'o': - s = '(0[oO])?[0-7]+' + s = r'(0[oO])?[0-7]+' self._type_conversions[group] = int_convert(8) self._group_index += 1 elif type == 'x': - s = '(0[xX])?[0-9a-fA-F]+' + s = r'(0[xX])?[0-9a-fA-F]+' self._type_conversions[group] = int_convert(16) self._group_index += 1 elif type == '%': @@ -994,10 +1004,10 @@ class Parser(object): self._type_conversions[group] = lambda s, m: float(s) elif type == 'd': if format.get('width'): - width = '{1,%s}' % int(format['width']) + width = r'{1,%s}' % int(format['width']) else: width = '+' - s = '\\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) + s = r'\d{w}|0[xX][0-9a-fA-F]{w}|0[bB][01]{w}|0[oO][0-7]{w}'.format(w=width) self._type_conversions[group] = int_convert(10) elif type == 'ti': s = r'(\d{4}-\d\d-\d\d)((\s+|T)%s)?(Z|\s*[-+]\d\d:?\d\d)?' % \ @@ -1055,18 +1065,19 @@ class Parser(object): self._type_conversions[group] = partial(date_convert, mm=n+1, dd=n+3, hms=n + 5) self._group_index += 5 - + elif type == 'l': + s = r'[A-Za-z]+' elif type: s = r'\%s+' % type elif format.get('precision'): if format.get('width'): - s = '.{%s,%s}?' % (format['width'], format['precision']) + s = r'.{%s,%s}?' % (format['width'], format['precision']) else: - s = '.{1,%s}?' % format['precision'] + s = r'.{1,%s}?' % format['precision'] elif format.get('width'): - s = '.{%s,}?' % format['width'] + s = r'.{%s,}?' % format['width'] else: - s = '.+?' + s = r'.+?' align = format['align'] fill = format['fill'] @@ -1079,7 +1090,7 @@ class Parser(object): # configurable fill defaulting to "0" if not fill: fill = '0' - s = '%s*' % fill + s + s = r'%s*' % fill + s # allow numbers to be prefixed with a sign s = r'[-+ ]?' + s @@ -1101,7 +1112,7 @@ class Parser(object): if not align: align = '>' - if fill in '.\+?*[](){}^$': + if fill in r'.\+?*[](){}^$': fill = '\\' + fill # align "=" has been handled @@ -1118,8 +1129,11 @@ class Parser(object): class Result(object): '''The result of a parse() or search(). - Fixed results may be looked up using result[index]. Named results may be - looked up using result['name']. + Fixed results may be looked up using `result[index]`. + + Named results may be looked up using `result['name']`. + + Named results may be tested for existence using `'name' in result`. ''' def __init__(self, fixed, named, spans): self.fixed = fixed @@ -1135,6 +1149,9 @@ class Result(object): return '<%s %r %r>' % (self.__class__.__name__, self.fixed, self.named) + def __contains__(self, name): + return name in self.named + class Match(object): '''The result of a parse() or search() if no results are generated. @@ -1295,7 +1312,7 @@ def compile(format, extra_types=None, case_sensitive=False): return Parser(format, extra_types=extra_types) -# Copyright (c) 2012-2013 Richard Jones +# Copyright (c) 2012-2019 Richard Jones # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/pipenv/vendor/passa/cli/options.py b/pipenv/vendor/passa/cli/options.py index f8ba1fe7..f20b612a 100644 --- a/pipenv/vendor/passa/cli/options.py +++ b/pipenv/vendor/passa/cli/options.py @@ -20,13 +20,13 @@ class Project(passa.models.projects.Project): pipfile = root.joinpath("Pipfile") if not pipfile.is_file(): raise argparse.ArgumentError( - "{0!r} is not a Pipfile project".format(root), + "project", "{0!r} is not a Pipfile project".format(root), ) try: super(Project, self).__init__(root.as_posix(), *args, **kwargs) except tomlkit.exceptions.ParseError as e: raise argparse.ArgumentError( - "failed to parse Pipfile: {0!r}".format(str(e)), + "project", "failed to parse Pipfile: {0!r}".format(str(e)), ) def __name__(self): diff --git a/pipenv/vendor/pep517/LICENSE b/pipenv/vendor/pep517/LICENSE new file mode 100644 index 00000000..b0ae9dbc --- /dev/null +++ b/pipenv/vendor/pep517/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2017 Thomas Kluyver + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pep517/__init__.py b/pipenv/vendor/pep517/__init__.py new file mode 100644 index 00000000..9c1a098f --- /dev/null +++ b/pipenv/vendor/pep517/__init__.py @@ -0,0 +1,4 @@ +"""Wrappers to build Python packages using PEP 517 hooks +""" + +__version__ = '0.5.0' diff --git a/pipenv/vendor/pep517/_in_process.py b/pipenv/vendor/pep517/_in_process.py new file mode 100644 index 00000000..d6524b66 --- /dev/null +++ b/pipenv/vendor/pep517/_in_process.py @@ -0,0 +1,207 @@ +"""This is invoked in a subprocess to call the build backend hooks. + +It expects: +- Command line args: hook_name, control_dir +- Environment variable: PEP517_BUILD_BACKEND=entry.point:spec +- control_dir/input.json: + - {"kwargs": {...}} + +Results: +- control_dir/output.json + - {"return_val": ...} +""" +from glob import glob +from importlib import import_module +import os +from os.path import join as pjoin +import re +import shutil +import sys + +# This is run as a script, not a module, so it can't do a relative import +import compat + + +class BackendUnavailable(Exception): + """Raised if we cannot import the backend""" + + +def _build_backend(): + """Find and load the build backend""" + ep = os.environ['PEP517_BUILD_BACKEND'] + mod_path, _, obj_path = ep.partition(':') + try: + obj = import_module(mod_path) + except ImportError: + raise BackendUnavailable + if obj_path: + for path_part in obj_path.split('.'): + obj = getattr(obj, path_part) + return obj + + +def get_requires_for_build_wheel(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_wheel + except AttributeError: + return [] + else: + return hook(config_settings) + + +def prepare_metadata_for_build_wheel(metadata_directory, config_settings): + """Invoke optional prepare_metadata_for_build_wheel + + Implements a fallback by building a wheel if the hook isn't defined. + """ + backend = _build_backend() + try: + hook = backend.prepare_metadata_for_build_wheel + except AttributeError: + return _get_wheel_metadata_from_wheel(backend, metadata_directory, + config_settings) + else: + return hook(metadata_directory, config_settings) + + +WHEEL_BUILT_MARKER = 'PEP517_ALREADY_BUILT_WHEEL' + + +def _dist_info_files(whl_zip): + """Identify the .dist-info folder inside a wheel ZipFile.""" + res = [] + for path in whl_zip.namelist(): + m = re.match(r'[^/\\]+-[^/\\]+\.dist-info/', path) + if m: + res.append(path) + if res: + return res + raise Exception("No .dist-info folder found in wheel") + + +def _get_wheel_metadata_from_wheel( + backend, metadata_directory, config_settings): + """Build a wheel and extract the metadata from it. + + Fallback for when the build backend does not + define the 'get_wheel_metadata' hook. + """ + from zipfile import ZipFile + whl_basename = backend.build_wheel(metadata_directory, config_settings) + with open(os.path.join(metadata_directory, WHEEL_BUILT_MARKER), 'wb'): + pass # Touch marker file + + whl_file = os.path.join(metadata_directory, whl_basename) + with ZipFile(whl_file) as zipf: + dist_info = _dist_info_files(zipf) + zipf.extractall(path=metadata_directory, members=dist_info) + return dist_info[0].split('/')[0] + + +def _find_already_built_wheel(metadata_directory): + """Check for a wheel already built during the get_wheel_metadata hook. + """ + if not metadata_directory: + return None + metadata_parent = os.path.dirname(metadata_directory) + if not os.path.isfile(pjoin(metadata_parent, WHEEL_BUILT_MARKER)): + return None + + whl_files = glob(os.path.join(metadata_parent, '*.whl')) + if not whl_files: + print('Found wheel built marker, but no .whl files') + return None + if len(whl_files) > 1: + print('Found multiple .whl files; unspecified behaviour. ' + 'Will call build_wheel.') + return None + + # Exactly one .whl file + return whl_files[0] + + +def build_wheel(wheel_directory, config_settings, metadata_directory=None): + """Invoke the mandatory build_wheel hook. + + If a wheel was already built in the + prepare_metadata_for_build_wheel fallback, this + will copy it rather than rebuilding the wheel. + """ + prebuilt_whl = _find_already_built_wheel(metadata_directory) + if prebuilt_whl: + shutil.copy2(prebuilt_whl, wheel_directory) + return os.path.basename(prebuilt_whl) + + return _build_backend().build_wheel(wheel_directory, config_settings, + metadata_directory) + + +def get_requires_for_build_sdist(config_settings): + """Invoke the optional get_requires_for_build_wheel hook + + Returns [] if the hook is not defined. + """ + backend = _build_backend() + try: + hook = backend.get_requires_for_build_sdist + except AttributeError: + return [] + else: + return hook(config_settings) + + +class _DummyException(Exception): + """Nothing should ever raise this exception""" + + +class GotUnsupportedOperation(Exception): + """For internal use when backend raises UnsupportedOperation""" + + +def build_sdist(sdist_directory, config_settings): + """Invoke the mandatory build_sdist hook.""" + backend = _build_backend() + try: + return backend.build_sdist(sdist_directory, config_settings) + except getattr(backend, 'UnsupportedOperation', _DummyException): + raise GotUnsupportedOperation + + +HOOK_NAMES = { + 'get_requires_for_build_wheel', + 'prepare_metadata_for_build_wheel', + 'build_wheel', + 'get_requires_for_build_sdist', + 'build_sdist', +} + + +def main(): + if len(sys.argv) < 3: + sys.exit("Needs args: hook_name, control_dir") + hook_name = sys.argv[1] + control_dir = sys.argv[2] + if hook_name not in HOOK_NAMES: + sys.exit("Unknown hook: %s" % hook_name) + hook = globals()[hook_name] + + hook_input = compat.read_json(pjoin(control_dir, 'input.json')) + + json_out = {'unsupported': False, 'return_val': None} + try: + json_out['return_val'] = hook(**hook_input['kwargs']) + except BackendUnavailable: + json_out['no_backend'] = True + except GotUnsupportedOperation: + json_out['unsupported'] = True + + compat.write_json(json_out, pjoin(control_dir, 'output.json'), indent=2) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/build.py b/pipenv/vendor/pep517/build.py new file mode 100644 index 00000000..6fca39a8 --- /dev/null +++ b/pipenv/vendor/pep517/build.py @@ -0,0 +1,108 @@ +"""Build a project using PEP 517 hooks. +""" +import argparse +import logging +import os +import contextlib +import pytoml +import shutil +import errno +import tempfile + +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +@contextlib.contextmanager +def tempdir(): + td = tempfile.mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +def _do_build(hooks, env, dist, dest): + get_requires_name = 'get_requires_for_build_{dist}'.format(**locals()) + get_requires = getattr(hooks, get_requires_name) + reqs = get_requires({}) + log.info('Got build requires: %s', reqs) + + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + + with tempdir() as td: + log.info('Trying to build %s in %s', dist, td) + build_name = 'build_{dist}'.format(**locals()) + build = getattr(hooks, build_name) + filename = build(td, {}) + source = os.path.join(td, filename) + shutil.move(source, os.path.join(dest, os.path.basename(filename))) + + +def mkdir_p(*args, **kwargs): + """Like `mkdir`, but does not raise an exception if the + directory already exists. + """ + try: + return os.mkdir(*args, **kwargs) + except OSError as exc: + if exc.errno != errno.EEXIST: + raise + + +def build(source_dir, dist, dest=None): + pyproject = os.path.join(source_dir, 'pyproject.toml') + dest = os.path.join(source_dir, dest or 'dist') + mkdir_p(dest) + + with open(pyproject) as f: + pyproject_data = pytoml.load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + _do_build(hooks, env, dist, dest) + + +parser = argparse.ArgumentParser() +parser.add_argument( + 'source_dir', + help="A directory containing pyproject.toml", +) +parser.add_argument( + '--binary', '-b', + action='store_true', + default=False, +) +parser.add_argument( + '--source', '-s', + action='store_true', + default=False, +) +parser.add_argument( + '--out-dir', '-o', + help="Destination in which to save the builds relative to source dir", +) + + +def main(args): + # determine which dists to build + dists = list(filter(None, ( + 'sdist' if args.source or not args.binary else None, + 'wheel' if args.binary or not args.source else None, + ))) + + for dist in dists: + build(args.source_dir, dist, args.out_dir) + + +if __name__ == '__main__': + main(parser.parse_args()) diff --git a/pipenv/vendor/pep517/check.py b/pipenv/vendor/pep517/check.py new file mode 100644 index 00000000..fc82cca7 --- /dev/null +++ b/pipenv/vendor/pep517/check.py @@ -0,0 +1,202 @@ +"""Check a project and backend by attempting to build using PEP 517 hooks. +""" +import argparse +import logging +import os +from os.path import isfile, join as pjoin +from pytoml import TomlError, load as toml_load +import shutil +from subprocess import CalledProcessError +import sys +import tarfile +from tempfile import mkdtemp +import zipfile + +from .colorlog import enable_colourful_output +from .envbuild import BuildEnvironment +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +def check_build_sdist(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_sdist({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build sdist in %s', td) + try: + try: + filename = hooks.build_sdist(td, {}) + log.info('build_sdist returned %r', filename) + except Exception: + log.info('Failure in build_sdist', exc_info=True) + return False + + if not filename.endswith('.tar.gz'): + log.error( + "Filename %s doesn't have .tar.gz extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if tarfile.is_tarfile(path): + log.info("Output file is a tar file") + else: + log.error("Output file is not a tar file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check_build_wheel(hooks, build_sys_requires): + with BuildEnvironment() as env: + try: + env.pip_install(build_sys_requires) + log.info('Installed static build dependencies') + except CalledProcessError: + log.error('Failed to install static build dependencies') + return False + + try: + reqs = hooks.get_requires_for_build_wheel({}) + log.info('Got build requires: %s', reqs) + except Exception: + log.error('Failure in get_requires_for_build_sdist', exc_info=True) + return False + + try: + env.pip_install(reqs) + log.info('Installed dynamic build dependencies') + except CalledProcessError: + log.error('Failed to install dynamic build dependencies') + return False + + td = mkdtemp() + log.info('Trying to build wheel in %s', td) + try: + try: + filename = hooks.build_wheel(td, {}) + log.info('build_wheel returned %r', filename) + except Exception: + log.info('Failure in build_wheel', exc_info=True) + return False + + if not filename.endswith('.whl'): + log.error("Filename %s doesn't have .whl extension", filename) + return False + + path = pjoin(td, filename) + if isfile(path): + log.info("Output file %s exists", path) + else: + log.error("Output file %s does not exist", path) + return False + + if zipfile.is_zipfile(path): + log.info("Output file is a zip file") + else: + log.error("Output file is not a zip file") + return False + + finally: + shutil.rmtree(td) + + return True + + +def check(source_dir): + pyproject = pjoin(source_dir, 'pyproject.toml') + if isfile(pyproject): + log.info('Found pyproject.toml') + else: + log.error('Missing pyproject.toml') + return False + + try: + with open(pyproject) as f: + pyproject_data = toml_load(f) + # Ensure the mandatory data can be loaded + buildsys = pyproject_data['build-system'] + requires = buildsys['requires'] + backend = buildsys['build-backend'] + log.info('Loaded pyproject.toml') + except (TomlError, KeyError): + log.error("Invalid pyproject.toml", exc_info=True) + return False + + hooks = Pep517HookCaller(source_dir, backend) + + sdist_ok = check_build_sdist(hooks, requires) + wheel_ok = check_build_wheel(hooks, requires) + + if not sdist_ok: + log.warning('Sdist checks failed; scroll up to see') + if not wheel_ok: + log.warning('Wheel checks failed') + + return sdist_ok + + +def main(argv=None): + ap = argparse.ArgumentParser() + ap.add_argument( + 'source_dir', + help="A directory containing pyproject.toml") + args = ap.parse_args(argv) + + enable_colourful_output() + + ok = check(args.source_dir) + + if ok: + print(ansi('Checks passed', 'green')) + else: + print(ansi('Checks failed', 'red')) + sys.exit(1) + + +ansi_codes = { + 'reset': '\x1b[0m', + 'bold': '\x1b[1m', + 'red': '\x1b[31m', + 'green': '\x1b[32m', +} + + +def ansi(s, attr): + if os.name != 'nt' and sys.stdout.isatty(): + return ansi_codes[attr] + str(s) + ansi_codes['reset'] + else: + return str(s) + + +if __name__ == '__main__': + main() diff --git a/pipenv/vendor/pep517/colorlog.py b/pipenv/vendor/pep517/colorlog.py new file mode 100644 index 00000000..69c8a59d --- /dev/null +++ b/pipenv/vendor/pep517/colorlog.py @@ -0,0 +1,115 @@ +"""Nicer log formatting with colours. + +Code copied from Tornado, Apache licensed. +""" +# Copyright 2012 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +try: + import curses +except ImportError: + curses = None + + +def _stderr_supports_color(): + color = False + if curses and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except Exception: + pass + return color + + +class LogFormatter(logging.Formatter): + """Log formatter with colour support + """ + DEFAULT_COLORS = { + logging.INFO: 2, # Green + logging.WARNING: 3, # Yellow + logging.ERROR: 1, # Red + logging.CRITICAL: 1, + } + + def __init__(self, color=True, datefmt=None): + r""" + :arg bool color: Enables color support. + :arg string fmt: Log message format. + It will be applied to the attributes dict of log records. The + text between ``%(color)s`` and ``%(end_color)s`` will be colored + depending on the level if color support is on. + :arg dict colors: color mappings from logging level to terminal color + code + :arg string datefmt: Datetime format. + Used for formatting ``(asctime)`` placeholder in ``prefix_fmt``. + .. versionchanged:: 3.2 + Added ``fmt`` and ``datefmt`` arguments. + """ + logging.Formatter.__init__(self, datefmt=datefmt) + self._colors = {} + if color and _stderr_supports_color(): + # The curses module has some str/bytes confusion in + # python3. Until version 3.2.3, most methods return + # bytes, but only accept strings. In addition, we want to + # output these strings with the logging module, which + # works with unicode strings. The explicit calls to + # unicode() below are harmless in python2 but will do the + # right conversion in python 3. + fg_color = (curses.tigetstr("setaf") or + curses.tigetstr("setf") or "") + if (3, 0) < sys.version_info < (3, 2, 3): + fg_color = str(fg_color, "ascii") + + for levelno, code in self.DEFAULT_COLORS.items(): + self._colors[levelno] = str( + curses.tparm(fg_color, code), "ascii") + self._normal = str(curses.tigetstr("sgr0"), "ascii") + + scr = curses.initscr() + self.termwidth = scr.getmaxyx()[1] + curses.endwin() + else: + self._normal = '' + # Default width is usually 80, but too wide is + # worse than too narrow + self.termwidth = 70 + + def formatMessage(self, record): + mlen = len(record.message) + right_text = '{initial}-{name}'.format(initial=record.levelname[0], + name=record.name) + if mlen + len(right_text) < self.termwidth: + space = ' ' * (self.termwidth - (mlen + len(right_text))) + else: + space = ' ' + + if record.levelno in self._colors: + start_color = self._colors[record.levelno] + end_color = self._normal + else: + start_color = end_color = '' + + return record.message + space + start_color + right_text + end_color + + +def enable_colourful_output(level=logging.INFO): + handler = logging.StreamHandler() + handler.setFormatter(LogFormatter()) + logging.root.addHandler(handler) + logging.root.setLevel(level) diff --git a/pipenv/vendor/pep517/compat.py b/pipenv/vendor/pep517/compat.py new file mode 100644 index 00000000..01c66fc7 --- /dev/null +++ b/pipenv/vendor/pep517/compat.py @@ -0,0 +1,23 @@ +"""Handle reading and writing JSON in UTF-8, on Python 3 and 2.""" +import json +import sys + +if sys.version_info[0] >= 3: + # Python 3 + def write_json(obj, path, **kwargs): + with open(path, 'w', encoding='utf-8') as f: + json.dump(obj, f, **kwargs) + + def read_json(path): + with open(path, 'r', encoding='utf-8') as f: + return json.load(f) + +else: + # Python 2 + def write_json(obj, path, **kwargs): + with open(path, 'wb') as f: + json.dump(obj, f, encoding='utf-8', **kwargs) + + def read_json(path): + with open(path, 'rb') as f: + return json.load(f) diff --git a/pipenv/vendor/pep517/envbuild.py b/pipenv/vendor/pep517/envbuild.py new file mode 100644 index 00000000..61253f4d --- /dev/null +++ b/pipenv/vendor/pep517/envbuild.py @@ -0,0 +1,158 @@ +"""Build wheels/sdists by installing build deps to a temporary environment. +""" + +import os +import logging +import pytoml +import shutil +from subprocess import check_call +import sys +from sysconfig import get_paths +from tempfile import mkdtemp + +from .wrappers import Pep517HookCaller + +log = logging.getLogger(__name__) + + +def _load_pyproject(source_dir): + with open(os.path.join(source_dir, 'pyproject.toml')) as f: + pyproject_data = pytoml.load(f) + buildsys = pyproject_data['build-system'] + return buildsys['requires'], buildsys['build-backend'] + + +class BuildEnvironment(object): + """Context manager to install build deps in a simple temporary environment + + Based on code I wrote for pip, which is MIT licensed. + """ + # Copyright (c) 2008-2016 The pip developers (see AUTHORS.txt file) + # + # Permission is hereby granted, free of charge, to any person obtaining + # a copy of this software and associated documentation files (the + # "Software"), to deal in the Software without restriction, including + # without limitation the rights to use, copy, modify, merge, publish, + # distribute, sublicense, and/or sell copies of the Software, and to + # permit persons to whom the Software is furnished to do so, subject to + # the following conditions: + # + # The above copyright notice and this permission notice shall be + # included in all copies or substantial portions of the Software. + # + # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND + # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE + # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION + # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + path = None + + def __init__(self, cleanup=True): + self._cleanup = cleanup + + def __enter__(self): + self.path = mkdtemp(prefix='pep517-build-env-') + log.info('Temporary build environment: %s', self.path) + + self.save_path = os.environ.get('PATH', None) + self.save_pythonpath = os.environ.get('PYTHONPATH', None) + + install_scheme = 'nt' if (os.name == 'nt') else 'posix_prefix' + install_dirs = get_paths(install_scheme, vars={ + 'base': self.path, + 'platbase': self.path, + }) + + scripts = install_dirs['scripts'] + if self.save_path: + os.environ['PATH'] = scripts + os.pathsep + self.save_path + else: + os.environ['PATH'] = scripts + os.pathsep + os.defpath + + if install_dirs['purelib'] == install_dirs['platlib']: + lib_dirs = install_dirs['purelib'] + else: + lib_dirs = install_dirs['purelib'] + os.pathsep + \ + install_dirs['platlib'] + if self.save_pythonpath: + os.environ['PYTHONPATH'] = lib_dirs + os.pathsep + \ + self.save_pythonpath + else: + os.environ['PYTHONPATH'] = lib_dirs + + return self + + def pip_install(self, reqs): + """Install dependencies into this env by calling pip in a subprocess""" + if not reqs: + return + log.info('Calling pip to install %s', reqs) + check_call([ + sys.executable, '-m', 'pip', 'install', '--ignore-installed', + '--prefix', self.path] + list(reqs)) + + def __exit__(self, exc_type, exc_val, exc_tb): + needs_cleanup = ( + self._cleanup and + self.path is not None and + os.path.isdir(self.path) + ) + if needs_cleanup: + shutil.rmtree(self.path) + + if self.save_path is None: + os.environ.pop('PATH', None) + else: + os.environ['PATH'] = self.save_path + + if self.save_pythonpath is None: + os.environ.pop('PYTHONPATH', None) + else: + os.environ['PYTHONPATH'] = self.save_pythonpath + + +def build_wheel(source_dir, wheel_dir, config_settings=None): + """Build a wheel from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str wheel_dir: Target directory to create wheel in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_wheel(config_settings) + env.pip_install(reqs) + return hooks.build_wheel(wheel_dir, config_settings) + + +def build_sdist(source_dir, sdist_dir, config_settings=None): + """Build an sdist from a source directory using PEP 517 hooks. + + :param str source_dir: Source directory containing pyproject.toml + :param str sdist_dir: Target directory to place sdist in + :param dict config_settings: Options to pass to build backend + + This is a blocking function which will run pip in a subprocess to install + build requirements. + """ + if config_settings is None: + config_settings = {} + requires, backend = _load_pyproject(source_dir) + hooks = Pep517HookCaller(source_dir, backend) + + with BuildEnvironment() as env: + env.pip_install(requires) + reqs = hooks.get_requires_for_build_sdist(config_settings) + env.pip_install(reqs) + return hooks.build_sdist(sdist_dir, config_settings) diff --git a/pipenv/vendor/pep517/wrappers.py b/pipenv/vendor/pep517/wrappers.py new file mode 100644 index 00000000..b14b8991 --- /dev/null +++ b/pipenv/vendor/pep517/wrappers.py @@ -0,0 +1,163 @@ +from contextlib import contextmanager +import os +from os.path import dirname, abspath, join as pjoin +import shutil +from subprocess import check_call +import sys +from tempfile import mkdtemp + +from . import compat + +_in_proc_script = pjoin(dirname(abspath(__file__)), '_in_process.py') + + +@contextmanager +def tempdir(): + td = mkdtemp() + try: + yield td + finally: + shutil.rmtree(td) + + +class BackendUnavailable(Exception): + """Will be raised if the backend cannot be imported in the hook process.""" + + +class UnsupportedOperation(Exception): + """May be raised by build_sdist if the backend indicates that it can't.""" + + +def default_subprocess_runner(cmd, cwd=None, extra_environ=None): + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + check_call(cmd, cwd=cwd, env=env) + + +class Pep517HookCaller(object): + """A wrapper around a source directory to be built with a PEP 517 backend. + + source_dir : The path to the source directory, containing pyproject.toml. + backend : The build backend spec, as per PEP 517, from pyproject.toml. + """ + def __init__(self, source_dir, build_backend): + self.source_dir = abspath(source_dir) + self.build_backend = build_backend + self._subprocess_runner = default_subprocess_runner + + # TODO: Is this over-engineered? Maybe frontends only need to + # set this when creating the wrapper, not on every call. + @contextmanager + def subprocess_runner(self, runner): + prev = self._subprocess_runner + self._subprocess_runner = runner + yield + self._subprocess_runner = prev + + def get_requires_for_build_wheel(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["wheel >= 0.25", "setuptools"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_wheel', { + 'config_settings': config_settings + }) + + def prepare_metadata_for_build_wheel( + self, metadata_directory, config_settings=None): + """Prepare a *.dist-info folder with metadata for this project. + + Returns the name of the newly created folder. + + If the build backend defines a hook with this name, it will be called + in a subprocess. If not, the backend will be asked to build a wheel, + and the dist-info extracted from that. + """ + return self._call_hook('prepare_metadata_for_build_wheel', { + 'metadata_directory': abspath(metadata_directory), + 'config_settings': config_settings, + }) + + def build_wheel( + self, wheel_directory, config_settings=None, + metadata_directory=None): + """Build a wheel from this project. + + Returns the name of the newly created file. + + In general, this will call the 'build_wheel' hook in the backend. + However, if that was previously called by + 'prepare_metadata_for_build_wheel', and the same metadata_directory is + used, the previously built wheel will be copied to wheel_directory. + """ + if metadata_directory is not None: + metadata_directory = abspath(metadata_directory) + return self._call_hook('build_wheel', { + 'wheel_directory': abspath(wheel_directory), + 'config_settings': config_settings, + 'metadata_directory': metadata_directory, + }) + + def get_requires_for_build_sdist(self, config_settings=None): + """Identify packages required for building a wheel + + Returns a list of dependency specifications, e.g.: + ["setuptools >= 26"] + + This does not include requirements specified in pyproject.toml. + It returns the result of calling the equivalently named hook in a + subprocess. + """ + return self._call_hook('get_requires_for_build_sdist', { + 'config_settings': config_settings + }) + + def build_sdist(self, sdist_directory, config_settings=None): + """Build an sdist from this project. + + Returns the name of the newly created file. + + This calls the 'build_sdist' backend hook in a subprocess. + """ + return self._call_hook('build_sdist', { + 'sdist_directory': abspath(sdist_directory), + 'config_settings': config_settings, + }) + + def _call_hook(self, hook_name, kwargs): + # On Python 2, pytoml returns Unicode values (which is correct) but the + # environment passed to check_call needs to contain string values. We + # convert here by encoding using ASCII (the backend can only contain + # letters, digits and _, . and : characters, and will be used as a + # Python identifier, so non-ASCII content is wrong on Python 2 in + # any case). + if sys.version_info[0] == 2: + build_backend = self.build_backend.encode('ASCII') + else: + build_backend = self.build_backend + + with tempdir() as td: + compat.write_json({'kwargs': kwargs}, pjoin(td, 'input.json'), + indent=2) + + # Run the hook in a subprocess + self._subprocess_runner( + [sys.executable, _in_proc_script, hook_name, td], + cwd=self.source_dir, + extra_environ={'PEP517_BUILD_BACKEND': build_backend} + ) + + data = compat.read_json(pjoin(td, 'output.json')) + if data.get('unsupported'): + raise UnsupportedOperation + if data.get('no_backend'): + raise BackendUnavailable + return data['return_val'] diff --git a/pipenv/vendor/pipdeptree.py b/pipenv/vendor/pipdeptree.py index 2082fc8a..899118cc 100644 --- a/pipenv/vendor/pipdeptree.py +++ b/pipenv/vendor/pipdeptree.py @@ -22,7 +22,7 @@ import pkg_resources # from graphviz import backend, Digraph -__version__ = '0.13.0' +__version__ = '0.13.2' flatten = chain.from_iterable @@ -127,6 +127,13 @@ def guess_version(pkg_key, default='?'): return getattr(m, '__version__', default) +def frozen_req_from_dist(dist): + try: + return FrozenRequirement.from_dist(dist) + except TypeError: + return FrozenRequirement.from_dist(dist, []) + + class Package(object): """Abstract class for wrappers around objects that pip returns. @@ -154,7 +161,7 @@ class Package(object): @staticmethod def frozen_repr(obj): - fr = FrozenRequirement.from_dist(obj, []) + fr = frozen_req_from_dist(obj) return str(fr).strip() def __getattr__(self, key): @@ -563,7 +570,7 @@ def _get_args(): def main(): args = _get_args() pkgs = get_installed_distributions(local_only=args.local_only, - user_only=args.user_only) + user_only=args.user_only) dist_index = build_dist_index(pkgs) tree = construct_tree(dist_index) diff --git a/pipenv/vendor/pyparsing.py b/pipenv/vendor/pyparsing.py index cf38419b..ab804d53 100644 --- a/pipenv/vendor/pyparsing.py +++ b/pipenv/vendor/pyparsing.py @@ -1,6 +1,7 @@ +#-*- coding: utf-8 -*- # module pyparsing.py # -# Copyright (c) 2003-2018 Paul T. McGuire +# Copyright (c) 2003-2019 Paul T. McGuire # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -27,15 +28,18 @@ __doc__ = \ pyparsing module - Classes and methods to define and execute parsing grammars ============================================================================= -The pyparsing module is an alternative approach to creating and executing simple grammars, -vs. the traditional lex/yacc approach, or the use of regular expressions. With pyparsing, you -don't need to learn a new syntax for defining grammars or matching expressions - the parsing module -provides a library of classes that you use to construct the grammar directly in Python. +The pyparsing module is an alternative approach to creating and +executing simple grammars, vs. the traditional lex/yacc approach, or the +use of regular expressions. With pyparsing, you don't need to learn +a new syntax for defining grammars or matching expressions - the parsing +module provides a library of classes that you use to construct the +grammar directly in Python. -Here is a program to parse "Hello, World!" (or any greeting of the form -C{", !"}), built up using L{Word}, L{Literal}, and L{And} elements -(L{'+'} operator gives L{And} expressions, strings are auto-converted to -L{Literal} expressions):: +Here is a program to parse "Hello, World!" (or any greeting of the form +``", !"``), built up using :class:`Word`, +:class:`Literal`, and :class:`And` elements +(the :class:`'+'` operators create :class:`And` expressions, +and the strings are auto-converted to :class:`Literal` expressions):: from pyparsing import Word, alphas @@ -49,33 +53,48 @@ The program outputs the following:: Hello, World! -> ['Hello', ',', 'World', '!'] -The Python representation of the grammar is quite readable, owing to the self-explanatory -class names, and the use of '+', '|' and '^' operators. +The Python representation of the grammar is quite readable, owing to the +self-explanatory class names, and the use of '+', '|' and '^' operators. -The L{ParseResults} object returned from L{ParserElement.parseString} can be accessed as a nested list, a dictionary, or an -object with named attributes. +The :class:`ParseResults` object returned from +:class:`ParserElement.parseString` can be +accessed as a nested list, a dictionary, or an object with named +attributes. -The pyparsing module handles some of the problems that are typically vexing when writing text parsers: - - extra or missing whitespace (the above program will also handle "Hello,World!", "Hello , World !", etc.) - - quoted strings - - embedded comments +The pyparsing module handles some of the problems that are typically +vexing when writing text parsers: + + - extra or missing whitespace (the above program will also handle + "Hello,World!", "Hello , World !", etc.) + - quoted strings + - embedded comments Getting Started - ----------------- -Visit the classes L{ParserElement} and L{ParseResults} to see the base classes that most other pyparsing +Visit the classes :class:`ParserElement` and :class:`ParseResults` to +see the base classes that most other pyparsing classes inherit from. Use the docstrings for examples of how to: - - construct literal match expressions from L{Literal} and L{CaselessLiteral} classes - - construct character word-group expressions using the L{Word} class - - see how to create repetitive expressions using L{ZeroOrMore} and L{OneOrMore} classes - - use L{'+'}, L{'|'}, L{'^'}, and L{'&'} operators to combine simple expressions into more complex ones - - associate names with your parsed results using L{ParserElement.setResultsName} - - find some helpful expression short-cuts like L{delimitedList} and L{oneOf} - - find more useful common expressions in the L{pyparsing_common} namespace class + + - construct literal match expressions from :class:`Literal` and + :class:`CaselessLiteral` classes + - construct character word-group expressions using the :class:`Word` + class + - see how to create repetitive expressions using :class:`ZeroOrMore` + and :class:`OneOrMore` classes + - use :class:`'+'`, :class:`'|'`, :class:`'^'`, + and :class:`'&'` operators to combine simple expressions into + more complex ones + - associate names with your parsed results using + :class:`ParserElement.setResultsName` + - find some helpful expression short-cuts like :class:`delimitedList` + and :class:`oneOf` + - find more useful common expressions in the :class:`pyparsing_common` + namespace class """ -__version__ = "2.2.2" -__versionTime__ = "29 Sep 2018 15:58 UTC" +__version__ = "2.3.1" +__versionTime__ = "09 Jan 2019 23:26 UTC" __author__ = "Paul McGuire " import string @@ -91,6 +110,12 @@ import traceback import types from datetime import datetime +try: + # Python 3 + from itertools import filterfalse +except ImportError: + from itertools import ifilterfalse as filterfalse + try: from _thread import RLock except ImportError: @@ -113,27 +138,33 @@ except ImportError: except ImportError: _OrderedDict = None +try: + from types import SimpleNamespace +except ImportError: + class SimpleNamespace: pass + + #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) ) __all__ = [ 'And', 'CaselessKeyword', 'CaselessLiteral', 'CharsNotIn', 'Combine', 'Dict', 'Each', 'Empty', 'FollowedBy', 'Forward', 'GoToColumn', 'Group', 'Keyword', 'LineEnd', 'LineStart', 'Literal', -'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', +'PrecededBy', 'MatchFirst', 'NoMatch', 'NotAny', 'OneOrMore', 'OnlyOnce', 'Optional', 'Or', 'ParseBaseException', 'ParseElementEnhance', 'ParseException', 'ParseExpression', 'ParseFatalException', 'ParseResults', 'ParseSyntaxException', 'ParserElement', 'QuotedString', 'RecursiveGrammarException', -'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', -'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', +'Regex', 'SkipTo', 'StringEnd', 'StringStart', 'Suppress', 'Token', 'TokenConverter', +'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore', 'Char', 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col', 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString', 'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums', 'htmlComment', 'javaStyleComment', 'line', 'lineEnd', 'lineStart', 'lineno', 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral', 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables', -'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', +'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd', 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute', 'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr', 'withClass', -'CloseMatch', 'tokenMap', 'pyparsing_common', +'CloseMatch', 'tokenMap', 'pyparsing_common', 'pyparsing_unicode', 'unicode_set', ] system_version = tuple(sys.version_info)[:3] @@ -142,6 +173,7 @@ if PY_3: _MAX_INT = sys.maxsize basestring = str unichr = chr + unicode = str _ustr = str # build list of single arg builtins, that can be used as parse actions @@ -152,9 +184,11 @@ else: range = xrange def _ustr(obj): - """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries - str(obj). If that fails with a UnicodeEncodeError, then it tries unicode(obj). It - then < returns the unicode object | encodes it with the default encoding | ... >. + """Drop-in replacement for str(obj) that tries to be Unicode + friendly. It first tries str(obj). If that fails with + a UnicodeEncodeError, then it tries unicode(obj). It then + < returns the unicode object | encodes it with the default + encoding | ... >. """ if isinstance(obj,unicode): return obj @@ -179,9 +213,9 @@ else: singleArgBuiltins.append(getattr(__builtin__,fname)) except AttributeError: continue - + _generatorType = type((y for y in range(1))) - + def _xml_escape(data): """Escape &, <, >, ", ', etc. in a string of data.""" @@ -192,9 +226,6 @@ def _xml_escape(data): data = data.replace(from_, to_) return data -class _Constants(object): - pass - alphas = string.ascii_uppercase + string.ascii_lowercase nums = "0123456789" hexnums = nums + "ABCDEFabcdef" @@ -220,16 +251,16 @@ class ParseBaseException(Exception): @classmethod def _from_exception(cls, pe): """ - internal factory method to simplify creating one type of ParseException + internal factory method to simplify creating one type of ParseException from another - avoids having __init__ signature conflicts among subclasses """ return cls(pe.pstr, pe.loc, pe.msg, pe.parserElement) def __getattr__( self, aname ): """supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text """ if( aname == "lineno" ): return lineno( self.loc, self.pstr ) @@ -262,22 +293,94 @@ class ParseException(ParseBaseException): """ Exception thrown when parse expressions don't match class; supported attributes by name are: - - lineno - returns the line number of the exception text - - col - returns the column number of the exception text - - line - returns the line containing the exception text - + - lineno - returns the line number of the exception text + - col - returns the column number of the exception text + - line - returns the line containing the exception text + Example:: + try: Word(nums).setName("integer").parseString("ABC") except ParseException as pe: print(pe) print("column: {}".format(pe.col)) - + prints:: + Expected integer (at char 0), (line:1, col:1) column: 1 + """ - pass + + @staticmethod + def explain(exc, depth=16): + """ + Method to take an exception and translate the Python internal traceback into a list + of the pyparsing expressions that caused the exception to be raised. + + Parameters: + + - exc - exception raised during parsing (need not be a ParseException, in support + of Python exceptions that might be raised in a parse action) + - depth (default=16) - number of levels back in the stack trace to list expression + and function names; if None, the full stack trace names will be listed; if 0, only + the failing input line, marker, and exception string will be shown + + Returns a multi-line string listing the ParserElements and/or function names in the + exception's stack trace. + + Note: the diagnostic output will include string representations of the expressions + that failed to parse. These representations will be more helpful if you use `setName` to + give identifiable names to your expressions. Otherwise they will use the default string + forms, which may be cryptic to read. + + explain() is only supported under Python 3. + """ + import inspect + + if depth is None: + depth = sys.getrecursionlimit() + ret = [] + if isinstance(exc, ParseBaseException): + ret.append(exc.line) + ret.append(' ' * (exc.col - 1) + '^') + ret.append("{0}: {1}".format(type(exc).__name__, exc)) + + if depth > 0: + callers = inspect.getinnerframes(exc.__traceback__, context=depth) + seen = set() + for i, ff in enumerate(callers[-depth:]): + frm = ff.frame + + f_self = frm.f_locals.get('self', None) + if isinstance(f_self, ParserElement): + if frm.f_code.co_name not in ('parseImpl', '_parseNoCache'): + continue + if f_self in seen: + continue + seen.add(f_self) + + self_type = type(f_self) + ret.append("{0}.{1} - {2}".format(self_type.__module__, + self_type.__name__, + f_self)) + elif f_self is not None: + self_type = type(f_self) + ret.append("{0}.{1}".format(self_type.__module__, + self_type.__name__)) + else: + code = frm.f_code + if code.co_name in ('wrapper', ''): + continue + + ret.append("{0}".format(code.co_name)) + + depth -= 1 + if not depth: + break + + return '\n'.join(ret) + class ParseFatalException(ParseBaseException): """user-throwable exception thrown when inconsistent parse content @@ -285,9 +388,11 @@ class ParseFatalException(ParseBaseException): pass class ParseSyntaxException(ParseFatalException): - """just like L{ParseFatalException}, but thrown internally when an - L{ErrorStop} ('-' operator) indicates that parsing is to stop - immediately because an unbacktrackable syntax error has been found""" + """just like :class:`ParseFatalException`, but thrown internally + when an :class:`ErrorStop` ('-' operator) indicates + that parsing is to stop immediately because an unbacktrackable + syntax error has been found. + """ pass #~ class ReparseException(ParseBaseException): @@ -304,7 +409,9 @@ class ParseSyntaxException(ParseFatalException): #~ self.reparseLoc = restartLoc class RecursiveGrammarException(Exception): - """exception thrown by L{ParserElement.validate} if the grammar could be improperly recursive""" + """exception thrown by :class:`ParserElement.validate` if the + grammar could be improperly recursive + """ def __init__( self, parseElementList ): self.parseElementTrace = parseElementList @@ -322,16 +429,18 @@ class _ParseResultsWithOffset(object): self.tup = (self.tup[0],i) class ParseResults(object): - """ - Structured parse results, to provide multiple means of access to the parsed data: - - as a list (C{len(results)}) - - by list index (C{results[0], results[1]}, etc.) - - by attribute (C{results.} - see L{ParserElement.setResultsName}) + """Structured parse results, to provide multiple means of access to + the parsed data: + + - as a list (``len(results)``) + - by list index (``results[0], results[1]``, etc.) + - by attribute (``results.`` - see :class:`ParserElement.setResultsName`) Example:: + integer = Word(nums) - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: # date_str = integer("year") + '/' + integer("month") + '/' + integer("day") @@ -348,7 +457,9 @@ class ParseResults(object): test("'month' in result") test("'minutes' in result") test("result.dump()", str) + prints:: + list(result) -> ['1999', '/', '12', '/', '31'] result[0] -> '1999' result['month'] -> '12' @@ -398,7 +509,7 @@ class ParseResults(object): toklist = [ toklist ] if asList: if isinstance(toklist,ParseResults): - self[name] = _ParseResultsWithOffset(toklist.copy(),0) + self[name] = _ParseResultsWithOffset(ParseResults(toklist.__toklist), 0) else: self[name] = _ParseResultsWithOffset(ParseResults(toklist[0]),0) self[name].__name = name @@ -467,19 +578,19 @@ class ParseResults(object): def _itervalues( self ): return (self[k] for k in self._iterkeys()) - + def _iteritems( self ): return ((k, self[k]) for k in self._iterkeys()) if PY_3: - keys = _iterkeys - """Returns an iterator of all named result keys (Python 3.x only).""" + keys = _iterkeys + """Returns an iterator of all named result keys.""" values = _itervalues - """Returns an iterator of all named result values (Python 3.x only).""" + """Returns an iterator of all named result values.""" items = _iteritems - """Returns an iterator of all named result key-value tuples (Python 3.x only).""" + """Returns an iterator of all named result key-value tuples.""" else: iterkeys = _iterkeys @@ -498,7 +609,7 @@ class ParseResults(object): def values( self ): """Returns all named result values (as a list in Python 2.x, as an iterator in Python 3.x).""" return list(self.itervalues()) - + def items( self ): """Returns all named result key-values (as a list of tuples in Python 2.x, as an iterator in Python 3.x).""" return list(self.iteritems()) @@ -507,19 +618,20 @@ class ParseResults(object): """Since keys() returns an iterator, this method is helpful in bypassing code that looks for the existence of any defined results names.""" return bool(self.__tokdict) - + def pop( self, *args, **kwargs): """ - Removes and returns item at specified index (default=C{last}). - Supports both C{list} and C{dict} semantics for C{pop()}. If passed no - argument or an integer argument, it will use C{list} semantics - and pop tokens from the list of parsed tokens. If passed a - non-integer argument (most likely a string), it will use C{dict} - semantics and pop the corresponding value from any defined - results names. A second default return value argument is - supported, just as in C{dict.pop()}. + Removes and returns item at specified index (default= ``last``). + Supports both ``list`` and ``dict`` semantics for ``pop()``. If + passed no argument or an integer argument, it will use ``list`` + semantics and pop tokens from the list of parsed tokens. If passed + a non-integer argument (most likely a string), it will use ``dict`` + semantics and pop the corresponding value from any defined results + names. A second default return value argument is supported, just as in + ``dict.pop()``. Example:: + def remove_first(tokens): tokens.pop(0) print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] @@ -536,7 +648,9 @@ class ParseResults(object): return tokens patt.addParseAction(remove_LABEL) print(patt.parseString("AAB 123 321").dump()) + prints:: + ['AAB', '123', '321'] - LABEL: AAB @@ -549,8 +663,8 @@ class ParseResults(object): args = (args[0], v) else: raise TypeError("pop() got an unexpected keyword argument '%s'" % k) - if (isinstance(args[0], int) or - len(args) == 1 or + if (isinstance(args[0], int) or + len(args) == 1 or args[0] in self): index = args[0] ret = self[index] @@ -563,14 +677,15 @@ class ParseResults(object): def get(self, key, defaultValue=None): """ Returns named result matching the given key, or if there is no - such name, then returns the given C{defaultValue} or C{None} if no - C{defaultValue} is specified. + such name, then returns the given ``defaultValue`` or ``None`` if no + ``defaultValue`` is specified. + + Similar to ``dict.get()``. - Similar to C{dict.get()}. - Example:: + integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") result = date_str.parseString("1999/12/31") print(result.get("year")) # -> '1999' @@ -585,10 +700,11 @@ class ParseResults(object): def insert( self, index, insStr ): """ Inserts new element at location index in the list of parsed tokens. - - Similar to C{list.insert()}. + + Similar to ``list.insert()``. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] # use a parse action to insert the parse location in the front of the parsed results @@ -607,8 +723,9 @@ class ParseResults(object): Add single element to end of ParseResults list of elements. Example:: + print(OneOrMore(Word(nums)).parseString("0 123 321")) # -> ['0', '123', '321'] - + # use a parse action to compute the sum of the parsed integers, and add it to the end def append_sum(tokens): tokens.append(sum(map(int, tokens))) @@ -621,8 +738,9 @@ class ParseResults(object): Add sequence of elements to end of ParseResults list of elements. Example:: + patt = OneOrMore(Word(alphas)) - + # use a parse action to append the reverse of the matched strings, to make a palindrome def make_palindrome(tokens): tokens.extend(reversed([t[::-1] for t in tokens])) @@ -646,7 +764,7 @@ class ParseResults(object): return self[name] except KeyError: return "" - + if name in self.__tokdict: if name not in self.__accumNames: return self.__tokdict[name][-1][0] @@ -671,7 +789,7 @@ class ParseResults(object): self[k] = v if isinstance(v[0],ParseResults): v[0].__parent = wkref(self) - + self.__toklist += other.__toklist self.__accumNames.update( other.__accumNames ) return self @@ -683,7 +801,7 @@ class ParseResults(object): else: # this may raise a TypeError - so be it return other + self - + def __repr__( self ): return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) ) @@ -706,11 +824,12 @@ class ParseResults(object): Returns the parse results as a nested list of matching tokens, all converted to strings. Example:: + patt = OneOrMore(Word(alphas)) result = patt.parseString("sldkj lsdkj sldkj") # even though the result prints in string-like form, it is actually a pyparsing ParseResults print(type(result), result) # -> ['sldkj', 'lsdkj', 'sldkj'] - + # Use asList() to create an actual list result_list = result.asList() print(type(result_list), result_list) # -> ['sldkj', 'lsdkj', 'sldkj'] @@ -722,12 +841,13 @@ class ParseResults(object): Returns the named parse results as a nested dictionary. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(type(result), repr(result)) # -> (['12', '/', '31', '/', '1999'], {'day': [('1999', 4)], 'year': [('12', 0)], 'month': [('31', 2)]}) - + result_dict = result.asDict() print(type(result_dict), repr(result_dict)) # -> {'day': '1999', 'year': '12', 'month': '31'} @@ -740,7 +860,7 @@ class ParseResults(object): item_fn = self.items else: item_fn = self.iteritems - + def toItem(obj): if isinstance(obj, ParseResults): if obj.haskeys(): @@ -749,15 +869,15 @@ class ParseResults(object): return [toItem(v) for v in obj] else: return obj - + return dict((k,toItem(v)) for k,v in item_fn()) def copy( self ): """ - Returns a new copy of a C{ParseResults} object. + Returns a new copy of a :class:`ParseResults` object. """ ret = ParseResults( self.__toklist ) - ret.__tokdict = self.__tokdict.copy() + ret.__tokdict = dict(self.__tokdict.items()) ret.__parent = self.__parent ret.__accumNames.update( self.__accumNames ) ret.__name = self.__name @@ -833,22 +953,25 @@ class ParseResults(object): def getName(self): r""" - Returns the results name for this token expression. Useful when several + Returns the results name for this token expression. Useful when several different expressions might match at a particular location. Example:: + integer = Word(nums) ssn_expr = Regex(r"\d\d\d-\d\d-\d\d\d\d") house_number_expr = Suppress('#') + Word(nums, alphanums) - user_data = (Group(house_number_expr)("house_number") + user_data = (Group(house_number_expr)("house_number") | Group(ssn_expr)("ssn") | Group(integer)("age")) user_info = OneOrMore(user_data) - + result = user_info.parseString("22 111-22-3333 #221B") for item in result: print(item.getName(), ':', item[0]) + prints:: + age : 22 ssn : 111-22-3333 house_number : 221B @@ -870,17 +993,20 @@ class ParseResults(object): def dump(self, indent='', depth=0, full=True): """ - Diagnostic method for listing out the contents of a C{ParseResults}. - Accepts an optional C{indent} argument so that this string can be embedded - in a nested display of other data. + Diagnostic method for listing out the contents of + a :class:`ParseResults`. Accepts an optional ``indent`` argument so + that this string can be embedded in a nested display of other data. Example:: + integer = Word(nums) date_str = integer("year") + '/' + integer("month") + '/' + integer("day") - + result = date_str.parseString('12/31/1999') print(result.dump()) + prints:: + ['12', '/', '31', '/', '1999'] - day: 1999 - month: 31 @@ -910,16 +1036,18 @@ class ParseResults(object): out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),vv.dump(indent,depth+1) )) else: out.append("\n%s%s[%d]:\n%s%s%s" % (indent,(' '*(depth)),i,indent,(' '*(depth+1)),_ustr(vv))) - + return "".join(out) def pprint(self, *args, **kwargs): """ - Pretty-printer for parsed results as a list, using the C{pprint} module. - Accepts additional positional or keyword args as defined for the - C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint}) + Pretty-printer for parsed results as a list, using the + `pprint `_ module. + Accepts additional positional or keyword args as defined for + `pprint.pprint `_ . Example:: + ident = Word(alphas, alphanums) num = Word(nums) func = Forward() @@ -927,7 +1055,9 @@ class ParseResults(object): func <<= ident + Group(Optional(delimitedList(term))) result = func.parseString("fna a,b,(fnb c,d,200),100") result.pprint(width=40) + prints:: + ['fna', ['a', 'b', @@ -970,24 +1100,25 @@ def col (loc,strg): The first column is number 1. Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{ParserElement.parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. + before starting the parsing process. See + :class:`ParserElement.parseString` for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. """ s = strg return 1 if 0} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - """ + Note - the default parsing behavior is to expand tabs in the input string + before starting the parsing process. See :class:`ParserElement.parseString` + for more information on parsing strings containing ```` s, and + suggested methods to maintain a consistent view of the parsed string, the + parse location, and line and column positions within the parsed string. + """ return strg.count("\n",0,loc) + 1 def line( loc, strg ): @@ -1041,7 +1172,7 @@ def _trim_arity(func, maxargs=2): return lambda s,l,t: func(t) limit = [0] foundArity = [False] - + # traceback return data structure changed in Py3.5 - normalize back to plain tuples if system_version[:2] >= (3,5): def extract_stack(limit=0): @@ -1056,12 +1187,12 @@ def _trim_arity(func, maxargs=2): else: extract_stack = traceback.extract_stack extract_tb = traceback.extract_tb - - # synthesize what would be returned by traceback.extract_stack at the call to + + # synthesize what would be returned by traceback.extract_stack at the call to # user's parse action 'func', so that we don't incur call penalty at parse time - + LINE_DIFF = 6 - # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND + # IF ANY CODE CHANGES, EVEN JUST COMMENTS OR BLANK LINES, BETWEEN THE NEXT LINE AND # THE CALL TO FUNC INSIDE WRAPPER, LINE_DIFF MUST BE MODIFIED!!!! this_line = extract_stack(limit=2)[-1] pa_call_line_synth = (this_line[0], this_line[1]+LINE_DIFF) @@ -1092,7 +1223,7 @@ def _trim_arity(func, maxargs=2): # copy func name to wrapper for sensible debug output func_name = "" try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -1111,9 +1242,10 @@ class ParserElement(object): Overrides the default whitespace chars Example:: + # default whitespace chars are space, and newline OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def', 'ghi', 'jkl'] - + # change to just treat newline as significant ParserElement.setDefaultWhitespaceChars(" \t") OneOrMore(Word(alphas)).parseString("abc def\nghi jkl") # -> ['abc', 'def'] @@ -1124,18 +1256,19 @@ class ParserElement(object): def inlineLiteralsUsing(cls): """ Set class to be used for inclusion of string literals into a parser. - + Example:: + # default literal class used is Literal integer = Word(nums) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '/', '12', '/', '31'] # change to Suppress ParserElement.inlineLiteralsUsing(Suppress) - date_str = integer("year") + '/' + integer("month") + '/' + integer("day") + date_str = integer("year") + '/' + integer("month") + '/' + integer("day") date_str.parseString("1999/12/31") # -> ['1999', '12', '31'] """ @@ -1149,7 +1282,7 @@ class ParserElement(object): self.resultsName = None self.saveAsList = savelist self.skipWhitespace = True - self.whiteChars = ParserElement.DEFAULT_WHITE_CHARS + self.whiteChars = set(ParserElement.DEFAULT_WHITE_CHARS) self.copyDefaultWhiteChars = True self.mayReturnEmpty = False # used when checking for left-recursion self.keepTabs = False @@ -1166,18 +1299,24 @@ class ParserElement(object): def copy( self ): """ - Make a copy of this C{ParserElement}. Useful for defining different parse actions - for the same parsing pattern, using copies of the original parse element. - + Make a copy of this :class:`ParserElement`. Useful for defining + different parse actions for the same parsing pattern, using copies of + the original parse element. + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) integerK = integer.copy().addParseAction(lambda toks: toks[0]*1024) + Suppress("K") integerM = integer.copy().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") - + print(OneOrMore(integerK | integerM | integer).parseString("5K 100 640K 256M")) + prints:: + [5120, 100, 655360, 268435456] - Equivalent form of C{expr.copy()} is just C{expr()}:: + + Equivalent form of ``expr.copy()`` is just ``expr()``:: + integerM = integer().addParseAction(lambda toks: toks[0]*1024*1024) + Suppress("M") """ cpy = copy.copy( self ) @@ -1190,8 +1329,9 @@ class ParserElement(object): def setName( self, name ): """ Define name for this expression, makes debugging and exception messages clearer. - + Example:: + Word(nums).parseString("ABC") # -> Exception: Expected W:(0123...) (at char 0), (line:1, col:1) Word(nums).setName("integer").parseString("ABC") # -> Exception: Expected integer (at char 0), (line:1, col:1) """ @@ -1205,17 +1345,18 @@ class ParserElement(object): """ Define name for referencing matching tokens as a nested attribute of the returned parse results. - NOTE: this returns a *copy* of the original C{ParserElement} object; + NOTE: this returns a *copy* of the original :class:`ParserElement` object; this is so that the client can define a basic element, such as an integer, and reference it in multiple places with different names. You can also set results names using the abbreviated syntax, - C{expr("name")} in place of C{expr.setResultsName("name")} - - see L{I{__call__}<__call__>}. + ``expr("name")`` in place of ``expr.setResultsName("name")`` + - see :class:`__call__`. Example:: - date_str = (integer.setResultsName("year") + '/' - + integer.setResultsName("month") + '/' + + date_str = (integer.setResultsName("year") + '/' + + integer.setResultsName("month") + '/' + integer.setResultsName("day")) # equivalent form: @@ -1231,7 +1372,7 @@ class ParserElement(object): def setBreak(self,breakFlag = True): """Method to invoke the Python pdb debugger when this element is - about to be parsed. Set C{breakFlag} to True to enable, False to + about to be parsed. Set ``breakFlag`` to True to enable, False to disable. """ if breakFlag: @@ -1250,25 +1391,28 @@ class ParserElement(object): def setParseAction( self, *fns, **kwargs ): """ Define one or more actions to perform when successfully matching parse element definition. - Parse action fn is a callable method with 0-3 arguments, called as C{fn(s,loc,toks)}, - C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where: - - s = the original string being parsed (see note below) - - loc = the location of the matching substring - - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object + Parse action fn is a callable method with 0-3 arguments, called as ``fn(s,loc,toks)`` , + ``fn(loc,toks)`` , ``fn(toks)`` , or just ``fn()`` , where: + + - s = the original string being parsed (see note below) + - loc = the location of the matching substring + - toks = a list of the matched tokens, packaged as a :class:`ParseResults` object + If the functions in fns modify the tokens, they can return them as the return value from fn, and the modified list of tokens will replace the original. Otherwise, fn does not need to return any value. Optional keyword arguments: - - callDuringTry = (default=C{False}) indicate if parse action should be run during lookaheads and alternate testing + - callDuringTry = (default= ``False`` ) indicate if parse action should be run during lookaheads and alternate testing Note: the default parsing behavior is to expand tabs in the input string - before starting the parsing process. See L{I{parseString}} for more information - on parsing strings containing C{}s, and suggested methods to maintain a - consistent view of the parsed string, the parse location, and line and column - positions within the parsed string. - + before starting the parsing process. See :class:`parseString for more + information on parsing strings containing ```` s, and suggested + methods to maintain a consistent view of the parsed string, the parse + location, and line and column positions within the parsed string. + Example:: + integer = Word(nums) date_str = integer + '/' + integer + '/' + integer @@ -1287,24 +1431,25 @@ class ParserElement(object): def addParseAction( self, *fns, **kwargs ): """ - Add one or more parse actions to expression's list of parse actions. See L{I{setParseAction}}. - - See examples in L{I{copy}}. + Add one or more parse actions to expression's list of parse actions. See :class:`setParseAction`. + + See examples in :class:`copy`. """ self.parseAction += list(map(_trim_arity, list(fns))) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) return self def addCondition(self, *fns, **kwargs): - """Add a boolean predicate function to expression's list of parse actions. See - L{I{setParseAction}} for function call signatures. Unlike C{setParseAction}, - functions passed to C{addCondition} need to return boolean success/fail of the condition. + """Add a boolean predicate function to expression's list of parse actions. See + :class:`setParseAction` for function call signatures. Unlike ``setParseAction``, + functions passed to ``addCondition`` need to return boolean success/fail of the condition. Optional keyword arguments: - - message = define a custom message to be used in the raised exception - - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException - + - message = define a custom message to be used in the raised exception + - fatal = if True, will raise ParseFatalException to stop parsing immediately; otherwise will raise ParseException + Example:: + integer = Word(nums).setParseAction(lambda toks: int(toks[0])) year_int = integer.copy() year_int.addCondition(lambda toks: toks[0] >= 2000, message="Only support years 2000 and later") @@ -1315,8 +1460,9 @@ class ParserElement(object): msg = kwargs.get("message", "failed user-defined condition") exc_type = ParseFatalException if kwargs.get("fatal", False) else ParseException for fn in fns: + fn = _trim_arity(fn) def pa(s,l,t): - if not bool(_trim_arity(fn)(s,l,t)): + if not bool(fn(s,l,t)): raise exc_type(s,l,msg) self.parseAction.append(pa) self.callDuringTry = self.callDuringTry or kwargs.get("callDuringTry", False) @@ -1325,12 +1471,12 @@ class ParserElement(object): def setFailAction( self, fn ): """Define action to perform if parsing fails at this expression. Fail acton fn is a callable function that takes the arguments - C{fn(s,loc,expr,err)} where: - - s = string being parsed - - loc = location where expression match was attempted and failed - - expr = the parse expression that failed - - err = the exception thrown - The function returns no value. It may throw C{L{ParseFatalException}} + ``fn(s,loc,expr,err)`` where: + - s = string being parsed + - loc = location where expression match was attempted and failed + - expr = the parse expression that failed + - err = the exception thrown + The function returns no value. It may throw :class:`ParseFatalException` if it is desired to stop parsing immediately.""" self.failAction = fn return self @@ -1412,8 +1558,14 @@ class ParserElement(object): if debugging: try: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1425,8 +1577,14 @@ class ParserElement(object): raise else: for fn in self.parseAction: - tokens = fn( instring, tokensStart, retTokens ) - if tokens is not None: + try: + tokens = fn( instring, tokensStart, retTokens ) + except IndexError as parse_action_exc: + exc = ParseException("exception raised in parse action") + exc.__cause__ = parse_action_exc + raise exc + + if tokens is not None and tokens is not retTokens: retTokens = ParseResults( tokens, self.resultsName, asList=self.saveAsList and isinstance(tokens,(ParseResults,list)), @@ -1443,7 +1601,7 @@ class ParserElement(object): return self._parse( instring, loc, doActions=False )[0] except ParseFatalException: raise ParseException( instring, loc, self.errmsg, self) - + def canParseNext(self, instring, loc): try: self.tryParse(instring, loc) @@ -1465,7 +1623,7 @@ class ParserElement(object): def clear(self): cache.clear() - + def cache_len(self): return len(cache) @@ -1577,23 +1735,23 @@ class ParserElement(object): often in many complex grammars) can immediately return a cached value, instead of re-executing parsing/validating code. Memoizing is done of both valid results and parsing exceptions. - + Parameters: - - cache_size_limit - (default=C{128}) - if an integer value is provided - will limit the size of the packrat cache; if None is passed, then - the cache size will be unbounded; if 0 is passed, the cache will - be effectively disabled. - + + - cache_size_limit - (default= ``128``) - if an integer value is provided + will limit the size of the packrat cache; if None is passed, then + the cache size will be unbounded; if 0 is passed, the cache will + be effectively disabled. + This speedup may break existing programs that use parse actions that have side-effects. For this reason, packrat parsing is disabled when you first import pyparsing. To activate the packrat feature, your - program must call the class method C{ParserElement.enablePackrat()}. If - your program uses C{psyco} to "compile as you go", you must call - C{enablePackrat} before calling C{psyco.full()}. If you do not do this, - Python will crash. For best results, call C{enablePackrat()} immediately - after importing pyparsing. - + program must call the class method :class:`ParserElement.enablePackrat`. + For best results, call ``enablePackrat()`` immediately after + importing pyparsing. + Example:: + import pyparsing pyparsing.ParserElement.enablePackrat() """ @@ -1612,23 +1770,25 @@ class ParserElement(object): expression has been built. If you want the grammar to require that the entire input string be - successfully parsed, then set C{parseAll} to True (equivalent to ending - the grammar with C{L{StringEnd()}}). + successfully parsed, then set ``parseAll`` to True (equivalent to ending + the grammar with ``StringEnd()``). - Note: C{parseString} implicitly calls C{expandtabs()} on the input string, + Note: ``parseString`` implicitly calls ``expandtabs()`` on the input string, in order to report proper column numbers in parse actions. If the input string contains tabs and - the grammar uses parse actions that use the C{loc} argument to index into the + the grammar uses parse actions that use the ``loc`` argument to index into the string being parsed, you can ensure you have a consistent view of the input string by: - - calling C{parseWithTabs} on your grammar before calling C{parseString} - (see L{I{parseWithTabs}}) - - define your parse action using the full C{(s,loc,toks)} signature, and - reference the input string using the parse action's C{s} argument - - explictly expand the tabs in your input string before calling - C{parseString} - + + - calling ``parseWithTabs`` on your grammar before calling ``parseString`` + (see :class:`parseWithTabs`) + - define your parse action using the full ``(s,loc,toks)`` signature, and + reference the input string using the parse action's ``s`` argument + - explictly expand the tabs in your input string before calling + ``parseString`` + Example:: + Word('a').parseString('aaaaabaaa') # -> ['aaaaa'] Word('a').parseString('aaaaabaaa', parseAll=True) # -> Exception: Expected end of text """ @@ -1659,22 +1819,23 @@ class ParserElement(object): """ Scan the input string for expression matches. Each match will return the matching tokens, start location, and end location. May be called with optional - C{maxMatches} argument, to clip scanning after 'n' matches are found. If - C{overlap} is specified, then overlapping matches will be reported. + ``maxMatches`` argument, to clip scanning after 'n' matches are found. If + ``overlap`` is specified, then overlapping matches will be reported. Note that the start and end locations are reported relative to the string - being parsed. See L{I{parseString}} for more information on parsing + being parsed. See :class:`parseString` for more information on parsing strings with embedded tabs. Example:: + source = "sldjf123lsdjjkf345sldkjf879lkjsfd987" print(source) for tokens,start,end in Word(alphas).scanString(source): print(' '*start + '^'*(end-start)) print(' '*start + tokens[0]) - + prints:: - + sldjf123lsdjjkf345sldkjf879lkjsfd987 ^^^^^ sldjf @@ -1728,19 +1889,22 @@ class ParserElement(object): def transformString( self, instring ): """ - Extension to C{L{scanString}}, to modify matching text with modified tokens that may - be returned from a parse action. To use C{transformString}, define a grammar and + Extension to :class:`scanString`, to modify matching text with modified tokens that may + be returned from a parse action. To use ``transformString``, define a grammar and attach a parse action to it that modifies the returned token list. - Invoking C{transformString()} on a target string will then scan for matches, + Invoking ``transformString()`` on a target string will then scan for matches, and replace the matched text patterns according to the logic in the parse - action. C{transformString()} returns the resulting transformed string. - + action. ``transformString()`` returns the resulting transformed string. + Example:: + wd = Word(alphas) wd.setParseAction(lambda toks: toks[0].title()) - + print(wd.transformString("now is the winter of our discontent made glorious summer by this sun of york.")) - Prints:: + + prints:: + Now Is The Winter Of Our Discontent Made Glorious Summer By This Sun Of York. """ out = [] @@ -1771,19 +1935,22 @@ class ParserElement(object): def searchString( self, instring, maxMatches=_MAX_INT ): """ - Another extension to C{L{scanString}}, simplifying the access to the tokens found + Another extension to :class:`scanString`, simplifying the access to the tokens found to match the given parse expression. May be called with optional - C{maxMatches} argument, to clip searching after 'n' matches are found. - + ``maxMatches`` argument, to clip searching after 'n' matches are found. + Example:: + # a capitalized word starts with an uppercase letter, followed by zero or more lowercase letters cap_word = Word(alphas.upper(), alphas.lower()) - + print(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity")) # the sum() builtin can be used to merge results into a single ParseResults object print(sum(cap_word.searchString("More than Iron, more than Lead, more than Gold I need Electricity"))) + prints:: + [['More'], ['Iron'], ['Lead'], ['Gold'], ['I'], ['Electricity']] ['More', 'Iron', 'Lead', 'Gold', 'I', 'Electricity'] """ @@ -1799,14 +1966,17 @@ class ParserElement(object): def split(self, instring, maxsplit=_MAX_INT, includeSeparators=False): """ Generator method to split a string using the given expression as a separator. - May be called with optional C{maxsplit} argument, to limit the number of splits; - and the optional C{includeSeparators} argument (default=C{False}), if the separating + May be called with optional ``maxsplit`` argument, to limit the number of splits; + and the optional ``includeSeparators`` argument (default= ``False``), if the separating matching text should be included in the split results. - - Example:: + + Example:: + punc = oneOf(list(".,;:/-!?")) print(list(punc.split("This, this?, this sentence, is badly punctuated!"))) + prints:: + ['This', ' this', '', ' this sentence', ' is badly punctuated', ''] """ splits = 0 @@ -1820,14 +1990,17 @@ class ParserElement(object): def __add__(self, other ): """ - Implementation of + operator - returns C{L{And}}. Adding strings to a ParserElement - converts them to L{Literal}s by default. - + Implementation of + operator - returns :class:`And`. Adding strings to a ParserElement + converts them to :class:`Literal`s by default. + Example:: + greet = Word(alphas) + "," + Word(alphas) + "!" hello = "Hello, World!" print (hello, "->", greet.parseString(hello)) - Prints:: + + prints:: + Hello, World! -> ['Hello', ',', 'World', '!'] """ if isinstance( other, basestring ): @@ -1840,7 +2013,7 @@ class ParserElement(object): def __radd__(self, other ): """ - Implementation of + operator when left operand is not a C{L{ParserElement}} + Implementation of + operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1852,7 +2025,7 @@ class ParserElement(object): def __sub__(self, other): """ - Implementation of - operator, returns C{L{And}} with error stop + Implementation of - operator, returns :class:`And` with error stop """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1864,7 +2037,7 @@ class ParserElement(object): def __rsub__(self, other ): """ - Implementation of - operator when left operand is not a C{L{ParserElement}} + Implementation of - operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1876,23 +2049,23 @@ class ParserElement(object): def __mul__(self,other): """ - Implementation of * operator, allows use of C{expr * 3} in place of - C{expr + expr + expr}. Expressions may also me multiplied by a 2-integer - tuple, similar to C{{min,max}} multipliers in regular expressions. Tuples - may also include C{None} as in: - - C{expr*(n,None)} or C{expr*(n,)} is equivalent - to C{expr*n + L{ZeroOrMore}(expr)} - (read as "at least n instances of C{expr}") - - C{expr*(None,n)} is equivalent to C{expr*(0,n)} - (read as "0 to n instances of C{expr}") - - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)} - - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)} + Implementation of * operator, allows use of ``expr * 3`` in place of + ``expr + expr + expr``. Expressions may also me multiplied by a 2-integer + tuple, similar to ``{min,max}`` multipliers in regular expressions. Tuples + may also include ``None`` as in: + - ``expr*(n,None)`` or ``expr*(n,)`` is equivalent + to ``expr*n + ZeroOrMore(expr)`` + (read as "at least n instances of ``expr``") + - ``expr*(None,n)`` is equivalent to ``expr*(0,n)`` + (read as "0 to n instances of ``expr``") + - ``expr*(None,None)`` is equivalent to ``ZeroOrMore(expr)`` + - ``expr*(1,None)`` is equivalent to ``OneOrMore(expr)`` - Note that C{expr*(None,n)} does not raise an exception if + Note that ``expr*(None,n)`` does not raise an exception if more than n exprs exist in the input stream; that is, - C{expr*(None,n)} does not enforce a maximum number of expr + ``expr*(None,n)`` does not enforce a maximum number of expr occurrences. If this behavior is desired, then write - C{expr*(None,n) + ~expr} + ``expr*(None,n) + ~expr`` """ if isinstance(other,int): minElements, optElements = other,0 @@ -1947,7 +2120,7 @@ class ParserElement(object): def __or__(self, other ): """ - Implementation of | operator - returns C{L{MatchFirst}} + Implementation of | operator - returns :class:`MatchFirst` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1959,7 +2132,7 @@ class ParserElement(object): def __ror__(self, other ): """ - Implementation of | operator when left operand is not a C{L{ParserElement}} + Implementation of | operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1971,7 +2144,7 @@ class ParserElement(object): def __xor__(self, other ): """ - Implementation of ^ operator - returns C{L{Or}} + Implementation of ^ operator - returns :class:`Or` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1983,7 +2156,7 @@ class ParserElement(object): def __rxor__(self, other ): """ - Implementation of ^ operator when left operand is not a C{L{ParserElement}} + Implementation of ^ operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -1995,7 +2168,7 @@ class ParserElement(object): def __and__(self, other ): """ - Implementation of & operator - returns C{L{Each}} + Implementation of & operator - returns :class:`Each` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2007,7 +2180,7 @@ class ParserElement(object): def __rand__(self, other ): """ - Implementation of & operator when left operand is not a C{L{ParserElement}} + Implementation of & operator when left operand is not a :class:`ParserElement` """ if isinstance( other, basestring ): other = ParserElement._literalStringClass( other ) @@ -2019,23 +2192,24 @@ class ParserElement(object): def __invert__( self ): """ - Implementation of ~ operator - returns C{L{NotAny}} + Implementation of ~ operator - returns :class:`NotAny` """ return NotAny( self ) def __call__(self, name=None): """ - Shortcut for C{L{setResultsName}}, with C{listAllMatches=False}. - - If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be - passed as C{True}. - - If C{name} is omitted, same as calling C{L{copy}}. + Shortcut for :class:`setResultsName`, with ``listAllMatches=False``. + + If ``name`` is given with a trailing ``'*'`` character, then ``listAllMatches`` will be + passed as ``True``. + + If ``name` is omitted, same as calling :class:`copy`. Example:: + # these are equivalent userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno") - userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") + userdata = Word(alphas)("name") + Word(nums+"-")("socsecno") """ if name is not None: return self.setResultsName(name) @@ -2044,7 +2218,7 @@ class ParserElement(object): def suppress( self ): """ - Suppresses the output of this C{ParserElement}; useful to keep punctuation from + Suppresses the output of this :class:`ParserElement`; useful to keep punctuation from cluttering up returned output. """ return Suppress( self ) @@ -2052,7 +2226,7 @@ class ParserElement(object): def leaveWhitespace( self ): """ Disables the skipping of whitespace before matching the characters in the - C{ParserElement}'s defined pattern. This is normally only used internally by + :class:`ParserElement`'s defined pattern. This is normally only used internally by the pyparsing module, but may be needed in some whitespace-sensitive grammars. """ self.skipWhitespace = False @@ -2069,9 +2243,9 @@ class ParserElement(object): def parseWithTabs( self ): """ - Overrides default behavior to expand C{}s to spaces before parsing the input string. - Must be called before C{parseString} when the input grammar contains elements that - match C{} characters. + Overrides default behavior to expand ````s to spaces before parsing the input string. + Must be called before ``parseString`` when the input grammar contains elements that + match ```` characters. """ self.keepTabs = True return self @@ -2081,11 +2255,12 @@ class ParserElement(object): Define expression to be ignored (e.g., comments) while doing pattern matching; may be called repeatedly, to define multiple comment or other ignorable patterns. - + Example:: + patt = OneOrMore(Word(alphas)) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj'] - + patt.ignore(cStyleComment) patt.parseString('ablaj /* comment */ lskjd') # -> ['ablaj', 'lskjd'] """ @@ -2112,19 +2287,21 @@ class ParserElement(object): def setDebug( self, flag=True ): """ Enable display of debugging messages while doing pattern matching. - Set C{flag} to True to enable, False to disable. + Set ``flag`` to True to enable, False to disable. Example:: + wd = Word(alphas).setName("alphaword") integer = Word(nums).setName("numword") term = wd | integer - + # turn on debugging for wd wd.setDebug() OneOrMore(term).parseString("abc 123 xyz 890") - + prints:: + Match alphaword at loc 0(1,1) Matched alphaword -> ['abc'] Match alphaword at loc 3(1,4) @@ -2137,12 +2314,12 @@ class ParserElement(object): Exception raised:Expected alphaword (at char 15), (line:1, col:16) The output shown is that produced by the default debug actions - custom debug actions can be - specified using L{setDebugActions}. Prior to attempting - to match the C{wd} expression, the debugging message C{"Match at loc (,)"} - is shown. Then if the parse succeeds, a C{"Matched"} message is shown, or an C{"Exception raised"} - message is shown. Also note the use of L{setName} to assign a human-readable name to the expression, + specified using :class:`setDebugActions`. Prior to attempting + to match the ``wd`` expression, the debugging message ``"Match at loc (,)"`` + is shown. Then if the parse succeeds, a ``"Matched"`` message is shown, or an ``"Exception raised"`` + message is shown. Also note the use of :class:`setName` to assign a human-readable name to the expression, which makes debugging and exception messages easier to understand - for instance, the default - name created for the C{Word} expression without calling C{setName} is C{"W:(ABCD...)"}. + name created for the :class:`Word` expression without calling ``setName`` is ``"W:(ABCD...)"``. """ if flag: self.setDebugActions( _defaultStartDebugAction, _defaultSuccessDebugAction, _defaultExceptionDebugAction ) @@ -2212,14 +2389,15 @@ class ParserElement(object): def matches(self, testString, parseAll=True): """ - Method for quick testing of a parser against a test string. Good for simple + Method for quick testing of a parser against a test string. Good for simple inline microtests of sub expressions while building up larger parser. - + Parameters: - testString - to test against this expression for a match - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + Example:: + expr = Word(nums) assert expr.matches("100") """ @@ -2228,28 +2406,32 @@ class ParserElement(object): return True except ParseBaseException: return False - - def runTests(self, tests, parseAll=True, comment='#', fullDump=True, printResults=True, failureTests=False): + + def runTests(self, tests, parseAll=True, comment='#', + fullDump=True, printResults=True, failureTests=False, postParse=None): """ Execute the parse expression on a series of test strings, showing each test, the parsed results or where the parse failed. Quick and easy way to run a parse expression against a list of sample strings. - + Parameters: - tests - a list of separate test strings, or a multiline string of test strings - - parseAll - (default=C{True}) - flag to pass to C{L{parseString}} when running tests - - comment - (default=C{'#'}) - expression for indicating embedded comments in the test + - parseAll - (default= ``True``) - flag to pass to :class:`parseString` when running tests + - comment - (default= ``'#'``) - expression for indicating embedded comments in the test string; pass None to disable comment filtering - - fullDump - (default=C{True}) - dump results as list followed by results names in nested outline; + - fullDump - (default= ``True``) - dump results as list followed by results names in nested outline; if False, only dump nested list - - printResults - (default=C{True}) prints test output to stdout - - failureTests - (default=C{False}) indicates if these tests are expected to fail parsing + - printResults - (default= ``True``) prints test output to stdout + - failureTests - (default= ``False``) indicates if these tests are expected to fail parsing + - postParse - (default= ``None``) optional callback for successful parse results; called as + `fn(test_string, parse_results)` and returns a string to be added to the test output Returns: a (success, results) tuple, where success indicates that all tests succeeded - (or failed if C{failureTests} is True), and the results contain a list of lines of each + (or failed if ``failureTests`` is True), and the results contain a list of lines of each test's output - + Example:: + number_expr = pyparsing_common.number.copy() result = number_expr.runTests(''' @@ -2273,7 +2455,9 @@ class ParserElement(object): 3.14.159 ''', failureTests=True) print("Success" if result[0] else "Failed!") + prints:: + # unsigned integer 100 [100] @@ -2291,7 +2475,7 @@ class ParserElement(object): [1e-12] Success - + # stray character 100Z ^ @@ -2313,7 +2497,7 @@ class ParserElement(object): lines, create a test like this:: expr.runTest(r"this is a test\\n of strings that spans \\n 3 lines") - + (Note that this is a raw string literal, you must include the leading 'r'.) """ if isinstance(tests, basestring): @@ -2332,10 +2516,18 @@ class ParserElement(object): out = ['\n'.join(comments), t] comments = [] try: - t = t.replace(r'\n','\n') + # convert newline marks to actual newlines, and strip leading BOM if present + t = t.replace(r'\n','\n').lstrip('\ufeff') result = self.parseString(t, parseAll=parseAll) out.append(result.dump(full=fullDump)) success = success and not failureTests + if postParse is not None: + try: + pp_value = postParse(t, result) + if pp_value is not None: + out.append(str(pp_value)) + except Exception as e: + out.append("{0} failed: {1}: {2}".format(postParse.__name__, type(e).__name__, e)) except ParseBaseException as pe: fatal = "(FATAL)" if isinstance(pe, ParseFatalException) else "" if '\n' in t: @@ -2357,21 +2549,20 @@ class ParserElement(object): print('\n'.join(out)) allResults.append((t, result)) - + return success, allResults - + class Token(ParserElement): - """ - Abstract C{ParserElement} subclass, for defining atomic matching patterns. + """Abstract :class:`ParserElement` subclass, for defining atomic + matching patterns. """ def __init__( self ): super(Token,self).__init__( savelist=False ) class Empty(Token): - """ - An empty token, will always match. + """An empty token, will always match. """ def __init__( self ): super(Empty,self).__init__() @@ -2381,8 +2572,7 @@ class Empty(Token): class NoMatch(Token): - """ - A token that will never match. + """A token that will never match. """ def __init__( self ): super(NoMatch,self).__init__() @@ -2396,18 +2586,18 @@ class NoMatch(Token): class Literal(Token): - """ - Token to exactly match a specified string. - + """Token to exactly match a specified string. + Example:: + Literal('blah').parseString('blah') # -> ['blah'] Literal('blah').parseString('blahfooblah') # -> ['blah'] Literal('blah').parseString('bla') # -> Exception: Expected "blah" - - For case-insensitive matching, use L{CaselessLiteral}. - + + For case-insensitive matching, use :class:`CaselessLiteral`. + For keyword matching (force word break before and after the matched string), - use L{Keyword} or L{CaselessKeyword}. + use :class:`Keyword` or :class:`CaselessKeyword`. """ def __init__( self, matchString ): super(Literal,self).__init__() @@ -2437,21 +2627,29 @@ _L = Literal ParserElement._literalStringClass = Literal class Keyword(Token): - """ - Token to exactly match a specified string as a keyword, that is, it must be - immediately followed by a non-keyword character. Compare with C{L{Literal}}: - - C{Literal("if")} will match the leading C{'if'} in C{'ifAndOnlyIf'}. - - C{Keyword("if")} will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'} - Accepts two optional constructor arguments in addition to the keyword string: - - C{identChars} is a string of characters that would be valid identifier characters, - defaulting to all alphanumerics + "_" and "$" - - C{caseless} allows case-insensitive matching, default is C{False}. - + """Token to exactly match a specified string as a keyword, that is, + it must be immediately followed by a non-keyword character. Compare + with :class:`Literal`: + + - ``Literal("if")`` will match the leading ``'if'`` in + ``'ifAndOnlyIf'``. + - ``Keyword("if")`` will not; it will only match the leading + ``'if'`` in ``'if x=1'``, or ``'if(y==2)'`` + + Accepts two optional constructor arguments in addition to the + keyword string: + + - ``identChars`` is a string of characters that would be valid + identifier characters, defaulting to all alphanumerics + "_" and + "$" + - ``caseless`` allows case-insensitive matching, default is ``False``. + Example:: + Keyword("start").parseString("start") # -> ['start'] Keyword("start").parseString("starting") # -> Exception - For case-insensitive matching, use L{CaselessKeyword}. + For case-insensitive matching, use :class:`CaselessKeyword`. """ DEFAULT_KEYWORD_CHARS = alphanums+"_$" @@ -2502,15 +2700,15 @@ class Keyword(Token): Keyword.DEFAULT_KEYWORD_CHARS = chars class CaselessLiteral(Literal): - """ - Token to match a specified string, ignoring case of letters. + """Token to match a specified string, ignoring case of letters. Note: the matched results will always be in the case of the given match string, NOT the case of the input text. Example:: + OneOrMore(CaselessLiteral("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD', 'CMD'] - - (Contrast with example for L{CaselessKeyword}.) + + (Contrast with example for :class:`CaselessKeyword`.) """ def __init__( self, matchString ): super(CaselessLiteral,self).__init__( matchString.upper() ) @@ -2526,36 +2724,39 @@ class CaselessLiteral(Literal): class CaselessKeyword(Keyword): """ - Caseless version of L{Keyword}. + Caseless version of :class:`Keyword`. Example:: + OneOrMore(CaselessKeyword("CMD")).parseString("cmd CMD Cmd10") # -> ['CMD', 'CMD'] - - (Contrast with example for L{CaselessLiteral}.) + + (Contrast with example for :class:`CaselessLiteral`.) """ def __init__( self, matchString, identChars=None ): super(CaselessKeyword,self).__init__( matchString, identChars, caseless=True ) - def parseImpl( self, instring, loc, doActions=True ): - if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and - (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ): - return loc+self.matchLen, self.match - raise ParseException(instring, loc, self.errmsg, self) - class CloseMatch(Token): - """ - A variation on L{Literal} which matches "close" matches, that is, - strings with at most 'n' mismatching characters. C{CloseMatch} takes parameters: - - C{match_string} - string to be matched - - C{maxMismatches} - (C{default=1}) maximum number of mismatches allowed to count as a match - - The results from a successful parse will contain the matched text from the input string and the following named results: - - C{mismatches} - a list of the positions within the match_string where mismatches were found - - C{original} - the original match_string used to compare against the input string - - If C{mismatches} is an empty list, then the match was an exact match. - + """A variation on :class:`Literal` which matches "close" matches, + that is, strings with at most 'n' mismatching characters. + :class:`CloseMatch` takes parameters: + + - ``match_string`` - string to be matched + - ``maxMismatches`` - (``default=1``) maximum number of + mismatches allowed to count as a match + + The results from a successful parse will contain the matched text + from the input string and the following named results: + + - ``mismatches`` - a list of the positions within the + match_string where mismatches were found + - ``original`` - the original match_string used to compare + against the input string + + If ``mismatches`` is an empty list, then the match was an exact + match. + Example:: + patt = CloseMatch("ATCATCGAATGGA") patt.parseString("ATCATCGAAXGGA") # -> (['ATCATCGAAXGGA'], {'mismatches': [[9]], 'original': ['ATCATCGAATGGA']}) patt.parseString("ATCAXCGAAXGGA") # -> Exception: Expected 'ATCATCGAATGGA' (with up to 1 mismatches) (at char 0), (line:1, col:1) @@ -2604,49 +2805,55 @@ class CloseMatch(Token): class Word(Token): - """ - Token for matching words composed of allowed character sets. - Defined with string containing all allowed initial characters, - an optional string containing allowed body characters (if omitted, + """Token for matching words composed of allowed character sets. + Defined with string containing all allowed initial characters, an + optional string containing allowed body characters (if omitted, defaults to the initial character set), and an optional minimum, - maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. An optional - C{excludeChars} parameter can list characters that might be found in - the input C{bodyChars} string; useful to define a word of all printables - except for one or two characters, for instance. - - L{srange} is useful for defining custom character set strings for defining - C{Word} expressions, using range notation from regular expression character sets. - - A common mistake is to use C{Word} to match a specific literal string, as in - C{Word("Address")}. Remember that C{Word} uses the string argument to define - I{sets} of matchable characters. This expression would match "Add", "AAA", - "dAred", or any other word made up of the characters 'A', 'd', 'r', 'e', and 's'. - To match an exact literal string, use L{Literal} or L{Keyword}. + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. An optional ``excludeChars`` parameter can + list characters that might be found in the input ``bodyChars`` + string; useful to define a word of all printables except for one or + two characters, for instance. + + :class:`srange` is useful for defining custom character set strings + for defining ``Word`` expressions, using range notation from + regular expression character sets. + + A common mistake is to use :class:`Word` to match a specific literal + string, as in ``Word("Address")``. Remember that :class:`Word` + uses the string argument to define *sets* of matchable characters. + This expression would match "Add", "AAA", "dAred", or any other word + made up of the characters 'A', 'd', 'r', 'e', and 's'. To match an + exact literal string, use :class:`Literal` or :class:`Keyword`. pyparsing includes helper strings for building Words: - - L{alphas} - - L{nums} - - L{alphanums} - - L{hexnums} - - L{alphas8bit} (alphabetic characters in ASCII range 128-255 - accented, tilded, umlauted, etc.) - - L{punc8bit} (non-alphabetic characters in ASCII range 128-255 - currency, symbols, superscripts, diacriticals, etc.) - - L{printables} (any non-whitespace character) + + - :class:`alphas` + - :class:`nums` + - :class:`alphanums` + - :class:`hexnums` + - :class:`alphas8bit` (alphabetic characters in ASCII range 128-255 + - accented, tilded, umlauted, etc.) + - :class:`punc8bit` (non-alphabetic characters in ASCII range + 128-255 - currency, symbols, superscripts, diacriticals, etc.) + - :class:`printables` (any non-whitespace character) Example:: + # a word composed of digits integer = Word(nums) # equivalent to Word("0123456789") or Word(srange("0-9")) - + # a word with a leading capital, and zero or more lowercase capital_word = Word(alphas.upper(), alphas.lower()) # hostnames are alphanumeric, with leading alpha, and '-' hostname = Word(alphas, alphanums+'-') - + # roman numeral (not a strict parser, accepts invalid mix of characters) roman = Word("IVXLCDM") - + # any string of non-whitespace characters, except for ',' csv_value = Word(printables, excludeChars=",") """ @@ -2762,22 +2969,38 @@ class Word(Token): return self.strRepr +class Char(Word): + """A short-cut class for defining ``Word(characters, exact=1)``, + when defining a match of any single character in a string of + characters. + """ + def __init__(self, charset): + super(Char, self).__init__(charset, exact=1) + self.reString = "[%s]" % _escapeRegexRangeChars(self.initCharsOrig) + self.re = re.compile( self.reString ) + + class Regex(Token): - r""" - Token for matching strings that match a given regular expression. - Defined with string specifying the regular expression in a form recognized by the inbuilt Python re module. - If the given regex contains named groups (defined using C{(?P...)}), these will be preserved as - named parse results. + r"""Token for matching strings that match a given regular + expression. Defined with string specifying the regular expression in + a form recognized by the stdlib Python `re module `_. + If the given regex contains named groups (defined using ``(?P...)``), + these will be preserved as named parse results. Example:: + realnum = Regex(r"[+-]?\d+\.\d*") date = Regex(r'(?P\d{4})-(?P\d\d?)-(?P\d\d?)') - # ref: http://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression - roman = Regex(r"M{0,4}(CM|CD|D?C{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") + # ref: https://stackoverflow.com/questions/267399/how-do-you-match-only-valid-roman-numerals-with-a-regular-expression + roman = Regex(r"M{0,4}(CM|CD|D?{0,3})(XC|XL|L?X{0,3})(IX|IV|V?I{0,3})") """ compiledREtype = type(re.compile("[A-Z]")) def __init__( self, pattern, flags=0, asGroupList=False, asMatch=False): - """The parameters C{pattern} and C{flags} are passed to the C{re.compile()} function as-is. See the Python C{re} module for an explanation of the acceptable patterns and flags.""" + """The parameters ``pattern`` and ``flags`` are passed + to the ``re.compile()`` function as-is. See the Python + `re module `_ module for an + explanation of the acceptable patterns and flags. + """ super(Regex,self).__init__() if isinstance(pattern, basestring): @@ -2801,7 +3024,7 @@ class Regex(Token): self.pattern = \ self.reString = str(pattern) self.flags = flags - + else: raise ValueError("Regex may only be constructed with a string or a compiled RE object") @@ -2818,16 +3041,16 @@ class Regex(Token): raise ParseException(instring, loc, self.errmsg, self) loc = result.end() - d = result.groupdict() if self.asMatch: ret = result elif self.asGroupList: ret = result.groups() else: ret = ParseResults(result.group()) + d = result.groupdict() if d: - for k in d: - ret[k] = d[k] + for k, v in d.items(): + ret[k] = v return loc,ret def __str__( self ): @@ -2844,17 +3067,23 @@ class Regex(Token): def sub(self, repl): """ Return Regex with an attached parse action to transform the parsed - result as if called using C{re.sub(expr, repl, string)}. + result as if called using `re.sub(expr, repl, string) `_. + + Example:: + + make_html = Regex(r"(\w+):(.*?):").sub(r"<\1>\2") + print(make_html.transformString("h1:main title:")) + # prints "

main title

" """ if self.asGroupList: - warnings.warn("cannot use sub() with Regex(asGroupList=True)", + warnings.warn("cannot use sub() with Regex(asGroupList=True)", SyntaxWarning, stacklevel=2) raise SyntaxError() if self.asMatch and callable(repl): - warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", + warnings.warn("cannot use sub() with a callable with Regex(asMatch=True)", SyntaxWarning, stacklevel=2) - raise SyntaxError() + raise SyntaxError() if self.asMatch: def pa(tokens): @@ -2867,24 +3096,38 @@ class Regex(Token): class QuotedString(Token): r""" Token for matching strings that are delimited by quoting characters. - + Defined with the following parameters: - - quoteChar - string of one or more characters defining the quote delimiting string - - escChar - character to escape quotes, typically backslash (default=C{None}) - - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=C{None}) - - multiline - boolean indicating whether quotes can span multiple lines (default=C{False}) - - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True}) - - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar) - - convertWhitespaceEscapes - convert escaped whitespace (C{'\t'}, C{'\n'}, etc.) to actual whitespace (default=C{True}) + + - quoteChar - string of one or more characters defining the + quote delimiting string + - escChar - character to escape quotes, typically backslash + (default= ``None`` ) + - escQuote - special quote sequence to escape an embedded quote + string (such as SQL's ``""`` to escape an embedded ``"``) + (default= ``None`` ) + - multiline - boolean indicating whether quotes can span + multiple lines (default= ``False`` ) + - unquoteResults - boolean indicating whether the matched text + should be unquoted (default= ``True`` ) + - endQuoteChar - string of one or more characters defining the + end of the quote delimited string (default= ``None`` => same as + quoteChar) + - convertWhitespaceEscapes - convert escaped whitespace + (``'\t'``, ``'\n'``, etc.) to actual whitespace + (default= ``True`` ) Example:: + qs = QuotedString('"') print(qs.searchString('lsjdf "This is the quote" sldjf')) complex_qs = QuotedString('{{', endQuoteChar='}}') print(complex_qs.searchString('lsjdf {{This is the "quote"}} sldjf')) sql_qs = QuotedString('"', escQuote='""') print(sql_qs.searchString('lsjdf "This is the quote with ""embedded"" quotes" sldjf')) + prints:: + [['This is the quote']] [['This is the "quote"']] [['This is the quote with "embedded" quotes']] @@ -3002,19 +3245,23 @@ class QuotedString(Token): class CharsNotIn(Token): - """ - Token for matching words composed of characters I{not} in a given set (will - include whitespace in matched characters if not listed in the provided exclusion set - see example). - Defined with string containing all disallowed characters, and an optional - minimum, maximum, and/or exact length. The default value for C{min} is 1 (a - minimum value < 1 is not valid); the default values for C{max} and C{exact} - are 0, meaning no maximum or exact length restriction. + """Token for matching words composed of characters *not* in a given + set (will include whitespace in matched characters if not listed in + the provided exclusion set - see example). Defined with string + containing all disallowed characters, and an optional minimum, + maximum, and/or exact length. The default value for ``min`` is + 1 (a minimum value < 1 is not valid); the default values for + ``max`` and ``exact`` are 0, meaning no maximum or exact + length restriction. Example:: + # define a comma-separated-value as anything that is not a ',' csv_value = CharsNotIn(',') print(delimitedList(csv_value).parseString("dkls,lsdkjf,s12 34,@!#,213")) + prints:: + ['dkls', 'lsdkjf', 's12 34', '@!#', '213'] """ def __init__( self, notChars, min=1, max=0, exact=0 ): @@ -3023,7 +3270,9 @@ class CharsNotIn(Token): self.notChars = notChars if min < 1: - raise ValueError("cannot specify a minimum length < 1; use Optional(CharsNotIn()) if zero-length char group is permitted") + raise ValueError( + "cannot specify a minimum length < 1; use " + + "Optional(CharsNotIn()) if zero-length char group is permitted") self.minLen = min @@ -3073,19 +3322,38 @@ class CharsNotIn(Token): return self.strRepr class White(Token): - """ - Special matching class for matching whitespace. Normally, whitespace is ignored - by pyparsing grammars. This class is included when some whitespace structures - are significant. Define with a string containing the whitespace characters to be - matched; default is C{" \\t\\r\\n"}. Also takes optional C{min}, C{max}, and C{exact} arguments, - as defined for the C{L{Word}} class. + """Special matching class for matching whitespace. Normally, + whitespace is ignored by pyparsing grammars. This class is included + when some whitespace structures are significant. Define with + a string containing the whitespace characters to be matched; default + is ``" \\t\\r\\n"``. Also takes optional ``min``, + ``max``, and ``exact`` arguments, as defined for the + :class:`Word` class. """ whiteStrs = { - " " : "", - "\t": "", - "\n": "", - "\r": "", - "\f": "", + ' ' : '', + '\t': '', + '\n': '', + '\r': '', + '\f': '', + 'u\00A0': '', + 'u\1680': '', + 'u\180E': '', + 'u\2000': '', + 'u\2001': '', + 'u\2002': '', + 'u\2003': '', + 'u\2004': '', + 'u\2005': '', + 'u\2006': '', + 'u\2007': '', + 'u\2008': '', + 'u\2009': '', + 'u\200A': '', + 'u\200B': '', + 'u\202F': '', + 'u\205F': '', + 'u\3000': '', } def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0): super(White,self).__init__() @@ -3131,8 +3399,8 @@ class _PositionToken(Token): self.mayIndexError = False class GoToColumn(_PositionToken): - """ - Token to advance to a specific column of input text; useful for tabular report scraping. + """Token to advance to a specific column of input text; useful for + tabular report scraping. """ def __init__( self, colno ): super(GoToColumn,self).__init__() @@ -3157,11 +3425,11 @@ class GoToColumn(_PositionToken): class LineStart(_PositionToken): - """ - Matches if current position is at the beginning of a line within the parse string - + """Matches if current position is at the beginning of a line within + the parse string + Example:: - + test = '''\ AAA this line AAA and this line @@ -3171,10 +3439,11 @@ class LineStart(_PositionToken): for t in (LineStart() + 'AAA' + restOfLine).searchString(test): print(t) - - Prints:: + + prints:: + ['AAA', ' this line'] - ['AAA', ' and this line'] + ['AAA', ' and this line'] """ def __init__( self ): @@ -3187,8 +3456,8 @@ class LineStart(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class LineEnd(_PositionToken): - """ - Matches if current position is at the end of a line within the parse string + """Matches if current position is at the end of a line within the + parse string """ def __init__( self ): super(LineEnd,self).__init__() @@ -3207,8 +3476,8 @@ class LineEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class StringStart(_PositionToken): - """ - Matches if current position is at the beginning of the parse string + """Matches if current position is at the beginning of the parse + string """ def __init__( self ): super(StringStart,self).__init__() @@ -3222,8 +3491,7 @@ class StringStart(_PositionToken): return loc, [] class StringEnd(_PositionToken): - """ - Matches if current position is at the end of the parse string + """Matches if current position is at the end of the parse string """ def __init__( self ): super(StringEnd,self).__init__() @@ -3240,12 +3508,13 @@ class StringEnd(_PositionToken): raise ParseException(instring, loc, self.errmsg, self) class WordStart(_PositionToken): - """ - Matches if the current position is at the beginning of a Word, and - is not preceded by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordStart(alphanums)}. C{WordStart} will also match at the beginning of - the string being parsed, or at the beginning of a line. + """Matches if the current position is at the beginning of a Word, + and is not preceded by any character in a given set of + ``wordChars`` (default= ``printables``). To emulate the + ``\b`` behavior of regular expressions, use + ``WordStart(alphanums)``. ``WordStart`` will also match at + the beginning of the string being parsed, or at the beginning of + a line. """ def __init__(self, wordChars = printables): super(WordStart,self).__init__() @@ -3260,12 +3529,12 @@ class WordStart(_PositionToken): return loc, [] class WordEnd(_PositionToken): - """ - Matches if the current position is at the end of a Word, and - is not followed by any character in a given set of C{wordChars} - (default=C{printables}). To emulate the C{\b} behavior of regular expressions, - use C{WordEnd(alphanums)}. C{WordEnd} will also match at the end of - the string being parsed, or at the end of a line. + """Matches if the current position is at the end of a Word, and is + not followed by any character in a given set of ``wordChars`` + (default= ``printables``). To emulate the ``\b`` behavior of + regular expressions, use ``WordEnd(alphanums)``. ``WordEnd`` + will also match at the end of the string being parsed, or at the end + of a line. """ def __init__(self, wordChars = printables): super(WordEnd,self).__init__() @@ -3283,8 +3552,8 @@ class WordEnd(_PositionToken): class ParseExpression(ParserElement): - """ - Abstract subclass of ParserElement, for combining and post-processing parsed tokens. + """Abstract subclass of ParserElement, for combining and + post-processing parsed tokens. """ def __init__( self, exprs, savelist = False ): super(ParseExpression,self).__init__(savelist) @@ -3315,7 +3584,7 @@ class ParseExpression(ParserElement): return self def leaveWhitespace( self ): - """Extends C{leaveWhitespace} defined in base class, and also invokes C{leaveWhitespace} on + """Extends ``leaveWhitespace`` defined in base class, and also invokes ``leaveWhitespace`` on all contained expressions.""" self.skipWhitespace = False self.exprs = [ e.copy() for e in self.exprs ] @@ -3376,7 +3645,7 @@ class ParseExpression(ParserElement): self.mayIndexError |= other.mayIndexError self.errmsg = "Expected " + _ustr(self) - + return self def setResultsName( self, name, listAllMatches=False ): @@ -3388,7 +3657,7 @@ class ParseExpression(ParserElement): for e in self.exprs: e.validate(tmp) self.checkRecursion( [] ) - + def copy(self): ret = super(ParseExpression,self).copy() ret.exprs = [e.copy() for e in self.exprs] @@ -3396,12 +3665,14 @@ class ParseExpression(ParserElement): class And(ParseExpression): """ - Requires all given C{ParseExpression}s to be found in the given order. + Requires all given :class:`ParseExpression` s to be found in the given order. Expressions may be separated by whitespace. - May be constructed using the C{'+'} operator. - May also be constructed using the C{'-'} operator, which will suppress backtracking. + May be constructed using the ``'+'`` operator. + May also be constructed using the ``'-'`` operator, which will + suppress backtracking. Example:: + integer = Word(nums) name_expr = OneOrMore(Word(alphas)) @@ -3423,6 +3694,11 @@ class And(ParseExpression): self.skipWhitespace = self.exprs[0].skipWhitespace self.callPreparse = True + def streamline(self): + super(And, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): # pass False as last arg to _parse for first element, since we already # pre-parsed the string as part of our And pre-parsing @@ -3471,17 +3747,20 @@ class And(ParseExpression): class Or(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the expression that matches the longest string will be used. - May be constructed using the C{'^'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the expression that matches the longest + string will be used. May be constructed using the ``'^'`` + operator. Example:: + # construct Or using '^' operator - + number = Word(nums) ^ Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) + prints:: + [['123'], ['3.1416'], ['789']] """ def __init__( self, exprs, savelist = False ): @@ -3491,6 +3770,11 @@ class Or(ParseExpression): else: self.mayReturnEmpty = True + def streamline(self): + super(Or, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3550,14 +3834,14 @@ class Or(ParseExpression): class MatchFirst(ParseExpression): - """ - Requires that at least one C{ParseExpression} is found. - If two expressions match, the first one listed is the one that will match. - May be constructed using the C{'|'} operator. + """Requires that at least one :class:`ParseExpression` is found. If + two expressions match, the first one listed is the one that will + match. May be constructed using the ``'|'`` operator. Example:: + # construct MatchFirst using '|' operator - + # watch the order of expressions to match number = Word(nums) | Combine(Word(nums) + '.' + Word(nums)) print(number.searchString("123 3.1416 789")) # Fail! -> [['123'], ['3'], ['1416'], ['789']] @@ -3570,9 +3854,15 @@ class MatchFirst(ParseExpression): super(MatchFirst,self).__init__(exprs, savelist) if self.exprs: self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs) + # self.saveAsList = any(e.saveAsList for e in self.exprs) else: self.mayReturnEmpty = True + def streamline(self): + super(MatchFirst, self).streamline() + self.saveAsList = any(e.saveAsList for e in self.exprs) + return self + def parseImpl( self, instring, loc, doActions=True ): maxExcLoc = -1 maxException = None @@ -3618,12 +3908,13 @@ class MatchFirst(ParseExpression): class Each(ParseExpression): - """ - Requires all given C{ParseExpression}s to be found, but in any order. - Expressions may be separated by whitespace. - May be constructed using the C{'&'} operator. + """Requires all given :class:`ParseExpression` s to be found, but in + any order. Expressions may be separated by whitespace. + + May be constructed using the ``'&'`` operator. Example:: + color = oneOf("RED ORANGE YELLOW GREEN BLUE PURPLE BLACK WHITE BROWN") shape_type = oneOf("SQUARE CIRCLE TRIANGLE STAR HEXAGON OCTAGON") integer = Word(nums) @@ -3632,7 +3923,7 @@ class Each(ParseExpression): color_attr = "color:" + color("color") size_attr = "size:" + integer("size") - # use Each (using operator '&') to accept attributes in any order + # use Each (using operator '&') to accept attributes in any order # (shape and posn are required, color and size are optional) shape_spec = shape_attr & posn_attr & Optional(color_attr) & Optional(size_attr) @@ -3642,7 +3933,9 @@ class Each(ParseExpression): color:GREEN size:20 shape:TRIANGLE posn:20,40 ''' ) + prints:: + shape: SQUARE color: BLACK posn: 100, 120 ['shape:', 'SQUARE', 'color:', 'BLACK', 'posn:', ['100', ',', '120']] - color: BLACK @@ -3676,6 +3969,12 @@ class Each(ParseExpression): self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) self.skipWhitespace = True self.initExprGroups = True + self.saveAsList = True + + def streamline(self): + super(Each, self).streamline() + self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs) + return self def parseImpl( self, instring, loc, doActions=True ): if self.initExprGroups: @@ -3742,8 +4041,8 @@ class Each(ParseExpression): class ParseElementEnhance(ParserElement): - """ - Abstract subclass of C{ParserElement}, for combining and post-processing parsed tokens. + """Abstract subclass of :class:`ParserElement`, for combining and + post-processing parsed tokens. """ def __init__( self, expr, savelist=False ): super(ParseElementEnhance,self).__init__(savelist) @@ -3819,20 +4118,25 @@ class ParseElementEnhance(ParserElement): class FollowedBy(ParseElementEnhance): - """ - Lookahead matching of the given parse expression. C{FollowedBy} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression matches at the current - position. C{FollowedBy} always returns a null token list. + """Lookahead matching of the given parse expression. + ``FollowedBy`` does *not* advance the parsing position within + the input string, it only verifies that the specified parse + expression matches at the current position. ``FollowedBy`` + always returns a null token list. If any results names are defined + in the lookahead expression, those *will* be returned for access by + name. Example:: + # use FollowedBy to match a label only if it is followed by a ':' data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + OneOrMore(attr_expr).parseString("shape: SQUARE color: BLACK posn: upper left").pprint() + prints:: + [['shape', 'SQUARE'], ['color', 'BLACK'], ['posn', 'upper left']] """ def __init__( self, expr ): @@ -3840,20 +4144,108 @@ class FollowedBy(ParseElementEnhance): self.mayReturnEmpty = True def parseImpl( self, instring, loc, doActions=True ): - self.expr.tryParse( instring, loc ) - return loc, [] + _, ret = self.expr._parse(instring, loc, doActions=doActions) + del ret[:] + return loc, ret + + +class PrecededBy(ParseElementEnhance): + """Lookbehind matching of the given parse expression. + ``PrecededBy`` does not advance the parsing position within the + input string, it only verifies that the specified parse expression + matches prior to the current position. ``PrecededBy`` always + returns a null token list, but if a results name is defined on the + given expression, it is returned. + + Parameters: + + - expr - expression that must match prior to the current parse + location + - retreat - (default= ``None``) - (int) maximum number of characters + to lookbehind prior to the current parse location + + If the lookbehind expression is a string, Literal, Keyword, or + a Word or CharsNotIn with a specified exact or maximum length, then + the retreat parameter is not required. Otherwise, retreat must be + specified to give a maximum number of characters to look back from + the current parse position for a lookbehind match. + + Example:: + + # VB-style variable names with type prefixes + int_var = PrecededBy("#") + pyparsing_common.identifier + str_var = PrecededBy("$") + pyparsing_common.identifier + + """ + def __init__(self, expr, retreat=None): + super(PrecededBy, self).__init__(expr) + self.expr = self.expr().leaveWhitespace() + self.mayReturnEmpty = True + self.mayIndexError = False + self.exact = False + if isinstance(expr, str): + retreat = len(expr) + self.exact = True + elif isinstance(expr, (Literal, Keyword)): + retreat = expr.matchLen + self.exact = True + elif isinstance(expr, (Word, CharsNotIn)) and expr.maxLen != _MAX_INT: + retreat = expr.maxLen + self.exact = True + elif isinstance(expr, _PositionToken): + retreat = 0 + self.exact = True + self.retreat = retreat + self.errmsg = "not preceded by " + str(expr) + self.skipWhitespace = False + + def parseImpl(self, instring, loc=0, doActions=True): + if self.exact: + if loc < self.retreat: + raise ParseException(instring, loc, self.errmsg) + start = loc - self.retreat + _, ret = self.expr._parse(instring, start) + else: + # retreat specified a maximum lookbehind window, iterate + test_expr = self.expr + StringEnd() + instring_slice = instring[:loc] + last_expr = ParseException(instring, loc, self.errmsg) + for offset in range(1, min(loc, self.retreat+1)): + try: + _, ret = test_expr._parse(instring_slice, loc-offset) + except ParseBaseException as pbe: + last_expr = pbe + else: + break + else: + raise last_expr + # return empty list of tokens, but preserve any defined results names + del ret[:] + return loc, ret class NotAny(ParseElementEnhance): - """ - Lookahead to disallow matching with the given parse expression. C{NotAny} - does I{not} advance the parsing position within the input string, it only - verifies that the specified parse expression does I{not} match at the current - position. Also, C{NotAny} does I{not} skip over leading whitespace. C{NotAny} - always returns a null token list. May be constructed using the '~' operator. + """Lookahead to disallow matching with the given parse expression. + ``NotAny`` does *not* advance the parsing position within the + input string, it only verifies that the specified parse expression + does *not* match at the current position. Also, ``NotAny`` does + *not* skip over leading whitespace. ``NotAny`` always returns + a null token list. May be constructed using the '~' operator. Example:: - + + AND, OR, NOT = map(CaselessKeyword, "AND OR NOT".split()) + + # take care not to mistake keywords for identifiers + ident = ~(AND | OR | NOT) + Word(alphas) + boolean_term = Optional(NOT) + ident + + # very crude boolean expression - to support parenthesis groups and + # operation hierarchy, use infixNotation + boolean_expr = boolean_term + ZeroOrMore((AND | OR) + boolean_term) + + # integers that are followed by "." are actually floats + integer = Word(nums) + ~Char(".") """ def __init__( self, expr ): super(NotAny,self).__init__(expr) @@ -3891,7 +4283,7 @@ class _MultipleMatch(ParseElementEnhance): check_ender = self.not_ender is not None if check_ender: try_not_ender = self.not_ender.tryParse - + # must be at least one (but first see if we are the stopOn sentinel; # if so, fail) if check_ender: @@ -3913,18 +4305,18 @@ class _MultipleMatch(ParseElementEnhance): pass return loc, tokens - + class OneOrMore(_MultipleMatch): - """ - Repetition of one or more of the given expression. - + """Repetition of one or more of the given expression. + Parameters: - expr - expression that must match one or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) @@ -3935,7 +4327,7 @@ class OneOrMore(_MultipleMatch): # use stopOn attribute for OneOrMore to avoid reading label string as part of the data attr_expr = Group(label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) OneOrMore(attr_expr).parseString(text).pprint() # Better -> [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'BLACK']] - + # could also be written as (attr_expr * (1,)).parseString(text).pprint() """ @@ -3950,21 +4342,20 @@ class OneOrMore(_MultipleMatch): return self.strRepr class ZeroOrMore(_MultipleMatch): - """ - Optional repetition of zero or more of the given expression. - + """Optional repetition of zero or more of the given expression. + Parameters: - expr - expression that must match zero or more times - - stopOn - (default=C{None}) - expression for a terminating sentinel - (only required if the sentinel would ordinarily match the repetition - expression) + - stopOn - (default= ``None``) - expression for a terminating sentinel + (only required if the sentinel would ordinarily match the repetition + expression) - Example: similar to L{OneOrMore} + Example: similar to :class:`OneOrMore` """ def __init__( self, expr, stopOn=None): super(ZeroOrMore,self).__init__(expr, stopOn=stopOn) self.mayReturnEmpty = True - + def parseImpl( self, instring, loc, doActions=True ): try: return super(ZeroOrMore, self).parseImpl(instring, loc, doActions) @@ -3989,27 +4380,29 @@ class _NullToken(object): _optionalNotMatched = _NullToken() class Optional(ParseElementEnhance): - """ - Optional matching of the given expression. + """Optional matching of the given expression. Parameters: - expr - expression that must match zero or more times - default (optional) - value to be returned if the optional expression is not found. Example:: + # US postal code can be a 5-digit zip, plus optional 4-digit qualifier zip = Combine(Word(nums, exact=5) + Optional('-' + Word(nums, exact=4))) zip.runTests(''' # traditional ZIP code 12345 - + # ZIP+4 form 12101-0001 - + # invalid ZIP 98765- ''') + prints:: + # traditional ZIP code 12345 ['12345'] @@ -4053,20 +4446,21 @@ class Optional(ParseElementEnhance): return self.strRepr class SkipTo(ParseElementEnhance): - """ - Token for skipping over all undefined text until the matched expression is found. + """Token for skipping over all undefined text until the matched + expression is found. Parameters: - expr - target expression marking the end of the data to be skipped - - include - (default=C{False}) if True, the target expression is also parsed + - include - (default= ``False``) if True, the target expression is also parsed (the skipped text and target expression are returned as a 2-element list). - - ignore - (default=C{None}) used to define grammars (typically quoted strings and + - ignore - (default= ``None``) used to define grammars (typically quoted strings and comments) that might contain false matches to the target expression - - failOn - (default=C{None}) define expressions that are not allowed to be - included in the skipped test; if found before the target expression is found, + - failOn - (default= ``None``) define expressions that are not allowed to be + included in the skipped test; if found before the target expression is found, the SkipTo is not a match Example:: + report = ''' Outstanding Issues Report - 1 Jan 2000 @@ -4083,14 +4477,16 @@ class SkipTo(ParseElementEnhance): # - parse action will call token.strip() for each matched token, i.e., the description body string_data = SkipTo(SEP, ignore=quotedString) string_data.setParseAction(tokenMap(str.strip)) - ticket_expr = (integer("issue_num") + SEP - + string_data("sev") + SEP - + string_data("desc") + SEP + ticket_expr = (integer("issue_num") + SEP + + string_data("sev") + SEP + + string_data("desc") + SEP + integer("days_open")) - + for tkt in ticket_expr.searchString(report): print tkt.dump() + prints:: + ['101', 'Critical', 'Intermittent system crash', '6'] - days_open: 6 - desc: Intermittent system crash @@ -4127,14 +4523,14 @@ class SkipTo(ParseElementEnhance): expr_parse = self.expr._parse self_failOn_canParseNext = self.failOn.canParseNext if self.failOn is not None else None self_ignoreExpr_tryParse = self.ignoreExpr.tryParse if self.ignoreExpr is not None else None - + tmploc = loc while tmploc <= instrlen: if self_failOn_canParseNext is not None: # break if failOn expression matches if self_failOn_canParseNext(instring, tmploc): break - + if self_ignoreExpr_tryParse is not None: # advance past ignore expressions while 1: @@ -4142,7 +4538,7 @@ class SkipTo(ParseElementEnhance): tmploc = self_ignoreExpr_tryParse(instring, tmploc) except ParseBaseException: break - + try: expr_parse(instring, tmploc, doActions=False, callPreParse=False) except (ParseException, IndexError): @@ -4160,7 +4556,7 @@ class SkipTo(ParseElementEnhance): loc = tmploc skiptext = instring[startloc:loc] skipresult = ParseResults(skiptext) - + if self.includeMatch: loc, mat = expr_parse(instring,loc,doActions,callPreParse=False) skipresult += mat @@ -4168,23 +4564,31 @@ class SkipTo(ParseElementEnhance): return loc, skipresult class Forward(ParseElementEnhance): - """ - Forward declaration of an expression to be defined later - + """Forward declaration of an expression to be defined later - used for recursive grammars, such as algebraic infix notation. - When the expression is known, it is assigned to the C{Forward} variable using the '<<' operator. + When the expression is known, it is assigned to the ``Forward`` + variable using the '<<' operator. + + Note: take care when assigning to ``Forward`` not to overlook + precedence of operators. - Note: take care when assigning to C{Forward} not to overlook precedence of operators. Specifically, '|' has a lower precedence than '<<', so that:: + fwdExpr << a | b | c + will actually be evaluated as:: + (fwdExpr << a) | b | c + thereby leaving b and c out as parseable alternatives. It is recommended that you - explicitly group the values inserted into the C{Forward}:: + explicitly group the values inserted into the ``Forward``:: + fwdExpr << (a | b | c) + Converting to use the '<<=' operator instead will avoid this problem. - See L{ParseResults.pprint} for an example of a recursive parser created using - C{Forward}. + See :class:`ParseResults.pprint` for an example of a recursive + parser created using ``Forward``. """ def __init__( self, other=None ): super(Forward,self).__init__( other, savelist=False ) @@ -4201,10 +4605,10 @@ class Forward(ParseElementEnhance): self.saveAsList = self.expr.saveAsList self.ignoreExprs.extend(self.expr.ignoreExprs) return self - + def __ilshift__(self, other): return self << other - + def leaveWhitespace( self ): self.skipWhitespace = False return self @@ -4254,19 +4658,20 @@ class _ForwardNoRecurse(Forward): class TokenConverter(ParseElementEnhance): """ - Abstract subclass of C{ParseExpression}, for converting parsed results. + Abstract subclass of :class:`ParseExpression`, for converting parsed results. """ def __init__( self, expr, savelist=False ): super(TokenConverter,self).__init__( expr )#, savelist ) self.saveAsList = False class Combine(TokenConverter): - """ - Converter to concatenate all matching tokens to a single string. - By default, the matching patterns must also be contiguous in the input string; - this can be disabled by specifying C{'adjacent=False'} in the constructor. + """Converter to concatenate all matching tokens to a single string. + By default, the matching patterns must also be contiguous in the + input string; this can be disabled by specifying + ``'adjacent=False'`` in the constructor. Example:: + real = Word(nums) + '.' + Word(nums) print(real.parseString('3.1416')) # -> ['3', '.', '1416'] # will also erroneously match the following @@ -4305,10 +4710,11 @@ class Combine(TokenConverter): return retToks class Group(TokenConverter): - """ - Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions. + """Converter to return the matched tokens as a list - useful for + returning tokens of :class:`ZeroOrMore` and :class:`OneOrMore` expressions. Example:: + ident = Word(alphas) num = Word(nums) term = ident | num @@ -4320,38 +4726,40 @@ class Group(TokenConverter): """ def __init__( self, expr ): super(Group,self).__init__( expr ) - self.saveAsList = True + self.saveAsList = expr.saveAsList def postParse( self, instring, loc, tokenlist ): return [ tokenlist ] class Dict(TokenConverter): - """ - Converter to return a repetitive expression as a list, but also as a dictionary. - Each element can also be referenced using the first token in the expression as its key. - Useful for tabular report scraping when the first column can be used as a item key. + """Converter to return a repetitive expression as a list, but also + as a dictionary. Each element can also be referenced using the first + token in the expression as its key. Useful for tabular report + scraping when the first column can be used as a item key. Example:: + data_word = Word(alphas) label = data_word + FollowedBy(':') attr_expr = Group(label + Suppress(':') + OneOrMore(data_word).setParseAction(' '.join)) text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) - + # print attributes as plain groups print(OneOrMore(attr_expr).parseString(text).dump()) - + # instead of OneOrMore(expr), parse using Dict(OneOrMore(Group(expr))) - Dict will auto-assign names result = Dict(OneOrMore(Group(attr_expr))).parseString(text) print(result.dump()) - - # access named fields as dict entries, or output as dict - print(result['shape']) - print(result.asDict()) - prints:: - ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] + # access named fields as dict entries, or output as dict + print(result['shape']) + print(result.asDict()) + + prints:: + + ['shape', 'SQUARE', 'posn', 'upper left', 'color', 'light blue', 'texture', 'burlap'] [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4359,7 +4767,8 @@ class Dict(TokenConverter): - texture: burlap SQUARE {'color': 'light blue', 'posn': 'upper left', 'texture': 'burlap', 'shape': 'SQUARE'} - See more examples at L{ParseResults} of accessing fields by results name. + + See more examples at :class:`ParseResults` of accessing fields by results name. """ def __init__( self, expr ): super(Dict,self).__init__( expr ) @@ -4391,10 +4800,10 @@ class Dict(TokenConverter): class Suppress(TokenConverter): - """ - Converter for ignoring the results of a parsed expression. + """Converter for ignoring the results of a parsed expression. Example:: + source = "a, b, c,d" wd = Word(alphas) wd_list1 = wd + ZeroOrMore(',' + wd) @@ -4404,10 +4813,13 @@ class Suppress(TokenConverter): # way afterward - use Suppress to keep them out of the parsed output wd_list2 = wd + ZeroOrMore(Suppress(',') + wd) print(wd_list2.parseString(source)) + prints:: + ['a', ',', 'b', ',', 'c', ',', 'd'] ['a', 'b', 'c', 'd'] - (See also L{delimitedList}.) + + (See also :class:`delimitedList`.) """ def postParse( self, instring, loc, tokenlist ): return [] @@ -4417,8 +4829,7 @@ class Suppress(TokenConverter): class OnlyOnce(object): - """ - Wrapper for parse actions, to ensure they are only called once. + """Wrapper for parse actions, to ensure they are only called once. """ def __init__(self, methodCall): self.callable = _trim_arity(methodCall) @@ -4433,13 +4844,15 @@ class OnlyOnce(object): self.called = False def traceParseAction(f): - """ - Decorator for debugging parse actions. - - When the parse action is called, this decorator will print C{">> entering I{method-name}(line:I{current_source_line}, I{parse_location}, I{matched_tokens})".} - When the parse action completes, the decorator will print C{"<<"} followed by the returned value, or any exception that the parse action raised. + """Decorator for debugging parse actions. + + When the parse action is called, this decorator will print + ``">> entering method-name(line:, , )"``. + When the parse action completes, the decorator will print + ``"<<"`` followed by the returned value, or any exception that the parse action raised. Example:: + wd = Word(alphas) @traceParseAction @@ -4448,7 +4861,9 @@ def traceParseAction(f): wds = OneOrMore(wd).setParseAction(remove_duplicate_chars) print(wds.parseString("slkdjs sld sldd sdlf sdljf")) + prints:: + >>entering remove_duplicate_chars(line: 'slkdjs sld sldd sdlf sdljf', 0, (['slkdjs', 'sld', 'sldd', 'sdlf', 'sdljf'], {})) < ['aa', 'bb', 'cc'] delimitedList(Word(hexnums), delim=':', combine=True).parseString("AA:BB:CC:DD:EE") # -> ['AA:BB:CC:DD:EE'] """ @@ -4496,16 +4913,21 @@ def delimitedList( expr, delim=",", combine=False ): return ( expr + ZeroOrMore( Suppress( delim ) + expr ) ).setName(dlName) def countedArray( expr, intExpr=None ): - """ - Helper to define a counted list of expressions. + """Helper to define a counted list of expressions. + This helper defines a pattern of the form:: + integer expr expr expr... + where the leading integer tells how many expr expressions follow. - The matched tokens returns the array of expr tokens as a list - the leading count token is suppressed. - - If C{intExpr} is specified, it should be a pyparsing expression that produces an integer value. + The matched tokens returns the array of expr tokens as a list - the + leading count token is suppressed. + + If ``intExpr`` is specified, it should be a pyparsing expression + that produces an integer value. Example:: + countedArray(Word(alphas)).parseString('2 ab cd ef') # -> ['ab', 'cd'] # in this parser, the leading integer value is given in binary, @@ -4536,17 +4958,19 @@ def _flatten(L): return ret def matchPreviousLiteral(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousLiteral(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches a - previous literal, will also match the leading C{"1:1"} in C{"1:10"}. - If this is not desired, use C{matchPreviousExpr}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches a previous literal, will also match the leading + ``"1:1"`` in ``"1:10"``. If this is not desired, use + :class:`matchPreviousExpr`. Do *not* use with packrat parsing + enabled. """ rep = Forward() def copyTokenToRepeater(s,l,t): @@ -4564,18 +4988,19 @@ def matchPreviousLiteral(expr): return rep def matchPreviousExpr(expr): - """ - Helper to define an expression that is indirectly defined from - the tokens matched in a previous expression, that is, it looks - for a 'repeat' of a previous expression. For example:: + """Helper to define an expression that is indirectly defined from + the tokens matched in a previous expression, that is, it looks for + a 'repeat' of a previous expression. For example:: + first = Word(nums) second = matchPreviousExpr(first) matchExpr = first + ":" + second - will match C{"1:1"}, but not C{"1:2"}. Because this matches by - expressions, will I{not} match the leading C{"1:1"} in C{"1:10"}; - the expressions are evaluated first, and then compared, so - C{"1"} is compared with C{"10"}. - Do I{not} use with packrat parsing enabled. + + will match ``"1:1"``, but not ``"1:2"``. Because this + matches by expressions, will *not* match the leading ``"1:1"`` + in ``"1:10"``; the expressions are evaluated first, and then + compared, so ``"1"`` is compared with ``"10"``. Do *not* use + with packrat parsing enabled. """ rep = Forward() e2 = expr.copy() @@ -4600,26 +5025,33 @@ def _escapeRegexRangeChars(s): return _ustr(s) def oneOf( strs, caseless=False, useRegex=True ): - """ - Helper to quickly define a set of alternative Literals, and makes sure to do - longest-first testing when there is a conflict, regardless of the input order, - but returns a C{L{MatchFirst}} for best performance. + """Helper to quickly define a set of alternative Literals, and makes + sure to do longest-first testing when there is a conflict, + regardless of the input order, but returns + a :class:`MatchFirst` for best performance. Parameters: - - strs - a string of space-delimited literals, or a collection of string literals - - caseless - (default=C{False}) - treat all literals as caseless - - useRegex - (default=C{True}) - as an optimization, will generate a Regex - object; otherwise, will generate a C{MatchFirst} object (if C{caseless=True}, or - if creating a C{Regex} raises an exception) + + - strs - a string of space-delimited literals, or a collection of + string literals + - caseless - (default= ``False``) - treat all literals as + caseless + - useRegex - (default= ``True``) - as an optimization, will + generate a Regex object; otherwise, will generate + a :class:`MatchFirst` object (if ``caseless=True``, or if + creating a :class:`Regex` raises an exception) Example:: + comp_oper = oneOf("< = > <= >= !=") var = Word(alphas) number = Word(nums) term = var | number comparison_expr = term + comp_oper + term print(comparison_expr.searchString("B = 12 AA=23 B<=AA AA>12")) + prints:: + [['B', '=', '12'], ['AA', '=', '23'], ['B', '<=', 'AA'], ['AA', '>', '12']] """ if caseless: @@ -4673,19 +5105,21 @@ def oneOf( strs, caseless=False, useRegex=True ): return MatchFirst(parseElementClass(sym) for sym in symbols).setName(' | '.join(symbols)) def dictOf( key, value ): - """ - Helper to easily and clearly define a dictionary by specifying the respective patterns - for the key and value. Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens - in the proper order. The key pattern can include delimiting markers or punctuation, - as long as they are suppressed, thereby leaving the significant key text. The value - pattern can include named results, so that the C{Dict} results can include named token - fields. + """Helper to easily and clearly define a dictionary by specifying + the respective patterns for the key and value. Takes care of + defining the :class:`Dict`, :class:`ZeroOrMore`, and + :class:`Group` tokens in the proper order. The key pattern + can include delimiting markers or punctuation, as long as they are + suppressed, thereby leaving the significant key text. The value + pattern can include named results, so that the :class:`Dict` results + can include named token fields. Example:: + text = "shape: SQUARE posn: upper left color: light blue texture: burlap" attr_expr = (label + Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join)) print(OneOrMore(attr_expr).parseString(text).dump()) - + attr_label = label attr_value = Suppress(':') + OneOrMore(data_word, stopOn=label).setParseAction(' '.join) @@ -4695,7 +5129,9 @@ def dictOf( key, value ): print(result['shape']) print(result.shape) # object attribute access works too print(result.asDict()) + prints:: + [['shape', 'SQUARE'], ['posn', 'upper left'], ['color', 'light blue'], ['texture', 'burlap']] - color: light blue - posn: upper left @@ -4705,29 +5141,34 @@ def dictOf( key, value ): SQUARE {'color': 'light blue', 'shape': 'SQUARE', 'posn': 'upper left', 'texture': 'burlap'} """ - return Dict( ZeroOrMore( Group ( key + value ) ) ) + return Dict(OneOrMore(Group(key + value))) def originalTextFor(expr, asString=True): - """ - Helper to return the original, untokenized text for a given expression. Useful to - restore the parsed fields of an HTML start tag into the raw tag text itself, or to - revert separate tokens with intervening whitespace back to the original matching - input text. By default, returns astring containing the original parsed text. - - If the optional C{asString} argument is passed as C{False}, then the return value is a - C{L{ParseResults}} containing any results names that were originally matched, and a - single token containing the original matched text from the input string. So if - the expression passed to C{L{originalTextFor}} contains expressions with defined - results names, you must set C{asString} to C{False} if you want to preserve those - results name values. + """Helper to return the original, untokenized text for a given + expression. Useful to restore the parsed fields of an HTML start + tag into the raw tag text itself, or to revert separate tokens with + intervening whitespace back to the original matching input text. By + default, returns astring containing the original parsed text. + + If the optional ``asString`` argument is passed as + ``False``, then the return value is + a :class:`ParseResults` containing any results names that + were originally matched, and a single token containing the original + matched text from the input string. So if the expression passed to + :class:`originalTextFor` contains expressions with defined + results names, you must set ``asString`` to ``False`` if you + want to preserve those results name values. Example:: + src = "this is test bold text normal text " for tag in ("b","i"): opener,closer = makeHTMLTags(tag) patt = originalTextFor(opener + SkipTo(closer) + closer) print(patt.searchString(src)[0]) + prints:: + [' bold text '] ['text'] """ @@ -4744,29 +5185,33 @@ def originalTextFor(expr, asString=True): matchExpr.ignoreExprs = expr.ignoreExprs return matchExpr -def ungroup(expr): - """ - Helper to undo pyparsing's default grouping of And expressions, even - if all but one are non-empty. +def ungroup(expr): + """Helper to undo pyparsing's default grouping of And expressions, + even if all but one are non-empty. """ return TokenConverter(expr).setParseAction(lambda t:t[0]) def locatedExpr(expr): - """ - Helper to decorate a returned token with its starting and ending locations in the input string. + """Helper to decorate a returned token with its starting and ending + locations in the input string. + This helper adds the following results names: + - locn_start = location where matched expression begins - locn_end = location where matched expression ends - value = the actual parsed results - Be careful if the input text contains C{} characters, you may want to call - C{L{ParserElement.parseWithTabs}} + Be careful if the input text contains ```` characters, you + may want to call :class:`ParserElement.parseWithTabs` Example:: + wd = Word(alphas) for match in locatedExpr(wd).searchString("ljsdf123lksdjjf123lkkjj1222"): print(match) + prints:: + [[0, 'ljsdf', 5]] [[8, 'lksdjjf', 15]] [[18, 'lkkjj', 23]] @@ -4790,22 +5235,30 @@ _charRange = Group(_singleChar + Suppress("-") + _singleChar) _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]" def srange(s): - r""" - Helper to easily define string ranges for use in Word construction. Borrows - syntax from regexp '[]' string range definitions:: + r"""Helper to easily define string ranges for use in Word + construction. Borrows syntax from regexp '[]' string range + definitions:: + srange("[0-9]") -> "0123456789" srange("[a-z]") -> "abcdefghijklmnopqrstuvwxyz" srange("[a-z$_]") -> "abcdefghijklmnopqrstuvwxyz$_" - The input string must be enclosed in []'s, and the returned string is the expanded - character set joined into a single string. - The values enclosed in the []'s may be: + + The input string must be enclosed in []'s, and the returned string + is the expanded character set joined into a single string. The + values enclosed in the []'s may be: + - a single character - - an escaped character with a leading backslash (such as C{\-} or C{\]}) - - an escaped hex character with a leading C{'\x'} (C{\x21}, which is a C{'!'} character) - (C{\0x##} is also supported for backwards compatibility) - - an escaped octal character with a leading C{'\0'} (C{\041}, which is a C{'!'} character) - - a range of any of the above, separated by a dash (C{'a-z'}, etc.) - - any combination of the above (C{'aeiouy'}, C{'a-zA-Z0-9_$'}, etc.) + - an escaped character with a leading backslash (such as ``\-`` + or ``\]``) + - an escaped hex character with a leading ``'\x'`` + (``\x21``, which is a ``'!'`` character) (``\0x##`` + is also supported for backwards compatibility) + - an escaped octal character with a leading ``'\0'`` + (``\041``, which is a ``'!'`` character) + - a range of any of the above, separated by a dash (``'a-z'``, + etc.) + - any combination of the above (``'aeiouy'``, + ``'a-zA-Z0-9_$'``, etc.) """ _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1)) try: @@ -4814,9 +5267,8 @@ def srange(s): return "" def matchOnlyAtCol(n): - """ - Helper method for defining parse actions that require matching at a specific - column in the input text. + """Helper method for defining parse actions that require matching at + a specific column in the input text. """ def verifyCol(strg,locn,toks): if col(locn,strg) != n: @@ -4824,24 +5276,26 @@ def matchOnlyAtCol(n): return verifyCol def replaceWith(replStr): - """ - Helper method for common parse actions that simply return a literal value. Especially - useful when used with C{L{transformString}()}. + """Helper method for common parse actions that simply return + a literal value. Especially useful when used with + :class:`transformString` (). Example:: + num = Word(nums).setParseAction(lambda toks: int(toks[0])) na = oneOf("N/A NA").setParseAction(replaceWith(math.nan)) term = na | num - + OneOrMore(term).parseString("324 234 N/A 234") # -> [324, 234, nan, 234] """ return lambda s,l,t: [replStr] def removeQuotes(s,l,t): - """ - Helper parse action for removing quotation marks from parsed quoted strings. + """Helper parse action for removing quotation marks from parsed + quoted strings. Example:: + # by default, quotation marks are included in parsed results quotedString.parseString("'Now is the Winter of our Discontent'") # -> ["'Now is the Winter of our Discontent'"] @@ -4852,18 +5306,20 @@ def removeQuotes(s,l,t): return t[0][1:-1] def tokenMap(func, *args): - """ - Helper to define a parse action by mapping a function to all elements of a ParseResults list.If any additional - args are passed, they are forwarded to the given function as additional arguments after - the token, as in C{hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))}, which will convert the - parsed data to an integer using base 16. + """Helper to define a parse action by mapping a function to all + elements of a ParseResults list. If any additional args are passed, + they are forwarded to the given function as additional arguments + after the token, as in + ``hex_integer = Word(hexnums).setParseAction(tokenMap(int, 16))``, + which will convert the parsed data to an integer using base 16. + + Example (compare the last to example in :class:`ParserElement.transformString`:: - Example (compare the last to example in L{ParserElement.transformString}:: hex_ints = OneOrMore(Word(hexnums)).setParseAction(tokenMap(int, 16)) hex_ints.runTests(''' 00 11 22 aa FF 0a 0d 1a ''') - + upperword = Word(alphas).setParseAction(tokenMap(str.upper)) OneOrMore(upperword).runTests(''' my kingdom for a horse @@ -4873,7 +5329,9 @@ def tokenMap(func, *args): OneOrMore(wd).setParseAction(' '.join).runTests(''' now is the winter of our discontent made glorious summer by this sun of york ''') + prints:: + 00 11 22 aa FF 0a 0d 1a [0, 17, 34, 170, 255, 10, 13, 26] @@ -4887,7 +5345,7 @@ def tokenMap(func, *args): return [func(tokn, *args) for tokn in t] try: - func_name = getattr(func, '__name__', + func_name = getattr(func, '__name__', getattr(func, '__class__').__name__) except Exception: func_name = str(func) @@ -4896,11 +5354,13 @@ def tokenMap(func, *args): return pa upcaseTokens = tokenMap(lambda t: _ustr(t).upper()) -"""(Deprecated) Helper parse action to convert tokens to upper case. Deprecated in favor of L{pyparsing_common.upcaseTokens}""" +"""(Deprecated) Helper parse action to convert tokens to upper case. +Deprecated in favor of :class:`pyparsing_common.upcaseTokens`""" downcaseTokens = tokenMap(lambda t: _ustr(t).lower()) -"""(Deprecated) Helper parse action to convert tokens to lower case. Deprecated in favor of L{pyparsing_common.downcaseTokens}""" - +"""(Deprecated) Helper parse action to convert tokens to lower case. +Deprecated in favor of :class:`pyparsing_common.downcaseTokens`""" + def _makeTags(tagStr, xml): """Internal helper to construct opening and closing tag expressions, given a tag name""" if isinstance(tagStr,basestring): @@ -4931,55 +5391,63 @@ def _makeTags(tagStr, xml): return openTag, closeTag def makeHTMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for HTML, given a tag name. Matches - tags in either upper or lower case, attributes with namespaces and with quoted or unquoted values. + """Helper to construct opening and closing tag expressions for HTML, + given a tag name. Matches tags in either upper or lower case, + attributes with namespaces and with quoted or unquoted values. Example:: - text = 'More info at the pyparsing wiki page' - # makeHTMLTags returns pyparsing expressions for the opening and closing tags as a 2-tuple + + text = 'More info at the pyparsing wiki page' + # makeHTMLTags returns pyparsing expressions for the opening and + # closing tags as a 2-tuple a,a_end = makeHTMLTags("A") link_expr = a + SkipTo(a_end)("link_text") + a_end - + for link in link_expr.searchString(text): - # attributes in the tag (like "href" shown here) are also accessible as named results + # attributes in the tag (like "href" shown here) are + # also accessible as named results print(link.link_text, '->', link.href) + prints:: - pyparsing -> http://pyparsing.wikispaces.com + + pyparsing -> https://github.com/pyparsing/pyparsing/wiki """ return _makeTags( tagStr, False ) def makeXMLTags(tagStr): - """ - Helper to construct opening and closing tag expressions for XML, given a tag name. Matches - tags only in the given upper/lower case. + """Helper to construct opening and closing tag expressions for XML, + given a tag name. Matches tags only in the given upper/lower case. - Example: similar to L{makeHTMLTags} + Example: similar to :class:`makeHTMLTags` """ return _makeTags( tagStr, True ) def withAttribute(*args,**attrDict): - """ - Helper to create a validating parse action to be used with start tags created - with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag - with a required attribute value, to avoid false matches on common tags such as - C{} or C{
}. + """Helper to create a validating parse action to be used with start + tags created with :class:`makeXMLTags` or + :class:`makeHTMLTags`. Use ``withAttribute`` to qualify + a starting tag with a required attribute value, to avoid false + matches on common tags such as ```` or ``
``. - Call C{withAttribute} with a series of attribute names and values. Specify the list - of filter attributes names and values as: - - keyword arguments, as in C{(align="right")}, or - - as an explicit dict with C{**} operator, when an attribute name is also a Python - reserved word, as in C{**{"class":"Customer", "align":"right"}} - - a list of name-value tuples, as in ( ("ns1:class", "Customer"), ("ns2:align","right") ) - For attribute names with a namespace prefix, you must use the second form. Attribute - names are matched insensitive to upper/lower case. - - If just testing for C{class} (with or without a namespace), use C{L{withClass}}. + Call ``withAttribute`` with a series of attribute names and + values. Specify the list of filter attributes names and values as: - To verify that the attribute exists, but without specifying a value, pass - C{withAttribute.ANY_VALUE} as the value. + - keyword arguments, as in ``(align="right")``, or + - as an explicit dict with ``**`` operator, when an attribute + name is also a Python reserved word, as in ``**{"class":"Customer", "align":"right"}`` + - a list of name-value tuples, as in ``(("ns1:class", "Customer"), ("ns2:align","right"))`` + + For attribute names with a namespace prefix, you must use the second + form. Attribute names are matched insensitive to upper/lower case. + + If just testing for ``class`` (with or without a namespace), use + :class:`withClass`. + + To verify that the attribute exists, but without specifying a value, + pass ``withAttribute.ANY_VALUE`` as the value. Example:: + html = '''
Some text @@ -4987,7 +5455,7 @@ def withAttribute(*args,**attrDict):
1,3 2,3 1,1
this has no type
- + ''' div,div_end = makeHTMLTags("div") @@ -4996,13 +5464,15 @@ def withAttribute(*args,**attrDict): grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + # construct a match with any div tag having a type attribute, regardless of the value div_any_type = div().setParseAction(withAttribute(type=withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 @@ -5024,11 +5494,12 @@ def withAttribute(*args,**attrDict): withAttribute.ANY_VALUE = object() def withClass(classname, namespace=''): - """ - Simplified version of C{L{withAttribute}} when matching on a div class - made - difficult because C{class} is a reserved word in Python. + """Simplified version of :class:`withAttribute` when + matching on a div class - made difficult because ``class`` is + a reserved word in Python. Example:: + html = '''
Some text @@ -5036,84 +5507,96 @@ def withClass(classname, namespace=''):
1,3 2,3 1,1
this <div> has no class
- + ''' div,div_end = makeHTMLTags("div") div_grid = div().setParseAction(withClass("grid")) - + grid_expr = div_grid + SkipTo(div | div_end)("body") for grid_header in grid_expr.searchString(html): print(grid_header.body) - + div_any_type = div().setParseAction(withClass(withAttribute.ANY_VALUE)) div_expr = div_any_type + SkipTo(div | div_end)("body") for div_header in div_expr.searchString(html): print(div_header.body) + prints:: + 1 4 0 1 0 1 4 0 1 0 1,3 2,3 1,1 """ classattr = "%s:class" % namespace if namespace else "class" - return withAttribute(**{classattr : classname}) + return withAttribute(**{classattr : classname}) -opAssoc = _Constants() +opAssoc = SimpleNamespace() opAssoc.LEFT = object() opAssoc.RIGHT = object() def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): - """ - Helper method for constructing grammars of expressions made up of - operators working in a precedence hierarchy. Operators may be unary or - binary, left- or right-associative. Parse actions can also be attached - to operator expressions. The generated parser will also recognize the use - of parentheses to override operator precedences (see example below). - - Note: if you define a deep operator list, you may see performance issues - when using infixNotation. See L{ParserElement.enablePackrat} for a - mechanism to potentially improve your parser performance. + """Helper method for constructing grammars of expressions made up of + operators working in a precedence hierarchy. Operators may be unary + or binary, left- or right-associative. Parse actions can also be + attached to operator expressions. The generated parser will also + recognize the use of parentheses to override operator precedences + (see example below). + + Note: if you define a deep operator list, you may see performance + issues when using infixNotation. See + :class:`ParserElement.enablePackrat` for a mechanism to potentially + improve your parser performance. Parameters: - - baseExpr - expression representing the most basic element for the nested - - opList - list of tuples, one for each operator precedence level in the - expression grammar; each tuple is of the form - (opExpr, numTerms, rightLeftAssoc, parseAction), where: - - opExpr is the pyparsing expression for the operator; - may also be a string, which will be converted to a Literal; - if numTerms is 3, opExpr is a tuple of two expressions, for the - two operators separating the 3 terms - - numTerms is the number of terms for this operator (must - be 1, 2, or 3) - - rightLeftAssoc is the indicator whether the operator is - right or left associative, using the pyparsing-defined - constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}. + - baseExpr - expression representing the most basic element for the + nested + - opList - list of tuples, one for each operator precedence level + in the expression grammar; each tuple is of the form ``(opExpr, + numTerms, rightLeftAssoc, parseAction)``, where: + + - opExpr is the pyparsing expression for the operator; may also + be a string, which will be converted to a Literal; if numTerms + is 3, opExpr is a tuple of two expressions, for the two + operators separating the 3 terms + - numTerms is the number of terms for this operator (must be 1, + 2, or 3) + - rightLeftAssoc is the indicator whether the operator is right + or left associative, using the pyparsing-defined constants + ``opAssoc.RIGHT`` and ``opAssoc.LEFT``. - parseAction is the parse action to be associated with - expressions matching this operator expression (the - parse action tuple member may be omitted); if the parse action - is passed a tuple or list of functions, this is equivalent to - calling C{setParseAction(*fn)} (L{ParserElement.setParseAction}) - - lpar - expression for matching left-parentheses (default=C{Suppress('(')}) - - rpar - expression for matching right-parentheses (default=C{Suppress(')')}) + expressions matching this operator expression (the parse action + tuple member may be omitted); if the parse action is passed + a tuple or list of functions, this is equivalent to calling + ``setParseAction(*fn)`` + (:class:`ParserElement.setParseAction`) + - lpar - expression for matching left-parentheses + (default= ``Suppress('(')``) + - rpar - expression for matching right-parentheses + (default= ``Suppress(')')``) Example:: - # simple example of four-function arithmetic with ints and variable names + + # simple example of four-function arithmetic with ints and + # variable names integer = pyparsing_common.signed_integer - varname = pyparsing_common.identifier - + varname = pyparsing_common.identifier + arith_expr = infixNotation(integer | varname, [ ('-', 1, opAssoc.RIGHT), (oneOf('* /'), 2, opAssoc.LEFT), (oneOf('+ -'), 2, opAssoc.LEFT), ]) - + arith_expr.runTests(''' 5+3*6 (5+3)*6 -2--11 ''', fullDump=False) + prints:: + 5+3*6 [[5, '+', [3, '*', 6]]] @@ -5123,6 +5606,12 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): -2--11 [[['-', 2], '-', ['-', 11]]] """ + # captive version of FollowedBy that does not do parse actions or capture results names + class _FB(FollowedBy): + def parseImpl(self, instring, loc, doActions=True): + self.expr.tryParse(instring, loc) + return loc, [] + ret = Forward() lastExpr = baseExpr | ( lpar + ret + rpar ) for i,operDef in enumerate(opList): @@ -5130,19 +5619,20 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): termName = "%s term" % opExpr if arity < 3 else "%s%s term" % opExpr if arity == 3: if opExpr is None or len(opExpr) != 2: - raise ValueError("if numterms=3, opExpr must be a tuple or list of two expressions") + raise ValueError( + "if numterms=3, opExpr must be a tuple or list of two expressions") opExpr1, opExpr2 = opExpr thisExpr = Forward().setName(termName) if rightLeftAssoc == opAssoc.LEFT: if arity == 1: - matchExpr = FollowedBy(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) + matchExpr = _FB(lastExpr + opExpr) + Group( lastExpr + OneOrMore( opExpr ) ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) + matchExpr = _FB(lastExpr + opExpr + lastExpr) + Group( lastExpr + OneOrMore( opExpr + lastExpr ) ) else: - matchExpr = FollowedBy(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) + matchExpr = _FB(lastExpr+lastExpr) + Group( lastExpr + OneOrMore(lastExpr) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr) + \ Group( lastExpr + opExpr1 + lastExpr + opExpr2 + lastExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5151,14 +5641,14 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): # try to avoid LR with this extra test if not isinstance(opExpr, Optional): opExpr = Optional(opExpr) - matchExpr = FollowedBy(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) + matchExpr = _FB(opExpr.expr + thisExpr) + Group( opExpr + thisExpr ) elif arity == 2: if opExpr is not None: - matchExpr = FollowedBy(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) + matchExpr = _FB(lastExpr + opExpr + thisExpr) + Group( lastExpr + OneOrMore( opExpr + thisExpr ) ) else: - matchExpr = FollowedBy(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) + matchExpr = _FB(lastExpr + thisExpr) + Group( lastExpr + OneOrMore( thisExpr ) ) elif arity == 3: - matchExpr = FollowedBy(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ + matchExpr = _FB(lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr) + \ Group( lastExpr + opExpr1 + thisExpr + opExpr2 + thisExpr ) else: raise ValueError("operator must be unary (1), binary (2), or ternary (3)") @@ -5175,7 +5665,8 @@ def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ): return ret operatorPrecedence = infixNotation -"""(Deprecated) Former name of C{L{infixNotation}}, will be dropped in a future release.""" +"""(Deprecated) Former name of :class:`infixNotation`, will be +dropped in a future release.""" dblQuotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*')+'"').setName("string enclosed in double quotes") sglQuotedString = Combine(Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\(?:[^x]|x[0-9a-fA-F]+)))*")+"'").setName("string enclosed in single quotes") @@ -5184,28 +5675,33 @@ quotedString = Combine(Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\(?:[^x]|x[0-9a-fA-F]+) unicodeString = Combine(_L('u') + quotedString.copy()).setName("unicode string literal") def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.copy()): - """ - Helper method for defining nested lists enclosed in opening and closing - delimiters ("(" and ")" are the default). + """Helper method for defining nested lists enclosed in opening and + closing delimiters ("(" and ")" are the default). Parameters: - - opener - opening character for a nested list (default=C{"("}); can also be a pyparsing expression - - closer - closing character for a nested list (default=C{")"}); can also be a pyparsing expression - - content - expression for items within the nested lists (default=C{None}) - - ignoreExpr - expression for ignoring opening and closing delimiters (default=C{quotedString}) + - opener - opening character for a nested list + (default= ``"("``); can also be a pyparsing expression + - closer - closing character for a nested list + (default= ``")"``); can also be a pyparsing expression + - content - expression for items within the nested lists + (default= ``None``) + - ignoreExpr - expression for ignoring opening and closing + delimiters (default= :class:`quotedString`) - If an expression is not provided for the content argument, the nested - expression will capture all whitespace-delimited content between delimiters - as a list of separate values. + If an expression is not provided for the content argument, the + nested expression will capture all whitespace-delimited content + between delimiters as a list of separate values. - Use the C{ignoreExpr} argument to define expressions that may contain - opening or closing characters that should not be treated as opening - or closing characters for nesting, such as quotedString or a comment - expression. Specify multiple expressions using an C{L{Or}} or C{L{MatchFirst}}. - The default is L{quotedString}, but if no expressions are to be ignored, - then pass C{None} for this argument. + Use the ``ignoreExpr`` argument to define expressions that may + contain opening or closing characters that should not be treated as + opening or closing characters for nesting, such as quotedString or + a comment expression. Specify multiple expressions using an + :class:`Or` or :class:`MatchFirst`. The default is + :class:`quotedString`, but if no expressions are to be ignored, then + pass ``None`` for this argument. Example:: + data_type = oneOf("void int short long char float double") decl_data_type = Combine(data_type + Optional(Word('*'))) ident = Word(alphas+'_', alphanums+'_') @@ -5215,29 +5711,31 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop code_body = nestedExpr('{', '}', ignoreExpr=(quotedString | cStyleComment)) - c_function = (decl_data_type("type") + c_function = (decl_data_type("type") + ident("name") - + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + + LPAR + Optional(delimitedList(arg), [])("args") + RPAR + code_body("body")) c_function.ignore(cStyleComment) - + source_code = ''' - int is_odd(int x) { - return (x%2); + int is_odd(int x) { + return (x%2); } - - int dec_to_hex(char hchar) { - if (hchar >= '0' && hchar <= '9') { - return (ord(hchar)-ord('0')); - } else { + + int dec_to_hex(char hchar) { + if (hchar >= '0' && hchar <= '9') { + return (ord(hchar)-ord('0')); + } else { return (10+ord(hchar)-ord('A')); - } + } } ''' for func in c_function.searchString(source_code): print("%(name)s (%(type)s) args: %(args)s" % func) + prints:: + is_odd (int) args: [['int', 'x']] dec_to_hex (int) args: [['char', 'hchar']] """ @@ -5255,7 +5753,7 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop ).setParseAction(lambda t:t[0].strip())) else: if ignoreExpr is not None: - content = (Combine(OneOrMore(~ignoreExpr + + content = (Combine(OneOrMore(~ignoreExpr + ~Literal(opener) + ~Literal(closer) + CharsNotIn(ParserElement.DEFAULT_WHITE_CHARS,exact=1)) ).setParseAction(lambda t:t[0].strip())) @@ -5274,23 +5772,24 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop return ret def indentedBlock(blockStatementExpr, indentStack, indent=True): - """ - Helper method for defining space-delimited indentation blocks, such as - those used to define block statements in Python source code. + """Helper method for defining space-delimited indentation blocks, + such as those used to define block statements in Python source code. Parameters: - - blockStatementExpr - expression defining syntax of statement that - is repeated within the indented block - - indentStack - list created by caller to manage indentation stack - (multiple statementWithIndentedBlock expressions within a single grammar - should share a common indentStack) - - indent - boolean indicating whether block must be indented beyond the - the current level; set to False for block of left-most statements - (default=C{True}) - A valid block must contain at least one C{blockStatement}. + - blockStatementExpr - expression defining syntax of statement that + is repeated within the indented block + - indentStack - list created by caller to manage indentation stack + (multiple statementWithIndentedBlock expressions within a single + grammar should share a common indentStack) + - indent - boolean indicating whether block must be indented beyond + the the current level; set to False for block of left-most + statements (default= ``True``) + + A valid block must contain at least one ``blockStatement``. Example:: + data = ''' def A(z): A1 @@ -5331,7 +5830,9 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): parseTree = module_body.parseString(data) parseTree.pprint() + prints:: + [['def', 'A', ['(', 'z', ')'], @@ -5349,7 +5850,7 @@ def indentedBlock(blockStatementExpr, indentStack, indent=True): 'spam', ['(', 'x', 'y', ')'], ':', - [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] + [[['def', 'eggs', ['(', 'z', ')'], ':', [['pass']]]]]]] """ def checkPeerIndent(s,l,t): if l >= len(s): return @@ -5399,51 +5900,61 @@ def replaceHTMLEntity(t): # it's easy to get these comment structures wrong - they're very common, so may as well make them available cStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/').setName("C style comment") -"Comment of the form C{/* ... */}" +"Comment of the form ``/* ... */``" htmlComment = Regex(r"").setName("HTML comment") -"Comment of the form C{}" +"Comment of the form ````" restOfLine = Regex(r".*").leaveWhitespace().setName("rest of line") dblSlashComment = Regex(r"//(?:\\\n|[^\n])*").setName("// comment") -"Comment of the form C{// ... (to end of line)}" +"Comment of the form ``// ... (to end of line)``" cppStyleComment = Combine(Regex(r"/\*(?:[^*]|\*(?!/))*") + '*/'| dblSlashComment).setName("C++ style comment") -"Comment of either form C{L{cStyleComment}} or C{L{dblSlashComment}}" +"Comment of either form :class:`cStyleComment` or :class:`dblSlashComment`" javaStyleComment = cppStyleComment -"Same as C{L{cppStyleComment}}" +"Same as :class:`cppStyleComment`" pythonStyleComment = Regex(r"#.*").setName("Python style comment") -"Comment of the form C{# ... (to end of line)}" +"Comment of the form ``# ... (to end of line)``" _commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') + Optional( Word(" \t") + ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem") commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList") -"""(Deprecated) Predefined expression of 1 or more printable words or quoted strings, separated by commas. - This expression is deprecated in favor of L{pyparsing_common.comma_separated_list}.""" +"""(Deprecated) Predefined expression of 1 or more printable words or +quoted strings, separated by commas. + +This expression is deprecated in favor of :class:`pyparsing_common.comma_separated_list`. +""" # some other useful expressions - using lower-case class name since we are really using this as a namespace class pyparsing_common: - """ - Here are some common low-level expressions that may be useful in jump-starting parser development: - - numeric forms (L{integers}, L{reals}, L{scientific notation}) - - common L{programming identifiers} - - network addresses (L{MAC}, L{IPv4}, L{IPv6}) - - ISO8601 L{dates} and L{datetime} - - L{UUID} - - L{comma-separated list} + """Here are some common low-level expressions that may be useful in + jump-starting parser development: + + - numeric forms (:class:`integers`, :class:`reals`, + :class:`scientific notation`) + - common :class:`programming identifiers` + - network addresses (:class:`MAC`, + :class:`IPv4`, :class:`IPv6`) + - ISO8601 :class:`dates` and + :class:`datetime` + - :class:`UUID` + - :class:`comma-separated list` + Parse actions: - - C{L{convertToInteger}} - - C{L{convertToFloat}} - - C{L{convertToDate}} - - C{L{convertToDatetime}} - - C{L{stripHTMLTags}} - - C{L{upcaseTokens}} - - C{L{downcaseTokens}} + + - :class:`convertToInteger` + - :class:`convertToFloat` + - :class:`convertToDate` + - :class:`convertToDatetime` + - :class:`stripHTMLTags` + - :class:`upcaseTokens` + - :class:`downcaseTokens` Example:: + pyparsing_common.number.runTests(''' # any int or real number, returned as the appropriate type 100 @@ -5490,7 +6001,9 @@ class pyparsing_common: # uuid 12345678-1234-5678-1234-567812345678 ''') + prints:: + # any int or real number, returned as the appropriate type 100 [100] @@ -5592,7 +6105,8 @@ class pyparsing_common: """expression that parses a floating point number and returns a float""" sci_real = Regex(r'[+-]?\d+([eE][+-]?\d+|\.\d*([eE][+-]?\d+)?)').setName("real number with scientific notation").setParseAction(convertToFloat) - """expression that parses a floating point number with optional scientific notation and returns a float""" + """expression that parses a floating point number with optional + scientific notation and returns a float""" # streamlining this expression makes the docs nicer-looking number = (sci_real | real | signed_integer).streamline() @@ -5600,12 +6114,12 @@ class pyparsing_common: fnumber = Regex(r'[+-]?\d+\.?\d*([eE][+-]?\d+)?').setName("fnumber").setParseAction(convertToFloat) """any int or real number, returned as float""" - + identifier = Word(alphas+'_', alphanums+'_').setName("identifier") """typical code identifier (leading alpha or '_', followed by 0 or more alphas, nums, or '_')""" - + ipv4_address = Regex(r'(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})(\.(25[0-5]|2[0-4][0-9]|1?[0-9]{1,2})){3}').setName("IPv4 address") - "IPv4 address (C{0.0.0.0 - 255.255.255.255})" + "IPv4 address (``0.0.0.0 - 255.255.255.255``)" _ipv6_part = Regex(r'[0-9a-fA-F]{1,4}').setName("hex_integer") _full_ipv6_address = (_ipv6_part + (':' + _ipv6_part)*7).setName("full IPv6 address") @@ -5614,7 +6128,7 @@ class pyparsing_common: _mixed_ipv6_address = ("::ffff:" + ipv4_address).setName("mixed IPv6 address") ipv6_address = Combine((_full_ipv6_address | _mixed_ipv6_address | _short_ipv6_address).setName("IPv6 address")).setName("IPv6 address") "IPv6 address (long, short, or mixed form)" - + mac_address = Regex(r'[0-9a-fA-F]{2}([:.-])[0-9a-fA-F]{2}(?:\1[0-9a-fA-F]{2}){4}').setName("MAC address") "MAC address xx:xx:xx:xx:xx (may also have '-' or '.' delimiters)" @@ -5624,13 +6138,16 @@ class pyparsing_common: Helper to create a parse action for converting parsed date string to Python datetime.date Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%d"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%d"``) Example:: + date_expr = pyparsing_common.iso8601_date.copy() date_expr.setParseAction(pyparsing_common.convertToDate()) print(date_expr.parseString("1999-12-31")) + prints:: + [datetime.date(1999, 12, 31)] """ def cvt_fn(s,l,t): @@ -5642,17 +6159,20 @@ class pyparsing_common: @staticmethod def convertToDatetime(fmt="%Y-%m-%dT%H:%M:%S.%f"): - """ - Helper to create a parse action for converting parsed datetime string to Python datetime.datetime + """Helper to create a parse action for converting parsed + datetime string to Python datetime.datetime Params - - - fmt - format to be passed to datetime.strptime (default=C{"%Y-%m-%dT%H:%M:%S.%f"}) + - fmt - format to be passed to datetime.strptime (default= ``"%Y-%m-%dT%H:%M:%S.%f"``) Example:: + dt_expr = pyparsing_common.iso8601_datetime.copy() dt_expr.setParseAction(pyparsing_common.convertToDatetime()) print(dt_expr.parseString("1999-12-31T23:59:59.999")) + prints:: + [datetime.datetime(1999, 12, 31, 23, 59, 59, 999000)] """ def cvt_fn(s,l,t): @@ -5663,31 +6183,34 @@ class pyparsing_common: return cvt_fn iso8601_date = Regex(r'(?P\d{4})(?:-(?P\d\d)(?:-(?P\d\d))?)?').setName("ISO8601 date") - "ISO8601 date (C{yyyy-mm-dd})" + "ISO8601 date (``yyyy-mm-dd``)" iso8601_datetime = Regex(r'(?P\d{4})-(?P\d\d)-(?P\d\d)[T ](?P\d\d):(?P\d\d)(:(?P\d\d(\.\d*)?)?)?(?PZ|[+-]\d\d:?\d\d)?').setName("ISO8601 datetime") - "ISO8601 datetime (C{yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)}) - trailing seconds, milliseconds, and timezone optional; accepts separating C{'T'} or C{' '}" + "ISO8601 datetime (``yyyy-mm-ddThh:mm:ss.s(Z|+-00:00)``) - trailing seconds, milliseconds, and timezone optional; accepts separating ``'T'`` or ``' '``" uuid = Regex(r'[0-9a-fA-F]{8}(-[0-9a-fA-F]{4}){3}-[0-9a-fA-F]{12}').setName("UUID") - "UUID (C{xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx})" + "UUID (``xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx``)" _html_stripper = anyOpenTag.suppress() | anyCloseTag.suppress() @staticmethod def stripHTMLTags(s, l, tokens): - """ - Parse action to remove HTML tags from web page HTML source + """Parse action to remove HTML tags from web page HTML source Example:: - # strip HTML links from normal text - text = 'More info at the pyparsing wiki page' + + # strip HTML links from normal text + text = 'More info at the pyparsing wiki page' td,td_end = makeHTMLTags("TD") table_text = td + SkipTo(td_end).setParseAction(pyparsing_common.stripHTMLTags)("body") + td_end - - print(table_text.parseString(text).body) # -> 'More info at the pyparsing wiki page' + print(table_text.parseString(text).body) + + Prints:: + + More info at the pyparsing wiki page """ return pyparsing_common._html_stripper.transformString(tokens[0]) - _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + _commasepitem = Combine(OneOrMore(~Literal(",") + ~LineEnd() + Word(printables, excludeChars=',') + Optional( White(" \t") ) ) ).streamline().setName("commaItem") comma_separated_list = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("comma separated list") """Predefined expression of 1 or more printable words or quoted strings, separated by commas.""" @@ -5699,6 +6222,164 @@ class pyparsing_common: """Parse action to convert tokens to lower case.""" +class _lazyclassproperty(object): + def __init__(self, fn): + self.fn = fn + self.__doc__ = fn.__doc__ + self.__name__ = fn.__name__ + + def __get__(self, obj, cls): + if cls is None: + cls = type(obj) + if not hasattr(cls, '_intern') or any(cls._intern is getattr(superclass, '_intern', []) for superclass in cls.__mro__[1:]): + cls._intern = {} + attrname = self.fn.__name__ + if attrname not in cls._intern: + cls._intern[attrname] = self.fn(cls) + return cls._intern[attrname] + + +class unicode_set(object): + """ + A set of Unicode characters, for language-specific strings for + ``alphas``, ``nums``, ``alphanums``, and ``printables``. + A unicode_set is defined by a list of ranges in the Unicode character + set, in a class attribute ``_ranges``, such as:: + + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + A unicode set can also be defined using multiple inheritance of other unicode sets:: + + class CJK(Chinese, Japanese, Korean): + pass + """ + _ranges = [] + + @classmethod + def _get_chars_for_ranges(cls): + ret = [] + for cc in cls.__mro__: + if cc is unicode_set: + break + for rr in cc._ranges: + ret.extend(range(rr[0], rr[-1]+1)) + return [unichr(c) for c in sorted(set(ret))] + + @_lazyclassproperty + def printables(cls): + "all non-whitespace characters in this range" + return u''.join(filterfalse(unicode.isspace, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphas(cls): + "all alphabetic characters in this range" + return u''.join(filter(unicode.isalpha, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def nums(cls): + "all numeric digit characters in this range" + return u''.join(filter(unicode.isdigit, cls._get_chars_for_ranges())) + + @_lazyclassproperty + def alphanums(cls): + "all alphanumeric characters in this range" + return cls.alphas + cls.nums + + +class pyparsing_unicode(unicode_set): + """ + A namespace class for defining common language unicode_sets. + """ + _ranges = [(32, sys.maxunicode)] + + class Latin1(unicode_set): + "Unicode set for Latin-1 Unicode Character Range" + _ranges = [(0x0020, 0x007e), (0x00a0, 0x00ff),] + + class LatinA(unicode_set): + "Unicode set for Latin-A Unicode Character Range" + _ranges = [(0x0100, 0x017f),] + + class LatinB(unicode_set): + "Unicode set for Latin-B Unicode Character Range" + _ranges = [(0x0180, 0x024f),] + + class Greek(unicode_set): + "Unicode set for Greek Unicode Character Ranges" + _ranges = [ + (0x0370, 0x03ff), (0x1f00, 0x1f15), (0x1f18, 0x1f1d), (0x1f20, 0x1f45), (0x1f48, 0x1f4d), + (0x1f50, 0x1f57), (0x1f59,), (0x1f5b,), (0x1f5d,), (0x1f5f, 0x1f7d), (0x1f80, 0x1fb4), (0x1fb6, 0x1fc4), + (0x1fc6, 0x1fd3), (0x1fd6, 0x1fdb), (0x1fdd, 0x1fef), (0x1ff2, 0x1ff4), (0x1ff6, 0x1ffe), + ] + + class Cyrillic(unicode_set): + "Unicode set for Cyrillic Unicode Character Range" + _ranges = [(0x0400, 0x04ff)] + + class Chinese(unicode_set): + "Unicode set for Chinese Unicode Character Range" + _ranges = [(0x4e00, 0x9fff), (0x3000, 0x303f), ] + + class Japanese(unicode_set): + "Unicode set for Japanese Unicode Character Range, combining Kanji, Hiragana, and Katakana ranges" + _ranges = [ ] + + class Kanji(unicode_set): + "Unicode set for Kanji Unicode Character Range" + _ranges = [(0x4E00, 0x9Fbf), (0x3000, 0x303f), ] + + class Hiragana(unicode_set): + "Unicode set for Hiragana Unicode Character Range" + _ranges = [(0x3040, 0x309f), ] + + class Katakana(unicode_set): + "Unicode set for Katakana Unicode Character Range" + _ranges = [(0x30a0, 0x30ff), ] + + class Korean(unicode_set): + "Unicode set for Korean Unicode Character Range" + _ranges = [(0xac00, 0xd7af), (0x1100, 0x11ff), (0x3130, 0x318f), (0xa960, 0xa97f), (0xd7b0, 0xd7ff), (0x3000, 0x303f), ] + + class CJK(Chinese, Japanese, Korean): + "Unicode set for combined Chinese, Japanese, and Korean (CJK) Unicode Character Range" + pass + + class Thai(unicode_set): + "Unicode set for Thai Unicode Character Range" + _ranges = [(0x0e01, 0x0e3a), (0x0e3f, 0x0e5b), ] + + class Arabic(unicode_set): + "Unicode set for Arabic Unicode Character Range" + _ranges = [(0x0600, 0x061b), (0x061e, 0x06ff), (0x0700, 0x077f), ] + + class Hebrew(unicode_set): + "Unicode set for Hebrew Unicode Character Range" + _ranges = [(0x0590, 0x05ff), ] + + class Devanagari(unicode_set): + "Unicode set for Devanagari Unicode Character Range" + _ranges = [(0x0900, 0x097f), (0xa8e0, 0xa8ff)] + +pyparsing_unicode.Japanese._ranges = (pyparsing_unicode.Japanese.Kanji._ranges + + pyparsing_unicode.Japanese.Hiragana._ranges + + pyparsing_unicode.Japanese.Katakana._ranges) + +# define ranges in language character sets +if PY_3: + setattr(pyparsing_unicode, "العربية", pyparsing_unicode.Arabic) + setattr(pyparsing_unicode, "中文", pyparsing_unicode.Chinese) + setattr(pyparsing_unicode, "кириллица", pyparsing_unicode.Cyrillic) + setattr(pyparsing_unicode, "Ελληνικά", pyparsing_unicode.Greek) + setattr(pyparsing_unicode, "עִברִית", pyparsing_unicode.Hebrew) + setattr(pyparsing_unicode, "日本語", pyparsing_unicode.Japanese) + setattr(pyparsing_unicode.Japanese, "漢字", pyparsing_unicode.Japanese.Kanji) + setattr(pyparsing_unicode.Japanese, "カタカナ", pyparsing_unicode.Japanese.Katakana) + setattr(pyparsing_unicode.Japanese, "ひらがな", pyparsing_unicode.Japanese.Hiragana) + setattr(pyparsing_unicode, "한국어", pyparsing_unicode.Korean) + setattr(pyparsing_unicode, "ไทย", pyparsing_unicode.Thai) + setattr(pyparsing_unicode, "देवनागरी", pyparsing_unicode.Devanagari) + + if __name__ == "__main__": selectToken = CaselessLiteral("select") @@ -5712,7 +6393,7 @@ if __name__ == "__main__": tableName = delimitedList(ident, ".", combine=True).setParseAction(upcaseTokens) tableNameList = Group(delimitedList(tableName)).setName("tables") - + simpleSQL = selectToken("command") + columnSpec("columns") + fromToken + tableNameList("tables") # demo runTests method, including embedded comments in test string diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index 8e12a198..f3a981bd 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,16 +1,19 @@ -from __future__ import print_function, absolute_import - -__version__ = '1.1.10' +from __future__ import absolute_import, print_function # Add NullHandler to "pythonfinder" logger, because Python2's default root # logger has no handler and warnings like this would be reported: # # > No handlers could be found for logger "pythonfinder.models.pyenv" import logging + +from .exceptions import InvalidPythonVersion +from .models import SystemPath, WindowsFinder +from .pythonfinder import Finder + +__version__ = "1.2.0" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] -from .pythonfinder import Finder -from .models import SystemPath, WindowsFinder -from .exceptions import InvalidPythonVersion diff --git a/pipenv/vendor/pythonfinder/__main__.py b/pipenv/vendor/pythonfinder/__main__.py index c804d573..3083e72d 100644 --- a/pipenv/vendor/pythonfinder/__main__.py +++ b/pipenv/vendor/pythonfinder/__main__.py @@ -1,12 +1,17 @@ +#!env python +# -*- coding=utf-8 -*- + from __future__ import absolute_import import os import sys +from pythonfinder.cli import cli + + PYTHONFINDER_MAIN = os.path.dirname(os.path.abspath(__file__)) PYTHONFINDER_PACKAGE = os.path.dirname(PYTHONFINDER_MAIN) -from pythonfinder import cli as cli if __name__ == "__main__": sys.exit(cli()) diff --git a/pipenv/vendor/pythonfinder/cli.py b/pipenv/vendor/pythonfinder/cli.py index 221cb2fd..eb2e603a 100644 --- a/pipenv/vendor/pythonfinder/cli.py +++ b/pipenv/vendor/pythonfinder/cli.py @@ -1,9 +1,11 @@ -#!/usr/bin/env python # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function, unicode_literals + +import sys + import click import crayons -import sys + from . import __version__ from .pythonfinder import Finder @@ -11,16 +13,22 @@ from .pythonfinder import Finder @click.command() @click.option("--find", default=False, nargs=1, help="Find a specific python version.") @click.option("--which", default=False, nargs=1, help="Run the which command.") -@click.option( - "--findall", is_flag=True, default=False, help="Find all python versions." -) +@click.option("--findall", is_flag=True, default=False, help="Find all python versions.") @click.option( "--version", is_flag=True, default=False, help="Display PythonFinder version." ) -@click.option("--ignore-unsupported/--no-unsupported", is_flag=True, default=True, envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", help="Ignore unsupported python versions.") -@click.version_option(prog_name='pyfinder', version=__version__) +@click.option( + "--ignore-unsupported/--no-unsupported", + is_flag=True, + default=True, + envvar="PYTHONFINDER_IGNORE_UNSUPPORTED", + help="Ignore unsupported python versions.", +) +@click.version_option(prog_name="pyfinder", version=__version__) @click.pass_context -def cli(ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True): +def cli( + ctx, find=False, which=False, findall=False, version=False, ignore_unsupported=True +): if version: click.echo( "{0} version {1}".format( diff --git a/pipenv/vendor/pythonfinder/environment.py b/pipenv/vendor/pythonfinder/environment.py index e8878403..eb000438 100644 --- a/pipenv/vendor/pythonfinder/environment.py +++ b/pipenv/vendor/pythonfinder/environment.py @@ -1,5 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function + import os import platform import sys @@ -34,3 +35,15 @@ else: IGNORE_UNSUPPORTED = bool(os.environ.get("PYTHONFINDER_IGNORE_UNSUPPORTED", False)) MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) + + +def get_shim_paths(): + shim_paths = [] + if ASDF_INSTALLED: + shim_paths.append(os.path.join(ASDF_DATA_DIR, "shims")) + if PYENV_INSTALLED: + shim_paths.append(os.path.join(PYENV_ROOT, "shims")) + return [os.path.normpath(os.path.normcase(p)) for p in shim_paths] + + +SHIM_PATHS = get_shim_paths() diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index df381daf..adfac16b 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import +from __future__ import absolute_import, print_function class InvalidPythonVersion(Exception): diff --git a/pipenv/vendor/pythonfinder/models/mixins.py b/pipenv/vendor/pythonfinder/models/mixins.py index 7d406548..c1e7312a 100644 --- a/pipenv/vendor/pythonfinder/models/mixins.py +++ b/pipenv/vendor/pythonfinder/models/mixins.py @@ -2,16 +2,62 @@ from __future__ import absolute_import, unicode_literals import abc -import attr import operator -import six +from collections import defaultdict -from ..utils import ensure_path, KNOWN_EXTS, unnest +import attr +import six +from cached_property import cached_property +from vistir.compat import fs_str + +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ( + KNOWN_EXTS, + Sequence, + expand_paths, + looks_like_python, + path_is_known_executable, +) + +if MYPY_RUNNING: + from .path import PathEntry + from .python import PythonVersion + from typing import ( + Optional, + Union, + Any, + Dict, + Iterator, + List, + DefaultDict, + Generator, + Tuple, + TypeVar, + Type, + ) + from vistir.compat import Path + + BaseFinderType = TypeVar("BaseFinderType") @attr.s class BasePath(object): + path = attr.ib(default=None) # type: Path + _children = attr.ib(default=attr.Factory(dict)) # type: Dict[str, PathEntry] + only_python = attr.ib(default=False) # type: bool + name = attr.ib(type=str) + _py_version = attr.ib(default=None) # type: Optional[PythonVersion] + _pythons = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, PathEntry] + + def __str__(self): + # type: () -> str + return fs_str("{0}".format(self.path.as_posix())) + def which(self, name): + # type: (str) -> Optional[PathEntry] """Search in this path for an executable. :param executable: The name of an executable to search for. @@ -24,26 +70,166 @@ class BasePath(object): for ext in KNOWN_EXTS ] children = self.children - found = next( - ( - children[(self.path / child).as_posix()] - for child in valid_names - if (self.path / child).as_posix() in children - ), - None, - ) + found = None + if self.path is not None: + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found + def __del__(self): + for key in ["as_python", "is_dir", "is_python", "is_executable", "py_version"]: + if key in self.__dict__: + del self.__dict__[key] + self._children = {} + for key in list(self._pythons.keys()): + del self._pythons[key] + + @property + def children(self): + # type: () -> Dict[str, PathEntry] + if not self.is_dir: + return {} + return self._children + + @cached_property + def as_python(self): + # type: () -> PythonVersion + py_version = None + if self.py_version: + return self.py_version + if not self.is_dir and self.is_python: + try: + from .python import PythonVersion + + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (ValueError, InvalidPythonVersion): + pass + if py_version is None: + pass + return py_version # type: ignore + + @name.default + def get_name(self): + # type: () -> Optional[str] + if self.path: + return self.path.name + return None + + @cached_property + def is_dir(self): + # type: () -> bool + if not self.path: + return False + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + return ret_val + + @cached_property + def is_executable(self): + # type: () -> bool + if not self.path: + return False + return path_is_known_executable(self.path) + + @cached_property + def is_python(self): + # type: () -> bool + if not self.path: + return False + return self.is_executable and (looks_like_python(self.path.name)) + + def get_py_version(self): + # type: () -> Optional[PythonVersion] + from ..environment import IGNORE_UNSUPPORTED + + if self.is_dir: + return None + if self.is_python: + py_version = None + from .python import PythonVersion + + try: + py_version = PythonVersion.from_path( # type: ignore + path=self, name=self.name + ) + except (InvalidPythonVersion, ValueError): + py_version = None + except Exception: + if not IGNORE_UNSUPPORTED: + raise + return py_version + return None + + @cached_property + def py_version(self): + # type: () -> Optional[PythonVersion] + if not self._py_version: + py_version = self.get_py_version() + self._py_version = py_version + else: + py_version = self._py_version + return py_version + + def _iter_pythons(self): + # type: () -> Iterator + if self.is_dir: + for entry in self.children.values(): + if entry is None: + continue + elif entry.is_dir: + for python in entry._iter_pythons(): + yield python + elif entry.is_python and entry.as_python is not None: + yield entry + elif self.is_python and self.as_python is not None: + yield self # type: ignore + + @property + def pythons(self): + # type: () -> DefaultDict[Union[str, Path], PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + def __iter__(self): + # type: () -> Iterator + for entry in self.children.values(): + yield entry + + def __next__(self): + # type: () -> Generator + return next(iter(self)) + + def next(self): + # type: () -> Generator + return self.__next__() + def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -58,35 +244,29 @@ class BasePath(object): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = ( - "find_all_python_versions" if self.is_dir else "find_python_version" - ) + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( - call_method, - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method, major, minor, patch, pre, dev, arch, name ) if not self.is_dir: return sub_finder(self) - path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + unnested = [sub_finder(path) for path in expand_paths(self)] version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + unnested = [p for p in unnested if p is not None and p.as_python is not None] + paths = sorted(unnested, key=version_sort, reverse=True) + return list(paths) def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -101,55 +281,62 @@ class BasePath(object): """ version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "matches", major, minor, patch, pre, dev, arch, python_name=name ) - is_py = operator.attrgetter("is_python") - py_version = operator.attrgetter("as_python") if not self.is_dir: if self.is_python and self.as_python and version_matcher(self.py_version): - return attr.evolve(self) - return - finder = ( - (child, child.as_python) - for child in unnest(self.pythons.values()) - if child.as_python - ) - py_filter = filter( - None, filter(lambda child: version_matcher(child[1]), finder) - ) - version_sort = operator.attrgetter("version_sort") - return next( - ( - c[0] - for c in sorted( - py_filter, key=lambda child: child[1].version_sort, reverse=True - ) - ), - None, - ) + return self # type: ignore + + matching_pythons = [ + [entry, entry.as_python.version_sort] + for entry in self._iter_pythons() + if ( + entry is not None + and entry.as_python is not None + and version_matcher(entry.py_version) + ) + ] + results = sorted(matching_pythons, key=operator.itemgetter(1, 0), reverse=True) + return next(iter(r[0] for r in results if r is not None), None) @six.add_metaclass(abc.ABCMeta) class BaseFinder(object): + def __init__(self): + #: Maps executable paths to PathEntries + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + self._versions = defaultdict(PathEntry) # type: Dict[Tuple, PathEntry] + def get_versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] """Return the available versions from the finder""" raise NotImplementedError @classmethod - def create(cls): + def create( + cls, *args, **kwargs # type: Type[BaseFinderType] # type: Any # type: Any + ): + # type: (...) -> BaseFinderType raise NotImplementedError @property def version_paths(self): - return self.versions.values() + # type: () -> Any + return self._versions.values() @property def expanded_paths(self): + # type: () -> Any return (p.paths.values() for p in self.version_paths) + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 221d892f..9e099b59 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -5,62 +5,125 @@ import copy import operator import os import sys - from collections import defaultdict from itertools import chain import attr import six - from cached_property import cached_property - from vistir.compat import Path, fs_str -from .mixins import BasePath -from ..environment import PYENV_INSTALLED, PYENV_ROOT, ASDF_INSTALLED, ASDF_DATA_DIR +from .mixins import BaseFinder, BasePath +from .python import PythonVersion +from ..environment import ( + ASDF_DATA_DIR, + ASDF_INSTALLED, + MYPY_RUNNING, + PYENV_INSTALLED, + PYENV_ROOT, + SHIM_PATHS, +) from ..exceptions import InvalidPythonVersion from ..utils import ( + Iterable, + Sequence, ensure_path, + expand_paths, filter_pythons, + is_in_path, looks_like_python, + normalize_path, optional_instance_of, + parse_asdf_version_order, + parse_pyenv_version_order, path_is_known_executable, unnest, - normalize_path, - parse_pyenv_version_order, - parse_asdf_version_order ) -from .python import PythonVersion +if MYPY_RUNNING: + from typing import ( + Optional, + Dict, + DefaultDict, + Iterator, + List, + Union, + Tuple, + Generator, + Callable, + Type, + Any, + TypeVar, + ) + from .python import PythonFinder + from .windows import WindowsFinder -ASDF_SHIM_PATH = normalize_path(os.path.join(ASDF_DATA_DIR, "shims")) -PYENV_SHIM_PATH = normalize_path(os.path.join(PYENV_ROOT, "shims")) -SHIM_PATHS = [ASDF_SHIM_PATH, PYENV_SHIM_PATH] + FinderType = TypeVar("FinderType", BaseFinder, PythonFinder, WindowsFinder) + ChildType = Union[PythonFinder, "PathEntry"] + PathType = Union[PythonFinder, "PathEntry"] @attr.s class SystemPath(object): global_search = attr.ib(default=True) - paths = attr.ib(default=attr.Factory(defaultdict)) - _executables = attr.ib(default=attr.Factory(list)) - _python_executables = attr.ib(default=attr.Factory(list)) - path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib(default=attr.Factory(defaultdict)) - only_python = attr.ib(default=False) - pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) - asdf_finder = attr.ib(default=None) - system = attr.ib(default=False) - _version_dict = attr.ib(default=attr.Factory(defaultdict)) - ignore_unsupported = attr.ib(default=False) + paths = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + _executables = attr.ib(default=attr.Factory(list)) # type: List[PathEntry] + _python_executables = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, PathEntry] + path_order = attr.ib(default=attr.Factory(list)) # type: List[str] + python_version_dict = attr.ib() # type: DefaultDict[Tuple, List[PythonVersion]] + only_python = attr.ib(default=False, type=bool) + pyenv_finder = attr.ib( + default=None, validator=optional_instance_of("PythonFinder") + ) # type: Optional[PythonFinder] + asdf_finder = attr.ib(default=None) # type: Optional[PythonFinder] + system = attr.ib(default=False, type=bool) + _version_dict = attr.ib( + default=attr.Factory(defaultdict) + ) # type: DefaultDict[Tuple, List[PathEntry]] + ignore_unsupported = attr.ib(default=False, type=bool) - __finders = attr.ib(default=attr.Factory(dict)) + __finders = attr.ib( + default=attr.Factory(dict) + ) # type: Dict[str, Union[WindowsFinder, PythonFinder]] def _register_finder(self, finder_name, finder): + # type: (str, Union[WindowsFinder, PythonFinder]) -> None if finder_name not in self.__finders: self.__finders[finder_name] = finder + def clear_caches(self): + for key in ["executables", "python_executables", "version_dict", "path_entries"]: + if key in self.__dict__: + del self.__dict__[key] + self._executables = [] + self._python_executables = {} + self.python_version_dict = defaultdict(list) + self._version_dict = defaultdict(list) + + def __del__(self): + self.clear_caches() + self.path_order = [] + self.pyenv_finder = None + self.asdf_finder = None + self.paths = defaultdict(PathEntry) + + @property + def finders(self): + # type: () -> List[str] + return [k for k in self.__finders.keys()] + + @python_version_dict.default + def create_python_version_dict(self): + # type: () -> DefaultDict[Tuple, List[PythonVersion]] + return defaultdict(list) + @cached_property def executables(self): + # type: () -> List[PathEntry] self.executables = [ p for p in chain(*(child.children.values() for child in self.paths.values())) @@ -70,6 +133,7 @@ class SystemPath(object): @cached_property def python_executables(self): + # type: () -> Dict[str, PathEntry] python_executables = {} for child in self.paths.values(): if child.pythons: @@ -82,31 +146,30 @@ class SystemPath(object): @cached_property def version_dict(self): - self._version_dict = defaultdict(list) + # type: () -> DefaultDict[Tuple, List[PathEntry]] + self._version_dict = defaultdict( + list + ) # type: DefaultDict[Tuple, List[PathEntry]] for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue - if type(entry).__name__ == "VersionPath": - for path in entry.paths.values(): - if path not in self._version_dict[version] and path.is_python: - self._version_dict[version].append(path) - continue - continue - elif entry not in self._version_dict[version] and entry.is_python: + if entry not in self._version_dict[version] and entry.is_python: self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python if not version: continue - version = version.version_tuple + if not isinstance(version, tuple): + version = version.version_tuple if version and entry not in self._version_dict[version]: self._version_dict[version].append(entry) return self._version_dict def __attrs_post_init__(self): + # type: () -> None #: slice in pyenv if not self.__class__ == SystemPath: return @@ -124,7 +187,7 @@ class SystemPath(object): if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) + self.paths[p] = self.get_path(p.joinpath(bin_dir)) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent @@ -136,35 +199,37 @@ class SystemPath(object): ) def _get_last_instance(self, path): + # type: (str) -> int reversed_paths = reversed(self.path_order) paths = [normalize_path(p) for p in reversed_paths] normalized_target = normalize_path(path) - last_instance = next( - iter(p for p in paths if normalized_target in p), None - ) - try: - path_index = self.path_order.index(last_instance) - except ValueError: - return + last_instance = next(iter(p for p in paths if normalized_target in p), None) + if last_instance is None: + raise ValueError("No instance found on path for target: {0!s}".format(path)) + path_index = self.path_order.index(last_instance) return path_index def _slice_in_paths(self, start_idx, paths): - before_path = self.path_order[: start_idx + 1] - after_path = self.path_order[start_idx + 2 :] - self.path_order = ( - before_path + [p.as_posix() for p in paths] + after_path - ) + # type: (int, List[Path]) -> None + before_path = [] # type: List[str] + after_path = [] # type: List[str] + if start_idx == 0: + after_path = self.path_order[:] + elif start_idx == -1: + before_path = self.path_order[:] + else: + before_path = self.path_order[: start_idx + 1] + after_path = self.path_order[start_idx + 2 :] + self.path_order = before_path + [p.as_posix() for p in paths] + after_path def _remove_path(self, path): + # type: (str) -> None path_copy = [p for p in reversed(self.path_order[:])] new_order = [] target = normalize_path(path) - path_map = { - normalize_path(pth): pth - for pth in self.paths.keys() - } + path_map = {normalize_path(pth): pth for pth in self.paths.keys()} if target in path_map: - del self.paths[path_map.get(target)] + del self.paths[path_map[target]] for current_path in path_copy: normalized = normalize_path(current_path) if normalized != target: @@ -173,41 +238,91 @@ class SystemPath(object): self.path_order = new_order def _setup_asdf(self): + # type: () -> None from .python import PythonFinder + + os_path = os.environ["PATH"].split(os.pathsep) self.asdf_finder = PythonFinder.create( - root=ASDF_DATA_DIR, ignore_unsupported=True, - sort_function=parse_asdf_version_order, version_glob_path="installs/python/*") - asdf_index = self._get_last_instance(ASDF_DATA_DIR) - if not asdf_index: + root=ASDF_DATA_DIR, + ignore_unsupported=True, + sort_function=parse_asdf_version_order, + version_glob_path="installs/python/*", + ) + asdf_index = None + try: + asdf_index = self._get_last_instance(ASDF_DATA_DIR) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if asdf_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return root_paths = [p for p in self.asdf_finder.roots] - self._slice_in_paths(asdf_index, root_paths) + self._slice_in_paths(asdf_index, [self.asdf_finder.root]) + self.paths[self.asdf_finder.root] = self.asdf_finder self.paths.update(self.asdf_finder.roots) self._remove_path(normalize_path(os.path.join(ASDF_DATA_DIR, "shims"))) self._register_finder("asdf", self.asdf_finder) + def reload_finder(self, finder_name): + # type: (str) -> None + if finder_name is None: + raise TypeError("Must pass a string as the name of the target finder") + finder_attr = "{0}_finder".format(finder_name) + setup_attr = "_setup_{0}".format(finder_name) + try: + current_finder = getattr(self, finder_attr) # type: Any + except AttributeError: + raise ValueError("Must pass a valid finder to reload.") + try: + setup_fn = getattr(self, setup_attr) + except AttributeError: + raise ValueError("Finder has no valid setup function: %s" % finder_name) + if current_finder is None: + # TODO: This is called 'reload', should we load a new finder for the first + # time here? lets just skip that for now to avoid unallowed finders + pass + if (finder_name == "pyenv" and not PYENV_INSTALLED) or ( + finder_name == "asdf" and not ASDF_INSTALLED + ): + # Don't allow loading of finders that aren't explicitly 'installed' as it were + pass + setattr(self, finder_attr, None) + if finder_name in self.__finders: + del self.__finders[finder_name] + setup_fn() + def _setup_pyenv(self): + # type: () -> None from .python import PythonFinder + os_path = os.environ["PATH"].split(os.pathsep) + self.pyenv_finder = PythonFinder.create( - root=PYENV_ROOT, sort_function=parse_pyenv_version_order, - version_glob_path="versions/*", ignore_unsupported=self.ignore_unsupported + root=PYENV_ROOT, + sort_function=parse_pyenv_version_order, + version_glob_path="versions/*", + ignore_unsupported=self.ignore_unsupported, ) - pyenv_index = self._get_last_instance(PYENV_ROOT) - if not pyenv_index: + pyenv_index = None + try: + pyenv_index = self._get_last_instance(PYENV_ROOT) + except ValueError: + pyenv_index = 0 if is_in_path(next(iter(os_path), ""), PYENV_ROOT) else -1 + if pyenv_index is None: # we are in a virtualenv without global pyenv on the path, so we should # not write pyenv to the path here return - root_paths = [p for p in self.pyenv_finder.roots] - self._slice_in_paths(pyenv_index, root_paths) + root_paths = [p for p in self.pyenv_finder.roots] + self._slice_in_paths(pyenv_index, [self.pyenv_finder.root]) + self.paths[self.pyenv_finder.root] = self.pyenv_finder self.paths.update(self.pyenv_finder.roots) self._remove_path(os.path.join(PYENV_ROOT, "shims")) self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): + # type: () -> None from .windows import WindowsFinder self.windows_finder = WindowsFinder.create() @@ -218,6 +333,9 @@ class SystemPath(object): self._register_finder("windows", self.windows_finder) def get_path(self, path): + # type: (Union[str, Path]) -> PathType + if path is None: + raise TypeError("A path must be provided in order to generate a path entry.") path = ensure_path(path) _path = self.paths.get(path) if not _path: @@ -227,69 +345,90 @@ class SystemPath(object): path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path + if not _path: + raise ValueError("Path not found or generated: {0!r}".format(path)) return _path def _get_paths(self): - return (self.get_path(k) for k in self.path_order) + # type: () -> Iterator + for path in self.path_order: + try: + entry = self.get_path(path) + except ValueError: + continue + else: + yield entry @cached_property def path_entries(self): - paths = self._get_paths() + # type: () -> List[Union[PathEntry, FinderType]] + paths = list(self._get_paths()) return paths def find_all(self, executable): - """Search the path for an executable. Return all copies. + # type: (str) -> List[Union[PathEntry, FinderType]] + """ + Search the path for an executable. Return all copies. :param executable: Name of the executable :type executable: str :returns: List[PathEntry] """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return list(filtered) def which(self, executable): - """Search for an executable on the path. + # type: (str) -> Union[PathEntry, None] + """ + Search for an executable on the path. :param executable: Name of the executable to be located. :type executable: str :returns: :class:`~pythonfinder.models.PathEntry` object. """ - sub_which = operator.methodcaller("which", name=executable) + + sub_which = operator.methodcaller("which", executable) filtered = (sub_which(self.get_path(k)) for k in self.path_order) return next(iter(f for f in filtered if f is not None), None) def _filter_paths(self, finder): - return ( - pth for pth in unnest(finder(p) for p in self.path_entries if p is not None) - if pth is not None - ) + # type: (Callable) -> Iterator + for path in self._get_paths(): + if path is None: + continue + python_versions = finder(path) + if python_versions is not None: + for python in python_versions: + if python is not None: + yield python def _get_all_pythons(self, finder): - paths = {p.path.as_posix(): p for p in self._filter_paths(finder)} - paths.update(self.python_executables) - return (p for p in paths.values() if p is not None) + # type: (Callable) -> Iterator + for python in self._filter_paths(finder): + if python is not None and python.is_python: + yield python def get_pythons(self, finder): + # type: (Callable) -> Iterator sort_key = operator.attrgetter("as_python.version_sort") - return ( - k for k in sorted( - (p for p in self._filter_paths(finder) if p.is_python), - key=sort_key, - reverse=True - ) if k is not None - ) + pythons = [entry for entry in self._get_all_pythons(finder)] + for python in sorted(pythons, key=sort_key, reverse=True): + if python is not None: + yield python def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -305,21 +444,12 @@ class SystemPath(object): """ sub_finder = operator.methodcaller( - "find_all_python_versions", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_all_python_versions", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None if major and not (minor or patch or pre or dev or arch or name): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, major ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -332,14 +462,15 @@ class SystemPath(object): def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[Union[str, int]] + patch=None, # type: Optional[Union[str, int]] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> PathEntry """Search for a specific python version on the path. :param major: Major python version to search for. @@ -356,33 +487,30 @@ class SystemPath(object): if isinstance(major, six.string_types) and not minor and not patch: # Only proceed if this is in the format "x.y.z" or similar - if major.count(".") > 0 and major[0].isdigit(): + if major.isdigit() or (major.count(".") > 0 and major[0].isdigit()): version = major.split(".", 2) - if len(version) > 3: - major, minor, patch, rest = version - elif len(version) == 3: - major, minor, patch = version + if isinstance(version, (tuple, list)): + if len(version) > 3: + major, minor, patch, rest = version + elif len(version) == 3: + major, minor, patch = version + elif len(version) == 2: + major, minor = version + else: + major = major[0] else: - major, minor = version + major = major + name = None else: name = "{0!s}".format(major) major = None sub_finder = operator.methodcaller( - "find_python_version", - major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + "find_python_version", major, minor, patch, pre, dev, arch, name ) alternate_sub_finder = None - if major and not (minor or patch or pre or dev or arch or name): + if name and not (minor or patch or pre or dev or arch or major): alternate_sub_finder = operator.methodcaller( - "find_all_python_versions", - major=None, - name=major + "find_all_python_versions", None, None, None, None, None, None, name ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -406,12 +534,13 @@ class SystemPath(object): @classmethod def create( cls, - path=None, - system=False, - only_python=False, - global_search=True, - ignore_unsupported=True, + path=None, # type: str + system=False, # type: bool + only_python=False, # type: bool + global_search=True, # type: bool + ignore_unsupported=True, # type: bool ): + # type: (...) -> SystemPath """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -423,14 +552,18 @@ class SystemPath(object): :rtype: :class:`pythonfinder.models.SystemPath` """ - path_entries = defaultdict(PathEntry) - paths = [] + path_entries = defaultdict( + PathEntry + ) # type: DefaultDict[str, Union[PythonFinder, PathEntry]] + paths = [] # type: List[str] if ignore_unsupported: os.environ["PYTHONFINDER_IGNORE_UNSUPPORTED"] = fs_str("1") if global_search: - paths = os.environ.get("PATH").split(os.pathsep) + if "PATH" in os.environ: + paths = os.environ["PATH"].split(os.pathsep) if path: paths = [path] + paths + paths = [p for p in paths if not any(is_in_path(p, shim) for shim in SHIM_PATHS)] _path_objects = [ensure_path(p.strip('"')) for p in paths] paths = [p.as_posix() for p in _path_objects] path_entries.update( @@ -439,7 +572,6 @@ class SystemPath(object): path=p.absolute(), is_root=True, only_python=only_python ) for p in _path_objects - if not any(shim in normalize_path(str(p)) for shim in SHIM_PATHS) } ) return cls( @@ -454,18 +586,15 @@ class SystemPath(object): @attr.s(slots=True) class PathEntry(BasePath): - path = attr.ib(default=None, validator=optional_instance_of(Path)) - _children = attr.ib(default=attr.Factory(dict)) - is_root = attr.ib(default=True) - only_python = attr.ib(default=False) - name = attr.ib() - py_version = attr.ib() - _pythons = attr.ib(default=attr.Factory(defaultdict)) + is_root = attr.ib(default=True, type=bool) - def __str__(self): - return fs_str("{0}".format(self.path.as_posix())) + def __del__(self): + if "_children" in self.__dict__: + del self.__dict__["_children"] + BasePath.__del__(self) def _filter_children(self): + # type: () -> Iterator[Path] if self.only_python: children = filter_pythons(self.path) else: @@ -473,86 +602,47 @@ class PathEntry(BasePath): return children def _gen_children(self): + # type: () -> Iterator + from ..environment import get_shim_paths + + shim_paths = get_shim_paths() pass_name = self.name != self.path.name pass_args = {"is_root": False, "only_python": self.only_python} if pass_name: - pass_args["name"] = self.name + if self.name is not None and isinstance(self.name, six.string_types): + pass_args["name"] = self.name # type: ignore + elif self.path is not None and isinstance(self.path.name, six.string_types): + pass_args["name"] = self.path.name # type: ignore if not self.is_dir: - yield (self.path.as_posix(), copy.deepcopy(self)) + yield (self.path.as_posix(), self) elif self.is_root: for child in self._filter_children(): - if any(shim in normalize_path(str(child)) for shim in SHIM_PATHS): + if any(is_in_path(str(child), shim) for shim in shim_paths): continue if self.only_python: try: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore except (InvalidPythonVersion, ValueError): continue else: - entry = PathEntry.create(path=child, **pass_args) + entry = PathEntry.create(path=child, **pass_args) # type: ignore yield (child.as_posix(), entry) return @cached_property def children(self): - if not self._children: - children = {} + # type: () -> Dict[str, PathEntry] + children = getattr(self, "_children", {}) # type: Dict[str, PathEntry] + if not children: for child_key, child_val in self._gen_children(): children[child_key] = child_val self._children = children return self._children - @name.default - def get_name(self): - return self.path.name - - @py_version.default - def get_py_version(self): - from ..environment import IGNORE_UNSUPPORTED - if self.is_dir: - return None - if self.is_python: - py_version = None - try: - py_version = PythonVersion.from_path(path=self, name=self.name) - except (InvalidPythonVersion, ValueError): - py_version = None - except Exception: - if not IGNORE_UNSUPPORTED: - raise - return py_version - return - - @property - def pythons(self): - if not self._pythons: - if self.is_dir: - for path, entry in self.children.items(): - _path = ensure_path(entry.path) - if entry.is_python: - self._pythons[_path.as_posix()] = entry - else: - if self.is_python: - _path = ensure_path(self.path) - self._pythons[_path.as_posix()] = self - return self._pythons - - @cached_property - def as_python(self): - py_version = None - if self.py_version: - return self.py_version - if not self.is_dir and self.is_python: - try: - from .python import PythonVersion - py_version = PythonVersion.from_path(path=attr.evolve(self), name=self.name) - except (ValueError, InvalidPythonVersion): - py_version = None - return py_version - @classmethod def create(cls, path, is_root=False, only_python=False, pythons=None, name=None): + # type: (Union[str, Path], bool, bool, Dict[str, PythonVersion], Optional[str]) -> PathEntry """Helper method for creating new :class:`pythonfinder.models.PathEntry` instances. :param str path: Path to the specified location. @@ -569,53 +659,35 @@ class PathEntry(BasePath): if not name: guessed_name = True name = target.name - creation_args = {"path": target, "is_root": is_root, "only_python": only_python, "name": name} + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python, + "name": name, + } if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) if pythons and only_python: children = {} - child_creation_args = { - "is_root": False, - "only_python": only_python - } + child_creation_args = {"is_root": False, "only_python": only_python} if not guessed_name: - child_creation_args["name"] = name + child_creation_args["name"] = _new.name # type: ignore for pth, python in pythons.items(): if any(shim in normalize_path(str(pth)) for shim in SHIM_PATHS): continue pth = ensure_path(pth) - children[pth.as_posix()] = PathEntry( - py_version=python, - path=pth, - **child_creation_args + children[pth.as_posix()] = PathEntry( # type: ignore + py_version=python, path=pth, **child_creation_args ) _new._children = children return _new - @cached_property - def is_dir(self): - try: - ret_val = self.path.is_dir() - except OSError: - ret_val = False - return ret_val - - @cached_property - def is_executable(self): - return path_is_known_executable(self.path) - - @cached_property - def is_python(self): - return self.is_executable and ( - looks_like_python(self.path.name) - ) - @attr.s class VersionPath(SystemPath): - base = attr.ib(default=None, validator=optional_instance_of(Path)) - name = attr.ib(default=None) + base = attr.ib(default=None, validator=optional_instance_of(Path)) # type: Path + name = attr.ib(default=None) # type: str @classmethod def create(cls, path, only_python=True, pythons=None, name=None): @@ -623,6 +695,7 @@ class VersionPath(SystemPath): Generates the version listings for it""" from .path import PathEntry + path = ensure_path(path) path_entries = defaultdict(PathEntry) bin_ = "{base}/bin" diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4fcbbca6..25a12d66 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,76 +2,117 @@ from __future__ import absolute_import, print_function import copy -import platform -import operator import logging - +import operator +import platform +import sys from collections import defaultdict import attr +import six +from packaging.version import Version +from vistir.compat import Path, lru_cache -from packaging.version import Version, LegacyVersion -from packaging.version import parse as parse_version -from vistir.compat import Path - -from ..environment import SYSTEM_ARCH, PYENV_ROOT, ASDF_DATA_DIR -from ..exceptions import InvalidPythonVersion from .mixins import BaseFinder, BasePath +from ..environment import ASDF_DATA_DIR, MYPY_RUNNING, PYENV_ROOT, SYSTEM_ARCH +from ..exceptions import InvalidPythonVersion from ..utils import ( + RE_MATCHER, _filter_none, ensure_path, get_python_version, - optional_instance_of, - unnest, is_in_path, - parse_pyenv_version_order, + looks_like_python, + optional_instance_of, parse_asdf_version_order, + parse_pyenv_version_order, parse_python_version, + unnest, ) +if MYPY_RUNNING: + from typing import ( + DefaultDict, + Optional, + Callable, + Generator, + Any, + Union, + Tuple, + List, + Dict, + Type, + TypeVar, + Iterator, + ) + from .path import PathEntry + from .._vendor.pep514tools.environment import Environment + + logger = logging.getLogger(__name__) @attr.s(slots=True) class PythonFinder(BaseFinder, BasePath): - root = attr.ib(default=None, validator=optional_instance_of(Path)) - #: ignore_unsupported should come before versions, because its value is used - #: in versions's default initializer. - ignore_unsupported = attr.ib(default=True) - #: The function to use to sort version order when returning an ordered verion set - sort_function = attr.ib(default=None) - paths = attr.ib(default=attr.Factory(list)) - roots = attr.ib(default=attr.Factory(defaultdict)) + root = attr.ib(default=None, validator=optional_instance_of(Path), type=Path) + # should come before versions, because its value is used in versions's default initializer. + #: Whether to ignore any paths which raise exceptions and are not actually python + ignore_unsupported = attr.ib(default=True, type=bool) #: Glob path for python versions off of the root directory - version_glob_path = attr.ib(default="versions/*") - versions = attr.ib() - pythons = attr.ib() + version_glob_path = attr.ib(default="versions/*", type=str) + #: The function to use to sort version order when returning an ordered verion set + sort_function = attr.ib(default=None) # type: Callable + #: The root locations used for discovery + roots = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + #: List of paths discovered during search + paths = attr.ib(type=list) + #: shim directory + shim_dir = attr.ib(default="shims", type=str) + #: Versions discovered in the specified paths + _versions = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + _pythons = attr.ib(default=attr.Factory(defaultdict), type=defaultdict) + + def __del__(self): + # type: () -> None + self._versions = defaultdict() + self._pythons = defaultdict() + self.roots = defaultdict() + self.paths = [] @property def expanded_paths(self): + # type: () -> Generator return ( - path for path in unnest(p for p in self.versions.values()) - if path is not None + path for path in unnest(p for p in self.versions.values()) if path is not None ) @property def is_pyenv(self): + # type: () -> bool return is_in_path(str(self.root), PYENV_ROOT) @property def is_asdf(self): + # type: () -> bool return is_in_path(str(self.root), ASDF_DATA_DIR) def get_version_order(self): + # type: () -> List[Path] version_paths = [ - p for p in self.root.glob(self.version_glob_path) + p + for p in self.root.glob(self.version_glob_path) if not (p.parent.name == "envs" or p.name == "envs") ] versions = {v.name: v for v in version_paths} + version_order = [] # type: List[Path] if self.is_pyenv: - version_order = [versions[v] for v in parse_pyenv_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_pyenv_version_order() if v in versions + ] elif self.is_asdf: - version_order = [versions[v] for v in parse_asdf_version_order() if v in versions] + version_order = [ + versions[v] for v in parse_asdf_version_order() if v in versions + ] for version in version_order: version_paths.remove(version) if version_order: @@ -80,91 +121,140 @@ class PythonFinder(BaseFinder, BasePath): version_order = version_paths return version_order + def get_bin_dir(self, base): + # type: (Union[Path, str]) -> Path + if isinstance(base, six.string_types): + base = Path(base) + return base / "bin" + @classmethod - def version_from_bin_dir(cls, base_dir, name=None): - from .path import PathEntry + def version_from_bin_dir(cls, entry): + # type: (PathEntry) -> Optional[PathEntry] py_version = None - version_path = PathEntry.create( - path=base_dir.absolute().as_posix(), - only_python=True, - name=base_dir.parent.name, - ) - py_version = next(iter(version_path.find_all_python_versions()), None) + py_version = next(iter(entry.find_all_python_versions()), None) return py_version - @versions.default - def get_versions(self): + def _iter_version_bases(self): + # type: () -> Iterator[Tuple[Path, PathEntry]] from .path import PathEntry - versions = defaultdict() - bin_ = "{base}/bin" + for p in self.get_version_order(): - bin_dir = Path(bin_.format(base=p.as_posix())) - version_path = None - if bin_dir.exists(): - version_path = PathEntry.create( - path=bin_dir.absolute().as_posix(), - only_python=False, - name=p.name, - is_root=True, + bin_dir = self.get_bin_dir(p) + if bin_dir.exists() and bin_dir.is_dir(): + entry = PathEntry.create( + path=bin_dir.absolute(), only_python=False, name=p.name, is_root=True ) + self.roots[p] = entry + yield (p, entry) + + def _iter_versions(self): + # type: () -> Iterator[Tuple[Path, PathEntry, Tuple]] + for base_path, entry in self._iter_version_bases(): version = None + version_entry = None try: - version = PythonVersion.parse(p.name) + version = PythonVersion.parse(entry.name) except (ValueError, InvalidPythonVersion): - entry = next(iter(version_path.find_all_python_versions()), None) - if not entry: - if self.ignore_unsupported: - continue - raise - else: - version = entry.py_version.as_dict() + version_entry = next(iter(entry.find_all_python_versions()), None) + if version is None: + if not self.ignore_unsupported: + raise + continue + if version_entry is not None: + version = version_entry.py_version.as_dict() except Exception: if not self.ignore_unsupported: raise logger.warning( - "Unsupported Python version %r, ignoring...", p.name, exc_info=True + "Unsupported Python version %r, ignoring...", + base_path.name, + exc_info=True, ) continue - if not version: - continue - version_tuple = ( - version.get("major"), - version.get("minor"), - version.get("patch"), - version.get("is_prerelease"), - version.get("is_devrelease"), - version.get("is_debug"), - ) - self.roots[p] = version_path - versions[version_tuple] = version_path - self.paths.append(version_path) - return versions + if version is not None: + version_tuple = ( + version.get("major"), + version.get("minor"), + version.get("patch"), + version.get("is_prerelease"), + version.get("is_devrelease"), + version.get("is_debug"), + ) + yield (base_path, entry, version_tuple) + + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + for base_path, entry, version_tuple in self._iter_versions(): + self._versions[version_tuple] = entry + return self._versions + + def _iter_pythons(self): + # type: () -> Iterator + for path, entry, version_tuple in self._iter_versions(): + if path.as_posix() in self._pythons: + yield self._pythons[path.as_posix()] + elif version_tuple not in self.versions: + for python in entry.find_all_python_versions(): + yield python + else: + yield self.versions[version_tuple] + + @paths.default + def get_paths(self): + # type: () -> List[PathEntry] + _paths = [base for _, base in self._iter_version_bases()] + return _paths + + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + if not self._pythons: + from .path import PathEntry + + self._pythons = defaultdict(PathEntry) # type: DefaultDict[str, PathEntry] + for python in self._iter_pythons(): + python_path = python.path.as_posix() # type: ignore + self._pythons[python_path] = python + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value - @pythons.default def get_pythons(self): - pythons = defaultdict() - for p in self.paths: - pythons.update(p.pythons) - return pythons + # type: () -> DefaultDict[str, PathEntry] + return self.pythons @classmethod - def create(cls, root, sort_function=None, version_glob_path=None, ignore_unsupported=True): + def create( + cls, root, sort_function, version_glob_path=None, ignore_unsupported=True + ): # type: ignore + # type: (Type[PythonFinder], str, Callable, Optional[str], bool) -> PythonFinder root = ensure_path(root) if not version_glob_path: version_glob_path = "versions/*" - return cls(root=root, ignore_unsupported=ignore_unsupported, - sort_function=sort_function, version_glob_path=version_glob_path) + return cls( + root=root, + path=root, + ignore_unsupported=ignore_unsupported, # type: ignore + sort_function=sort_function, + version_glob_path=version_glob_path, + ) def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> List[PathEntry] """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -179,36 +269,37 @@ class PythonFinder(BaseFinder, BasePath): :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + sub_finder = operator.methodcaller( + call_method, major, minor, patch, pre, dev, arch, name ) - py = operator.attrgetter("as_python") - pythons = ( - py_ver for py_ver in (py(p) for p in self.pythons.values() if p is not None) - if py_ver is not None - ) - # pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return sorted(matching_versions, key=version_sort, reverse=True) + if not any([major, minor, patch, name]): + pythons = [ + next(iter(py for py in base.find_all_python_versions()), None) + for _, base in self._iter_version_bases() + ] + else: + pythons = [sub_finder(path) for path in self.paths] + pythons = [p for p in pythons if p and p.is_python and p.as_python is not None] + version_sort = operator.attrgetter("as_python.version_sort") + paths = [ + p + for p in sorted(list(pythons), key=version_sort, reverse=True) + if p is not None + ] + return paths def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -222,40 +313,75 @@ class PythonFinder(BaseFinder, BasePath): :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ - version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, + sub_finder = operator.methodcaller( + "find_python_version", major, minor, patch, pre, dev, arch, name ) - pythons = filter(None, [p.as_python for p in self.pythons.values()]) - matching_versions = filter(lambda py: version_matcher(py), pythons) - version_sort = operator.attrgetter("version_sort") - return next(iter(c for c in sorted(matching_versions, key=version_sort, reverse=True)), None) + version_sort = operator.attrgetter("as_python.version_sort") + unnested = [sub_finder(self.roots[path]) for path in self.roots] + unnested = [ + p + for p in unnested + if p is not None and p.is_python and p.as_python is not None + ] + paths = sorted(list(unnested), key=version_sort, reverse=True) + return next(iter(p for p in paths if p is not None), None) + + def which(self, name): + # type: (str) -> Optional[PathEntry] + """Search in this path for an executable. + + :param executable: The name of an executable to search for. + :type executable: str + :returns: :class:`~pythonfinder.models.PathEntry` instance. + """ + + matches = (p.which(name) for p in self.paths) + non_empty_match = next(iter(m for m in matches if m is not None), None) + return non_empty_match @attr.s(slots=True) class PythonVersion(object): - major = attr.ib(default=0) - minor = attr.ib(default=None) - patch = attr.ib(default=0) - is_prerelease = attr.ib(default=False) - is_postrelease = attr.ib(default=False) - is_devrelease = attr.ib(default=False) - is_debug = attr.ib(default=False) - version = attr.ib(default=None) - architecture = attr.ib(default=None) - comes_from = attr.ib(default=None) - executable = attr.ib(default=None) - name = attr.ib(default=None) + major = attr.ib(default=0, type=int) + minor = attr.ib(default=None) # type: Optional[int] + patch = attr.ib(default=None) # type: Optional[int] + is_prerelease = attr.ib(default=False, type=bool) + is_postrelease = attr.ib(default=False, type=bool) + is_devrelease = attr.ib(default=False, type=bool) + is_debug = attr.ib(default=False, type=bool) + version = attr.ib(default=None) # type: Version + architecture = attr.ib(default=None) # type: Optional[str] + comes_from = attr.ib(default=None) # type: Optional[PathEntry] + executable = attr.ib(default=None) # type: Optional[str] + name = attr.ib(default=None, type=str) + + def __getattribute__(self, key): + result = super(PythonVersion, self).__getattribute__(key) + if key in ["minor", "patch"] and result is None: + executable = None # type: Optional[str] + if self.executable: + executable = self.executable + elif self.comes_from: + executable = self.comes_from.path.as_posix() + if executable is not None: + if not isinstance(executable, six.string_types): + executable = executable.as_posix() + instance_dict = self.parse_executable(executable) + for k in instance_dict.keys(): + try: + super(PythonVersion, self).__getattribute__(k) + except AttributeError: + continue + else: + setattr(self, k, instance_dict[k]) + result = instance_dict.get(key) + return result @property def version_sort(self): - """version_sort tuple for sorting against other instances of the same class. + # type: () -> Tuple[Optional[int], Optional[int], int, int] + """ + A tuple for sorting against other instances of the same class. Returns a tuple of the python version but includes a point for non-dev, and a point for non-prerelease versions. So released versions will have 2 points @@ -275,7 +401,9 @@ class PythonVersion(object): @property def version_tuple(self): - """Provides a version tuple for using as a dictionary key. + # type: () -> Tuple[int, Optional[int], Optional[int], bool, bool, bool] + """ + Provides a version tuple for using as a dictionary key. :return: A tuple describing the python version meetadata contained. :rtype: tuple @@ -292,45 +420,52 @@ class PythonVersion(object): def matches( self, - major=None, - minor=None, - patch=None, - pre=False, - dev=False, - arch=None, - debug=False, - name=None, + major=None, # type: Optional[int] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=False, # type: bool + dev=False, # type: bool + arch=None, # type: Optional[str] + debug=False, # type: bool + python_name=None, # type: Optional[str] ): + # type: (...) -> bool + result = False if arch: own_arch = self.get_architecture() if arch.isdigit(): arch = "{0}bit".format(arch) - return ( - (major is None or self.major == major) - and (minor is None or self.minor == minor) - and (patch is None or self.patch == patch) + if ( + (major is None or self.major and self.major == major) + and (minor is None or self.minor and self.minor == minor) + and (patch is None or self.patch and self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) and (arch is None or own_arch == arch) and (debug is None or self.is_debug == debug) and ( - name is None - or (name and self.name) - and (self.name == name or self.name.startswith(name)) + python_name is None + or (python_name and self.name) + and (self.name == python_name or self.name.startswith(python_name)) ) - ) + ): + result = True + return result def as_major(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"minor": None, "patch": None}) return self.create(**self_dict) def as_minor(self): + # type: () -> PythonVersion self_dict = attr.asdict(self, recurse=False, filter=_filter_none).copy() self_dict.update({"patch": None}) return self.create(**self_dict) def as_dict(self): + # type: () -> Dict[str, Union[int, bool, Version, None]] return { "major": self.major, "minor": self.minor, @@ -342,35 +477,67 @@ class PythonVersion(object): "version": self.version, } + def update_metadata(self, metadata): + # type: (Dict[str, Union[str, int, Version]]) -> None + """ + Update the metadata on the current :class:`pythonfinder.models.python.PythonVersion` + + Given a parsed version dictionary from :func:`pythonfinder.utils.parse_python_version`, + update the instance variables of the current version instance to reflect the newly + supplied values. + """ + + for key in metadata: + try: + current_value = getattr(self, key) + except AttributeError: + continue + else: + setattr(self, key, metadata[key]) + @classmethod + @lru_cache(maxsize=1024) def parse(cls, version): - """Parse a valid version string into a dictionary + # type: (str) -> Dict[str, Union[str, int, Version]] + """ + Parse a valid version string into a dictionary Raises: ValueError -- Unable to parse version string ValueError -- Not a valid python version + TypeError -- NoneType or unparseable type passed in - :param version: A valid version string - :type version: str + :param str version: A valid version string :return: A dictionary with metadata about the specified python version. - :rtype: dict. + :rtype: dict """ + if version is None: + raise TypeError("Must pass a value to parse!") version_dict = parse_python_version(str(version)) if not version_dict: raise ValueError("Not a valid python version: %r" % version) return version_dict def get_architecture(self): + # type: () -> str if self.architecture: return self.architecture - arch, _ = platform.architecture(self.comes_from.path.as_posix()) + arch = None + if self.comes_from is not None: + arch, _ = platform.architecture(self.comes_from.path.as_posix()) + elif self.executable is not None: + arch, _ = platform.architecture(self.executable) + if arch is None: + arch, _ = platform.architecture(sys.executable) self.architecture = arch return self.architecture @classmethod def from_path(cls, path, name=None, ignore_unsupported=True): - """Parses a python version from a system path. + # type: (Union[str, PathEntry], Optional[str], bool) -> PythonVersion + """ + Parses a python version from a system path. Raises: ValueError -- Not a valid python path @@ -388,23 +555,56 @@ class PythonVersion(object): if not isinstance(path, PathEntry): path = PathEntry.create(path, is_root=False, only_python=True, name=name) from ..environment import IGNORE_UNSUPPORTED + ignore_unsupported = ignore_unsupported or IGNORE_UNSUPPORTED + path_name = getattr(path, "name", path.path.name) # str if not path.is_python: if not (ignore_unsupported or IGNORE_UNSUPPORTED): raise ValueError("Not a valid python path: %s" % path.path) - py_version = get_python_version(path.path.absolute().as_posix()) - instance_dict = cls.parse(py_version.strip()) - if not isinstance(instance_dict.get("version"), Version) and not ignore_unsupported: - raise ValueError("Not a valid python path: %s" % path.path) - if not name: - name = path.name + try: + instance_dict = cls.parse(path_name) + except Exception: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + else: + if instance_dict.get("minor") is None and looks_like_python(path.path.name): + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + + if ( + not isinstance(instance_dict.get("version"), Version) + and not ignore_unsupported + ): + raise ValueError("Not a valid python path: %s" % path) + if instance_dict.get("patch") is None: + instance_dict = cls.parse_executable(path.path.absolute().as_posix()) + if name is None: + name = path_name instance_dict.update( - {"comes_from": path, "name": name} + {"comes_from": path, "name": name, "executable": path.path.as_posix()} ) - return cls(**instance_dict) + return cls(**instance_dict) # type: ignore + + @classmethod + @lru_cache(maxsize=1024) + def parse_executable(cls, path): + # type: (str) -> Dict[str, Optional[Union[str, int, Version]]] + result_dict = {} # type: Dict[str, Optional[Union[str, int, Version]]] + result_version = None # type: Optional[str] + if path is None: + raise TypeError("Must pass a valid path to parse.") + if not isinstance(path, six.string_types): + path = path.as_posix() + try: + result_version = get_python_version(path) + except Exception: + raise ValueError("Not a valid python path: %r" % path) + if result_version is None: + raise ValueError("Not a valid python path: %s" % path) + result_dict = cls.parse(result_version.strip()) + return result_dict @classmethod def from_windows_launcher(cls, launcher_entry, name=None): + # type: (Environment, Optional[str]) -> PythonVersion """Create a new PythonVersion instance from a Windows Launcher Entry :param launcher_entry: A python launcher environment object. @@ -428,18 +628,18 @@ class PythonVersion(object): launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, - "name": name + "name": name, } ) py_version = cls.create(**creation_dict) comes_from = PathEntry.create(exe_path, only_python=True, name=name) - comes_from.py_version = copy.deepcopy(py_version) py_version.comes_from = comes_from py_version.name = comes_from.name return py_version @classmethod def create(cls, **kwargs): + # type: (...) -> PythonVersion if "architecture" in kwargs: if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) @@ -448,10 +648,13 @@ class PythonVersion(object): @attr.s class VersionMap(object): - versions = attr.ib(default=attr.Factory(defaultdict(list))) + versions = attr.ib( + factory=defaultdict + ) # type: DefaultDict[Tuple[int, Optional[int], Optional[int], bool, bool, bool], List[PathEntry]] def add_entry(self, entry): - version = entry.as_python + # type: (...) -> None + version = entry.as_python # type: PythonVersion if version: entries = self.versions[version.version_tuple] paths = {p.path for p in self.versions.get(version.version_tuple, [])} @@ -459,13 +662,18 @@ class VersionMap(object): self.versions[version.version_tuple].append(entry) def merge(self, target): + # type: (VersionMap) -> None for version, entries in target.versions.items(): if version not in self.versions: self.versions[version] = entries else: - current_entries = {p.path for p in self.versions.get(version)} + current_entries = { + p.path + for p in self.versions[version] # type: ignore + if version in self.versions + } new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append( + self.versions[version].extend( [e for e in entries if e.path in new_entries] ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f985630f..e96e1081 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -2,63 +2,61 @@ from __future__ import absolute_import, print_function import operator - from collections import defaultdict import attr -from ..exceptions import InvalidPythonVersion -from ..utils import ensure_path from .mixins import BaseFinder from .path import PathEntry from .python import PythonVersion, VersionMap +from ..environment import MYPY_RUNNING +from ..exceptions import InvalidPythonVersion +from ..utils import ensure_path + +if MYPY_RUNNING: + from typing import DefaultDict, Tuple, List, Optional, Union, TypeVar, Type, Any + + FinderType = TypeVar("FinderType") @attr.s class WindowsFinder(BaseFinder): - paths = attr.ib(default=attr.Factory(list)) - version_list = attr.ib(default=attr.Factory(list)) - versions = attr.ib() - pythons = attr.ib() + paths = attr.ib(default=attr.Factory(list), type=list) + version_list = attr.ib(default=attr.Factory(list), type=list) + _versions = attr.ib() # type: DefaultDict[Tuple, PathEntry] + _pythons = attr.ib() # type: DefaultDict[str, PathEntry] def find_all_python_versions( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type (...) -> List[PathEntry] version_matcher = operator.methodcaller( - "matches", - major=major, - minor=minor, - patch=patch, - pre=pre, - dev=dev, - arch=arch, - name=name, - ) - py_filter = filter( - None, filter(lambda c: version_matcher(c), self.version_list) + "matches", major, minor, patch, pre, dev, arch, python_name=name ) + pythons = [py for py in self.version_list if version_matcher(py)] version_sort = operator.attrgetter("version_sort") - return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] + return [c.comes_from for c in sorted(pythons, key=version_sort, reverse=True)] def find_python_version( self, - major=None, - minor=None, - patch=None, - pre=None, - dev=None, - arch=None, - name=None, + major=None, # type: Optional[Union[str, int]] + minor=None, # type: Optional[int] + patch=None, # type: Optional[int] + pre=None, # type: Optional[bool] + dev=None, # type: Optional[bool] + arch=None, # type: Optional[str] + name=None, # type: Optional[str] ): + # type: (...) -> Optional[PathEntry] return next( - ( + iter( v for v in self.find_all_python_versions( major=major, @@ -67,15 +65,16 @@ class WindowsFinder(BaseFinder): pre=pre, dev=dev, arch=arch, - name=None, + name=name, ) ), None, ) - @versions.default + @_versions.default def get_versions(self): - versions = defaultdict(PathEntry) + # type: () -> DefaultDict[Tuple, PathEntry] + versions = defaultdict(PathEntry) # type: DefaultDict[Tuple, PathEntry] from pythonfinder._vendor.pep514tools import environment as pep514env env_versions = pep514env.findall() @@ -92,25 +91,49 @@ class WindowsFinder(BaseFinder): py_version = PythonVersion.from_windows_launcher(version_object) except InvalidPythonVersion: continue + if py_version is None: + continue self.version_list.append(py_version) + python_path = ( + py_version.comes_from.path + if py_version.comes_from + else py_version.executable + ) + python_kwargs = {python_path: py_version} if python_path is not None else {} base_dir = PathEntry.create( - path, - is_root=True, - only_python=True, - pythons={py_version.comes_from.path: py_version}, + path, is_root=True, only_python=True, pythons=python_kwargs ) versions[py_version.version_tuple[:5]] = base_dir self.paths.append(base_dir) return versions - @pythons.default + @property + def versions(self): + # type: () -> DefaultDict[Tuple, PathEntry] + if not self._versions: + self._versions = self.get_versions() + return self._versions + + @_pythons.default def get_pythons(self): - pythons = defaultdict() + # type: () -> DefaultDict[str, PathEntry] + pythons = defaultdict() # type: DefaultDict[str, PathEntry] for version in self.version_list: _path = ensure_path(version.comes_from.path) pythons[_path.as_posix()] = version.comes_from return pythons + @property + def pythons(self): + # type: () -> DefaultDict[str, PathEntry] + return self._pythons + + @pythons.setter + def pythons(self, value): + # type: (DefaultDict[str, PathEntry]) -> None + self._pythons = value + @classmethod - def create(cls): + def create(cls, *args, **kwargs): + # type: (Type[FinderType], Any, Any) -> FinderType return cls() diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 011754ea..d5f38fb4 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -1,26 +1,46 @@ # -*- coding=utf-8 -*- -from __future__ import print_function, absolute_import -import os -import six +from __future__ import absolute_import, print_function + import operator -from .models import SystemPath +import os + +import six +from click import secho from vistir.compat import lru_cache +from . import environment +from .exceptions import InvalidPythonVersion +from .models import path as pyfinder_path +from .utils import Iterable, filter_pythons, version_re + +if environment.MYPY_RUNNING: + from typing import Optional, Dict, Any, Union, List, Iterator + from .models.path import Path, PathEntry + from .models.windows import WindowsFinder + from .models.path import SystemPath + class Finder(object): - def __init__(self, path=None, system=False, global_search=True, ignore_unsupported=True): - """ - Finder A cross-platform Finder for locating python and other executables. - Searches for python and other specified binaries starting in `path`, if supplied, - but searching the bin path of `sys.executable` if `system=True`, and then - searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` - is `False`, this search operation is restricted to the allowed locations of - `path` and `system`. + """ + A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in *path*, if supplied, + but searching the bin path of ``sys.executable`` if *system* is ``True``, and then + searching in the ``os.environ['PATH']`` if *global_search* is ``True``. When *global_search* + is ``False``, this search operation is restricted to the allowed locations of + *path* and *system*. + """ + + def __init__( + self, path=None, system=False, global_search=True, ignore_unsupported=True + ): + # type: (Optional[str], bool, bool, bool) -> None + """Create a new :class:`~pythonfinder.pythonfinder.Finder` instance. :param path: A bin-directory search location, defaults to None :param path: str, optional - :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: Whether to include the bin-dir of ``sys.executable``, defaults to False :param system: bool, optional :param global_search: Whether to search the global path from os.environ, defaults to True :param global_search: bool, optional @@ -29,34 +49,65 @@ class Finder(object): :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ - self.path_prepend = path - self.global_search = global_search - self.system = system - self.ignore_unsupported = ignore_unsupported - self._system_path = None - self._windows_finder = None + self.path_prepend = path # type: Optional[str] + self.global_search = global_search # type: bool + self.system = system # type: bool + self.ignore_unsupported = ignore_unsupported # type: bool + self._system_path = None # type: Optional[SystemPath] + self._windows_finder = None # type: Optional[WindowsFinder] def __hash__(self): + # type: () -> int return hash( (self.path_prepend, self.system, self.global_search, self.ignore_unsupported) ) def __eq__(self, other): + # type: (Any) -> bool return self.__hash__() == other.__hash__() + def create_system_path(self): + # type: () -> SystemPath + return pyfinder_path.SystemPath.create( + path=self.path_prepend, + system=self.system, + global_search=self.global_search, + ignore_unsupported=self.ignore_unsupported, + ) + + def reload_system_path(self): + # type: () -> None + """ + Rebuilds the base system path and all of the contained finders within it. + + This will re-apply any changes to the environment or any version changes on the system. + """ + + if self._system_path is not None: + self._system_path.clear_caches() + self._system_path = None + six.moves.reload_module(pyfinder_path) + self._system_path = self.create_system_path() + + def rehash(self): + # type: () -> None + if not self._system_path: + self._system_path = self.create_system_path() + self.find_all_python_versions.cache_clear() + self.find_python_version.cache_clear() + self.reload_system_path() + filter_pythons.cache_clear() + @property def system_path(self): - if not self._system_path: - self._system_path = SystemPath.create( - path=self.path_prepend, - system=self.system, - global_search=self.global_search, - ignore_unsupported=self.ignore_unsupported, - ) + # type: () -> SystemPath + if self._system_path is None: + self._system_path = self.create_system_path() return self._system_path @property def windows_finder(self): + # type: () -> Optional[WindowsFinder] if os.name == "nt" and not self._windows_finder: from .models import WindowsFinder @@ -64,14 +115,38 @@ class Finder(object): return self._windows_finder def which(self, exe): + # type: (str) -> Optional[PathEntry] return self.system_path.which(exe) @lru_cache(maxsize=1024) def find_python_version( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): + # type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> PathEntry + """ + Find the python version which corresponds most closely to the version requested. + + :param Union[str, int] major: The major version to look for, or the full version, or the name of the target version. + :param Optional[int] minor: The minor version. If provided, disables string-based lookups from the major version field. + :param Optional[int] patch: The patch version. + :param Optional[bool] pre: If provided, specifies whether to search pre-releases. + :param Optional[bool] dev: If provided, whether to search dev-releases. + :param Optional[str] arch: If provided, which architecture to search. + :param Optional[str] name: *Name* of the target python, e.g. ``anaconda3-5.3.0`` + :return: A new *PathEntry* pointer at a matching python version, if one can be located. + :rtype: :class:`pythonfinder.models.path.PathEntry` + """ + from .models import PythonVersion + minor = int(minor) if minor is not None else minor + patch = int(patch) if patch is not None else patch + + version_dict = { + "minor": minor, + "patch": patch, + } # type: Dict[str, Union[str, int, Any]] + if ( isinstance(major, six.string_types) and pre is None @@ -79,7 +154,7 @@ class Finder(object): and dev is None and patch is None ): - if arch is None and "-" in major: + if arch is None and "-" in major and major[0].isdigit(): orig_string = "{0!s}".format(major) major, _, arch = major.rpartition("-") if arch.startswith("x"): @@ -91,22 +166,55 @@ class Finder(object): arch = None else: arch = "{0}bit".format(arch) - try: - version_dict = PythonVersion.parse(major) - except ValueError: - if name is None: - name = "{0!s}".format(major) - major = None - version_dict = {} - major = version_dict.get("major", major) - minor = version_dict.get("minor", minor) - patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is None else pre - dev = version_dict.get("is_devrelease", dev) if dev is None else dev - arch = version_dict.get("architecture", arch) if arch is None else arch - if os.name == "nt": + try: + version_dict = PythonVersion.parse(major) + except (ValueError, InvalidPythonVersion): + if name is None: + name = "{0!s}".format(major) + major = None + version_dict = {} + elif major[0].isalpha(): + name = "%s" % major + major = None + else: + if "." in major and all(part.isdigit() for part in major.split(".")[:2]): + match = version_re.match(major) + version_dict = match.groupdict() + version_dict["is_prerelease"] = bool( + version_dict.get("prerel", False) + ) + version_dict["is_devrelease"] = bool(version_dict.get("dev", False)) + else: + version_dict = { + "major": major, + "minor": minor, + "patch": patch, + "pre": pre, + "dev": dev, + "arch": arch, + } + if version_dict.get("minor") is not None: + minor = int(version_dict["minor"]) + if version_dict.get("patch") is not None: + patch = int(version_dict["patch"]) + if version_dict.get("major") is not None: + major = int(version_dict["major"]) + _pre = version_dict.get("is_prerelease", pre) + pre = bool(_pre) if _pre is not None else pre + _dev = version_dict.get("is_devrelease", dev) + dev = bool(_dev) if _dev is not None else dev + arch = ( + version_dict.get("architecture", None) if arch is None else arch + ) # type: ignore + if os.name == "nt" and self.windows_finder is not None: match = self.windows_finder.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, + name=name, ) if match: return match @@ -118,28 +226,26 @@ class Finder(object): def find_all_python_versions( self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None, name=None ): + # type: (Optional[Union[str, int]], Optional[int], Optional[int], Optional[bool], Optional[bool], Optional[str], Optional[str]) -> List[PathEntry] version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter( - None, - [ - path - for version in python_version_dict.values() - for path in version - if path.as_python - ], + paths = ( + path + for version in python_version_dict.values() + for path in version + if path is not None and path.as_python ) - paths = sorted(paths, key=version_sort, reverse=True) - return paths + path_list = sorted(paths, key=version_sort, reverse=True) + return path_list versions = self.system_path.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, name=name ) - if not isinstance(versions, list): + if not isinstance(versions, Iterable): versions = [versions] - paths = sorted(versions, key=version_sort, reverse=True) - path_map = {} - for path in paths: + path_list = sorted(versions, key=version_sort, reverse=True) + path_map = {} # type: Dict[str, PathEntry] + for path in path_list: try: resolved_path = path.path.resolve() except OSError: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 18441919..a82654f3 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -1,46 +1,74 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function +import io import itertools import os - +import re from fnmatch import fnmatch import attr -import io -import re import six - import vistir - from packaging.version import LegacyVersion, Version -from .environment import PYENV_ROOT, ASDF_DATA_DIR, MYPY_RUNNING +from .environment import MYPY_RUNNING, PYENV_ROOT from .exceptions import InvalidPythonVersion -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore # noqa +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa +# fmt: off +from six.moves import Iterable # type: ignore # noqa # isort:skip +from six.moves import Sequence # type: ignore # noqa # isort:skip +# fmt: on try: from functools import lru_cache except ImportError: - from backports.functools_lru_cache import lru_cache + from backports.functools_lru_cache import lru_cache # type: ignore # noqa if MYPY_RUNNING: - from typing import Any, Union, List, Callable, Iterable, Set, Tuple, Dict, Optional - from attr.validators import _OptionalValidator + from typing import Any, Union, List, Callable, Set, Tuple, Dict, Optional, Iterator + from attr.validators import _OptionalValidator # type: ignore + from .models.path import PathEntry -version_re = re.compile(r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" - r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" - r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?") +version_re = re.compile( + r"(?P\d+)(?:\.(?P\d+))?(?:\.(?P(?<=\.)[0-9]+))?\.?" + r"(?:(?P[abc]|rc|dev)(?:(?P\d+(?:\.\d+)*))?)" + r"?(?P(\.post(?P\d+))?(\.dev(?P\d+))?)?" +) PYTHON_IMPLEMENTATIONS = ( - "python", "ironpython", "jython", "pypy", "anaconda", "miniconda", - "stackless", "activepython", "micropython" + "python", + "ironpython", + "jython", + "pypy", + "anaconda", + "miniconda", + "stackless", + "activepython", + "micropython", ) -RULES_BASE = ["*{0}", "*{0}?", "*{0}?.?", "*{0}?.?m"] +RE_MATCHER = re.compile( + r"(({0})(?:\d?(?:\.\d[cpm]{{0,3}}))?(?:-?[\d\.]+)*[^z])".format( + "|".join(PYTHON_IMPLEMENTATIONS) + ) +) +RULES_BASE = [ + "*{0}", + "*{0}?", + "*{0}?.?", + "*{0}?.?m", + "{0}?-?.?", + "{0}?-?.?.?", + "{0}?.?-?.?.?", +] RULES = [rule.format(impl) for impl in PYTHON_IMPLEMENTATIONS for rule in RULES_BASE] KNOWN_EXTS = {"exe", "py", "fish", "sh", ""} @@ -51,10 +79,7 @@ KNOWN_EXTS = KNOWN_EXTS | set( MATCH_RULES = [] for rule in RULES: MATCH_RULES.extend( - [ - "{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) - for ext in KNOWN_EXTS - ] + ["{0}.{1}".format(rule, ext) if ext else "{0}".format(rule) for ext in KNOWN_EXTS] ) @@ -64,8 +89,14 @@ def get_python_version(path): """Get python version string using subprocess from a given path.""" version_cmd = [path, "-c", "import sys; print(sys.version.split()[0])"] try: - c = vistir.misc.run(version_cmd, block=True, nospin=True, return_object=True, - combine_stderr=False, write_to_stdout=False) + c = vistir.misc.run( + version_cmd, + block=True, + nospin=True, + return_object=True, + combine_stderr=False, + write_to_stdout=False, + ) except OSError: raise InvalidPythonVersion("%s is not a valid python path" % path) if not c.out: @@ -77,6 +108,7 @@ def get_python_version(path): def parse_python_version(version_str): # type: (str) -> Dict[str, Union[str, int, Version]] from packaging.version import parse as parse_version + is_debug = False if version_str.endswith("-debug"): is_debug = True @@ -117,7 +149,7 @@ def parse_python_version(version_str): "is_prerelease": is_prerelease, "is_devrelease": is_devrelease, "is_debug": is_debug, - "version": version + "version": version, } @@ -178,7 +210,10 @@ def looks_like_python(name): if not any(name.lower().startswith(py_name) for py_name in PYTHON_IMPLEMENTATIONS): return False - return any(fnmatch(name, rule) for rule in MATCH_RULES) + match = RE_MATCHER.match(name) + if match: + return any(fnmatch(name, rule) for rule in MATCH_RULES) + return False @lru_cache(maxsize=1024) @@ -198,7 +233,7 @@ def path_is_python(path): @lru_cache(maxsize=1024) def ensure_path(path): - # type: (Union[vistir.compat.Path, str]) -> bool + # type: (Union[vistir.compat.Path, str]) -> vistir.compat.Path """ Given a path (either a string or a Path object), expand variables and return a Path object. @@ -224,9 +259,11 @@ def _filter_none(k, v): # TODO: Reimplement in vistir def normalize_path(path): # type: (str) -> str - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) @lru_cache(maxsize=1024) @@ -248,13 +285,16 @@ def unnest(item): item, target = itertools.tee(item, 2) else: target = item - for el in target: - if isinstance(el, Iterable) and not isinstance(el, six.string_types): - el, el_copy = itertools.tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el + if getattr(target, "__iter__", None): + for el in target: + if isinstance(el, Iterable) and not isinstance(el, six.string_types): + el, el_copy = itertools.tee(el, 2) + for sub in unnest(el_copy): + yield sub + else: + yield el + else: + yield target def parse_pyenv_version_order(filename="version"): @@ -274,11 +314,13 @@ def parse_asdf_version_order(filename=".tool-versions"): if os.path.exists(version_order_file) and os.path.isfile(version_order_file): with io.open(version_order_file, encoding="utf-8") as fh: contents = fh.read() - python_section = next(iter( - line for line in contents.splitlines() if line.startswith("python") - ), None) + python_section = next( + iter(line for line in contents.splitlines() if line.startswith("python")), + None, + ) if python_section: - python_key, _, versions = python_section.partition(" ") + # python_key, _, versions + _, _, versions = python_section.partition(" ") if versions: return versions.split() return [] @@ -287,3 +329,37 @@ def parse_asdf_version_order(filename=".tool-versions"): # TODO: Reimplement in vistir def is_in_path(path, parent): return normalize_path(str(path)).startswith(normalize_path(str(parent))) + + +def expand_paths(path, only_python=True): + # type: (Union[Sequence, PathEntry], bool) -> Iterator + """ + Recursively expand a list or :class:`~pythonfinder.models.path.PathEntry` instance + + :param Union[Sequence, PathEntry] path: The path or list of paths to expand + :param bool only_python: Whether to filter to include only python paths, default True + :returns: An iterator over the expanded set of path entries + :rtype: Iterator[PathEntry] + """ + + if path is not None and ( + isinstance(path, Sequence) + and not getattr(path.__class__, "__name__", "") == "PathEntry" + ): + for p in unnest(path): + if p is None: + continue + for expanded in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield expanded + elif path is not None and path.is_dir: + for p in path.children.values(): + if p is not None and p.is_python and p.as_python is not None: + for sub_path in itertools.chain.from_iterable( + expand_paths(p, only_python=only_python) + ): + yield sub_path + else: + if path is not None and path.is_python and path.as_python is not None: + yield path diff --git a/pipenv/vendor/pytoml/LICENSE b/pipenv/vendor/pytoml/LICENSE new file mode 100644 index 00000000..9739fc67 --- /dev/null +++ b/pipenv/vendor/pytoml/LICENSE @@ -0,0 +1,16 @@ +No-notice MIT License + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/pipenv/vendor/pytoml/__init__.py b/pipenv/vendor/pytoml/__init__.py new file mode 100644 index 00000000..8ed060ff --- /dev/null +++ b/pipenv/vendor/pytoml/__init__.py @@ -0,0 +1,4 @@ +from .core import TomlError +from .parser import load, loads +from .test import translate_to_test +from .writer import dump, dumps \ No newline at end of file diff --git a/pipenv/vendor/pytoml/core.py b/pipenv/vendor/pytoml/core.py new file mode 100644 index 00000000..c182734e --- /dev/null +++ b/pipenv/vendor/pytoml/core.py @@ -0,0 +1,13 @@ +class TomlError(RuntimeError): + def __init__(self, message, line, col, filename): + RuntimeError.__init__(self, message, line, col, filename) + self.message = message + self.line = line + self.col = col + self.filename = filename + + def __str__(self): + return '{}({}, {}): {}'.format(self.filename, self.line, self.col, self.message) + + def __repr__(self): + return 'TomlError({!r}, {!r}, {!r}, {!r})'.format(self.message, self.line, self.col, self.filename) diff --git a/pipenv/vendor/pytoml/parser.py b/pipenv/vendor/pytoml/parser.py new file mode 100644 index 00000000..3493aa64 --- /dev/null +++ b/pipenv/vendor/pytoml/parser.py @@ -0,0 +1,341 @@ +import string, re, sys, datetime +from .core import TomlError +from .utils import rfc3339_re, parse_rfc3339_re + +if sys.version_info[0] == 2: + _chr = unichr +else: + _chr = chr + +def load(fin, translate=lambda t, x, v: v, object_pairs_hook=dict): + return loads(fin.read(), translate=translate, object_pairs_hook=object_pairs_hook, filename=getattr(fin, 'name', repr(fin))) + +def loads(s, filename='', translate=lambda t, x, v: v, object_pairs_hook=dict): + if isinstance(s, bytes): + s = s.decode('utf-8') + + s = s.replace('\r\n', '\n') + + root = object_pairs_hook() + tables = object_pairs_hook() + scope = root + + src = _Source(s, filename=filename) + ast = _p_toml(src, object_pairs_hook=object_pairs_hook) + + def error(msg): + raise TomlError(msg, pos[0], pos[1], filename) + + def process_value(v, object_pairs_hook): + kind, text, value, pos = v + if kind == 'str' and value.startswith('\n'): + value = value[1:] + if kind == 'array': + if value and any(k != value[0][0] for k, t, v, p in value[1:]): + error('array-type-mismatch') + value = [process_value(item, object_pairs_hook=object_pairs_hook) for item in value] + elif kind == 'table': + value = object_pairs_hook([(k, process_value(value[k], object_pairs_hook=object_pairs_hook)) for k in value]) + return translate(kind, text, value) + + for kind, value, pos in ast: + if kind == 'kv': + k, v = value + if k in scope: + error('duplicate_keys. Key "{0}" was used more than once.'.format(k)) + scope[k] = process_value(v, object_pairs_hook=object_pairs_hook) + else: + is_table_array = (kind == 'table_array') + cur = tables + for name in value[:-1]: + if isinstance(cur.get(name), list): + d, cur = cur[name][-1] + else: + d, cur = cur.setdefault(name, (None, object_pairs_hook())) + + scope = object_pairs_hook() + name = value[-1] + if name not in cur: + if is_table_array: + cur[name] = [(scope, object_pairs_hook())] + else: + cur[name] = (scope, object_pairs_hook()) + elif isinstance(cur[name], list): + if not is_table_array: + error('table_type_mismatch') + cur[name].append((scope, object_pairs_hook())) + else: + if is_table_array: + error('table_type_mismatch') + old_scope, next_table = cur[name] + if old_scope is not None: + error('duplicate_tables') + cur[name] = (scope, next_table) + + def merge_tables(scope, tables): + if scope is None: + scope = object_pairs_hook() + for k in tables: + if k in scope: + error('key_table_conflict') + v = tables[k] + if isinstance(v, list): + scope[k] = [merge_tables(sc, tbl) for sc, tbl in v] + else: + scope[k] = merge_tables(v[0], v[1]) + return scope + + return merge_tables(root, tables) + +class _Source: + def __init__(self, s, filename=None): + self.s = s + self._pos = (1, 1) + self._last = None + self._filename = filename + self.backtrack_stack = [] + + def last(self): + return self._last + + def pos(self): + return self._pos + + def fail(self): + return self._expect(None) + + def consume_dot(self): + if self.s: + self._last = self.s[0] + self.s = self[1:] + self._advance(self._last) + return self._last + return None + + def expect_dot(self): + return self._expect(self.consume_dot()) + + def consume_eof(self): + if not self.s: + self._last = '' + return True + return False + + def expect_eof(self): + return self._expect(self.consume_eof()) + + def consume(self, s): + if self.s.startswith(s): + self.s = self.s[len(s):] + self._last = s + self._advance(s) + return True + return False + + def expect(self, s): + return self._expect(self.consume(s)) + + def consume_re(self, re): + m = re.match(self.s) + if m: + self.s = self.s[len(m.group(0)):] + self._last = m + self._advance(m.group(0)) + return m + return None + + def expect_re(self, re): + return self._expect(self.consume_re(re)) + + def __enter__(self): + self.backtrack_stack.append((self.s, self._pos)) + + def __exit__(self, type, value, traceback): + if type is None: + self.backtrack_stack.pop() + else: + self.s, self._pos = self.backtrack_stack.pop() + return type == TomlError + + def commit(self): + self.backtrack_stack[-1] = (self.s, self._pos) + + def _expect(self, r): + if not r: + raise TomlError('msg', self._pos[0], self._pos[1], self._filename) + return r + + def _advance(self, s): + suffix_pos = s.rfind('\n') + if suffix_pos == -1: + self._pos = (self._pos[0], self._pos[1] + len(s)) + else: + self._pos = (self._pos[0] + s.count('\n'), len(s) - suffix_pos) + +_ews_re = re.compile(r'(?:[ \t]|#[^\n]*\n|#[^\n]*\Z|\n)*') +def _p_ews(s): + s.expect_re(_ews_re) + +_ws_re = re.compile(r'[ \t]*') +def _p_ws(s): + s.expect_re(_ws_re) + +_escapes = { 'b': '\b', 'n': '\n', 'r': '\r', 't': '\t', '"': '"', + '\\': '\\', 'f': '\f' } + +_basicstr_re = re.compile(r'[^"\\\000-\037]*') +_short_uni_re = re.compile(r'u([0-9a-fA-F]{4})') +_long_uni_re = re.compile(r'U([0-9a-fA-F]{8})') +_escapes_re = re.compile(r'[btnfr\"\\]') +_newline_esc_re = re.compile('\n[ \t\n]*') +def _p_basicstr_content(s, content=_basicstr_re): + res = [] + while True: + res.append(s.expect_re(content).group(0)) + if not s.consume('\\'): + break + if s.consume_re(_newline_esc_re): + pass + elif s.consume_re(_short_uni_re) or s.consume_re(_long_uni_re): + v = int(s.last().group(1), 16) + if 0xd800 <= v < 0xe000: + s.fail() + res.append(_chr(v)) + else: + s.expect_re(_escapes_re) + res.append(_escapes[s.last().group(0)]) + return ''.join(res) + +_key_re = re.compile(r'[0-9a-zA-Z-_]+') +def _p_key(s): + with s: + s.expect('"') + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return r + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return r + return s.expect_re(_key_re).group(0) + +_float_re = re.compile(r'[+-]?(?:0|[1-9](?:_?\d)*)(?:\.\d(?:_?\d)*)?(?:[eE][+-]?(?:\d(?:_?\d)*))?') + +_basicstr_ml_re = re.compile(r'(?:""?(?!")|[^"\\\000-\011\013-\037])*') +_litstr_re = re.compile(r"[^'\000\010\012-\037]*") +_litstr_ml_re = re.compile(r"(?:(?:|'|'')(?:[^'\000-\010\013-\037]))*") +def _p_value(s, object_pairs_hook): + pos = s.pos() + + if s.consume('true'): + return 'bool', s.last(), True, pos + if s.consume('false'): + return 'bool', s.last(), False, pos + + if s.consume('"'): + if s.consume('""'): + r = _p_basicstr_content(s, _basicstr_ml_re) + s.expect('"""') + else: + r = _p_basicstr_content(s, _basicstr_re) + s.expect('"') + return 'str', r, r, pos + + if s.consume('\''): + if s.consume('\'\''): + r = s.expect_re(_litstr_ml_re).group(0) + s.expect('\'\'\'') + else: + r = s.expect_re(_litstr_re).group(0) + s.expect('\'') + return 'str', r, r, pos + + if s.consume_re(rfc3339_re): + m = s.last() + return 'datetime', m.group(0), parse_rfc3339_re(m), pos + + if s.consume_re(_float_re): + m = s.last().group(0) + r = m.replace('_','') + if '.' in m or 'e' in m or 'E' in m: + return 'float', m, float(r), pos + else: + return 'int', m, int(r, 10), pos + + if s.consume('['): + items = [] + with s: + while True: + _p_ews(s) + items.append(_p_value(s, object_pairs_hook=object_pairs_hook)) + s.commit() + _p_ews(s) + s.expect(',') + s.commit() + _p_ews(s) + s.expect(']') + return 'array', None, items, pos + + if s.consume('{'): + _p_ws(s) + items = object_pairs_hook() + if not s.consume('}'): + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + while s.consume(','): + _p_ws(s) + k = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + items[k] = _p_value(s, object_pairs_hook=object_pairs_hook) + _p_ws(s) + s.expect('}') + return 'table', None, items, pos + + s.fail() + +def _p_stmt(s, object_pairs_hook): + pos = s.pos() + if s.consume( '['): + is_array = s.consume('[') + _p_ws(s) + keys = [_p_key(s)] + _p_ws(s) + while s.consume('.'): + _p_ws(s) + keys.append(_p_key(s)) + _p_ws(s) + s.expect(']') + if is_array: + s.expect(']') + return 'table_array' if is_array else 'table', keys, pos + + key = _p_key(s) + _p_ws(s) + s.expect('=') + _p_ws(s) + value = _p_value(s, object_pairs_hook=object_pairs_hook) + return 'kv', (key, value), pos + +_stmtsep_re = re.compile(r'(?:[ \t]*(?:#[^\n]*)?\n)+[ \t]*') +def _p_toml(s, object_pairs_hook): + stmts = [] + _p_ews(s) + with s: + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + while True: + s.commit() + s.expect_re(_stmtsep_re) + stmts.append(_p_stmt(s, object_pairs_hook=object_pairs_hook)) + _p_ews(s) + s.expect_eof() + return stmts diff --git a/pipenv/vendor/pytoml/test.py b/pipenv/vendor/pytoml/test.py new file mode 100644 index 00000000..ec8abfc6 --- /dev/null +++ b/pipenv/vendor/pytoml/test.py @@ -0,0 +1,30 @@ +import datetime +from .utils import format_rfc3339 + +try: + _string_types = (str, unicode) + _int_types = (int, long) +except NameError: + _string_types = str + _int_types = int + +def translate_to_test(v): + if isinstance(v, dict): + return { k: translate_to_test(v) for k, v in v.items() } + if isinstance(v, list): + a = [translate_to_test(x) for x in v] + if v and isinstance(v[0], dict): + return a + else: + return {'type': 'array', 'value': a} + if isinstance(v, datetime.datetime): + return {'type': 'datetime', 'value': format_rfc3339(v)} + if isinstance(v, bool): + return {'type': 'bool', 'value': 'true' if v else 'false'} + if isinstance(v, _int_types): + return {'type': 'integer', 'value': str(v)} + if isinstance(v, float): + return {'type': 'float', 'value': '{:.17}'.format(v)} + if isinstance(v, _string_types): + return {'type': 'string', 'value': v} + raise RuntimeError('unexpected value: {!r}'.format(v)) diff --git a/pipenv/vendor/pytoml/utils.py b/pipenv/vendor/pytoml/utils.py new file mode 100644 index 00000000..636a680b --- /dev/null +++ b/pipenv/vendor/pytoml/utils.py @@ -0,0 +1,67 @@ +import datetime +import re + +rfc3339_re = re.compile(r'(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2}):(\d{2})(\.\d+)?(?:Z|([+-]\d{2}):(\d{2}))') + +def parse_rfc3339(v): + m = rfc3339_re.match(v) + if not m or m.group(0) != v: + return None + return parse_rfc3339_re(m) + +def parse_rfc3339_re(m): + r = map(int, m.groups()[:6]) + if m.group(7): + micro = float(m.group(7)) + else: + micro = 0 + + if m.group(8): + g = int(m.group(8), 10) * 60 + int(m.group(9), 10) + tz = _TimeZone(datetime.timedelta(0, g * 60)) + else: + tz = _TimeZone(datetime.timedelta(0, 0)) + + y, m, d, H, M, S = r + return datetime.datetime(y, m, d, H, M, S, int(micro * 1000000), tz) + + +def format_rfc3339(v): + offs = v.utcoffset() + offs = int(offs.total_seconds()) // 60 if offs is not None else 0 + + if offs == 0: + suffix = 'Z' + else: + if offs > 0: + suffix = '+' + else: + suffix = '-' + offs = -offs + suffix = '{0}{1:02}:{2:02}'.format(suffix, offs // 60, offs % 60) + + if v.microsecond: + return v.strftime('%Y-%m-%dT%H:%M:%S.%f') + suffix + else: + return v.strftime('%Y-%m-%dT%H:%M:%S') + suffix + +class _TimeZone(datetime.tzinfo): + def __init__(self, offset): + self._offset = offset + + def utcoffset(self, dt): + return self._offset + + def dst(self, dt): + return None + + def tzname(self, dt): + m = self._offset.total_seconds() // 60 + if m < 0: + res = '-' + m = -m + else: + res = '+' + h = m // 60 + m = m - h * 60 + return '{}{:.02}{:.02}'.format(res, h, m) diff --git a/pipenv/vendor/pytoml/writer.py b/pipenv/vendor/pytoml/writer.py new file mode 100644 index 00000000..73b5089c --- /dev/null +++ b/pipenv/vendor/pytoml/writer.py @@ -0,0 +1,106 @@ +from __future__ import unicode_literals +import io, datetime, math, string, sys + +from .utils import format_rfc3339 + +if sys.version_info[0] == 3: + long = int + unicode = str + + +def dumps(obj, sort_keys=False): + fout = io.StringIO() + dump(obj, fout, sort_keys=sort_keys) + return fout.getvalue() + + +_escapes = {'\n': 'n', '\r': 'r', '\\': '\\', '\t': 't', '\b': 'b', '\f': 'f', '"': '"'} + + +def _escape_string(s): + res = [] + start = 0 + + def flush(): + if start != i: + res.append(s[start:i]) + return i + 1 + + i = 0 + while i < len(s): + c = s[i] + if c in '"\\\n\r\t\b\f': + start = flush() + res.append('\\' + _escapes[c]) + elif ord(c) < 0x20: + start = flush() + res.append('\\u%04x' % ord(c)) + i += 1 + + flush() + return '"' + ''.join(res) + '"' + + +_key_chars = string.digits + string.ascii_letters + '-_' +def _escape_id(s): + if any(c not in _key_chars for c in s): + return _escape_string(s) + return s + + +def _format_value(v): + if isinstance(v, bool): + return 'true' if v else 'false' + if isinstance(v, int) or isinstance(v, long): + return unicode(v) + if isinstance(v, float): + if math.isnan(v) or math.isinf(v): + raise ValueError("{0} is not a valid TOML value".format(v)) + else: + return repr(v) + elif isinstance(v, unicode) or isinstance(v, bytes): + return _escape_string(v) + elif isinstance(v, datetime.datetime): + return format_rfc3339(v) + elif isinstance(v, list): + return '[{0}]'.format(', '.join(_format_value(obj) for obj in v)) + elif isinstance(v, dict): + return '{{{0}}}'.format(', '.join('{} = {}'.format(_escape_id(k), _format_value(obj)) for k, obj in v.items())) + else: + raise RuntimeError(v) + + +def dump(obj, fout, sort_keys=False): + tables = [((), obj, False)] + + while tables: + name, table, is_array = tables.pop() + if name: + section_name = '.'.join(_escape_id(c) for c in name) + if is_array: + fout.write('[[{0}]]\n'.format(section_name)) + else: + fout.write('[{0}]\n'.format(section_name)) + + table_keys = sorted(table.keys()) if sort_keys else table.keys() + new_tables = [] + has_kv = False + for k in table_keys: + v = table[k] + if isinstance(v, dict): + new_tables.append((name + (k,), v, False)) + elif isinstance(v, list) and v and all(isinstance(o, dict) for o in v): + new_tables.extend((name + (k,), d, True) for d in v) + elif v is None: + # based on mojombo's comment: https://github.com/toml-lang/toml/issues/146#issuecomment-25019344 + fout.write( + '#{} = null # To use: uncomment and replace null with value\n'.format(_escape_id(k))) + has_kv = True + else: + fout.write('{0} = {1}\n'.format(_escape_id(k), _format_value(v))) + has_kv = True + + tables.extend(reversed(new_tables)) + + if (name or has_kv) and tables: + fout.write('\n') diff --git a/pipenv/vendor/requests/__version__.py b/pipenv/vendor/requests/__version__.py index 803773a0..f5b5d036 100644 --- a/pipenv/vendor/requests/__version__.py +++ b/pipenv/vendor/requests/__version__.py @@ -5,8 +5,8 @@ __title__ = 'requests' __description__ = 'Python HTTP for Humans.' __url__ = 'http://python-requests.org' -__version__ = '2.20.1' -__build__ = 0x022001 +__version__ = '2.21.0' +__build__ = 0x022100 __author__ = 'Kenneth Reitz' __author_email__ = 'me@kennethreitz.org' __license__ = 'Apache 2.0' diff --git a/pipenv/vendor/requests/models.py b/pipenv/vendor/requests/models.py index 3dded57e..62dcd0b7 100644 --- a/pipenv/vendor/requests/models.py +++ b/pipenv/vendor/requests/models.py @@ -781,7 +781,7 @@ class Response(object): return chunks - def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=None, delimiter=None): + def iter_lines(self, chunk_size=ITER_CHUNK_SIZE, decode_unicode=False, delimiter=None): """Iterates over the response data, one line at a time. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py index aca59917..c3e237ac 100644 --- a/pipenv/vendor/requirementslib/__init__.py +++ b/pipenv/vendor/requirementslib/__init__.py @@ -1,16 +1,21 @@ # -*- coding=utf-8 -*- -__version__ = '1.3.2' +from __future__ import absolute_import, print_function import logging import warnings + from vistir.compat import ResourceWarning +from .models.lockfile import Lockfile +from .models.pipfile import Pipfile +from .models.requirements import Requirement + +__version__ = "1.4.2" + + logger = logging.getLogger(__name__) logger.addHandler(logging.NullHandler()) warnings.filterwarnings("ignore", category=ResourceWarning) -from .models.requirements import Requirement -from .models.lockfile import Lockfile -from .models.pipfile import Pipfile __all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py index f87fd585..44f34edb 100644 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ b/pipenv/vendor/requirementslib/models/dependencies.py @@ -20,6 +20,7 @@ from vistir.contextmanagers import cd, temp_environ from vistir.misc import partialclass from vistir.path import create_tracked_tempdir +from ..environment import MYPY_RUNNING from ..utils import prepare_pip_source_args, _ensure_dir from .cache import CACHE_DIR, DependencyCache from .utils import ( @@ -29,6 +30,17 @@ from .utils import ( ) +if MYPY_RUNNING: + from typing import Any, Dict, List, Generator, Optional, Union, Tuple, TypeVar, Text, Set, AnyStr + from pip_shims.shims import InstallRequirement, InstallationCandidate, PackageFinder, Command + from packaging.requirements import Requirement as PackagingRequirement + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar('RequirementType', covariant=True, bound=PackagingRequirement) + MarkerType = TypeVar('MarkerType', covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + + PKGS_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "pkgs")) WHEEL_DOWNLOAD_DIR = fs_str(os.path.join(CACHE_DIR, "wheels")) @@ -43,6 +55,7 @@ def _get_filtered_versions(ireq, versions, prereleases): def find_all_matches(finder, ireq, pre=False): + # type: (PackageFinder, InstallRequirement, bool) -> List[InstallationCandidate] """Find all matching dependencies using the supplied finder and the given ireq. @@ -65,6 +78,7 @@ def find_all_matches(finder, ireq, pre=False): def get_pip_command(): + # type: () -> Command # Use pip's parser for pip.conf management and defaults. # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are defered to pip. @@ -89,7 +103,7 @@ def get_pip_command(): @attr.s class AbstractDependency(object): - name = attr.ib() + name = attr.ib() # type: STRING_TYPE specifiers = attr.ib() markers = attr.ib() candidates = attr.ib() @@ -284,6 +298,7 @@ def get_abstract_dependencies(reqs, sources=None, parent=None): def get_dependencies(ireq, sources=None, parent=None): + # type: (Union[InstallRequirement, InstallationCandidate], Optional[List[Dict[S, Union[S, bool]]]], Optional[AbstractDependency]) -> Set[S, ...] """Get all dependencies for a given install requirement. :param ireq: A single InstallRequirement @@ -556,6 +571,7 @@ def get_pip_options(args=[], sources=None, pip_command=None): def get_finder(sources=None, pip_command=None, pip_options=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[Command], Any) -> PackageFinder """Get a package finder for looking up candidates to install :param sources: A list of pipfile-formatted sources, defaults to None diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py index 54b2761c..42248868 100644 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ b/pipenv/vendor/requirementslib/models/lockfile.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import copy import os diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 84a4a26d..22eee048 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -18,16 +18,16 @@ from ..exceptions import RequirementError from ..utils import is_editable, is_vcs, merge_items from .project import ProjectFile from .requirements import Requirement -from .utils import optional_instance_of +from .utils import optional_instance_of, get_url_name from ..environment import MYPY_RUNNING if MYPY_RUNNING: - from typing import Union, Any, Dict, Iterable, Sequence, Mapping, List, NoReturn - package_type = Dict[str, Dict[str, Union[List[str], str]]] - source_type = Dict[str, Union[str, bool]] + from typing import Union, Any, Dict, Iterable, Mapping, List, Text + package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] + source_type = Dict[Text, Union[Text, bool]] sources_type = Iterable[source_type] - meta_type = Dict[str, Union[int, Dict[str, str], sources_type]] - lockfile_type = Dict[str, Union[package_type, meta_type]] + meta_type = Dict[Text, Union[int, Dict[Text, Text], sources_type]] + lockfile_type = Dict[Text, Union[package_type, meta_type]] # Let's start by patching plette to make sure we can validate data without being broken @@ -45,7 +45,7 @@ def patch_plette(): global VALIDATORS def validate(cls, data): - # type: (Any, Dict[str, Any]) -> None + # type: (Any, Dict[Text, Any]) -> None if not cerberus: # Skip validation if Cerberus is not available. return schema = cls.__SCHEMA__ @@ -87,9 +87,10 @@ def reorder_source_keys(data): sources = data["source"] # type: sources_type for i, entry in enumerate(sources): table = tomlkit.table() # type: Mapping - table["name"] = entry["name"] - table["url"] = entry["url"] - table["verify_ssl"] = entry["verify_ssl"] + source_entry = PipfileLoader.populate_source(entry.copy()) + table["name"] = source_entry["name"] + table["url"] = source_entry["url"] + table["verify_ssl"] = source_entry["verify_ssl"] data["source"][i] = table return data @@ -97,7 +98,7 @@ def reorder_source_keys(data): class PipfileLoader(plette.pipfiles.Pipfile): @classmethod def validate(cls, data): - # type: (Dict[str, Any]) -> None + # type: (Dict[Text, Any]) -> None for key, klass in plette.pipfiles.PIPFILE_SECTIONS.items(): if key not in data or key == "source": continue @@ -106,9 +107,21 @@ class PipfileLoader(plette.pipfiles.Pipfile): except Exception: pass + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = source["verify_ssl"].lower() == "true" + return source + @classmethod def load(cls, f, encoding=None): - # type: (Any, str) -> PipfileLoader + # type: (Any, Text) -> PipfileLoader content = f.read() if encoding is not None: content = content.decode(encoding) @@ -132,7 +145,7 @@ class PipfileLoader(plette.pipfiles.Pipfile): return instance def __getattribute__(self, key): - # type: (str) -> Any + # type: (Text) -> Any if key == "source": return self._data[key] return super(PipfileLoader, self).__getattribute__(key) @@ -169,8 +182,8 @@ class Pipfile(object): return self._pipfile def get_deps(self, dev=False, only=True): - # type: (bool, bool) -> Dict[str, Dict[str, Union[List[str], str]]] - deps = {} # type: Dict[str, Dict[str, Union[List[str], str]]] + # type: (bool, bool) -> Dict[Text, Dict[Text, Union[List[Text], Text]]] + deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] if dev: deps.update(self.pipfile._data["dev-packages"]) if only: @@ -178,11 +191,11 @@ class Pipfile(object): return merge_items([deps, self.pipfile._data["packages"]]) def get(self, k): - # type: (str) -> Any + # type: (Text) -> Any return self.__getitem__(k) def __contains__(self, k): - # type: (str) -> bool + # type: (Text) -> bool check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) if check_pipfile: return True @@ -234,10 +247,10 @@ class Pipfile(object): @classmethod def read_projectfile(cls, path): - # type: (str) -> ProjectFile + # type: (Text) -> ProjectFile """Read the specified project file and provide an interface for writing/updating. - :param str path: Path to the target file. + :param Text path: Path to the target file. :return: A project file with the model and location for interaction :rtype: :class:`~requirementslib.models.project.ProjectFile` """ @@ -250,10 +263,11 @@ class Pipfile(object): @classmethod def load_projectfile(cls, path, create=False): - # type: (str, bool) -> ProjectFile - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> ProjectFile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -275,10 +289,11 @@ class Pipfile(object): @classmethod def load(cls, path, create=False): - # type: (str, bool) -> Pipfile - """Given a path, load or create the necessary pipfile. + # type: (Text, bool) -> Pipfile + """ + Given a path, load or create the necessary pipfile. - :param str path: Path to the project root or pipfile + :param Text path: Path to the project root or pipfile :param bool create: Whether to create the pipfile if not found, defaults to True :raises OSError: Thrown if the project root directory doesn't exist :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` @@ -327,17 +342,17 @@ class Pipfile(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } self._build_system = build_system @property def build_requires(self): - # type: () -> List[str] + # type: () -> List[Text] return self.build_system.get("requires", []) @property def build_backend(self): - # type: () -> str + # type: () -> Text return self.build_system.get("build-backend", None) diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py index f6e037d6..28afcf0b 100644 --- a/pipenv/vendor/requirementslib/models/project.py +++ b/pipenv/vendor/requirementslib/models/project.py @@ -1,6 +1,6 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, unicode_literals, print_function import collections import io diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index af3d07ed..5fb219ac 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,85 +1,1165 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import collections import copy import hashlib import os - +import sys from contextlib import contextmanager +from distutils.sysconfig import get_python_lib +from functools import partial import attr import pip_shims - +import six +import vistir +from cached_property import cached_property from first import first from packaging.markers import Marker from packaging.requirements import Requirement as PackagingRequirement -from packaging.specifiers import Specifier, SpecifierSet, LegacySpecifier, InvalidSpecifier +from packaging.specifiers import ( + InvalidSpecifier, + LegacySpecifier, + Specifier, + SpecifierSet, +) from packaging.utils import canonicalize_name from six.moves.urllib import parse as urllib_parse from six.moves.urllib.parse import unquote -from vistir.compat import FileNotFoundError, Path +from vistir.compat import FileNotFoundError, Mapping, Path, lru_cache +from vistir.contextmanagers import temp_path from vistir.misc import dedup from vistir.path import ( create_tracked_tempdir, get_converted_relative_path, is_file_url, is_valid_url, + mkdir_p, + normalize_path, ) -from ..exceptions import RequirementError -from ..utils import ( - VCS_LIST, - is_installable_file, - is_vcs, - ensure_setup_py, - add_ssh_scheme_to_git_uri, - strip_ssh_from_git_uri, -) -from .setup_info import SetupInfo +from .setup_info import SetupInfo, _prepare_wheel_building_kwargs +from .url import URI from .utils import ( + DIRECT_URL_RE, HASH_STRING, - build_vcs_link, + URL_RE, + build_vcs_uri, + convert_direct_url_to_url, + create_link, extras_to_string, filter_none, format_requirement, + get_default_pyproject_backend, + get_pyproject, get_version, init_requirement, is_pinned_requirement, make_install_requirement, + normalize_name, parse_extras, specs_to_string, split_markers_from_line, + split_ref_from_uri, split_vcs_method_from_uri, validate_path, validate_specifiers, validate_vcs, - normalize_name, - create_link, - get_pyproject +) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError +from ..utils import ( + VCS_LIST, + add_ssh_scheme_to_git_uri, + get_setup_paths, + is_installable_file, + is_vcs, + strip_ssh_from_git_uri, ) +if MYPY_RUNNING: + from typing import ( + Optional, + TypeVar, + List, + Dict, + Union, + Any, + Tuple, + Sequence, + Set, + AnyStr, + Text, + Generator, + FrozenSet, + ) + from pip_shims.shims import ( + Link, + InstallRequirement, + PackageFinder, + InstallationCandidate, + ) -@attr.s(slots=True) + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + F = TypeVar("F", "FileRequirement", "VCSRequirement", covariant=True) + from six.moves.urllib.parse import SplitResult + from .vcs import VCSRepository + from .dependencies import AbstractDependency + + NON_STRING_ITERABLE = Union[List, Set, Tuple] + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + BASE_TYPES = Union[bool, STRING_TYPE, Tuple[STRING_TYPE, ...]] + CUSTOM_TYPES = Union[VCSRepository, RequirementType, SetupInfo, "Line"] + CREATION_ARG_TYPES = Union[BASE_TYPES, Link, CUSTOM_TYPES] + PIPFILE_ENTRY_TYPE = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PIPFILE_TYPE = Union[STRING_TYPE, Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE]] + TPIPFILE = Dict[STRING_TYPE, PIPFILE_ENTRY_TYPE] + + +SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) + + +run = partial(vistir.misc.run, combine_stderr=False, return_object=True, nospin=True) + + +class Line(object): + def __init__(self, line, extras=None): + # type: (AnyStr, Optional[Union[List[S], Set[S], Tuple[S, ...]]]) -> None + self.editable = False # type: bool + if line.startswith("-e "): + line = line[len("-e ") :] + self.editable = True + self.extras = () # type: Tuple[STRING_TYPE, ...] + if extras is not None: + self.extras = tuple(sorted(set(extras))) + self.line = line # type: STRING_TYPE + self.hashes = [] # type: List[STRING_TYPE] + self.markers = None # type: Optional[STRING_TYPE] + self.vcs = None # type: Optional[STRING_TYPE] + self.path = None # type: Optional[STRING_TYPE] + self.relpath = None # type: Optional[STRING_TYPE] + self.uri = None # type: Optional[STRING_TYPE] + self._link = None # type: Optional[Link] + self.is_local = False # type: bool + self._name = None # type: Optional[STRING_TYPE] + self._specifier = None # type: Optional[STRING_TYPE] + self.parsed_marker = None # type: Optional[Marker] + self.preferred_scheme = None # type: Optional[STRING_TYPE] + self._requirement = None # type: Optional[PackagingRequirement] + self.is_direct_url = False # type: bool + self._parsed_url = None # type: Optional[urllib_parse.ParseResult] + self._setup_cfg = None # type: Optional[STRING_TYPE] + self._setup_py = None # type: Optional[STRING_TYPE] + self._pyproject_toml = None # type: Optional[STRING_TYPE] + self._pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + self._pyproject_backend = None # type: Optional[STRING_TYPE] + self._wheel_kwargs = None # type: Optional[Dict[STRING_TYPE, STRING_TYPE]] + self._vcsrepo = None # type: Optional[VCSRepository] + self._setup_info = None # type: Optional[SetupInfo] + self._ref = None # type: Optional[STRING_TYPE] + self._ireq = None # type: Optional[InstallRequirement] + self._src_root = None # type: Optional[STRING_TYPE] + self.dist = None # type: Any + super(Line, self).__init__() + self.parse() + + def __hash__(self): + return hash( + ( + self.editable, + self.line, + self.markers, + tuple(self.extras), + tuple(self.hashes), + self.vcs, + self.ireq, + ) + ) + + def __repr__(self): + try: + return ( + "".format( + self=self + ) + ) + except Exception: + return "".format(self.__dict__.values()) + + @classmethod + def split_hashes(cls, line): + # type: (S) -> Tuple[S, List[S]] + if "--hash" not in line: + return line, [] + split_line = line.split() + line_parts = [] # type: List[S] + hashes = [] # type: List[S] + for part in split_line: + if part.startswith("--hash"): + param, _, value = part.partition("=") + hashes.append(value) + else: + line_parts.append(part) + line = " ".join(line_parts) + return line, hashes + + @property + def line_with_prefix(self): + # type: () -> STRING_TYPE + line = self.line + extras_str = extras_to_string(self.extras) + if self.is_direct_url: + line = self.link.url + elif extras_str: + if self.is_vcs: + line = self.link.url + if "git+file:/" in line and "git+file:///" not in line: + line = line.replace("git+file:/", "git+file:///") + else: + line = "{0}{1}".format(line, extras_str) + if self.editable: + return "-e {0}".format(line) + return line + + @property + def line_for_ireq(self): + # type: () -> STRING_TYPE + line = "" # type: STRING_TYPE + if self.is_file or self.is_url and not self.is_vcs: + scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" + local_line = next( + iter( + [ + os.path.dirname(os.path.abspath(f)) + for f in [self.setup_py, self.setup_cfg, self.pyproject_toml] + if f is not None + ] + ), + None, + ) + if local_line and self.extras: + local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) + line = local_line if local_line is not None else self.line + if scheme == "path": + if not line and self.base_path is not None: + line = os.path.abspath(self.base_path) + else: + if DIRECT_URL_RE.match(self.line): + uri = URI.parse(self.line) + line = uri.full_url + self._requirement = init_requirement(self.line) + line = convert_direct_url_to_url(self.line) + else: + if self.link: + line = self.link.url + else: + try: + uri = URI.parse(line) + except ValueError: + line = line + else: + line = uri.base_url + self._link = uri.as_link + + if self.editable: + if not line: + if self.is_path or self.is_file: + if not self.path: + line = pip_shims.shims.url_to_path(self.url) + else: + line = self.path + if self.extras: + line = "{0}{1}".format(line, extras_to_string(self.extras)) + else: + line = self.link.url + elif self.is_vcs and not self.editable: + line = add_ssh_scheme_to_git_uri(self.line) + if not line: + line = self.line + return line + + @property + def base_path(self): + # type: () -> Optional[S] + if not self.link and not self.path: + self.parse_link() + if not self.path: + pass + path = normalize_path(self.path) + if os.path.exists(path) and os.path.isdir(path): + path = path + elif os.path.exists(path) and os.path.isfile(path): + path = os.path.dirname(path) + else: + path = None + return path + + @property + def setup_py(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_py is None: + self.populate_setup_paths() + return self._setup_py + + @property + def setup_cfg(self): + # type: () -> Optional[STRING_TYPE] + if self._setup_cfg is None: + self.populate_setup_paths() + return self._setup_cfg + + @property + def pyproject_toml(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_toml is None: + self.populate_setup_paths() + return self._pyproject_toml + + @property + def specifier(self): + # type: () -> Optional[STRING_TYPE] + options = [self._specifier] + for req in (self.ireq, self.requirement): + if req is not None and getattr(req, "specifier", None): + options.append(req.specifier) + specifier = next( + iter(spec for spec in options if spec is not None), None + ) # type: Optional[Union[Specifier, SpecifierSet]] + spec_string = None # type: Optional[STRING_TYPE] + if specifier is not None: + spec_string = specs_to_string(specifier) + elif ( + specifier is None + and not self.is_named + and (self._setup_info is not None and self._setup_info.version) + ): + spec_string = "=={0}".format(self._setup_info.version) + if spec_string: + self._specifier = spec_string + return self._specifier + + @specifier.setter + def specifier(self, spec): + # type: (str) -> None + if not spec.startswith("=="): + spec = "=={0}".format(spec) + self._specifier = spec + self.specifiers = SpecifierSet(spec) + + @property + def specifiers(self): + # type: () -> Optional[SpecifierSet] + ireq_needs_specifier = False + req_needs_specifier = False + if self.ireq is None or self.ireq.req is None or not self.ireq.req.specifier: + ireq_needs_specifier = True + if self.requirement is None or not self.requirement.specifier: + req_needs_specifier = True + if any([ireq_needs_specifier, req_needs_specifier]): + # TODO: Should we include versions for VCS dependencies? IS there a reason not + # to? For now we are using hashes as the equivalent to pin + # note: we need versions for direct dependencies at the very least + if ( + self.is_file + or self.is_url + or self.is_path + or (self.is_vcs and not self.editable) + ): + if self.specifier is not None: + specifier = self.specifier + if not isinstance(specifier, SpecifierSet): + specifier = SpecifierSet(specifier) + self.specifiers = specifier + return specifier + if self.ireq is not None and self.ireq.req is not None: + return self.ireq.req.specifier + elif self.requirement is not None: + return self.requirement.specifier + return None + + @specifiers.setter + def specifiers(self, specifiers): + # type: (Union[Text, str, SpecifierSet]) -> None + if not isinstance(specifiers, SpecifierSet): + if isinstance(specifiers, six.string_types): + specifiers = SpecifierSet(specifiers) + else: + raise TypeError("Must pass a string or a SpecifierSet") + specs = self.get_requirement_specs(specifiers) + if self.ireq is not None and self._ireq and self._ireq.req is not None: + self._ireq.req.specifier = specifiers + self._ireq.req.specs = specs + if self.requirement is not None: + self.requirement.specifier = specifiers + self.requirement.specs = specs + + @classmethod + def get_requirement_specs(cls, specifierset): + # type: (SpecifierSet) -> List[Tuple[AnyStr, AnyStr]] + specs = [] + spec = next(iter(specifierset._specs), None) + if spec: + specs.append(spec._spec) + return specs + + @property + def requirement(self): + # type: () -> Optional[RequirementType] + if self._requirement is None: + self.parse_requirement() + if self._requirement is None and self._name is not None: + self._requirement = init_requirement(canonicalize_name(self.name)) + if self.is_file or self.is_url and self._requirement is not None: + self._requirement.url = self.url + if ( + self._requirement + and self._requirement.specifier + and not self._requirement.specs + ): + specs = self.get_requirement_specs(self._requirement.specifier) + self._requirement.specs = specs + return self._requirement + + def populate_setup_paths(self): + # type: () -> None + if not self.link and not self.path: + self.parse_link() + if not self.path: + return + base_path = self.base_path + if base_path is None: + return + setup_paths = get_setup_paths( + base_path, subdirectory=self.subdirectory + ) # type: Dict[STRING_TYPE, Optional[STRING_TYPE]] + self._setup_py = setup_paths.get("setup_py") + self._setup_cfg = setup_paths.get("setup_cfg") + self._pyproject_toml = setup_paths.get("pyproject_toml") + + @property + def pyproject_requires(self): + # type: () -> Optional[Tuple[STRING_TYPE, ...]] + if self._pyproject_requires is None and self.pyproject_toml is not None: + if self.path is not None: + pyproject_requires, pyproject_backend = None, None + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + self._pyproject_backend = pyproject_backend + return self._pyproject_requires + + @property + def pyproject_backend(self): + # type: () -> Optional[STRING_TYPE] + if self._pyproject_requires is None and self.pyproject_toml is not None: + pyproject_requires = None # type: Optional[Sequence[STRING_TYPE]] + pyproject_backend = None # type: Optional[STRING_TYPE] + pyproject_results = get_pyproject(self.path) # type: ignore + if pyproject_results: + pyproject_requires, pyproject_backend = pyproject_results + if not pyproject_backend and self.setup_cfg is not None: + setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) + pyproject_backend = get_default_pyproject_backend() + pyproject_requires = setup_dict.get( + "build_requires", ["setuptools", "wheel"] + ) # type: ignore + if pyproject_requires: + self._pyproject_requires = tuple(pyproject_requires) + if pyproject_backend: + self._pyproject_backend = pyproject_backend + return self._pyproject_backend + + def parse_hashes(self): + # type: () -> None + """ + Parse hashes from *self.line* and set them on the current object. + :returns: Nothing + :rtype: None + """ + + line, hashes = self.split_hashes(self.line) + self.hashes = hashes + self.line = line + + def parse_extras(self): + # type: () -> None + """ + Parse extras from *self.line* and set them on the current object + :returns: Nothing + :rtype: None + """ + + extras = None + if "@" in self.line or self.is_vcs or self.is_url: + line = "{0}".format(self.line) + uri = URI.parse(line) + name = uri.name + if name: + self._name = name + if uri.host and uri.path and uri.scheme: + self.line = uri.to_string( + escape_password=False, direct=False, strip_ssh=uri.is_implicit_ssh + ) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + else: + self.line, extras = pip_shims.shims._strip_extras(self.line) + extras_set = set() # type: Set[STRING_TYPE] + if extras is not None: + extras_set = set(parse_extras(extras)) + if self._name: + self._name, name_extras = pip_shims.shims._strip_extras(self._name) + if name_extras: + name_extras = set(parse_extras(name_extras)) + extras_set |= name_extras + if extras_set is not None: + self.extras = tuple(sorted(extras_set)) + + def get_url(self): + # type: () -> STRING_TYPE + """Sets ``self.name`` if given a **PEP-508** style URL""" + + line = self.line + try: + parsed = URI.parse(line) + line = parsed.to_string(escape_password=False, direct=False, strip_ref=True) + except ValueError: + pass + else: + self._parsed_url = parsed + return line + if self.vcs is not None and self.line.startswith("{0}+".format(self.vcs)): + _, _, _parseable = self.line.partition("+") + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(_parseable)) + line, _ = split_ref_from_uri(line) + else: + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line)) + if "@" in self.line and parsed.scheme == "": + name, _, url = self.line.partition("@") + if self._name is None: + url = url.strip() + self._name = name.strip() + if is_valid_url(url): + self.is_direct_url = True + line = url.strip() + parsed = urllib_parse.urlparse(line) + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + self._parsed_url = parsed + return line + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is None: + self.parse_name() + if self._name is None and not self.is_named and not self.is_wheel: + if self.setup_info: + self._name = self.setup_info.name + return self._name + + @name.setter + def name(self, name): + # type: (STRING_TYPE) -> None + self._name = name + if self._setup_info: + self._setup_info.name = name + if self.requirement and self._requirement: + self._requirement.name = name + if self.ireq and self._ireq and self._ireq.req: + self._ireq.req.name = name + + @property + def url(self): + # type: () -> Optional[STRING_TYPE] + if self.uri is not None: + url = add_ssh_scheme_to_git_uri(self.uri) + else: + url = getattr(self.link, "url_without_fragment", None) + if url is not None: + url = add_ssh_scheme_to_git_uri(unquote(url)) + if url is not None and self._parsed_url is None: + if self.vcs is not None: + _, _, _parseable = url.partition("+") + self._parsed_url = urllib_parse.urlparse(_parseable) + if self.is_vcs: + # strip the ref from the url + url, _ = split_ref_from_uri(url) + return url + + @property + def link(self): + # type: () -> Link + if self._link is None: + self.parse_link() + return self._link + + @property + def subdirectory(self): + # type: () -> Optional[STRING_TYPE] + if self.link is not None: + return self.link.subdirectory_fragment + return "" + + @property + def is_wheel(self): + # type: () -> bool + if self.link is None: + return False + return self.link.is_wheel + + @property + def is_artifact(self): + # type: () -> bool + if self.link is None: + return False + return self.link.is_artifact + + @property + def is_vcs(self): + # type: () -> bool + # Installable local files and installable non-vcs urls are handled + # as files, generally speaking + if is_vcs(self.line) or is_vcs(self.get_url()): + return True + return False + + @property + def is_url(self): + # type: () -> bool + url = self.get_url() + if is_valid_url(url) or is_file_url(url): + return True + return False + + @property + def is_path(self): + # type: () -> bool + if ( + self.path + and ( + self.path.startswith(".") + or os.path.isabs(self.path) + or os.path.exists(self.path) + ) + and is_installable_file(self.path) + ): + return True + elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( + os.path.exists(self.get_url()) and is_installable_file(self.get_url()) + ): + return True + return False + + @property + def is_file_url(self): + # type: () -> bool + url = self.get_url() + parsed_url_scheme = self._parsed_url.scheme if self._parsed_url else "" + if url and is_file_url(self.get_url()) or parsed_url_scheme == "file": + return True + return False + + @property + def is_file(self): + # type: () -> bool + if ( + self.is_path + or (is_file_url(self.get_url()) and is_installable_file(self.get_url())) + or ( + self._parsed_url + and self._parsed_url.scheme == "file" + and is_installable_file(urllib_parse.urlunparse(self._parsed_url)) + ) + ): + return True + return False + + @property + def is_named(self): + # type: () -> bool + return not (self.is_file_url or self.is_url or self.is_file or self.is_vcs) + + @property + def ref(self): + # type: () -> Optional[STRING_TYPE] + if self._ref is None and self.relpath is not None: + self.relpath, self._ref = split_ref_from_uri(self.relpath) + return self._ref + + @property + def ireq(self): + # type: () -> Optional[pip_shims.InstallRequirement] + if self._ireq is None: + self.parse_ireq() + return self._ireq + + @property + def is_installable(self): + # type: () -> bool + possible_paths = (self.line, self.get_url(), self.path, self.base_path) + return any(is_installable_file(p) for p in possible_paths if p is not None) + + @property + def wheel_kwargs(self): + if not self._wheel_kwargs: + self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) + return self._wheel_kwargs + + def get_setup_info(self): + # type: () -> SetupInfo + setup_info = SetupInfo.from_ireq(self.ireq) + if not setup_info.name: + setup_info.get_info() + return setup_info + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + if self._setup_info is None and not self.is_named and not self.is_wheel: + if self._setup_info: + if not self._setup_info.name: + self._setup_info.get_info() + else: + # make two attempts at this before failing to allow for stale data + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + try: + self.setup_info = self.get_setup_info() + except FileNotFoundError: + raise + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if setup_info.version: + self.specifier = setup_info.version + if setup_info.name and not self.name: + self.name = setup_info.name + + def _get_vcsrepo(self): + # type: () -> Optional[VCSRepository] + from .vcs import VCSRepository + + checkout_directory = self.wheel_kwargs["src_dir"] # type: ignore + if self.name is not None: + checkout_directory = os.path.join( + checkout_directory, self.name + ) # type: ignore + vcsrepo = VCSRepository( + url=self.link.url, + name=self.name, + ref=self.ref if self.ref else None, + checkout_directory=checkout_directory, + vcs_type=self.vcs, + subdirectory=self.subdirectory, + ) + if not (self.link.scheme.startswith("file") and self.editable): + vcsrepo.obtain() + return vcsrepo + + @property + def vcsrepo(self): + # type: () -> Optional[VCSRepository] + if self._vcsrepo is None and self.is_vcs: + self._vcsrepo = self._get_vcsrepo() + return self._vcsrepo + + @vcsrepo.setter + def vcsrepo(self, repo): + # type (VCSRepository) -> None + self._vcsrepo = repo + ireq = self.ireq + wheel_kwargs = self.wheel_kwargs.copy() + wheel_kwargs["src_dir"] = repo.checkout_directory + ireq.source_dir = wheel_kwargs["src_dir"] + ireq.build_location(wheel_kwargs["build_dir"]) + ireq._temp_build_dir.path = wheel_kwargs["build_dir"] + with temp_path(): + sys.path = [repo.checkout_directory, "", ".", get_python_lib(plat_specific=0)] + setupinfo = SetupInfo.create( + repo.checkout_directory, + ireq=ireq, + subdirectory=self.subdirectory, + kwargs=wheel_kwargs, + ) + self._setup_info = setupinfo + self._setup_info.reload() + + def get_ireq(self): + # type: () -> InstallRequirement + line = self.line_for_ireq + if self.editable: + ireq = pip_shims.shims.install_req_from_editable(line) + else: + ireq = pip_shims.shims.install_req_from_line(line) + if self.is_named: + ireq = pip_shims.shims.install_req_from_line(self.line) + if self.is_file or self.is_url: + ireq.link = self.link + if self.extras and not ireq.extras: + ireq.extras = set(self.extras) + if self.parsed_marker is not None and not ireq.markers: + ireq.markers = self.parsed_marker + if not ireq.req and self._requirement is not None: + ireq.req = copy.deepcopy(self._requirement) + return ireq + + def parse_ireq(self): + # type: () -> None + if self._ireq is None: + self._ireq = self.get_ireq() + if self._ireq is not None: + if self.requirement is not None and self._ireq.req is None: + self._ireq.req = self.requirement + + def _parse_wheel(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_wheel: + pass + from pip_shims.shims import Wheel + + _wheel = Wheel(self.link.filename) + name = _wheel.name + version = _wheel.version + self._specifier = "=={0}".format(version) + return name + + def _parse_name_from_link(self): + # type: () -> Optional[STRING_TYPE] + + if self.link is None: + return None + if getattr(self.link, "egg_fragment", None): + return self.link.egg_fragment + elif self.is_wheel: + return self._parse_wheel() + return None + + def _parse_name_from_line(self): + # type: () -> Optional[STRING_TYPE] + if not self.is_named: + pass + try: + self._requirement = init_requirement(self.line) + except Exception: + raise RequirementError( + "Failed parsing requirement from {0!r}".format(self.line) + ) + name = self._requirement.name + if not self._specifier and self._requirement and self._requirement.specifier: + self._specifier = specs_to_string(self._requirement.specifier) + if self._requirement.extras and not self.extras: + self.extras = self._requirement.extras + if not name: + name = self.line + specifier_match = next( + iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None + ) + specifier = None # type: Optional[STRING_TYPE] + if specifier_match: + specifier = "{0!s}".format(specifier_match) + if specifier is not None and specifier in name: + version = None # type: Optional[STRING_TYPE] + name, specifier, version = name.partition(specifier) + self._specifier = "{0}{1}".format(specifier, version) + return name + + def parse_name(self): + # type: () -> None + if self._name is None: + name = None + if self.link is not None: + name = self._parse_name_from_link() + if name is None and ( + (self.is_url or self.is_artifact or self.is_vcs) and self._parsed_url + ): + if self._parsed_url.fragment: + _, _, name = self._parsed_url.fragment.partition("egg=") + if "&" in name: + # subdirectory fragments might also be in here + name, _, _ = name.partition("&") + if self.is_named: + name = self._parse_name_from_line() + if name is not None: + name, extras = pip_shims.shims._strip_extras(name) + if extras is not None and not self.extras: + self.extras = tuple(sorted(set(parse_extras(extras)))) + self._name = name + + def _parse_requirement_from_vcs(self): + # type: () -> Optional[PackagingRequirement] + url = self.url if self.url else self.link.url + if url: + url = unquote(url) + if ( + url + and self.uri != url + and "git+ssh://" in url + and (self.uri is not None and "git+git@" in self.uri) + and self._requirement is not None + ): + self._requirement.line = self.uri + self._requirement.url = self.url + vcs_uri = build_vcs_uri( # type: ignore + vcs=self.vcs, + uri=self.url, + ref=self.ref, + subdirectory=self.subdirectory, + extras=self.extras, + name=self.name, + ) + if vcs_uri: + self._requirement.link = create_link(vcs_uri) + elif self.link: + self._requirement.link = self.link + # else: + # req.link = self.link + if self.ref and self._requirement is not None: + if self._vcsrepo is not None: + self._requirement.revision = self._vcsrepo.get_commit_hash() + else: + self._requirement.revision = self.ref + return self._requirement + + def parse_requirement(self): + # type: () -> None + if self._name is None: + self.parse_name() + if not self._name and not self.is_vcs and not self.is_named: + if self.setup_info and self.setup_info.name: + self._name = self.setup_info.name + name, extras, url = self.requirement_info + if name: + self._requirement = init_requirement(name) # type: PackagingRequirement + if extras: + self._requirement.extras = set(extras) + if url: + self._requirement.url = url + if self.is_direct_url: + url = self.link.url + if self.link: + self._requirement.link = self.link + self._requirement.editable = self.editable + if self.path and self.link and self.link.scheme.startswith("file"): + self._requirement.local_file = True + self._requirement.path = self.path + if self.is_vcs: + self._requirement.vcs = self.vcs + self._requirement.line = self.link.url + self._parse_requirement_from_vcs() + else: + self._requirement.line = self.line + if self.parsed_marker is not None: + self._requirement.marker = self.parsed_marker + if self.specifiers: + self._requirement.specifier = self.specifiers + specs = [] + spec = next(iter(s for s in self.specifiers._specs), None) + if spec: + specs.append(spec._spec) + self._requirement.spec = spec + else: + if self.is_vcs: + raise ValueError( + "pipenv requires an #egg fragment for version controlled " + "dependencies. Please install remote dependency " + "in the form {0}#egg=.".format(url) + ) + + def parse_link(self): + # type: () -> None + parsed_url = None # type: Optional[URI] + if not is_valid_url(self.line) and ( + self.line.startswith("./") + or (os.path.exists(self.line) or os.path.isabs(self.line)) + ): + url = pip_shims.shims.path_to_url(os.path.abspath(self.line)) + parsed_url = URI.parse(url) + elif is_valid_url(self.line) or is_vcs(self.line) or is_file_url(self.line): + parsed_url = URI.parse(self.line) + if parsed_url is not None: + line = parsed_url.to_string( + escape_password=False, direct=False, strip_ref=True, strip_ssh=False + ) + if parsed_url.is_vcs: + self.vcs, _ = parsed_url.scheme.split("+") + if parsed_url.is_file_url: + self.is_local = True + parsed_link = parsed_url.as_link + self._ref = parsed_url.ref + self.uri = parsed_url.bare_url + if parsed_url.name: + self._name = parsed_url.name + if parsed_url.extras: + self.extras = tuple(sorted(set(parsed_url.extras))) + self._link = parsed_link + vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( + self.line + ) + ref = None + if link is not None and "@" in unquote(link.path) and uri is not None: + uri, _, ref = unquote(uri).rpartition("@") + if relpath is not None and "@" in relpath: + relpath, _, ref = relpath.rpartition("@") + if path is not None and "@" in path: + path, _ = split_ref_from_uri(path) + link_url = link.url_without_fragment + if "@" in link_url: + link_url, _ = split_ref_from_uri(link_url) + self.preferred_scheme = prefer + self.relpath = relpath + self.path = path + # self.uri = uri + if prefer in ("path", "relpath") or uri.startswith("file"): + self.is_local = True + if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link: + self._link = parsed_link + else: + self._link = link + + def parse_markers(self): + # type: () -> None + if self.markers: + markers = PackagingRequirement("fakepkg; {0}".format(self.markers)).marker + self.parsed_marker = markers + + @property + def requirement_info(self): + # type: () -> Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a + :class:`~packaging.requirements.Requirement` out of. + + :return: A Tuple of an optional name, a Tuple of extras, and an optional URL. + :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] + """ + + # Direct URLs can be converted to packaging requirements directly, but + # only if they are `file://` (with only two slashes) + name = None # type: Optional[S] + extras = () # type: Tuple[Optional[S], ...] + url = None # type: Optional[STRING_TYPE] + # if self.is_direct_url: + if self._name: + name = canonicalize_name(self._name) + if self.is_file or self.is_url or self.is_path or self.is_file_url or self.is_vcs: + url = "" + if self.is_vcs: + url = self.url if self.url else self.uri + if self.is_direct_url: + url = self.link.url_without_fragment + else: + if self.link: + url = self.link.url_without_fragment + elif self.url: + url = self.url + if self.ref: + url = "{0}@{1}".format(url, self.ref) + else: + url = self.uri + if self.link and name is None: + self._name = self.link.egg_fragment + if self._name: + name = canonicalize_name(self._name) + return name, extras, url # type: ignore + + @property + def line_is_installable(self): + # type: () -> bool + """ + This is a safeguard against decoy requirements when a user installs a package + whose name coincides with the name of a folder in the cwd, e.g. install *alembic* + when there is a folder called *alembic* in the working directory. + + In this case we first need to check that the given requirement is a valid + URL, VCS requirement, or installable filesystem path before deciding to treat it + as a file requirement over a named requirement. + """ + line = self.line + if is_file_url(line): + link = create_link(line) + line = link.url_without_fragment + line, _ = split_ref_from_uri(line) + if ( + is_vcs(line) + or ( + is_valid_url(line) + and (not is_file_url(line) or is_installable_file(line)) + ) + or is_installable_file(line) + ): + return True + return False + + def parse(self): + # type: () -> None + self.parse_hashes() + self.line, self.markers = split_markers_from_line(self.line) + self.parse_extras() + self.line = self.line.strip('"').strip("'").strip() + if self.line.startswith("git+file:/") and not self.line.startswith( + "git+file:///" + ): + self.line = self.line.replace("git+file:/", "git+file:///") + self.parse_markers() + if self.is_file_url: + if self.line_is_installable: + self.populate_setup_paths() + else: + raise RequirementError( + "Supplied requirement is not installable: {0!r}".format(self.line) + ) + self.parse_link() + # self.parse_requirement() + # self.parse_ireq() + + +@attr.s(slots=True, hash=True) class NamedRequirement(object): - name = attr.ib() - version = attr.ib(validator=attr.validators.optional(validate_specifiers)) - req = attr.ib() - extras = attr.ib(default=attr.Factory(list)) - editable = attr.ib(default=False) + name = attr.ib() # type: STRING_TYPE + version = attr.ib() # type: Optional[STRING_TYPE] + req = attr.ib() # type: PackagingRequirement + extras = attr.ib(default=attr.Factory(list)) # type: Tuple[STRING_TYPE, ...] + editable = attr.ib(default=False) # type: bool + _parsed_line = attr.ib(default=None) # type: Optional[Line] @req.default def get_requirement(self): + # type: () -> RequirementType req = init_requirement( "{0}{1}".format(canonicalize_name(self.name), self.version) ) return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + @classmethod - def from_line(cls, line): + def from_line(cls, line, parsed_line=None): + # type: (AnyStr, Optional[Line]) -> NamedRequirement req = init_requirement(line) - specifiers = None + specifiers = None # type: Optional[STRING_TYPE] if req.specifier: specifiers = specs_to_string(req.specifier) req.line = line @@ -90,39 +1170,57 @@ class NamedRequirement(object): if not name: name = getattr(req, "key", line) req.name = name - extras = None + creation_kwargs = { + "name": name, + "version": specifiers, + "req": req, + "parsed_line": parsed_line, + "extras": None, + } + extras = None # type: Optional[Tuple[STRING_TYPE, ...]] if req.extras: - extras = list(req.extras) - return cls(name=name, version=specifiers, req=req, extras=extras) + extras = tuple(req.extras) + creation_kwargs["extras"] = extras + return cls(**creation_kwargs) @classmethod - def from_pipfile(cls, name, pipfile): - creation_args = {} + def from_pipfile(cls, name, pipfile): # type: S # type: TPIPFILE + # type: (...) -> NamedRequirement + creation_args = {} # type: TPIPFILE if hasattr(pipfile, "keys"): attr_fields = [field.name for field in attr.fields(cls)] - creation_args = {k: v for k, v in pipfile.items() if k in attr_fields} + creation_args = { + k: v for k, v in pipfile.items() if k in attr_fields + } # type: ignore creation_args["name"] = name - version = get_version(pipfile) + version = get_version(pipfile) # type: Optional[STRING_TYPE] extras = creation_args.get("extras", None) - creation_args["version"] = version + creation_args["version"] = version # type: ignore req = init_requirement("{0}{1}".format(name, version)) - if extras: - req.extras += tuple(extras) + if req and extras and req.extras and isinstance(req.extras, tuple): + if isinstance(extras, six.string_types): + req.extras = (extras) + tuple(["{0}".format(xtra) for xtra in req.extras]) + elif isinstance(extras, (tuple, list)): + req.extras += tuple(extras) creation_args["req"] = req - return cls(**creation_args) + return cls(**creation_args) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE # FIXME: This should actually be canonicalized but for now we have to # simply lowercase it and replace underscores, since full canonicalization # also replaces dots and that doesn't actually work when querying the index - return "{0}".format(normalize_name(self.name)) + return normalize_name(self.name) @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=filter_none).copy() + # type: () -> Dict[STRING_TYPE, Any] + pipfile_dict = attr.asdict(self, filter=filter_none).copy() # type: ignore if "version" not in pipfile_dict: pipfile_dict["version"] = "*" + if "_parsed_line" in pipfile_dict: + pipfile_dict.pop("_parsed_line") name = pipfile_dict.pop("name") return {name: pipfile_dict} @@ -132,40 +1230,46 @@ LinkInfo = collections.namedtuple( ) -@attr.s(slots=True) +@attr.s(slots=True, cmp=True, hash=True) class FileRequirement(object): """File requirements for tar.gz installable files or wheels or setup.py containing directories.""" #: Path to the relevant `setup.py` location - setup_path = attr.ib(default=None) + setup_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: path to hit - without any of the VCS prefixes (like git+ / http+ / etc) - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Whether the package is editable - editable = attr.ib(default=False) + editable = attr.ib(default=False, cmp=True) # type: bool #: Extras if applicable - extras = attr.ib(default=attr.Factory(list)) - _uri_scheme = attr.ib(default=None) + extras = attr.ib( + default=attr.Factory(tuple), cmp=True + ) # type: Tuple[STRING_TYPE, ...] + _uri_scheme = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: URI of the package - uri = attr.ib() + uri = attr.ib(cmp=True) # type: Optional[STRING_TYPE] #: Link object representing the package to clone - link = attr.ib() + link = attr.ib(cmp=True) # type: Optional[Link] #: PyProject Requirements - pyproject_requires = attr.ib(default=attr.Factory(list)) + pyproject_requires = attr.ib( + factory=tuple, cmp=True + ) # type: Optional[Tuple[STRING_TYPE, ...]] #: PyProject Build System - pyproject_backend = attr.ib(default=None) + pyproject_backend = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: PyProject Path - pyproject_path = attr.ib(default=None) - _has_hashed_name = attr.ib(default=False) - #: Package name - name = attr.ib() - #: A :class:`~pkg_resources.Requirement` isntance - req = attr.ib() + pyproject_path = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] #: Setup metadata e.g. dependencies - setup_info = attr.ib(default=None) + _setup_info = attr.ib(default=None, cmp=True) # type: Optional[SetupInfo] + _has_hashed_name = attr.ib(default=False, cmp=True) # type: bool + _parsed_line = attr.ib(default=None, cmp=False, hash=True) # type: Optional[Line] + #: Package name + name = attr.ib(cmp=True) # type: Optional[STRING_TYPE] + #: A :class:`~pkg_resources.Requirement` instance + req = attr.ib(cmp=True) # type: Optional[PackagingRequirement] @classmethod def get_link_from_line(cls, line): + # type: (STRING_TYPE) -> LinkInfo """Parse link information from given requirement line. Return a 6-tuple: @@ -199,15 +1303,16 @@ class FileRequirement(object): # Git allows `git@github.com...` lines that are not really URIs. # Add "ssh://" so we can parse correctly, and restore afterwards. - fixed_line = add_ssh_scheme_to_git_uri(line) - added_ssh_scheme = fixed_line != line + fixed_line = add_ssh_scheme_to_git_uri(line) # type: STRING_TYPE + added_ssh_scheme = fixed_line != line # type: bool # We can assume a lot of things if this is a local filesystem path. if "://" not in fixed_line: - p = Path(fixed_line).absolute() - path = p.as_posix() - uri = p.as_uri() - link = create_link(uri) + p = Path(fixed_line).absolute() # type: Path + path = p.as_posix() # type: Optional[STRING_TYPE] + uri = p.as_uri() # type: STRING_TYPE + link = create_link(uri) # type: Link + relpath = None # type: Optional[STRING_TYPE] try: relpath = get_converted_relative_path(path) except ValueError: @@ -216,19 +1321,17 @@ class FileRequirement(object): # This is an URI. We'll need to perform some elaborated parsing. - parsed_url = urllib_parse.urlsplit(fixed_line) - original_url = parsed_url._replace() - if added_ssh_scheme and ":" in parsed_url.netloc: - original_netloc, original_path_start = parsed_url.netloc.rsplit(":", 1) - uri_path = "/{0}{1}".format(original_path_start, parsed_url.path) - parsed_url = original_url._replace(netloc=original_netloc, path=uri_path) + parsed_url = urllib_parse.urlsplit(fixed_line) # type: SplitResult + original_url = parsed_url._replace() # type: SplitResult # Split the VCS part out if needed. - original_scheme = parsed_url.scheme + original_scheme = parsed_url.scheme # type: STRING_TYPE + vcs_type = None # type: Optional[STRING_TYPE] if "+" in original_scheme: - vcs_type, scheme = original_scheme.split("+", 1) - parsed_url = parsed_url._replace(scheme=scheme) - prefer = "uri" + scheme = None # type: Optional[STRING_TYPE] + vcs_type, _, scheme = original_scheme.partition("+") + parsed_url = parsed_url._replace(scheme=scheme) # type: ignore + prefer = "uri" # type: STRING_TYPE else: vcs_type = None prefer = "file" @@ -250,87 +1353,182 @@ class FileRequirement(object): relpath = None # Cut the fragment, but otherwise this is fixed_line. uri = urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme, fragment="") + parsed_url._replace(scheme=original_scheme, fragment="") # type: ignore ) if added_ssh_scheme: original_uri = urllib_parse.urlunsplit( - original_url._replace(scheme=original_scheme, fragment="") + original_url._replace(scheme=original_scheme, fragment="") # type: ignore ) uri = strip_ssh_from_git_uri(original_uri) # Re-attach VCS prefix to build a Link. link = create_link( - urllib_parse.urlunsplit(parsed_url._replace(scheme=original_scheme)) + urllib_parse.urlunsplit( + parsed_url._replace(scheme=original_scheme) + ) # type: ignore ) return LinkInfo(vcs_type, prefer, relpath, path, uri, link) @property def setup_py_dir(self): + # type: () -> Optional[STRING_TYPE] if self.setup_path: return os.path.dirname(os.path.abspath(self.setup_path)) + return None @property def dependencies(self): - build_deps = [] - setup_deps = [] - deps = {} + # type: () -> Tuple[Dict[S, PackagingRequirement], List[Union[S, PackagingRequirement]], List[S]] + build_deps = [] # type: List[Union[S, PackagingRequirement]] + setup_deps = [] # type: List[S] + deps = {} # type: Dict[S, PackagingRequirement] if self.setup_info: setup_info = self.setup_info.as_dict() deps.update(setup_info.get("requires", {})) setup_deps.extend(setup_info.get("setup_requires", [])) build_deps.extend(setup_info.get("build_requires", [])) + if self.extras and self.setup_info.extras: + for dep in self.extras: + if dep not in self.setup_info.extras: + continue + extras_list = self.setup_info.extras.get(dep, []) # type: ignore + for req_instance in extras_list: # type: ignore + deps[req_instance.key] = req_instance if self.pyproject_requires: - build_deps.extend(self.pyproject_requires) + build_deps.extend(list(self.pyproject_requires)) + setup_deps = list(set(setup_deps)) + build_deps = list(set(build_deps)) return deps, setup_deps, build_deps + def __attrs_post_init__(self): + # type: () -> None + if self.name is None and self.parsed_line: + if self.parsed_line.setup_info: + self._setup_info = self.parsed_line.setup_info + if self.parsed_line.setup_info.name: + self.name = self.parsed_line.setup_info.name + if self.req is None and ( + self._parsed_line is not None and self._parsed_line.requirement is not None + ): + self.req = self._parsed_line.requirement + if ( + self._parsed_line + and self._parsed_line.ireq + and not self._parsed_line.ireq.req + ): + if self.req is not None and self._parsed_line._ireq is not None: + self._parsed_line._ireq.req = self.req + + @property + def setup_info(self): + # type: () -> Optional[SetupInfo] + from .setup_info import SetupInfo + + if self._setup_info is None and self.parsed_line: + if self.parsed_line and self._parsed_line and self.parsed_line.setup_info: + if ( + self._parsed_line._setup_info + and not self._parsed_line._setup_info.name + ): + self._parsed_line._setup_info.get_info() + self._setup_info = self.parsed_line._setup_info + elif self.parsed_line and ( + self.parsed_line.ireq and not self.parsed_line.is_wheel + ): + self._setup_info = SetupInfo.from_ireq(self.parsed_line.ireq) + else: + if self.link and not self.link.is_wheel: + self._setup_info = Line(self.line_part).setup_info + if self._setup_info: + self._setup_info.get_info() + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + # type: (SetupInfo) -> None + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line._setup_info = setup_info + @uri.default def get_uri(self): + # type: () -> STRING_TYPE if self.path and not self.uri: self._uri_scheme = "path" return pip_shims.shims.path_to_url(os.path.abspath(self.path)) - elif getattr(self, "req", None) and getattr(self.req, "url"): + elif ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "url") + ): return self.req.url + elif self.link is not None: + return self.link.url_without_fragment + return "" @name.default def get_name(self): + # type: () -> STRING_TYPE loc = self.path or self.uri if loc and not self._uri_scheme: self._uri_scheme = "path" if self.path else "file" - name = None - if getattr(self, "req", None) and getattr(self.req, "name") and self.req.name is not None: - if self.is_direct_url: + name = None # type: Optional[STRING_TYPE] + hashed_loc = None # type: Optional[STRING_TYPE] + hashed_name = None # type: Optional[STRING_TYPE] + if loc: + hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() + hashed_name = hashed_loc[-7:] + if ( + getattr(self, "req", None) + and self.req is not None + and getattr(self.req, "name") + and self.req.name is not None + ): + if self.is_direct_url and self.req.name != hashed_name: return self.req.name - if self.link and self.link.egg_fragment and not self._has_hashed_name: + if self.link and self.link.egg_fragment and self.link.egg_fragment != hashed_name: return self.link.egg_fragment elif self.link and self.link.is_wheel: from pip_shims import Wheel + self._has_hashed_name = False return Wheel(self.link.filename).name - elif self.link and ((self.link.scheme == "file" or self.editable) or ( - self.path and self.setup_path and os.path.isfile(str(self.setup_path)) - )): + elif self.link and ( + (self.link.scheme == "file" or self.editable) + or (self.path and self.setup_path and os.path.isfile(str(self.setup_path))) + ): + _ireq = None # type: Optional[InstallRequirement] + target_path = "" # type: STRING_TYPE + if self.setup_py_dir: + target_path = Path(self.setup_py_dir).as_posix() + elif self.path: + target_path = Path(os.path.abspath(self.path)).as_posix() if self.editable: - line = pip_shims.shims.path_to_url(self.setup_py_dir) + line = pip_shims.shims.path_to_url(target_path) if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_editable(line) else: - line = Path(self.setup_py_dir).as_posix() + line = target_path if self.extras: line = "{0}[{1}]".format(line, ",".join(self.extras)) _ireq = pip_shims.shims.install_req_from_line(line) - if getattr(self, "req", None): + if getattr(self, "req", None) is not None: _ireq.req = copy.deepcopy(self.req) - else: - if self.extras: - _ireq.extras = set(self.extras) + if self.extras and _ireq and not _ireq.extras: + _ireq.extras = set(self.extras) from .setup_info import SetupInfo + subdir = getattr(self, "subdirectory", None) - setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) + if self.setup_info is not None: + setupinfo = self.setup_info + else: + setupinfo = SetupInfo.from_ireq(_ireq, subdir=subdir) if setupinfo: - self.setup_info = setupinfo + self._setup_info = setupinfo + self._setup_info.get_info() setupinfo_dict = setupinfo.as_dict() setup_name = setupinfo_dict.get("name", None) if setup_name: @@ -339,23 +1537,22 @@ class FileRequirement(object): build_requires = setupinfo_dict.get("build_requires") build_backend = setupinfo_dict.get("build_backend") if build_requires and not self.pyproject_requires: - self.pyproject_requires = build_requires + self.pyproject_requires = tuple(build_requires) if build_backend and not self.pyproject_backend: self.pyproject_backend = build_backend - hashed_loc = hashlib.sha256(loc.encode("utf-8")).hexdigest() - hashed_name = hashed_loc[-7:] if not name or name.lower() == "unknown": self._has_hashed_name = True name = hashed_name - else: - self._has_hashed_name = False name_in_link = getattr(self.link, "egg_fragment", "") if self.link else "" - if not self._has_hashed_name and name_in_link != name: + if not self._has_hashed_name and name_in_link != name and self.link is not None: self.link = create_link("{0}#egg={1}".format(self.link.url, name)) - return name + if name is not None: + return name + return "" @link.default def get_link(self): + # type: () -> pip_shims.shims.Link target = "{0}".format(self.uri) if hasattr(self, "name") and not self._has_hashed_name: target = "{0}#egg={1}".format(target, self.name) @@ -364,9 +1561,38 @@ class FileRequirement(object): @req.default def get_requirement(self): + # type: () -> RequirementType + if self.name is None: + if self._parsed_line is not None and self._parsed_line.name is not None: + self.name = self._parsed_line.name + else: + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format( + self + ) + ) + if self._parsed_line: + try: + # initialize specifiers to make sure we capture them + self._parsed_line.specifiers + except Exception: + pass + req = copy.deepcopy(self._parsed_line.requirement) + if req: + return req + req = init_requirement(normalize_name(self.name)) + if req is None: + raise ValueError( + "Failed to generate a requirement: missing name for {0!r}".format(self) + ) req.editable = False - req.line = self.link.url_without_fragment + if self.link is not None: + req.line = self.link.url_without_fragment + elif self.uri is not None: + req.line = self.uri + else: + req.line = self.name if self.path and self.link and self.link.scheme.startswith("file"): req.local_file = True req.path = self.path @@ -383,8 +1609,35 @@ class FileRequirement(object): req.link = self.link return req + @property + def parsed_line(self): + # type: () -> Optional[Line] + if self._parsed_line is None: + self._parsed_line = Line(self.line_part) + return self._parsed_line + + @property + def is_local(self): + # type: () -> bool + uri = getattr(self, "uri", None) + if uri is None: + if getattr(self, "path", None) and self.path is not None: + uri = pip_shims.shims.path_to_url(os.path.abspath(self.path)) + elif ( + getattr(self, "req", None) + and self.req is not None + and (getattr(self.req, "url") and self.req.url is not None) + ): + uri = self.req.url + if uri and is_file_url(uri): + return True + return False + @property def is_remote_artifact(self): + # type: () -> bool + if self.link is None: + return False return ( any( self.link.scheme.startswith(scheme) @@ -396,116 +1649,190 @@ class FileRequirement(object): @property def is_direct_url(self): + # type: () -> bool + if self._parsed_line is not None and self._parsed_line.is_direct_url: + return True return self.is_remote_artifact @property def formatted_path(self): + # type: () -> Optional[STRING_TYPE] if self.path: path = self.path if not isinstance(path, Path): path = Path(path) return path.as_posix() - return + return None @classmethod def create( - cls, path=None, uri=None, editable=False, extras=None, link=None, vcs_type=None, - name=None, req=None, line=None, uri_scheme=None, setup_path=None, relpath=None + cls, + path=None, # type: Optional[STRING_TYPE] + uri=None, # type: STRING_TYPE + editable=False, # type: bool + extras=None, # type: Optional[Tuple[STRING_TYPE, ...]] + link=None, # type: Link + vcs_type=None, # type: Optional[Any] + name=None, # type: Optional[STRING_TYPE] + req=None, # type: Optional[Any] + line=None, # type: Optional[STRING_TYPE] + uri_scheme=None, # type: STRING_TYPE + setup_path=None, # type: Optional[Any] + relpath=None, # type: Optional[Any] + parsed_line=None, # type: Optional[Line] ): + # type: (...) -> F + if parsed_line is None and line is not None: + parsed_line = Line(line) if relpath and not path: path = relpath - if not path and uri and link.scheme == "file": - path = os.path.abspath(pip_shims.shims.url_to_path(unquote(uri))) + if not path and uri and link is not None and link.scheme == "file": + path = os.path.abspath( + pip_shims.shims.url_to_path(unquote(uri)) + ) # type: ignore try: path = get_converted_relative_path(path) except ValueError: # Vistir raises a ValueError if it can't make a relpath path = path - if line and not (uri_scheme and uri and link): + if line is not None and not (uri_scheme and uri and link): vcs_type, uri_scheme, relpath, path, uri, link = cls.get_link_from_line(line) if not uri_scheme: uri_scheme = "path" if path else "file" if path and not uri: - uri = unquote(pip_shims.shims.path_to_url(os.path.abspath(path))) - if not link: - link = create_link(uri) - if not uri: + uri = unquote( + pip_shims.shims.path_to_url(os.path.abspath(path)) + ) # type: ignore + if not link and uri: + link = cls.get_link_from_line(uri).link + if not uri and link: uri = unquote(link.url_without_fragment) if not extras: - extras = [] + extras = () pyproject_path = None - if path is not None: - pyproject_requires = get_pyproject(os.path.abspath(path)) - pyproject_backend = None pyproject_requires = None - if pyproject_requires is not None: - pyproject_requires, pyproject_backend = pyproject_requires + pyproject_backend = None + pyproject_tuple = None # type: Optional[Tuple[STRING_TYPE]] + if path is not None: + pyproject_requires_and_backend = get_pyproject(path) + if pyproject_requires_and_backend is not None: + pyproject_requires, pyproject_backend = pyproject_requires_and_backend if path: - pyproject_path = Path(path).joinpath("pyproject.toml") - if not pyproject_path.exists(): - pyproject_path = None - if not setup_path and path is not None: - setup_path = Path(path).joinpath("setup.py") + setup_paths = get_setup_paths(path) + if isinstance(setup_paths, Mapping): + if "pyproject_toml" in setup_paths and setup_paths["pyproject_toml"]: + pyproject_path = Path(setup_paths["pyproject_toml"]) + if "setup_py" in setup_paths and setup_paths["setup_py"]: + setup_path = Path(setup_paths["setup_py"]).as_posix() if setup_path and isinstance(setup_path, Path): setup_path = setup_path.as_posix() creation_kwargs = { "editable": editable, "extras": extras, "pyproject_path": pyproject_path, - "setup_path": setup_path if setup_path else None, + "setup_path": setup_path, "uri_scheme": uri_scheme, "link": link, "uri": uri, - "pyproject_requires": pyproject_requires, - "pyproject_backend": pyproject_backend + "pyproject_requires": pyproject_tuple, + "pyproject_backend": pyproject_backend, + "path": path or relpath, + "parsed_line": parsed_line, } if vcs_type: - creation_kwargs["vcs_type"] = vcs_type - _line = None - if not name: - _line = unquote(link.url_without_fragment) if link.url else uri - if editable: - if extras: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) - ireq = pip_shims.shims.install_req_from_editable(_line) - else: - _line = path if (uri_scheme and uri_scheme == "path") else _line - if extras: - _line = "{0}[{1}]".format(_line, ",".join(sorted(set(extras)))) - ireq = pip_shims.shims.install_req_from_line(_line) - if extras and not ireq.extras: - ireq.extras = set(extras) - setup_info = SetupInfo.from_ireq(ireq) - setupinfo_dict = setup_info.as_dict() - setup_name = setupinfo_dict.get("name", None) - if setup_name: - name = setup_name - build_requires = setupinfo_dict.get("build_requires", []) - build_backend = setupinfo_dict.get("build_backend", []) - if not creation_kwargs.get("pyproject_requires") and build_requires: - creation_kwargs["pyproject_requires"] = build_requires - if not creation_kwargs.get("pyproject_backend") and build_backend: - creation_kwargs["pyproject_backend"] = build_backend - creation_kwargs["setup_info"] = setup_info - if path or relpath: - creation_kwargs["path"] = relpath if relpath else path - if req: - creation_kwargs["req"] = req - if creation_kwargs.get("req") and line and not getattr(creation_kwargs["req"], "line", None): - creation_kwargs["req"].line = line + creation_kwargs["vcs"] = vcs_type if name: creation_kwargs["name"] = name - cls_inst = cls(**creation_kwargs) - if not _line: - if editable and uri_scheme == "path": - _line = relpath if relpath else path + _line = None # type: Optional[STRING_TYPE] + ireq = None # type: Optional[InstallRequirement] + setup_info = None # type: Optional[SetupInfo] + if parsed_line: + if parsed_line.name: + name = parsed_line.name + if parsed_line.setup_info: + name = parsed_line.setup_info.as_dict().get("name", name) + if not name or not parsed_line: + if link is not None and link.url_without_fragment is not None: + _line = unquote(link.url_without_fragment) + if name: + _line = "{0}#egg={1}".format(_line, name) + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + elif isinstance(uri, six.string_types): + _line = unquote(uri) + elif line: + _line = unquote(line) + if editable: + if ( + _line + and extras + and extras_to_string(extras) not in _line + and ( + (link and link.scheme == "file") + or (uri and uri.startswith("file")) + or (not uri and not link) + ) + ): + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + if ireq is None: + ireq = pip_shims.shims.install_req_from_editable( + _line + ) # type: ignore else: - _line = unquote(cls_inst.link.url_without_fragment) or cls_inst.uri - _line = "{0}#egg={1}".format(line, cls_inst.name) if not cls_inst._has_hashed_name else _line - cls_inst.req.line = line if line else _line - return cls_inst + _line = path if (uri_scheme and uri_scheme == "path") else _line + if _line and extras and extras_to_string(extras) not in _line: + _line = "{0}[{1}]".format( + _line, ",".join(sorted(set(extras))) + ) # type: ignore + if ireq is None: + ireq = pip_shims.shims.install_req_from_line(_line) # type: ignore + if editable: + _line = "-e {0}".format(editable) + if _line: + parsed_line = Line(_line) + if ireq is None and parsed_line and parsed_line.ireq: + ireq = parsed_line.ireq + if extras and ireq is not None and not ireq.extras: + ireq.extras = set(extras) + if setup_info is None: + setup_info = SetupInfo.from_ireq(ireq) + setupinfo_dict = setup_info.as_dict() + setup_name = setupinfo_dict.get("name", None) + build_requires = () # type: Tuple[STRING_TYPE, ...] + build_backend = "" + if setup_name is not None: + name = setup_name + build_requires = setupinfo_dict.get("build_requires", build_requires) + build_backend = setupinfo_dict.get("build_backend", build_backend) + if "pyproject_requires" not in creation_kwargs and build_requires: + creation_kwargs["pyproject_requires"] = tuple(build_requires) + if "pyproject_backend" not in creation_kwargs and build_backend: + creation_kwargs["pyproject_backend"] = build_backend + if setup_info is None and parsed_line and parsed_line.setup_info: + setup_info = parsed_line.setup_info + creation_kwargs["setup_info"] = setup_info + if path or relpath: + creation_kwargs["path"] = relpath if relpath else path + if req is not None: + creation_kwargs["req"] = req + creation_req = creation_kwargs.get("req") + if creation_kwargs.get("req") is not None: + creation_req_line = getattr(creation_req, "line", None) + if creation_req_line is None and line is not None: + creation_kwargs["req"].line = line # type: ignore + if parsed_line and parsed_line.name: + if name and len(parsed_line.name) != 7 and len(name) == 7: + name = parsed_line.name + if name: + creation_kwargs["name"] = name + return cls(**creation_kwargs) # type: ignore @classmethod - def from_line(cls, line, extras=None): + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F line = line.strip('"').strip("'") link = None path = None @@ -515,7 +1842,9 @@ class FileRequirement(object): name = None req = None if not extras: - extras = [] + extras = () + else: + extras = tuple(extras) if not any([is_installable_file(line), is_valid_url(line), is_file_url(line)]): try: req = init_requirement(line) @@ -535,8 +1864,13 @@ class FileRequirement(object): "setup_path": setup_path, "uri_scheme": prefer, "line": line, - "extras": extras + "extras": extras, + # "name": name, } + if req is not None: + arg_dict["req"] = req + if parsed_line is not None: + arg_dict["parsed_line"] = parsed_line if link and link.is_wheel: from pip_shims import Wheel @@ -549,6 +1883,7 @@ class FileRequirement(object): @classmethod def from_pipfile(cls, name, pipfile): + # type: (STRING_TYPE, Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE, bool]]) -> F # Parse the values out. After this dance we should have two variables: # path - Local filesystem path. # uri - Absolute URI that is parsable with urlsplit. @@ -556,7 +1891,7 @@ class FileRequirement(object): uri = pipfile.get("uri") fil = pipfile.get("file") path = pipfile.get("path") - if path: + if path and isinstance(path, six.string_types): if isinstance(path, Path) and not path.is_absolute(): path = get_converted_relative_path(path.as_posix()) elif not os.path.isabs(path): @@ -580,124 +1915,196 @@ class FileRequirement(object): if not uri: uri = pip_shims.shims.path_to_url(path) - link = create_link(uri) + link_info = None # type: Optional[LinkInfo] + if uri and isinstance(uri, six.string_types): + link_info = cls.get_link_from_line(uri) + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + link = None # type: Optional[Link] + if link_info: + link = link_info.link + if link.url_without_fragment: + uri = unquote(link.url_without_fragment) + extras = () # type: Optional[Tuple[STRING_TYPE, ...]] + if "extras" in pipfile: + extras = tuple(pipfile["extras"]) # type: ignore + editable = pipfile["editable"] if "editable" in pipfile else False arg_dict = { "name": name, "path": path, - "uri": unquote(link.url_without_fragment), - "editable": pipfile.get("editable", False), + "uri": uri, + "editable": editable, "link": link, "uri_scheme": uri_scheme, + "extras": extras if extras else None, } - if link.scheme != "file" and not pipfile.get("editable", False): - arg_dict["line"] = "{0}@ {1}".format(name, link.url_without_fragment) - return cls.create(**arg_dict) + + line = "" # type: STRING_TYPE + extras_string = "" if not extras else extras_to_string(extras) + if editable and uri_scheme == "path": + line = "{0}{1}".format(path, extras_string) + else: + if name: + line_name = "{0}{1}".format(name, extras_string) + line = "{0}#egg={1}".format(unquote(link.url_without_fragment), line_name) + else: + if link: + line = unquote(link.url) + elif uri and isinstance(uri, six.string_types): + line = uri + else: + raise ValueError( + "Failed parsing requirement from pipfile: {0!r}".format(pipfile) + ) + line = "{0}{1}".format(line, extras_string) + if "subdirectory" in pipfile: + arg_dict["subdirectory"] = pipfile["subdirectory"] + line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) + if editable: + line = "-e {0}".format(line) + arg_dict["line"] = line + return cls.create(**arg_dict) # type: ignore @property def line_part(self): + # type: () -> STRING_TYPE + link_url = None # type: Optional[STRING_TYPE] + seed = None # type: Optional[STRING_TYPE] + if self.link is not None: + link_url = unquote(self.link.url_without_fragment) if self._uri_scheme and self._uri_scheme == "path": # We may need any one of these for passing to pip - seed = self.path or unquote(self.link.url_without_fragment) or self.uri + seed = self.path or link_url or self.uri elif (self._uri_scheme and self._uri_scheme == "file") or ( (self.link.is_artifact or self.link.is_wheel) and self.link.url ): - seed = unquote(self.link.url_without_fragment) or self.uri + seed = link_url or self.uri # add egg fragments to remote artifacts (valid urls only) - if not self._has_hashed_name and self.is_remote_artifact: + if not self._has_hashed_name and self.is_remote_artifact and seed is not None: seed += "#egg={0}".format(self.name) editable = "-e " if self.editable else "" + if seed is None: + raise ValueError("Could not calculate url for {0!r}".format(self)) return "{0}{1}".format(editable, seed) @property def pipfile_part(self): + # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] excludes = [ - "_base_line", "_has_hashed_name", "setup_path", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_base_line", + "_has_hashed_name", + "setup_path", + "pyproject_path", + "_uri_scheme", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes - pipfile_dict = attr.asdict(self, filter=filter_func).copy() - name = pipfile_dict.pop("name") + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa + pipfile_dict = attr.asdict(self, filter=filter_func).copy() # type: Dict + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "_uri_scheme" in pipfile_dict: pipfile_dict.pop("_uri_scheme") # For local paths and remote installable artifacts (zipfiles, etc) collision_keys = {"file", "uri", "path"} + collision_order = ["file", "uri", "path"] # type: List[STRING_TYPE] + collisions = [] # type: List[STRING_TYPE] + key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) if self._uri_scheme: dict_key = self._uri_scheme - target_key = ( - dict_key - if dict_key in pipfile_dict - else next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - ) - if target_key: + target_key = dict_key if dict_key in pipfile_dict else key_match + if target_key is not None: winning_value = pipfile_dict.pop(target_key) - collisions = (k for k in collision_keys if k in pipfile_dict) + collisions = [k for k in collision_keys if k in pipfile_dict] for key in collisions: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value elif ( self.is_remote_artifact - or self.link.is_artifact + or (self.link is not None and self.link.is_artifact) and (self._uri_scheme and self._uri_scheme == "file") ): dict_key = "file" # Look for uri first because file is a uri format and this is designed # to make sure we add file keys to the pipfile as a replacement of uri - target_key = next( - (k for k in ("file", "uri", "path") if k in pipfile_dict), None - ) - winning_value = pipfile_dict.pop(target_key) + if key_match is not None: + winning_value = pipfile_dict.pop(key_match) key_to_remove = (k for k in collision_keys if k in pipfile_dict) for key in key_to_remove: pipfile_dict.pop(key) pipfile_dict[dict_key] = winning_value else: - collisions = [key for key in ["path", "file", "uri"] if key in pipfile_dict] + collisions = [key for key in collision_order if key in pipfile_dict.keys()] if len(collisions) > 1: for k in collisions[1:]: pipfile_dict.pop(k) return {name: pipfile_dict} -@attr.s(slots=True) +@attr.s(slots=True, hash=True) class VCSRequirement(FileRequirement): #: Whether the repository is editable - editable = attr.ib(default=None) + editable = attr.ib(default=None) # type: Optional[bool] #: URI for the repository - uri = attr.ib(default=None) + uri = attr.ib(default=None) # type: Optional[STRING_TYPE] #: path to the repository, if it's local - path = attr.ib(default=None, validator=attr.validators.optional(validate_path)) + path = attr.ib( + default=None, validator=attr.validators.optional(validate_path) + ) # type: Optional[STRING_TYPE] #: vcs type, i.e. git/hg/svn - vcs = attr.ib(validator=attr.validators.optional(validate_vcs), default=None) + vcs = attr.ib( + validator=attr.validators.optional(validate_vcs), default=None + ) # type: Optional[STRING_TYPE] #: vcs reference name (branch / commit / tag) - ref = attr.ib(default=None) + ref = attr.ib(default=None) # type: Optional[STRING_TYPE] #: Subdirectory to use for installation if applicable - subdirectory = attr.ib(default=None) - _repo = attr.ib(default=None) - _base_line = attr.ib(default=None) - name = attr.ib() - link = attr.ib() - req = attr.ib() + subdirectory = attr.ib(default=None) # type: Optional[STRING_TYPE] + _repo = attr.ib(default=None) # type: Optional[VCSRepository] + _base_line = attr.ib(default=None) # type: Optional[STRING_TYPE] + name = attr.ib() # type: STRING_TYPE + link = attr.ib() # type: Optional[pip_shims.shims.Link] + req = attr.ib() # type: Optional[RequirementType] def __attrs_post_init__(self): + # type: () -> None if not self.uri: if self.path: self.uri = pip_shims.shims.path_to_url(self.path) - split = urllib_parse.urlsplit(self.uri) - scheme, rest = split[0], split[1:] - vcs_type = "" - if "+" in scheme: - vcs_type, scheme = scheme.split("+", 1) - vcs_type = "{0}+".format(vcs_type) - new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) - new_uri = "{0}{1}".format(vcs_type, new_uri) - self.uri = new_uri + if self.uri is not None: + split = urllib_parse.urlsplit(self.uri) + scheme, rest = split[0], split[1:] + vcs_type = "" + if "+" in scheme: + vcs_type, scheme = scheme.split("+", 1) + vcs_type = "{0}+".format(vcs_type) + new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) + new_uri = "{0}{1}".format(vcs_type, new_uri) + self.uri = new_uri + + @property + def url(self): + # type: () -> STRING_TYPE + if self.link and self.link.url: + return self.link.url + elif self.uri: + return self.uri + raise ValueError("No valid url found for requirement {0!r}".format(self)) @link.default def get_link(self): + # type: () -> pip_shims.shims.Link uri = self.uri if self.uri else pip_shims.shims.path_to_url(self.path) - return build_vcs_link( + vcs_uri = build_vcs_uri( self.vcs, add_ssh_scheme_to_git_uri(uri), name=self.name, @@ -705,26 +2112,65 @@ class VCSRequirement(FileRequirement): subdirectory=self.subdirectory, extras=self.extras, ) + return self.get_link_from_line(vcs_uri).link @name.default def get_name(self): - return ( - self.link.egg_fragment or self.req.name - if getattr(self, "req", None) - else super(VCSRequirement, self).get_name() - ) + # type: () -> STRING_TYPE + if self.link and self.link.egg_fragment: + return self.link.egg_fragment + if self.req and self.req.name: + return self.req.name + return super(VCSRequirement, self).get_name() @property def vcs_uri(self): + # type: () -> Optional[STRING_TYPE] uri = self.uri - if not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): - uri = "{0}+{1}".format(self.vcs, uri) + if uri and not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): + if self.vcs: + uri = "{0}+{1}".format(self.vcs, uri) return uri + @property + def setup_info(self): + if self._parsed_line and self._parsed_line.setup_info: + if not self._parsed_line.setup_info.name: + self._parsed_line._setup_info.get_info() + return self._parsed_line.setup_info + if self._repo: + from .setup_info import SetupInfo + + self._setup_info = SetupInfo.from_ireq( + Line(self._repo.checkout_directory).ireq + ) + self._setup_info.get_info() + return self._setup_info + ireq = self.parsed_line.ireq + from .setup_info import SetupInfo + + self._setup_info = SetupInfo.from_ireq(ireq) + return self._setup_info + + @setup_info.setter + def setup_info(self, setup_info): + self._setup_info = setup_info + if self._parsed_line: + self._parsed_line.setup_info = setup_info + @req.default def get_requirement(self): - name = self.name or self.link.egg_fragment - url = self.uri or self.link.url_without_fragment + # type: () -> PackagingRequirement + name = None # type: Optional[STRING_TYPE] + if self.name: + name = self.name + elif self.link and self.link.egg_fragment: + name = self.link.egg_fragment + url = None + if self.uri: + url = self.uri + elif self.link is not None: + url = self.link.url_without_fragment if not name: raise ValueError( "pipenv requires an #egg fragment for version controlled " @@ -733,9 +2179,25 @@ class VCSRequirement(FileRequirement): ) req = init_requirement(canonicalize_name(self.name)) req.editable = self.editable - if not getattr(req, "url") and self.uri: - req.url = self.uri - req.line = self.link.url + if not getattr(req, "url"): + if url is not None: + url = add_ssh_scheme_to_git_uri(url) + elif self.uri is not None: + link = self.get_link_from_line(self.uri).link + if link: + url = link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + if url: + req.url = url + line = url if url else self.vcs_uri + if self.editable: + line = "-e {0}".format(line) + req.line = line if self.ref: req.revision = self.ref if self.extras: @@ -746,27 +2208,38 @@ class VCSRequirement(FileRequirement): req.path = self.path req.link = self.link if ( - self.uri != unquote(self.link.url_without_fragment) + self.link + and self.link.url_without_fragment + and self.uri + and self.uri != unquote(self.link.url_without_fragment) and "git+ssh://" in self.link.url and "git+git@" in self.uri ): req.line = self.uri - req.url = self.uri + url = self.link.url_without_fragment + if ( + url + and url.startswith("git+file:/") + and not url.startswith("git+file:///") + ): + url = url.replace("git+file:/", "git+file:///") + req.url = url return req - @property - def is_local(self): - if is_file_url(self.uri): - return True - return False - @property def repo(self): + # type: () -> VCSRepository if self._repo is None: - self._repo = self.get_vcs_repo() + if self._parsed_line and self._parsed_line.vcsrepo: + self._repo = self._parsed_line.vcsrepo + else: + self._repo = self.get_vcs_repo() + if self._parsed_line: + self._parsed_line.vcsrepo = self._repo return self._repo def get_checkout_dir(self, src_dir=None): + # type: (Optional[S]) -> STRING_TYPE src_dir = os.environ.get("PIP_SRC", None) if not src_dir else src_dir checkout_dir = None if self.is_local: @@ -776,22 +2249,20 @@ class VCSRequirement(FileRequirement): if path and os.path.exists(path): checkout_dir = os.path.abspath(path) return checkout_dir + if src_dir is not None: + checkout_dir = os.path.join(os.path.abspath(src_dir), self.name) + mkdir_p(src_dir) + return checkout_dir return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - def get_vcs_repo(self, src_dir=None): + def get_vcs_repo(self, src_dir=None, checkout_dir=None): + # type: (Optional[STRING_TYPE], STRING_TYPE) -> VCSRepository from .vcs import VCSRepository - checkout_dir = self.get_checkout_dir(src_dir=src_dir) - link = build_vcs_link( - self.vcs, - self.uri, - name=self.name, - ref=self.ref, - subdirectory=self.subdirectory, - extras=self.extras, - ) + if checkout_dir is None: + checkout_dir = self.get_checkout_dir(src_dir=src_dir) vcsrepo = VCSRepository( - url=link.url, + url=self.url, name=self.name, ref=self.ref if self.ref else None, checkout_directory=checkout_dir, @@ -803,7 +2274,9 @@ class VCSRequirement(FileRequirement): pyproject_info = None if self.subdirectory: self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") - self.pyproject_path = os.path.join(checkout_dir, self.subdirectory, "pyproject.toml") + self.pyproject_path = os.path.join( + checkout_dir, self.subdirectory, "pyproject.toml" + ) pyproject_info = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) else: self.setup_path = os.path.join(checkout_dir, "setup.py") @@ -811,16 +2284,18 @@ class VCSRequirement(FileRequirement): pyproject_info = get_pyproject(checkout_dir) if pyproject_info is not None: pyproject_requires, pyproject_backend = pyproject_info - self.pyproject_requires = pyproject_requires + self.pyproject_requires = tuple(pyproject_requires) self.pyproject_backend = pyproject_backend return vcsrepo def get_commit_hash(self): + # type: () -> STRING_TYPE hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): + # type: (Optional[STRING_TYPE], Optional[STRING_TYPE]) -> STRING_TYPE if ref: self.ref = ref else: @@ -830,29 +2305,57 @@ class VCSRequirement(FileRequirement): if not self.is_local and ref is not None: self.repo.checkout_ref(ref) repo_hash = self.repo.get_commit_hash() - self.req.revision = repo_hash + if self.req: + self.req.revision = repo_hash return repo_hash @contextmanager def locked_vcs_repo(self, src_dir=None): + # type: (Optional[AnyStr]) -> Generator[VCSRepository, None, None] if not src_dir: src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") vcsrepo = self.get_vcs_repo(src_dir=src_dir) - self.req.revision = vcsrepo.get_commit_hash() + if not self.req: + if self.parsed_line is not None: + self.req = self.parsed_line.requirement + else: + self.req = self.get_requirement() + revision = self.req.revision = vcsrepo.get_commit_hash() # Remove potential ref in the end of uri after ref is parsed - if "@" in self.link.show_url and "@" in self.uri: - uri, ref = self.uri.rsplit("@", 1) - checkout = self.req.revision - if checkout and ref in checkout: + if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri: + uri, ref = split_ref_from_uri(self.uri) + checkout = revision + if checkout and ref and ref in checkout: self.uri = uri - - yield vcsrepo + orig_repo = self._repo self._repo = vcsrepo + if self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self._setup_info: + self._setup_info = attr.evolve( + self._setup_info, + requirements=(), + _extras_requirements=(), + build_requires=(), + setup_requires=(), + version=None, + metadata=None, + ) + if self.parsed_line and self._parsed_line: + self._parsed_line.vcsrepo = vcsrepo + if self.req: + self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) + try: + yield self._repo + except Exception: + self._repo = orig_repo + raise @classmethod def from_pipfile(cls, name, pipfile): - creation_args = {} + # type: (STRING_TYPE, Dict[S, Union[Tuple[S, ...], S, bool]]) -> F + creation_args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] pipfile_keys = [ k for k in ( @@ -868,54 +2371,102 @@ class VCSRequirement(FileRequirement): + VCS_LIST if k in pipfile ] + # extras = None # type: Optional[Tuple[STRING_TYPE, ...]] for key in pipfile_keys: - if key == "extras": - extras = pipfile.get(key, None) - if extras: - pipfile[key] = sorted(dedup([extra.lower() for extra in extras])) - if key in VCS_LIST: - creation_args["vcs"] = key - target = pipfile.get(key) - drive, path = os.path.splitdrive(target) - if ( - not drive - and not os.path.exists(target) - and ( - is_valid_url(target) - or is_file_url(target) - or target.startswith("git@") - ) - ): - creation_args["uri"] = target + if key == "extras" and key in pipfile: + extras = pipfile[key] + if isinstance(extras, (list, tuple)): + pipfile[key] = tuple(sorted({extra.lower() for extra in extras})) else: - creation_args["path"] = target - if os.path.isabs(target): - creation_args["uri"] = pip_shims.shims.path_to_url(target) - else: - creation_args[key] = pipfile.get(key) + pipfile[key] = extras + if key in VCS_LIST and key in pipfile_keys: + creation_args["vcs"] = key + target = pipfile[key] + if isinstance(target, six.string_types): + drive, path = os.path.splitdrive(target) + if ( + not drive + and not os.path.exists(target) + and ( + is_valid_url(target) + or is_file_url(target) + or target.startswith("git@") + ) + ): + creation_args["uri"] = target + else: + creation_args["path"] = target + if os.path.isabs(target): + creation_args["uri"] = pip_shims.shims.path_to_url(target) + elif key in pipfile_keys: + creation_args[key] = pipfile[key] creation_args["name"] = name - return cls(**creation_args) + cls_inst = cls(**creation_args) # type: ignore + return cls_inst @classmethod - def from_line(cls, line, editable=None, extras=None): + def from_line(cls, line, editable=None, extras=None, parsed_line=None): + # type: (AnyStr, Optional[bool], Optional[Tuple[AnyStr, ...]], Optional[Line]) -> F relpath = None + if parsed_line is None: + parsed_line = Line(line) + if editable: + parsed_line.editable = editable + if extras: + parsed_line.extras = extras if line.startswith("-e "): editable = True line = line.split(" ", 1)[1] + if "@" in line: + parsed = urllib_parse.urlparse(add_ssh_scheme_to_git_uri(line)) + if not parsed.scheme: + possible_name, _, line = line.partition("@") + possible_name = possible_name.strip() + line = line.strip() + possible_name, extras = pip_shims.shims._strip_extras(possible_name) + name = possible_name + line = "{0}#egg={1}".format(line, name) vcs_type, prefer, relpath, path, uri, link = cls.get_link_from_line(line) if not extras and link.egg_fragment: name, extras = pip_shims.shims._strip_extras(link.egg_fragment) - if extras: - extras = parse_extras(extras) else: - name = link.egg_fragment + name, _ = pip_shims.shims._strip_extras(link.egg_fragment) + parsed_extras = None # type: Optional[List[STRING_TYPE]] + extras_tuple = None # type: Optional[Tuple[STRING_TYPE, ...]] + if not extras: + line, extras = pip_shims.shims._strip_extras(line) + if extras: + if isinstance(extras, six.string_types): + parsed_extras = parse_extras(extras) + if parsed_extras: + extras_tuple = tuple(parsed_extras) subdirectory = link.subdirectory_fragment ref = None - if "@" in link.path and "@" in uri: - uri, _, ref = uri.rpartition("@") + if uri: + uri, ref = split_ref_from_uri(uri) + if path is not None and "@" in path: + path, _ref = split_ref_from_uri(path) + if ref is None: + ref = _ref if relpath and "@" in relpath: - relpath, ref = relpath.rsplit("@", 1) - return cls( + relpath, ref = split_ref_from_uri(relpath) + + creation_args = { + "name": name if name else parsed_line.name, + "path": relpath or path, + "editable": editable, + "extras": extras_tuple, + "link": link, + "vcs_type": vcs_type, + "line": line, + "uri": uri, + "uri_scheme": prefer, + "parsed_line": parsed_line, + } + if relpath: + creation_args["relpath"] = relpath + # return cls.create(**creation_args) + cls_inst = cls( name=name, ref=ref, vcs=vcs_type, @@ -924,97 +2475,192 @@ class VCSRequirement(FileRequirement): path=relpath or path, editable=editable, uri=uri, - extras=extras, + extras=extras_tuple if extras_tuple else tuple(), base_line=line, + parsed_line=parsed_line, ) + if cls_inst.req and ( + cls_inst._parsed_line.ireq and not cls_inst.parsed_line.ireq.req + ): + cls_inst._parsed_line._ireq.req = cls_inst.req + return cls_inst @property def line_part(self): + # type: () -> STRING_TYPE """requirements.txt compatible line part sans-extras""" + base = "" # type: STRING_TYPE if self.is_local: base_link = self.link if not self.link: base_link = self.get_link() - final_format = ( - "{{0}}#egg={0}".format(base_link.egg_fragment) - if base_link.egg_fragment - else "{0}" - ) + if base_link and base_link.egg_fragment: + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) + else: + final_format = "{0}" base = final_format.format(self.vcs_uri) - elif self._base_line: + elif self._parsed_line is not None and ( + self._parsed_line.is_direct_url and self._parsed_line.line_with_prefix + ): + return self._parsed_line.line_with_prefix + elif getattr(self, "_base_line", None) and ( + isinstance(self._base_line, six.string_types) + ): base = self._base_line else: - base = self.link.url - if base and self.extras and not extras_to_string(self.extras) in base: + base = getattr(self, "link", self.get_link()).url + if base and self.extras and extras_to_string(self.extras) not in base: if self.subdirectory: base = "{0}".format(self.get_link().url) else: base = "{0}{1}".format(base, extras_to_string(sorted(self.extras))) - if self.editable: + if "git+file:/" in base and "git+file:///" not in base: + base = base.replace("git+file:/", "git+file:///") + if self.editable and not base.startswith("-e "): base = "-e {0}".format(base) return base @staticmethod def _choose_vcs_source(pipfile): + # type: (Dict[S, Union[S, Any]]) -> Dict[S, Union[S, Any]] src_keys = [k for k in pipfile.keys() if k in ["path", "uri", "file"]] + vcs_type = "" # type: Optional[STRING_TYPE] + alt_type = "" # type: Optional[STRING_TYPE] + vcs_value = "" # type: STRING_TYPE if src_keys: chosen_key = first(src_keys) vcs_type = pipfile.pop("vcs") - _, pipfile_url = split_vcs_method_from_uri(pipfile.get(chosen_key)) - pipfile[vcs_type] = pipfile_url + if chosen_key in pipfile: + vcs_value = pipfile[chosen_key] + alt_type, pipfile_url = split_vcs_method_from_uri(vcs_value) + if vcs_type is None: + vcs_type = alt_type + if vcs_type and pipfile_url: + pipfile[vcs_type] = pipfile_url for removed in src_keys: pipfile.pop(removed) return pipfile @property def pipfile_part(self): + # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, RequirementType, pip_shims.shims.Link]]] excludes = [ - "_repo", "_base_line", "setup_path", "_has_hashed_name", "pyproject_path", - "pyproject_requires", "pyproject_backend", "setup_info" + "_repo", + "_base_line", + "setup_path", + "_has_hashed_name", + "pyproject_path", + "pyproject_requires", + "pyproject_backend", + "_setup_info", + "_parsed_line", + "_uri_scheme", ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes + filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa pipfile_dict = attr.asdict(self, filter=filter_func).copy() + name = pipfile_dict.pop("name", None) + if name is None: + if self.name: + name = self.name + elif self.parsed_line and self.parsed_line.name: + name = self.name = self.parsed_line.name + elif self.setup_info and self.setup_info.name: + name = self.name = self.setup_info.name if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = pip_shims.shims._strip_extras(pipfile_dict.pop("name")) - return {name: pipfile_dict} + name, _ = pip_shims.shims._strip_extras(name) + return {name: pipfile_dict} # type: ignore -@attr.s +@attr.s(cmp=True, hash=True) class Requirement(object): - name = attr.ib() - vcs = attr.ib(default=None, validator=attr.validators.optional(validate_vcs)) - req = attr.ib(default=None) - markers = attr.ib(default=None) - specifiers = attr.ib(validator=attr.validators.optional(validate_specifiers)) - index = attr.ib(default=None) - editable = attr.ib(default=None) - hashes = attr.ib(default=attr.Factory(list), converter=list) - extras = attr.ib(default=attr.Factory(list)) - abstract_dep = attr.ib(default=None) - _ireq = None + _name = attr.ib(cmp=True) # type: STRING_TYPE + vcs = attr.ib( + default=None, validator=attr.validators.optional(validate_vcs), cmp=True + ) # type: Optional[STRING_TYPE] + req = attr.ib( + default=None, cmp=True + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + markers = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + _specifiers = attr.ib( + validator=attr.validators.optional(validate_specifiers), cmp=True + ) # type: Optional[STRING_TYPE] + index = attr.ib(default=None, cmp=True) # type: Optional[STRING_TYPE] + editable = attr.ib(default=None, cmp=True) # type: Optional[bool] + hashes = attr.ib( + factory=frozenset, converter=frozenset, cmp=True + ) # type: FrozenSet[STRING_TYPE] + extras = attr.ib(factory=tuple, cmp=True) # type: Tuple[STRING_TYPE, ...] + abstract_dep = attr.ib(default=None, cmp=False) # type: Optional[AbstractDependency] + _line_instance = attr.ib(default=None, cmp=False) # type: Optional[Line] + _ireq = attr.ib( + default=None, cmp=False + ) # type: Optional[pip_shims.InstallRequirement] - @name.default + def __hash__(self): + return hash(self.as_line()) + + @_name.default def get_name(self): - return self.req.name + # type: () -> Optional[STRING_TYPE] + if self.req is not None: + return self.req.name + return None + + @property + def name(self): + # type: () -> Optional[STRING_TYPE] + if self._name is not None: + return self._name + name = None + if self.req and self.req.name: + name = self.req.name + elif self.req and self.is_file_or_url and self.req.setup_info: + name = self.req.setup_info.name + self._name = name + return name @property def requirement(self): - return self.req.req + # type: () -> Optional[PackagingRequirement] + if self.req: + return self.req.req + return None + + def add_hashes(self, hashes): + # type: (Union[S, List[S], Set[S], Tuple[S, ...]]) -> Requirement + new_hashes = set() # type: Set[STRING_TYPE] + if self.hashes is not None: + new_hashes |= set(self.hashes) + if isinstance(hashes, six.string_types): + new_hashes.add(hashes) + else: + new_hashes |= set(hashes) + return attr.evolve(self, hashes=tuple(new_hashes)) def get_hashes_as_pip(self, as_list=False): - if self.hashes: - if as_list: - return [HASH_STRING.format(h) for h in self.hashes] - return "".join([HASH_STRING.format(h) for h in self.hashes]) - return "" if not as_list else [] + # type: (bool) -> Union[STRING_TYPE, List[STRING_TYPE]] + hashes = "" # type: Union[STRING_TYPE, List[STRING_TYPE]] + if as_list: + hashes = [] + if self.hashes: + hashes = [HASH_STRING.format(h) for h in self.hashes] + else: + hashes = "" + if self.hashes: + hashes = "".join([HASH_STRING.format(h) for h in self.hashes]) + return hashes @property def hashes_as_pip(self): - self.get_hashes_as_pip() + # type: () -> STRING_TYPE + hashes = self.get_hashes_as_pip() + assert isinstance(hashes, six.string_types) + return hashes @property def markers_as_pip(self): + # type: () -> S if self.markers: return " ; {0}".format(self.markers).replace('"', "'") @@ -1022,141 +2668,240 @@ class Requirement(object): @property def extras_as_pip(self): + # type: () -> STRING_TYPE if self.extras: return "[{0}]".format( - ",".join(sorted([extra.lower() for extra in self.extras])) + ",".join(sorted([extra.lower() for extra in self.extras])) # type: ignore ) return "" - @property + @cached_property def commit_hash(self): - if not self.is_vcs: + # type: () -> Optional[S] + if self.req is None or not isinstance(self.req, VCSRequirement): return None commit_hash = None - with self.req.locked_vcs_repo() as repo: - commit_hash = repo.get_commit_hash() + if self.req is not None: + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() return commit_hash - @specifiers.default + @_specifiers.default def get_specifiers(self): - if self.req and self.req.req.specifier: + # type: () -> S + if self.req and self.req.req and self.req.req.specifier: return specs_to_string(self.req.req.specifier) - return + return "" + + def update_name_from_path(self, path): + from .setup_info import get_metadata + + metadata = get_metadata(path) + name = self.name + if metadata is not None: + name = metadata.get("name") + if name is not None: + if self.req.name is None: + self.req.name = name + if self.req.req and self.req.req.name is None: + self.req.req.name = name + if self._line_instance._name is None: + self._line_instance.name = name + if self.req._parsed_line._name is None: + self.req._parsed_line.name = name + if self.req._setup_info and self.req._setup_info.name is None: + self.req._setup_info.name = name + + @property + def line_instance(self): + # type: () -> Optional[Line] + if self._line_instance is None: + if self.req is not None and self.req._parsed_line is not None: + self._line_instance = self.req._parsed_line + else: + include_extras = True + include_specifiers = True + if self.is_vcs: + include_extras = False + if self.is_file_or_url or self.is_vcs or not self._specifiers: + include_specifiers = False + line_part = "" # type: STRING_TYPE + if self.req and self.req.line_part: + line_part = "{0!s}".format(self.req.line_part) + parts = [] # type: List[STRING_TYPE] + parts = [ + line_part, + self.extras_as_pip if include_extras else "", + self._specifiers if include_specifiers and self._specifiers else "", + self.markers_as_pip, + ] + line = "".join(parts) + self._line_instance = Line(line) + return self._line_instance + + @line_instance.setter + def line_instance(self, line_instance): + # type: (Line) -> None + if self.req and not self.req._parsed_line: + self.req._parsed_line = line_instance + self._line_instance = line_instance + + @property + def specifiers(self): + # type: () -> Optional[STRING_TYPE] + if self._specifiers: + return self._specifiers + else: + specs = self.get_specifiers() + if specs: + self._specifiers = specs + return specs + if not self._specifiers and ( + self.req is not None + and isinstance(self.req, NamedRequirement) + and self.req.version + ): + self._specifiers = self.req.version + elif ( + not self.editable + and self.req + and (not isinstance(self.req, NamedRequirement) and self.req.setup_info) + ): + if ( + self.line_instance + and self.line_instance.setup_info + and self.line_instance.setup_info.version + ): + self._specifiers = "=={0}".format(self.req.setup_info.version) + elif not self._specifiers: + if self.req and self.req.parsed_line and self.req.parsed_line.specifiers: + self._specifiers = specs_to_string(self.req.parsed_line.specifiers) + elif self.line_instance and self.line_instance.specifiers: + self._specifiers = specs_to_string(self.line_instance.specifiers) + elif self.is_file_or_url or self.is_vcs: + try: + setupinfo_dict = self.run_requires() + except Exception: + setupinfo_dict = None + if setupinfo_dict is not None: + self._specifiers = "=={0}".format(setupinfo_dict.get("version")) + if self._specifiers: + specset = SpecifierSet(self._specifiers) + if self.line_instance and not self.line_instance.specifiers: + self.line_instance.specifiers = specset + if self.req: + if self.req._parsed_line and not self.req._parsed_line.specifiers: + self.req._parsed_line.specifiers = specset + elif not self.req._parsed_line and self.line_instance: + self.req._parsed_line = self.line_instance + if self.req and self.req.req and not self.req.req.specifier: + self.req.req.specifier = specset + return self._specifiers @property def is_vcs(self): + # type: () -> bool return isinstance(self.req, VCSRequirement) + @property + def build_backend(self): + # type: () -> Optional[STRING_TYPE] + if self.req is not None and ( + not isinstance(self.req, NamedRequirement) and self.req.is_local + ): + setup_info = self.run_requires() + build_backend = setup_info.get("build_backend") + return build_backend + return "setuptools.build_meta" + + @property + def uses_pep517(self): + # type: () -> bool + if self.build_backend: + return True + return False + @property def is_file_or_url(self): + # type: () -> bool return isinstance(self.req, FileRequirement) @property def is_named(self): + # type: () -> bool return isinstance(self.req, NamedRequirement) + @property + def is_wheel(self): + # type: () -> bool + if ( + self.req + and not isinstance(self.req, NamedRequirement) + and (self.req.link is not None and self.req.link.is_wheel) + ): + return True + return False + @property def normalized_name(self): + # type: () -> S return canonicalize_name(self.name) def copy(self): return attr.evolve(self) @classmethod + @lru_cache() def from_line(cls, line): + # type: (AnyStr) -> Requirement if isinstance(line, pip_shims.shims.InstallRequirement): line = format_requirement(line) - hashes = None - if "--hash=" in line: - hashes = line.split(" --hash=") - line, hashes = hashes[0], hashes[1:] - editable = line.startswith("-e ") - line = line.split(" ", 1)[1] if editable else line - line, markers = split_markers_from_line(line) - line, extras = pip_shims.shims._strip_extras(line) - if extras: - extras = parse_extras(extras) - line = line.strip('"').strip("'").strip() - line_with_prefix = "-e {0}".format(line) if editable else line - vcs = None - # Installable local files and installable non-vcs urls are handled - # as files, generally speaking - line_is_vcs = is_vcs(line) - # check for pep-508 compatible requirements - name, _, possible_url = line.partition("@") - if is_installable_file(line) or ( - (is_valid_url(possible_url) or is_file_url(line) or is_valid_url(line)) and - not (line_is_vcs or is_vcs(possible_url)) - ): - r = FileRequirement.from_line(line_with_prefix, extras=extras) - elif line_is_vcs: - r = VCSRequirement.from_line(line_with_prefix, extras=extras) - vcs = r.vcs + parsed_line = Line(line) + r = ( + None + ) # type: Optional[Union[VCSRequirement, FileRequirement, NamedRequirement]] + if ( + (parsed_line.is_file and parsed_line.is_installable) or parsed_line.is_url + ) and not parsed_line.is_vcs: + r = file_req_from_parsed_line(parsed_line) + elif parsed_line.is_vcs: + r = vcs_req_from_parsed_line(parsed_line) elif line == "." and not is_installable_file(line): raise RequirementError( "Error parsing requirement %s -- are you sure it is installable?" % line ) else: - specs = "!=<>~" - spec_matches = set(specs) & set(line) - version = None - name = line - if spec_matches: - spec_idx = min((line.index(match) for match in spec_matches)) - name = line[:spec_idx] - version = line[spec_idx:] - if not extras: - name, extras = pip_shims.shims._strip_extras(name) - if extras: - extras = parse_extras(extras) - if version: - name = "{0}{1}".format(name, version) - r = NamedRequirement.from_line(line) + r = named_req_from_parsed_line(parsed_line) req_markers = None - if markers: - req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - r.req.local_file = getattr(r.req, "local_file", False) - name = getattr(r.req, "name", None) - if not name: - name = getattr(r.req, "project_name", None) - r.req.name = name - if not name: - name = getattr(r.req, "key", None) - if name: - r.req.name = name + if parsed_line.markers: + req_markers = PackagingRequirement("fakepkg; {0}".format(parsed_line.markers)) + if r is not None and r.req is not None: + r.req.marker = getattr(req_markers, "marker", None) if req_markers else None + args = {} # type: Dict[STRING_TYPE, CREATION_ARG_TYPES] args = { "name": r.name, - "vcs": vcs, + "vcs": parsed_line.vcs, "req": r, - "markers": markers, - "editable": editable, + "markers": parsed_line.markers, + "editable": parsed_line.editable, + "line_instance": parsed_line, } - if extras: - extras = sorted(dedup([extra.lower() for extra in extras])) + if parsed_line.extras: + extras = () # type: Tuple[STRING_TYPE, ...] + extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) args["extras"] = extras - r.req.extras = extras - r.extras = extras - elif r.extras: - args["extras"] = sorted(dedup([extra.lower() for extra in r.extras])) - if hashes: - args["hashes"] = hashes - cls_inst = cls(**args) - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) + if r is not None: + r.extras = extras + elif r is not None and r.extras is not None: + args["extras"] = tuple( + sorted(dedup([extra.lower() for extra in r.extras])) + ) # type: ignore + if r.req is not None: + r.req.extras = args["extras"] + if parsed_line.hashes: + args["hashes"] = tuple(parsed_line.hashes) # type: ignore + cls_inst = cls(**args) # type: ignore return cls_inst @classmethod @@ -1190,41 +2935,29 @@ class Requirement(object): if markers: markers = str(markers) req_markers = PackagingRequirement("fakepkg; {0}".format(markers)) - r.req.marker = getattr(req_markers, "marker", None) - r.req.specifier = SpecifierSet(_pipfile["version"]) + if r.req is not None: + r.req.marker = req_markers.marker extras = _pipfile.get("extras") - r.req.extras = ( - sorted(dedup([extra.lower() for extra in extras])) if extras else [] - ) + if r.req: + if r.req.specifier: + r.req.specifier = SpecifierSet(_pipfile["version"]) + r.req.extras = ( + tuple(sorted(dedup([extra.lower() for extra in extras]))) + if extras + else () + ) args = { "name": r.name, "vcs": vcs, "req": r, "markers": markers, - "extras": _pipfile.get("extras"), + "extras": tuple(_pipfile.get("extras", ())), "editable": _pipfile.get("editable", False), "index": _pipfile.get("index"), } if any(key in _pipfile for key in ["hash", "hashes"]): args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) cls_inst = cls(**args) - if cls_inst.is_named: - cls_inst.req.req.line = cls_inst.as_line() - old_name = cls_inst.req.req.name or cls_inst.req.name - if not cls_inst.is_named and not cls_inst.editable and not name: - if cls_inst.is_vcs: - ireq = pip_shims.shims.install_req_from_req(cls_inst.as_line(include_hashes=False)) - info = SetupInfo.from_ireq(ireq) - if info is not None: - info_dict = info.as_dict() - cls_inst.req.setup_info = info - else: - info_dict = {} - else: - info_dict = cls_inst.run_requires() - found_name = info_dict.get("name", old_name) - if old_name != found_name: - cls_inst.req.req.line.replace(old_name, found_name) return cls_inst def as_line( @@ -1264,7 +2997,9 @@ class Requirement(object): parts.extend(hashes) else: parts.append(hashes) - if sources and not (self.requirement.local_file or self.vcs): + + is_local = self.is_file_or_url and self.req and self.req.is_local + if sources and self.requirement and not (is_local or self.vcs): from ..utils import prepare_pip_source_args if self.index: @@ -1281,13 +3016,15 @@ class Requirement(object): return line def get_markers(self): + # type: () -> Marker markers = self.markers if markers: fake_pkg = PackagingRequirement("fakepkg; {0}".format(markers)) - markers = fake_pkg.markers + markers = fake_pkg.marker return markers def get_specifier(self): + # type: () -> Union[SpecifierSet, LegacySpecifier] try: return Specifier(self.specifiers) except InvalidSpecifier: @@ -1315,7 +3052,11 @@ class Requirement(object): @property def is_direct_url(self): - return self.is_file_or_url and self.req.is_direct_url + return ( + self.is_file_or_url + and self.req.is_direct_url + or (self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url) + ) def as_pipfile(self): good_keys = ( @@ -1334,10 +3075,20 @@ class Requirement(object): name = self.name if "markers" in req_dict and req_dict["markers"]: req_dict["markers"] = req_dict["markers"].replace('"', "'") + if not self.req.name: + name_carriers = (self.req, self, self.line_instance, self.req.parsed_line) + name_options = [ + getattr(carrier, "name", None) + for carrier in name_carriers + if carrier is not None + ] + req_name = next(iter(n for n in name_options if n is not None), None) + self.req.name = req_name + req_name, dict_from_subreq = self.req.pipfile_part.popitem() base_dict = { k: v - for k, v in self.req.pipfile_part[name].items() - if k not in ["req", "link"] + for k, v in dict_from_subreq.items() + if k not in ["req", "link", "_setup_info"] } base_dict.update(req_dict) conflicting_keys = ("file", "path", "uri") @@ -1354,24 +3105,31 @@ class Requirement(object): except AttributeError: hashes.append(_hash) base_dict["hashes"] = sorted(hashes) + if "extras" in base_dict: + base_dict["extras"] = list(base_dict["extras"]) if len(base_dict.keys()) == 1 and "version" in base_dict: base_dict = base_dict.get("version") return {name: base_dict} def as_ireq(self): - ireq_line = self.as_line(include_hashes=False) - if self.editable or self.req.editable: - if ireq_line.startswith("-e "): - ireq_line = ireq_line[len("-e ") :] - with ensure_setup_py(self.req.setup_path): - ireq = pip_shims.shims.install_req_from_editable(ireq_line) - else: - ireq = pip_shims.shims.install_req_from_line(ireq_line) + if self.line_instance and self.line_instance.ireq: + return self.line_instance.ireq + elif getattr(self.req, "_parsed_line", None) and self.req._parsed_line.ireq: + return self.req._parsed_line.ireq + kwargs = {"include_hashes": False} + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + kwargs["include_markers"] = False + ireq_line = self.as_line(**kwargs) + ireq = Line(ireq_line).ireq if not getattr(ireq, "req", None): ireq.req = self.req.req + if (self.is_file_or_url and self.req.is_local) or self.is_vcs: + if getattr(ireq, "req", None) and getattr(ireq.req, "marker", None): + ireq.req.marker = None else: ireq.req.extras = self.req.req.extras - ireq.req.marker = self.req.req.marker + if not ((self.is_file_or_url and self.req.is_local) or self.is_vcs): + ireq.req.marker = self.req.req.marker return ireq @property @@ -1431,17 +3189,17 @@ class Requirement(object): else: ireq = sorted(self.find_all_matches(), key=lambda k: k.version) deps = get_dependencies(ireq.pop(), sources=sources) - return get_abstract_dependencies( - deps, sources=sources, parent=self.abstract_dep - ) + return get_abstract_dependencies(deps, sources=sources, parent=self.abstract_dep) def find_all_matches(self, sources=None, finder=None): + # type: (Optional[List[Dict[S, Union[S, bool]]]], Optional[PackageFinder]) -> List[InstallationCandidate] """Find all matching candidates for the current requirement. Consults a finder to find all matching candidates. :param sources: Pipfile-formatted sources, defaults to None :param sources: list[dict], optional + :param PackageFinder finder: A **PackageFinder** instance from pip's repository implementation :return: A list of Installation Candidates :rtype: list[ :class:`~pip._internal.index.InstallationCandidate` ] """ @@ -1455,17 +3213,21 @@ class Requirement(object): def run_requires(self, sources=None, finder=None): if self.req and self.req.setup_info is not None: info_dict = self.req.setup_info.as_dict() + elif self.line_instance and self.line_instance.setup_info is not None: + info_dict = self.line_instance.setup_info.as_dict() else: from .setup_info import SetupInfo + if not finder: from .dependencies import get_finder + finder = get_finder(sources=sources) info = SetupInfo.from_requirement(self, finder=finder) if info is None: return {} info_dict = info.get_info() if self.req and not self.req.setup_info: - self.req.setup_info = info + self.req._setup_info = info if self.req._has_hashed_name and info_dict.get("name"): self.req.name = self.name = info_dict["name"] if self.req.req.name != info_dict["name"]: @@ -1473,10 +3235,102 @@ class Requirement(object): return info_dict def merge_markers(self, markers): + # type: (Union[AnyStr, Marker]) -> None if not isinstance(markers, Marker): markers = Marker(markers) - _markers = set(Marker(self.ireq.markers)) if self.ireq.markers else set(markers) + _markers = set() # type: Set[Marker] + if self.ireq and self.ireq.markers: + _markers.add(Marker(self.ireq.markers)) _markers.add(markers) new_markers = Marker(" or ".join([str(m) for m in sorted(_markers)])) self.markers = str(new_markers) - self.req.req.marker = new_markers + if self.req and self.req.req: + self.req.req.marker = new_markers + return + + +def file_req_from_parsed_line(parsed_line): + # type: (Line) -> FileRequirement + path = parsed_line.relpath if parsed_line.relpath else parsed_line.path + pyproject_requires = None # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + req_dict = { + "setup_path": parsed_line.setup_py, + "path": path, + "editable": parsed_line.editable, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": parsed_line.link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + } + if parsed_line.name is not None: + req_dict["name"] = parsed_line.name + return FileRequirement(**req_dict) # type: ignore + + +def vcs_req_from_parsed_line(parsed_line): + # type: (Line) -> VCSRequirement + line = "{0}".format(parsed_line.line) + if parsed_line.editable: + line = "-e {0}".format(line) + if parsed_line.url is not None: + link = create_link( + build_vcs_uri( + vcs=parsed_line.vcs, + uri=parsed_line.url, + name=parsed_line.name, + ref=parsed_line.ref, + subdirectory=parsed_line.subdirectory, + extras=list(parsed_line.extras), + ) + ) + else: + link = parsed_line.link + pyproject_requires = () # type: Optional[Tuple[STRING_TYPE, ...]] + if parsed_line.pyproject_requires is not None: + pyproject_requires = tuple(parsed_line.pyproject_requires) + vcs_dict = { + "setup_path": parsed_line.setup_py, + "path": parsed_line.path, + "editable": parsed_line.editable, + "vcs": parsed_line.vcs, + "ref": parsed_line.ref, + "subdirectory": parsed_line.subdirectory, + "extras": parsed_line.extras, + "uri_scheme": parsed_line.preferred_scheme, + "link": link, + "uri": parsed_line.uri, + "pyproject_requires": pyproject_requires, + "pyproject_backend": parsed_line.pyproject_backend, + "pyproject_path": Path(parsed_line.pyproject_toml) + if parsed_line.pyproject_toml + else None, + "parsed_line": parsed_line, + "req": parsed_line.requirement, + "base_line": line, + } + if parsed_line.name: + vcs_dict["name"] = parsed_line.name + return VCSRequirement(**vcs_dict) # type: ignore + + +def named_req_from_parsed_line(parsed_line): + # type: (Line) -> NamedRequirement + if parsed_line.name is not None: + return NamedRequirement( + name=parsed_line.name, + version=parsed_line.specifier, + req=parsed_line.requirement, + extras=parsed_line.extras, + editable=parsed_line.editable, + parsed_line=parsed_line, + ) + return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index ec631e67..0f7bc177 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -1,28 +1,45 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import atexit import contextlib import os +import shutil import sys import attr -import packaging.version import packaging.specifiers import packaging.utils +import packaging.version +import pep517.envbuild +import pep517.wrappers import six +from appdirs import user_cache_dir +from distlib.wheel import Wheel +from packaging.markers import Marker +from six.moves import configparser +from six.moves.urllib.parse import unquote, urlparse, urlunparse +from vistir.compat import Iterable, Path, lru_cache +from vistir.contextmanagers import cd, temp_path +from vistir.misc import run +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p, rmtree + +from .utils import ( + get_default_pyproject_backend, + get_name_variants, + get_pyproject, + init_requirement, + split_vcs_method_from_uri, + strip_extras_markers_from_requirement, +) +from ..environment import MYPY_RUNNING +from ..exceptions import RequirementError try: from setuptools.dist import distutils except ImportError: import distutils -from appdirs import user_cache_dir -from six.moves import configparser -from six.moves.urllib.parse import unquote -from vistir.compat import Path, Iterable -from vistir.contextmanagers import cd -from vistir.misc import run -from vistir.path import create_tracked_tempdir, ensure_mkdir_p, mkdir_p - -from .utils import init_requirement, get_pyproject try: from os import scandir @@ -30,6 +47,37 @@ except ImportError: from scandir import scandir +if MYPY_RUNNING: + from typing import ( + Any, + Dict, + List, + Generator, + Optional, + Union, + Tuple, + TypeVar, + Text, + Set, + AnyStr, + ) + from pip_shims.shims import InstallRequirement, PackageFinder + from pkg_resources import ( + PathMetadata, + DistInfoDistribution, + Requirement as PkgResourcesRequirement, + ) + from packaging.requirements import Requirement as PackagingRequirement + + TRequirement = TypeVar("TRequirement") + RequirementType = TypeVar( + "RequirementType", covariant=True, bound=PackagingRequirement + ) + MarkerType = TypeVar("MarkerType", covariant=True, bound=Marker) + STRING_TYPE = Union[str, bytes, Text] + S = TypeVar("S", bytes, str, Text) + + CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) # The following are necessary for people who like to use "if __name__" conditionals @@ -38,8 +86,56 @@ _setup_stop_after = None _setup_distribution = None +def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None): + # type: (List[AnyStr], Optional[AnyStr], Optional[Dict[AnyStr, AnyStr]]) -> None + """The default method of calling the wrapper subprocess.""" + env = os.environ.copy() + if extra_environ: + env.update(extra_environ) + + run( + cmd, + cwd=cwd, + env=env, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class BuildEnv(pep517.envbuild.BuildEnvironment): + def pip_install(self, reqs): + cmd = [ + sys.executable, + "-m", + "pip", + "install", + "--ignore-installed", + "--prefix", + self.path, + ] + list(reqs) + run( + cmd, + block=True, + combine_stderr=True, + return_object=False, + write_to_stdout=False, + nospin=True, + ) + + +class HookCaller(pep517.wrappers.Pep517HookCaller): + def __init__(self, source_dir, build_backend): + self.source_dir = os.path.abspath(source_dir) + self.build_backend = build_backend + self._subprocess_runner = pep517_subprocess_runner + + @contextlib.contextmanager def _suppress_distutils_logs(): + # type: () -> Generator[None, None, None] """Hack to hide noise generated by `setup.py develop`. There isn't a good way to suppress them now, so let's monky-patch. @@ -57,19 +153,47 @@ def _suppress_distutils_logs(): distutils.log.Log._log = f +def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel"): + if config_settings is None: + config_settings = {} + requires, backend = get_pyproject(source_dir) + hookcaller = HookCaller(source_dir, backend) + if dist_type == "sdist": + get_requires_fn = hookcaller.get_requires_for_build_sdist + build_fn = hookcaller.build_sdist + else: + get_requires_fn = hookcaller.get_requires_for_build_wheel + build_fn = hookcaller.build_wheel + + with BuildEnv() as env: + env.pip_install(requires) + reqs = get_requires_fn(config_settings) + env.pip_install(reqs) + return build_fn(build_dir, config_settings) + + @ensure_mkdir_p(mode=0o775) -def _get_src_dir(): +def _get_src_dir(root): + # type: (AnyStr) -> AnyStr src = os.environ.get("PIP_SRC") if src: return src virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env: + if virtual_env is not None: return os.path.join(virtual_env, "src") - return os.path.join(os.getcwd(), "src") # Match pip's behavior. + if root is not None: + # Intentionally don't match pip's behavior here -- this is a temporary copy + src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") + else: + src_dir = os.path.join(root, "src") + return src_dir +@lru_cache() def ensure_reqs(reqs): + # type: (List[Union[S, PkgResourcesRequirement]]) -> List[PkgResourcesRequirement] import pkg_resources + if not isinstance(reqs, Iterable): raise TypeError("Expecting an Iterable, got %r" % reqs) new_reqs = [] @@ -78,31 +202,35 @@ def ensure_reqs(reqs): continue if isinstance(req, six.string_types): req = pkg_resources.Requirement.parse("{0}".format(str(req))) + # req = strip_extras_markers_from_requirement(req) new_reqs.append(req) return new_reqs -def _prepare_wheel_building_kwargs(ireq): - download_dir = os.path.join(CACHE_DIR, "pkgs") +def _prepare_wheel_building_kwargs( + ireq=None, src_root=None, src_dir=None, editable=False +): + # type: (Optional[InstallRequirement], Optional[AnyStr], Optional[AnyStr], bool) -> Dict[AnyStr, AnyStr] + download_dir = os.path.join(CACHE_DIR, "pkgs") # type: STRING_TYPE mkdir_p(download_dir) - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") + wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: STRING_TYPE mkdir_p(wheel_download_dir) - if ireq.source_dir is not None: - src_dir = ireq.source_dir - elif ireq.editable: - src_dir = _get_src_dir() - else: - src_dir = create_tracked_tempdir(prefix="reqlib-src") + if src_dir is None: + if editable and src_root is not None: + src_dir = src_root + elif ireq is None and src_root is not None: + src_dir = _get_src_dir(root=src_root) # type: STRING_TYPE + elif ireq is not None and ireq.editable and src_root is not None: + src_dir = _get_src_dir(root=src_root) + else: + src_dir = create_tracked_tempdir(prefix="reqlib-src") - # This logic matches pip's behavior, although I don't fully understand the - # intention. I guess the idea is to build editables in-place, otherwise out - # of the source tree? - if ireq.editable: - build_dir = src_dir - else: - build_dir = create_tracked_tempdir(prefix="reqlib-build") + # Let's always resolve in isolation + if src_dir is None: + src_dir = create_tracked_tempdir(prefix="reqlib-src") + build_dir = create_tracked_tempdir(prefix="reqlib-build") return { "build_dir": build_dir, @@ -112,95 +240,288 @@ def _prepare_wheel_building_kwargs(ireq): } -def iter_egginfos(path, pkg_name=None): +def iter_metadata(path, pkg_name=None, metadata_type="egg-info"): + # type: (AnyStr, Optional[AnyStr], AnyStr) -> Generator + if pkg_name is not None: + pkg_variants = get_name_variants(pkg_name) + non_matching_dirs = [] for entry in scandir(path): if entry.is_dir(): - if not entry.name.endswith("egg-info"): - for dir_entry in iter_egginfos(entry.path, pkg_name=pkg_name): - yield dir_entry - elif pkg_name is None or entry.name.startswith(pkg_name.replace("-", "_")): - yield entry + entry_name, ext = os.path.splitext(entry.name) + if ext.endswith(metadata_type): + if pkg_name is None or entry_name.lower() in pkg_variants: + yield entry + elif not entry.name.endswith(metadata_type): + non_matching_dirs.append(entry) + for entry in non_matching_dirs: + for dir_entry in iter_metadata( + entry.path, pkg_name=pkg_name, metadata_type=metadata_type + ): + yield dir_entry def find_egginfo(target, pkg_name=None): - egg_dirs = (egg_dir for egg_dir in iter_egginfos(target, pkg_name=pkg_name)) + # type: (AnyStr, Optional[AnyStr]) -> Generator + egg_dirs = ( + egg_dir + for egg_dir in iter_metadata(target, pkg_name=pkg_name) + if egg_dir is not None + ) if pkg_name: - yield next(iter(egg_dirs), None) + yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None) else: for egg_dir in egg_dirs: yield egg_dir -def get_metadata(path, pkg_name=None): +def find_distinfo(target, pkg_name=None): + # type: (AnyStr, Optional[AnyStr]) -> Generator + dist_dirs = ( + dist_dir + for dist_dir in iter_metadata( + target, pkg_name=pkg_name, metadata_type="dist-info" + ) + if dist_dir is not None + ) if pkg_name: - pkg_name = packaging.utils.canonicalize_name(pkg_name) + yield next(iter(dist for dist in dist_dirs if dist is not None), None) + else: + for dist_dir in dist_dirs: + yield dist_dir + + +def get_metadata(path, pkg_name=None, metadata_type=None): + # type: (S, Optional[S], Optional[S]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + metadata_dirs = [] + wheel_allowed = metadata_type == "wheel" or metadata_type is None + egg_allowed = metadata_type == "egg" or metadata_type is None egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) - if egg_dir is not None: + dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) + if dist_dir and wheel_allowed: + metadata_dirs.append(dist_dir) + if egg_dir and egg_allowed: + metadata_dirs.append(egg_dir) + matched_dir = next(iter(d for d in metadata_dirs if d is not None), None) + metadata_dir = None + base_dir = None + if matched_dir is not None: import pkg_resources - egg_dir = os.path.abspath(egg_dir.path) - base_dir = os.path.dirname(egg_dir) - path_metadata = pkg_resources.PathMetadata(base_dir, egg_dir) - dist = next( - iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), - None, + metadata_dir = os.path.abspath(matched_dir.path) + base_dir = os.path.dirname(metadata_dir) + dist = None + distinfo_dist = None + egg_dist = None + if wheel_allowed and dist_dir is not None: + distinfo_dist = next(iter(pkg_resources.find_distributions(base_dir)), None) + if egg_allowed and egg_dir is not None: + path_metadata = pkg_resources.PathMetadata(base_dir, metadata_dir) + egg_dist = next( + iter(pkg_resources.distributions_from_metadata(path_metadata.egg_info)), + None, + ) + dist = next(iter(d for d in (distinfo_dist, egg_dist) if d is not None), None) + if dist is not None: + return get_metadata_from_dist(dist) + return {} + + +@lru_cache() +def get_extra_name_from_marker(marker): + # type: (MarkerType) -> Optional[S] + if not marker: + raise ValueError("Invalid value for marker: {0!r}".format(marker)) + if not getattr(marker, "_markers", None): + raise TypeError("Expecting a marker instance, received {0!r}".format(marker)) + for elem in marker._markers: + if isinstance(elem, tuple) and elem[0].value == "extra": + return elem[2].value + return None + + +def get_metadata_from_wheel(wheel_path): + # type: (S) -> Dict[Any, Any] + if not isinstance(wheel_path, six.string_types): + raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) + try: + dist = Wheel(wheel_path) + except Exception: + pass + metadata = dist.metadata + name = metadata.name + version = metadata.version + requires = [] + extras_keys = getattr(metadata, "extras", []) + extras = {k: [] for k in extras_keys} + for req in getattr(metadata, "run_requires", []): + parsed_req = init_requirement(req) + parsed_marker = parsed_req.marker + if parsed_marker: + extra = get_extra_name_from_marker(parsed_marker) + if extra is None: + requires.append(parsed_req) + continue + if extra not in extras: + extras[extra] = [] + parsed_req = strip_extras_markers_from_requirement(parsed_req) + extras[extra].append(parsed_req) + else: + requires.append(parsed_req) + return {"name": name, "version": version, "requires": requires, "extras": extras} + + +def get_metadata_from_dist(dist): + # type: (Union[PathMetadata, DistInfoDistribution]) -> Dict[S, Union[S, List[RequirementType], Dict[S, RequirementType]]] + try: + requires = dist.requires() + except Exception: + requires = [] + try: + dep_map = dist._build_dep_map() + except Exception: + dep_map = {} + deps = [] + extras = {} + for k in dep_map.keys(): + if k is None: + deps.extend(dep_map.get(k)) + continue + else: + extra = None + _deps = dep_map.get(k) + if k.startswith(":python_version"): + marker = k.replace(":", "; ") + else: + marker = "" + extra = "{0}".format(k) + _deps = ["{0}{1}".format(str(req), marker) for req in _deps] + _deps = ensure_reqs(tuple(_deps)) + if extra: + extras[extra] = _deps + else: + deps.extend(_deps) + return { + "name": dist.project_name, + "version": dist.version, + "requires": requires, + "extras": extras, + } + + +@attr.s(slots=True, frozen=True) +class BaseRequirement(object): + name = attr.ib(default="", cmp=True) # type: STRING_TYPE + requirement = attr.ib( + default=None, cmp=True + ) # type: Optional[PkgResourcesRequirement] + + def __str__(self): + # type: () -> S + return "{0}".format(str(self.requirement)) + + def as_dict(self): + # type: () -> Dict[S, Optional[PkgResourcesRequirement]] + return {self.name: self.requirement} + + def as_tuple(self): + # type: () -> Tuple[S, Optional[PkgResourcesRequirement]] + return (self.name, self.requirement) + + @classmethod + @lru_cache() + def from_string(cls, line): + # type: (S) -> BaseRequirement + line = line.strip() + req = init_requirement(line) + return cls.from_req(req) + + @classmethod + @lru_cache() + def from_req(cls, req): + # type: (PkgResourcesRequirement) -> BaseRequirement + name = None + key = getattr(req, "key", None) + name = getattr(req, "name", None) + project_name = getattr(req, "project_name", None) + if key is not None: + name = key + if name is None: + name = project_name + return cls(name=name, requirement=req) + + +@attr.s(slots=True, frozen=True) +class Extra(object): + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + requirements = attr.ib(factory=frozenset, cmp=True, type=frozenset) + + def __str__(self): + # type: () -> S + return "{0}: {{{1}}}".format( + self.section, ", ".join([r.name for r in self.requirements]) ) - if dist: - try: - requires = dist.requires() - except exception: - requires = [] - try: - dep_map = dist._build_dep_map() - except Exception: - dep_map = {} - deps = [] - extras = {} - for k in dep_map.keys(): - if k is None: - deps.extend(dep_map.get(k)) - continue - else: - extra = None - _deps = dep_map.get(k) - if k.startswith(":python_version"): - marker = k.replace(":", "; ") - else: - marker = "" - extra = "{0}".format(k) - _deps = ["{0}{1}".format(str(req), marker) for req in _deps] - _deps = ensure_reqs(_deps) - if extra: - extras[extra] = _deps - else: - deps.extend(_deps) - return { - "name": dist.project_name, - "version": dist.version, - "requires": requires, - "extras": extras - } + + def add(self, req): + # type: (BaseRequirement) -> None + if req not in self.requirements: + return attr.evolve( + self, requirements=frozenset(set(self.requirements).add(req)) + ) + return self + + def as_dict(self): + # type: () -> Dict[S, Tuple[RequirementType, ...]] + return {self.name: tuple([r.requirement for r in self.requirements])} -@attr.s(slots=True) +@attr.s(slots=True, cmp=True, hash=True) class SetupInfo(object): - name = attr.ib(type=str, default=None) - base_dir = attr.ib(type=Path, default=None) - version = attr.ib(type=packaging.version.Version, default=None) - requires = attr.ib(type=dict, default=attr.Factory(dict)) - build_requires = attr.ib(type=list, default=attr.Factory(list)) - build_backend = attr.ib(type=list, default=attr.Factory(list)) - setup_requires = attr.ib(type=dict, default=attr.Factory(list)) - python_requires = attr.ib(type=packaging.specifiers.SpecifierSet, default=None) - extras = attr.ib(type=dict, default=attr.Factory(dict)) - setup_cfg = attr.ib(type=Path, default=None) - setup_py = attr.ib(type=Path, default=None) - pyproject = attr.ib(type=Path, default=None) - ireq = attr.ib(default=None) - extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict) + name = attr.ib(default=None, cmp=True) # type: STRING_TYPE + base_dir = attr.ib(default=None, cmp=True, hash=False) # type: STRING_TYPE + version = attr.ib(default=None, cmp=True) # type: STRING_TYPE + _requirements = attr.ib(type=frozenset, factory=frozenset, cmp=True, hash=True) + build_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + build_backend = attr.ib(cmp=True) # type: STRING_TYPE + setup_requires = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + python_requires = attr.ib( + type=packaging.specifiers.SpecifierSet, default=None, cmp=True + ) + _extras_requirements = attr.ib(type=tuple, default=attr.Factory(tuple), cmp=True) + setup_cfg = attr.ib(type=Path, default=None, cmp=True, hash=False) + setup_py = attr.ib(type=Path, default=None, cmp=True, hash=False) + pyproject = attr.ib(type=Path, default=None, cmp=True, hash=False) + ireq = attr.ib( + default=None, cmp=True, hash=False + ) # type: Optional[InstallRequirement] + extra_kwargs = attr.ib(default=attr.Factory(dict), type=dict, cmp=False, hash=False) + metadata = attr.ib(default=None) # type: Optional[Tuple[STRING_TYPE]] - def parse_setup_cfg(self): - if self.setup_cfg is not None and self.setup_cfg.exists(): + @build_backend.default + def get_build_backend(self): + # type: () -> S + return get_default_pyproject_backend() + + @property + def requires(self): + # type: () -> Dict[S, RequirementType] + return {req.name: req.requirement for req in self._requirements} + + @property + def extras(self): + # type: () -> Dict[S, Optional[Any]] + extras_dict = {} + extras = set(self._extras_requirements) + for section, deps in extras: + if isinstance(deps, BaseRequirement): + extras_dict[section] = deps.requirement + elif isinstance(deps, (list, tuple)): + extras_dict[section] = [d.requirement for d in deps] + return extras_dict + + @classmethod + def get_setup_cfg(cls, setup_cfg_path): + # type: (S) -> Dict[S, Union[S, None, Set[BaseRequirement], List[S], Tuple[S, Tuple[BaseRequirement]]]] + if os.path.exists(setup_cfg_path): default_opts = { "metadata": {"name": "", "version": ""}, "options": { @@ -212,56 +533,110 @@ class SetupInfo(object): }, } parser = configparser.ConfigParser(default_opts) - parser.read(self.setup_cfg.as_posix()) + parser.read(setup_cfg_path) + results = {} if parser.has_option("metadata", "name"): - name = parser.get("metadata", "name") - if not self.name and name is not None: - self.name = name + results["name"] = parser.get("metadata", "name") if parser.has_option("metadata", "version"): - version = parser.get("metadata", "version") - if not self.version and version is not None: - self.version = version + results["version"] = parser.get("metadata", "version") + install_requires = set() # type: Set[BaseRequirement] if parser.has_option("options", "install_requires"): - self.requires.update( - { - dep.strip(): init_requirement(dep.strip()) + install_requires = set( + [ + BaseRequirement.from_string(dep) for dep in parser.get("options", "install_requires").split("\n") if dep - } + ] ) + results["install_requires"] = install_requires if parser.has_option("options", "python_requires"): - python_requires = parser.get("options", "python_requires") - if python_requires and not self.python_requires: - self.python_requires = python_requires + results["python_requires"] = parser.get("options", "python_requires") + if parser.has_option("options", "build_requires"): + results["build_requires"] = parser.get("options", "build_requires") + extras = [] if "options.extras_require" in parser.sections(): - self.extras.update( - { - section: [ - init_requirement(dep.strip()) - for dep in parser.get( - "options.extras_require", section - ).split("\n") - if dep - ] - for section in parser.options("options.extras_require") - if section not in ["options", "metadata"] - } + extras_require_section = parser.options("options.extras_require") + for section in extras_require_section: + if section in ["options", "metadata"]: + continue + section_contents = parser.get("options.extras_require", section) + section_list = section_contents.split("\n") + section_extras = [] + for extra_name in section_list: + if not extra_name or extra_name.startswith("#"): + continue + section_extras.append(BaseRequirement.from_string(extra_name)) + if section_extras: + extras.append(tuple([section, tuple(section_extras)])) + results["extras_require"] = tuple(extras) + return results + + @property + def egg_base(self): + # type: () -> S + base = None # type: Optional[STRING_TYPE] + if self.setup_py.exists(): + base = self.setup_py.parent + elif self.pyproject.exists(): + base = self.pyproject.parent + elif self.setup_cfg.exists(): + base = self.setup_cfg.parent + if base is None: + base = Path(self.base_dir) + if base is None: + base = Path(self.extra_kwargs["src_dir"]) + egg_base = base.joinpath("reqlib-metadata") + if not egg_base.exists(): + atexit.register(rmtree, egg_base.as_posix()) + egg_base.mkdir(parents=True, exist_ok=True) + return egg_base.as_posix() + + def parse_setup_cfg(self): + # type: () -> None + if self.setup_cfg is not None and self.setup_cfg.exists(): + parsed = self.get_setup_cfg(self.setup_cfg.as_posix()) + if self.name is None: + self.name = parsed.get("name") + if self.version is None: + self.version = parsed.get("version") + build_requires = parsed.get("build_requires", []) + if self.build_requires: + self.build_requires = tuple( + set(self.build_requires) | set(build_requires) ) - if self.ireq.extras: - self.requires.update({ - extra: self.extras[extra] - for extra in self.ireq.extras if extra in self.extras - }) + self._requirements = frozenset( + set(self._requirements) | set(parsed["install_requires"]) + ) + if self.python_requires is None: + self.python_requires = parsed.get("python_requires") + if not self._extras_requirements: + self._extras_requirements = parsed["extras_require"] + else: + self._extras_requirements = ( + self._extras_requirements + parsed["extras_require"] + ) + if self.ireq is not None and self.ireq.extras: + for extra in self.ireq.extras: + if extra in self.extras: + extras_tuple = tuple( + [BaseRequirement.from_req(req) for req in self.extras[extra]] + ) + self._extras_requirements += ((extra, extras_tuple),) + self._requirements = frozenset( + set(self._requirements) | set(list(extras_tuple)) + ) def run_setup(self): + # type: () -> None if self.setup_py is not None and self.setup_py.exists(): target_cwd = self.setup_py.parent.as_posix() - with cd(target_cwd), _suppress_distutils_logs(): + with temp_path(), cd(target_cwd), _suppress_distutils_logs(): # This is for you, Hynek # see https://github.com/hynek/environ_config/blob/69b1c8a/setup.py script_name = self.setup_py.as_posix() - args = ["egg_info"] + args = ["egg_info", "--egg-base", self.egg_base] g = {"__file__": script_name, "__name__": "__main__"} + sys.path.insert(0, os.path.dirname(os.path.abspath(script_name))) local_dict = {} if sys.version_info < (3, 5): save_argv = sys.argv @@ -272,16 +647,22 @@ class SetupInfo(object): _setup_stop_after = "run" sys.argv[0] = script_name sys.argv[1:] = args - with open(script_name, 'rb') as f: + with open(script_name, "rb") as f: if sys.version_info < (3, 5): exec(f.read(), g, local_dict) else: exec(f.read(), g) # We couldn't import everything needed to run setup except NameError: - python = os.environ.get('PIP_PYTHON_PATH', sys.executable) - out, _ = run([python, "setup.py"] + args, cwd=target_cwd, block=True, - combine_stderr=False, return_object=False, nospin=True) + python = os.environ.get("PIP_PYTHON_PATH", sys.executable) + out, _ = run( + [python, "setup.py"] + args, + cwd=target_cwd, + block=True, + combine_stderr=False, + return_object=False, + nospin=True, + ) finally: _setup_stop_after = None sys.argv = save_argv @@ -297,67 +678,218 @@ class SetupInfo(object): self.python_requires = packaging.specifiers.SpecifierSet( dist.python_requires ) + if not self._extras_requirements: + self._extras_requirements = () if dist.extras_require and not self.extras: - self.extras = dist.extras_require + for extra, extra_requires in dist.extras_require: + extras_tuple = tuple( + BaseRequirement.from_req(req) for req in extra_requires + ) + self._extras_requirements += ((extra, extras_tuple),) install_requires = dist.get_requires() if not install_requires: install_requires = dist.install_requires if install_requires and not self.requires: - requirements = [init_requirement(req) for req in install_requires] - self.requires.update({req.key: req for req in requirements}) + requirements = set( + [BaseRequirement.from_req(req) for req in install_requires] + ) + if getattr(self.ireq, "extras", None): + for extra in self.ireq.extras: + requirements |= set(list(self.extras.get(extra, []))) + self._requirements = frozenset(set(self._requirements) | requirements) if dist.setup_requires and not self.setup_requires: - self.setup_requires = dist.setup_requires + self.setup_requires = tuple(dist.setup_requires) if not self.version: self.version = dist.get_version() - def get_egg_metadata(self): - if self.setup_py is not None and self.setup_py.exists(): - metadata = get_metadata(self.setup_py.parent.as_posix(), pkg_name=self.name) - if metadata: - if not self.name: - self.name = metadata.get("name", self.name) - if not self.version: - self.version = metadata.get("version", self.version) - self.requires.update( - {req.key: req for req in metadata.get("requires", {})} - ) - if getattr(self.ireq, "extras", None): - for extra in self.ireq.extras: - extras = metadata.get("extras", {}).get(extra, []) - if extras: - extras = ensure_reqs(extras) - self.extras[extra] = set(extras) - self.requires.update( - {req.key: req for req in extras if req is not None} - ) + @property + @lru_cache() + def pep517_config(self): + config = {} + config.setdefault("--global-option", []) + return config + + def build_wheel(self): + # type: () -> S + if not self.pyproject.exists(): + build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) + self.pyproject.write_text( + u""" +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="wheel", + ) + + # noinspection PyPackageRequirements + def build_sdist(self): + # type: () -> S + if not self.pyproject.exists(): + build_requires = ", ".join(['"{0}"'.format(r) for r in self.build_requires]) + self.pyproject.write_text( + u""" +[build-system] +requires = [{0}] +build-backend = "{1}" + """.format( + build_requires, self.build_backend + ).strip() + ) + return build_pep517( + self.base_dir, + self.extra_kwargs["build_dir"], + config_settings=self.pep517_config, + dist_type="sdist", + ) + + def build(self): + # type: () -> None + dist_path = None + try: + dist_path = self.build_wheel() + except Exception: + try: + dist_path = self.build_sdist() + self.get_egg_metadata(metadata_type="egg") + except Exception: + pass + else: + self.get_metadata_from_wheel( + os.path.join(self.extra_kwargs["build_dir"], dist_path) + ) + if not self.metadata or not self.name: + self.get_egg_metadata() + if not self.metadata or not self.name: + self.run_setup() + return None + + def reload(self): + # type: () -> Dict[S, Any] + """ + Wipe existing distribution info metadata for rebuilding. + """ + for metadata_dir in os.listdir(self.egg_base): + shutil.rmtree(metadata_dir, ignore_errors=True) + self.metadata = None + self._requirements = frozenset() + self._extras_requirements = () + self.get_info() + + def get_metadata_from_wheel(self, wheel_path): + # type: (S) -> Dict[Any, Any] + metadata_dict = get_metadata_from_wheel(wheel_path) + if metadata_dict: + self.populate_metadata(metadata_dict) + + def get_egg_metadata(self, metadata_dir=None, metadata_type=None): + # type: (Optional[AnyStr], Optional[AnyStr]) -> None + package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] + # if self.setup_py is not None and self.setup_py.exists(): + metadata_dirs = [] + if any([fn is not None and fn.exists() for fn in package_indicators]): + metadata_dirs = [ + self.extra_kwargs["build_dir"], + self.egg_base, + self.extra_kwargs["src_dir"], + ] + if metadata_dir is not None: + metadata_dirs = [metadata_dir] + metadata_dirs + metadata = [ + get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) + for d in metadata_dirs + if os.path.exists(d) + ] + metadata = next(iter(d for d in metadata if d), None) + if metadata is not None: + self.populate_metadata(metadata) + + def populate_metadata(self, metadata): + # type: (Dict[Any, Any]) -> None + _metadata = () + for k, v in metadata.items(): + if k == "extras" and isinstance(v, dict): + extras = () + for extra, reqs in v.items(): + extras += ((extra, tuple(reqs)),) + _metadata += extras + elif isinstance(v, (list, tuple)): + _metadata += (k, tuple(v)) + else: + _metadata += (k, v) + self.metadata = _metadata + if self.name is None: + self.name = metadata.get("name", self.name) + if not self.version: + self.version = metadata.get("version", self.version) + self._requirements = frozenset( + set(self._requirements) + | set([BaseRequirement.from_req(req) for req in metadata.get("requires", [])]) + ) + if getattr(self.ireq, "extras", None): + for extra in self.ireq.extras: + extras = metadata.get("extras", {}).get(extra, []) + if extras: + extras_tuple = tuple( + [ + BaseRequirement.from_req(req) + for req in ensure_reqs(tuple(extras)) + if req is not None + ] + ) + self._extras_requirements += ((extra, extras_tuple),) + self._requirements = frozenset( + set(self._requirements) | set(extras_tuple) + ) def run_pyproject(self): + # type: () -> None if self.pyproject and self.pyproject.exists(): result = get_pyproject(self.pyproject.parent) if result is not None: requires, backend = result if backend: self.build_backend = backend - if requires and not self.build_requires: - self.build_requires = requires + else: + self.build_backend = get_default_pyproject_backend() + if requires: + self.build_requires = tuple(set(requires) | set(self.build_requires)) + else: + self.build_requires = ("setuptools", "wheel") def get_info(self): + # type: () -> Dict[S, Any] if self.setup_cfg and self.setup_cfg.exists(): - self.parse_setup_cfg() - if self.setup_py and self.setup_py.exists(): + with cd(self.base_dir): + self.parse_setup_cfg() + + with cd(self.base_dir): + self.run_pyproject() + self.build() + + if self.setup_py and self.setup_py.exists() and self.metadata is None: if not self.requires or not self.name: try: - self.run_setup() + with cd(self.base_dir): + self.run_setup() except Exception: - self.get_egg_metadata() - if not self.requires or not self.name: - self.get_egg_metadata() + with cd(self.base_dir): + self.get_egg_metadata() + if self.metadata is None or not self.name: + with cd(self.base_dir): + self.get_egg_metadata() - if self.pyproject and self.pyproject.exists(): - self.run_pyproject() return self.as_dict() def as_dict(self): + # type: () -> Dict[S, Any] prop_dict = { "name": self.name, "version": self.version, @@ -378,23 +910,40 @@ class SetupInfo(object): @classmethod def from_requirement(cls, requirement, finder=None): + # type: (TRequirement, Optional[PackageFinder]) -> Optional[SetupInfo] ireq = requirement.as_ireq() subdir = getattr(requirement.req, "subdirectory", None) return cls.from_ireq(ireq, subdir=subdir, finder=finder) @classmethod + @lru_cache() def from_ireq(cls, ireq, subdir=None, finder=None): + # type: (InstallRequirement, Optional[AnyStr], Optional[PackageFinder]) -> Optional[SetupInfo] import pip_shims.shims + if not ireq.link: + return if ireq.link.is_wheel: return if not finder: from .dependencies import get_finder finder = get_finder() + _, uri = split_vcs_method_from_uri(unquote(ireq.link.url_without_fragment)) + parsed = urlparse(uri) + if "file" in parsed.scheme: + url_path = parsed.path + if "@" in url_path: + url_path, _, _ = url_path.rpartition("@") + parsed = parsed._replace(path=url_path) + uri = urlunparse(parsed) + path = None + if ireq.link.scheme == "file" or uri.startswith("file://"): + if "file:/" in uri and "file:///" not in uri: + uri = uri.replace("file:/", "file:///") + path = pip_shims.shims.url_to_path(uri) kwargs = _prepare_wheel_building_kwargs(ireq) - ireq.populate_link(finder, False, False) - ireq.ensure_has_source_dir(kwargs["build_dir"]) + ireq.source_dir = kwargs["src_dir"] if not ( ireq.editable and pip_shims.shims.is_file_url(ireq.link) @@ -406,36 +955,30 @@ class SetupInfo(object): else: only_download = False download_dir = kwargs["download_dir"] - ireq_src_dir = None - if ireq.link.scheme == "file": - path = pip_shims.shims.url_to_path(unquote(ireq.link.url_without_fragment)) - if pip_shims.shims.is_installable_dir(path): - ireq_src_dir = path - if not ireq.editable or not (pip_shims.is_file_url(ireq.link) and ireq_src_dir): - pip_shims.shims.unpack_url( - ireq.link, - ireq.source_dir, - download_dir, - only_download=only_download, - session=finder.session, - hashes=ireq.hashes(False), - progress_bar="off", + elif path is not None and os.path.isdir(path): + raise RequirementError( + "The file URL points to a directory not installable: {}".format(ireq.link) ) - if ireq.editable: - created = cls.create( - ireq.source_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - else: - build_dir = ireq.build_location(kwargs["build_dir"]) - ireq._temp_build_dir.path = kwargs["build_dir"] - created = cls.create( - build_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs - ) - created.get_info() + ireq.build_location(kwargs["build_dir"]) + src_dir = ireq.ensure_has_source_dir(kwargs["src_dir"]) + ireq._temp_build_dir.path = kwargs["build_dir"] + + ireq.populate_link(finder, False, False) + pip_shims.shims.unpack_url( + ireq.link, + src_dir, + download_dir, + only_download=only_download, + session=finder.session, + hashes=ireq.hashes(False), + progress_bar="off", + ) + created = cls.create(src_dir, subdirectory=subdir, ireq=ireq, kwargs=kwargs) return created @classmethod def create(cls, base_dir, subdirectory=None, ireq=None, kwargs=None): + # type: (AnyStr, Optional[AnyStr], Optional[InstallRequirement], Optional[Dict[AnyStr, AnyStr]]) -> Optional[SetupInfo] if not base_dir or base_dir is None: return @@ -454,4 +997,6 @@ class SetupInfo(object): creation_kwargs["setup_cfg"] = setup_cfg if ireq: creation_kwargs["ireq"] = ireq - return cls(**creation_kwargs) + created = cls(**creation_kwargs) + created.get_info() + return created diff --git a/pipenv/vendor/requirementslib/models/url.py b/pipenv/vendor/requirementslib/models/url.py new file mode 100644 index 00000000..8a5337ff --- /dev/null +++ b/pipenv/vendor/requirementslib/models/url.py @@ -0,0 +1,398 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import attr +import pip_shims.shims +from orderedmultidict import omdict +from six.moves.urllib.parse import quote_plus, unquote_plus +from urllib3 import util as urllib3_util +from urllib3.util import parse_url as urllib3_parse +from urllib3.util.url import Url + +from ..environment import MYPY_RUNNING + +if MYPY_RUNNING: + from typing import List, Tuple, Text, Union, TypeVar, Optional + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + + +def _get_parsed_url(url): + # type: (S) -> Url + """ + This is a stand-in function for `urllib3.util.parse_url` + + The orignal function doesn't handle special characters very well, this simply splits + out the authentication section, creates the parsed url, then puts the authentication + section back in, bypassing validation. + + :return: The new, parsed URL object + :rtype: :class:`~urllib3.util.url.Url` + """ + + try: + parsed = urllib3_parse(url) + except ValueError: + scheme, _, url = url.partition("://") + auth, _, url = url.rpartition("@") + url = "{scheme}://{url}".format(scheme=scheme, url=url) + parsed = urllib3_parse(url)._replace(auth=auth) + return parsed + + +def remove_password_from_url(url): + # type: (S) -> S + """ + Given a url, remove the password and insert 4 dashes + + :param url: The url to replace the authentication in + :type url: S + :return: The new URL without authentication + :rtype: S + """ + + parsed = _get_parsed_url(url) + if parsed.auth: + auth, _, _ = parsed.auth.partition(":") + return parsed._replace(auth="{auth}:----".format(auth=auth)).url + return parsed.url + + +@attr.s +class URI(object): + #: The target hostname, e.g. `amazon.com` + host = attr.ib(type=str) + #: The URI Scheme, e.g. `salesforce` + scheme = attr.ib(default="https", type=str) + #: The numeric port of the url if specified + port = attr.ib(default=None, type=int) + #: The url path, e.g. `/path/to/endpoint` + path = attr.ib(default="", type=str) + #: Query parameters, e.g. `?variable=value...` + query = attr.ib(default="", type=str) + #: URL Fragments, e.g. `#fragment=value` + fragment = attr.ib(default="", type=str) + #: Subdirectory fragment, e.g. `&subdirectory=blah...` + subdirectory = attr.ib(default="", type=str) + #: VCS ref this URI points at, if available + ref = attr.ib(default="", type=str) + #: The username if provided, parsed from `user:password@hostname` + username = attr.ib(default="", type=str) + #: Password parsed from `user:password@hostname` + password = attr.ib(default="", type=str, repr=False) + #: An orderedmultidict representing query fragments + query_dict = attr.ib(factory=omdict, type=omdict) + #: The name of the specified package in case it is a VCS URI with an egg fragment + name = attr.ib(default="", type=str) + #: Any extras requested from the requirement + extras = attr.ib(factory=tuple, type=tuple) + #: Whether the url was parsed as a direct pep508-style URL + is_direct_url = attr.ib(default=False, type=bool) + #: Whether the url was an implicit `git+ssh` url (passed as `git+git@`) + is_implicit_ssh = attr.ib(default=False, type=bool) + _auth = attr.ib(default=None, type=str, repr=False) + _fragment_dict = attr.ib(factory=dict, type=dict) + + def _parse_query(self): + # type: () -> URI + query = self.query if self.query is not None else "" + query_dict = omdict() + queries = query.split("&") + query_items = [] + for q in queries: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + query_items.append((key, val)) + query_dict.load(query_items) + return attr.evolve(self, query_dict=query_dict, query=query) + + def _parse_fragment(self): + # type: () -> URI + subdirectory = self.subdirectory if self.subdirectory else "" + fragment = self.fragment if self.fragment else "" + if self.fragment is None: + return self + fragments = self.fragment.split("&") + fragment_items = {} + name = self.name if self.name else "" + extras = self.extras + for q in fragments: + key, _, val = q.partition("=") + val = unquote_plus(val.replace("+", " ")) + fragment_items[key] = val + if key == "egg": + from .utils import parse_extras + + name, stripped_extras = pip_shims.shims._strip_extras(val) + if stripped_extras: + extras = tuple(parse_extras(stripped_extras)) + elif key == "subdirectory": + subdirectory = val + return attr.evolve( + self, + fragment_dict=fragment_items, + subdirectory=subdirectory, + fragment=fragment, + extras=extras, + name=name, + ) + + def _parse_auth(self): + # type: () -> URI + if self._auth: + username, _, password = self._auth.partition(":") + password = quote_plus(password) + return attr.evolve(self, username=username, password=password) + return self + + def get_password(self, unquote=False, include_token=True): + # type: (bool, bool) -> str + password = self.password + if password and unquote: + password = unquote_plus(password) + else: + password = "" + return password + + @staticmethod + def parse_subdirectory(url_part): + # type: (str) -> Tuple[str, Optional[str]] + subdir = None + if "&subdirectory" in url_part: + url_part, _, subdir = url_part.rpartition("&") + subdir = "&{0}".format(subdir.strip()) + return url_part.strip(), subdir + + @classmethod + def parse(cls, url): + # type: (S) -> URI + from .utils import DIRECT_URL_RE, split_ref_from_uri + + is_direct_url = False + name_with_extras = None + is_implicit_ssh = url.strip().startswith("git+git@") + if is_implicit_ssh: + from ..utils import add_ssh_scheme_to_git_uri + + url = add_ssh_scheme_to_git_uri(url) + direct_match = DIRECT_URL_RE.match(url) + if direct_match is not None: + is_direct_url = True + name_with_extras, _, url = url.partition("@") + name_with_extras = name_with_extras.strip() + url, ref = split_ref_from_uri(url.strip()) + if "file:/" in url and "file:///" not in url: + url = url.replace("file:/", "file:///") + parsed = _get_parsed_url(url) + if not (parsed.scheme and parsed.host and parsed.path): + # check if this is a file uri + if not ( + parsed.scheme + and parsed.path + and (parsed.scheme == "file" or parsed.scheme.endswith("+file")) + ): + raise ValueError("Failed parsing URL {0!r} - Not a valid url".format(url)) + parsed_dict = dict(parsed._asdict()).copy() + parsed_dict["is_direct_url"] = is_direct_url + parsed_dict["is_implicit_ssh"] = is_implicit_ssh + if name_with_extras: + fragment = "" + if parsed_dict["fragment"] is not None: + fragment = "{0}".format(parsed_dict["fragment"]) + elif "&subdirectory" in parsed_dict["path"]: + path, fragment = cls.parse_subdirectory(parsed_dict["path"]) + parsed_dict["path"] = path + elif ref is not None and "&subdirectory" in ref: + ref, fragment = cls.parse_subdirectory(ref) + parsed_dict["fragment"] = "egg={0}{1}".format(name_with_extras, fragment) + if ref is not None: + parsed_dict["ref"] = ref.strip() + return cls(**parsed_dict)._parse_auth()._parse_query()._parse_fragment() + + def to_string( + self, + escape_password=True, # type: bool + unquote=True, # type: bool + direct=None, # type: Optional[bool] + strip_ssh=False, # type: bool + strip_ref=False, # type: bool + strip_name=False, # type: bool + strip_subdir=False, # type: bool + ): + # type: (...) -> str + """ + Converts the current URI to a string, unquoting or escaping the password as needed + + :param escape_password: Whether to replace password with ``----``, default True + :param escape_password: bool, optional + :param unquote: Whether to unquote url-escapes in the password, default False + :param unquote: bool, optional + :param bool direct: Whether to format as a direct URL + :param bool strip_ssh: Whether to strip the SSH scheme from the url (git only) + :param bool strip_ref: Whether to drop the VCS ref (if present) + :param bool strip_name: Whether to drop the name and extras (if present) + :param bool strip_subdir: Whether to drop the subdirectory (if present) + :return: The reconstructed string representing the URI + :rtype: str + """ + + if direct is None: + direct = self.is_direct_url + if escape_password: + password = "----" if (self.password or self.username) else "" + else: + password = self.get_password(unquote=unquote) + auth = "" + if self.username: + if password: + auth = "{self.username}:{password}@".format(password=password, self=self) + else: + auth = "{self.username}@".format(self=self) + query = "" + if self.query: + query = "{query}?{self.query}".format(query=query, self=self) + if not direct: + if self.name and not strip_name: + fragment = "#egg={self.name_with_extras}".format(self=self) + elif not strip_name and ( + self.extras and self.scheme and self.scheme.startswith("file") + ): + from .utils import extras_to_string + + fragment = extras_to_string(self.extras) + else: + fragment = "" + query = "{query}{fragment}".format(query=query, fragment=fragment) + if self.subdirectory and not strip_subdir: + query = "{query}&subdirectory={self.subdirectory}".format( + query=query, self=self + ) + host_port_path = self.get_host_port_path(strip_ref=strip_ref) + url = "{self.scheme}://{auth}{host_port_path}{query}".format( + self=self, auth=auth, host_port_path=host_port_path, query=query + ) + if strip_ssh: + from ..utils import strip_ssh_from_git_uri + + url = strip_ssh_from_git_uri(url) + if self.name and direct and not strip_name: + return "{self.name_with_extras}@ {url}".format(self=self, url=url) + return url + + def get_host_port_path(self, strip_ref=False): + # type: (bool) -> str + host = self.host if self.host else "" + if self.port: + host = "{host}:{self.port!s}".format(host=host, self=self) + path = "{self.path}".format(self=self) + if self.ref and not strip_ref: + path = "{path}@{self.ref}".format(path=path, self=self) + return "{host}{path}".format(host=host, path=path) + + @property + def name_with_extras(self): + # type: () -> str + from .utils import extras_to_string + + if not self.name: + return "" + extras = extras_to_string(self.extras) + return "{self.name}{extras}".format(self=self, extras=extras) + + @property + def as_link(self): + # type: () -> Link + link = pip_shims.shims.Link( + self.to_string(escape_password=False, strip_ssh=False, direct=False) + ) + return link + + @property + def bare_url(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + strip_subdir=True, + ) + + @property + def url_without_fragment_or_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + strip_ref=True, + ) + + @property + def url_without_fragment(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_name=True, + ) + + @property + def url_without_ref(self): + # type: () -> str + return self.to_string( + escape_password=False, + strip_ssh=self.is_implicit_ssh, + direct=False, + strip_ref=True, + ) + + @property + def base_url(self): + # type: () -> str + return self.to_string( + escape_password=False, strip_ssh=self.is_implicit_ssh, direct=False + ) + + @property + def full_url(self): + # type: () -> str + return self.to_string(escape_password=False, strip_ssh=False, direct=False) + + @property + def safe_string(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) + + @property + def unsafe_string(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=True) + + @property + def uri_escape(self): + # type: () -> str + return self.to_string(escape_password=False, unquote=False) + + @property + def is_vcs(self): + # type: () -> bool + from ..utils import VCS_SCHEMES + + return self.scheme in VCS_SCHEMES + + @property + def is_file_url(self): + # type: () -> bool + return all([self.scheme, self.scheme == "file"]) + + def __str__(self): + # type: () -> str + return self.to_string(escape_password=True, unquote=True) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index 0fac2aa3..6a68d6dc 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -1,47 +1,137 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import io import os +import re +import string import sys - from collections import defaultdict from itertools import chain, groupby from operator import attrgetter import six import tomlkit - from attr import validators from first import first from packaging.markers import InvalidMarker, Marker, Op, Value, Variable from packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet +from packaging.version import parse as parse_version +from six.moves.urllib import parse as urllib_parse +from urllib3 import util as urllib3_util +from vistir.compat import lru_cache from vistir.misc import dedup +from vistir.path import is_valid_url +from ..environment import MYPY_RUNNING +from ..utils import SCHEME_LIST, VCS_LIST, add_ssh_scheme_to_git_uri, is_star -from ..utils import SCHEME_LIST, VCS_LIST, is_star, add_ssh_scheme_to_git_uri +if MYPY_RUNNING: + from typing import ( + Union, + Optional, + List, + Set, + Any, + TypeVar, + Tuple, + Sequence, + Dict, + Text, + AnyStr, + Match, + Iterable, + ) + from attr import _ValidatorType + from packaging.requirements import Requirement as PackagingRequirement + from pkg_resources import Requirement as PkgResourcesRequirement + from pkg_resources.extern.packaging.markers import ( + Op as PkgResourcesOp, + Variable as PkgResourcesVariable, + Value as PkgResourcesValue, + Marker as PkgResourcesMarker, + ) + from pip_shims.shims import Link + from vistir.compat import Path + + _T = TypeVar("_T") + TMarker = Union[Marker, PkgResourcesMarker] + TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) + TValue = TypeVar("TValue", PkgResourcesValue, Value) + TOp = TypeVar("TOp", PkgResourcesOp, Op) + MarkerTuple = Tuple[TVariable, TOp, TValue] + TRequirement = Union[PackagingRequirement, PkgResourcesRequirement] + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) HASH_STRING = " --hash={0}" +ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits) +PUNCTUATION = r"[\-_\.]" +ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits) +NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC) +REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits) +EXTRAS = r"(?P\[{0}(?:,{0})*\])".format(NAME) +NAME_WITH_EXTRAS = r"(?P{0}){1}?".format(NAME, EXTRAS) +NAME_RE = re.compile(NAME_WITH_EXTRAS) +SUBDIR_RE = r"(?:[&#]subdirectory=(?P.*))" +URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) +REF_RE = r"(?:@(?P{0}+)?)".format(REF) +PATH_RE = r"(?P[:/])(?P[^ @]+){0}?".format(REF_RE) +PASS_RE = r"(?:(?<=:)(?P[^ ]+))" +AUTH_RE = r"(?:(?P[^ ]+)[:@]{0}?@)".format(PASS_RE) +HOST_RE = r"(?:{0}?(?P[^ ]+?\.?{1}+(?P:\d+)?))?".format( + AUTH_RE, ALPHA_NUMERIC +) +URL = r"(?P[^ ]+://){0}{1}".format(HOST_RE, PATH_RE) +URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) +DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) + def filter_none(k, v): + # type: (AnyStr, Any) -> bool if v: return True return False def optional_instance_of(cls): + # type: (Any) -> _ValidatorType[Optional[_T]] return validators.optional(validators.instance_of(cls)) def create_link(link): - from pip_shims import Link + # type: (AnyStr) -> Link + + if not isinstance(link, six.string_types): + raise TypeError("must provide a string to instantiate a new link") + from pip_shims.shims import Link + return Link(link) +def get_url_name(url): + # type: (AnyStr) -> AnyStr + """ + Given a url, derive an appropriate name to use in a pipfile. + + :param str url: A url to derive a string from + :returns: The name of the corresponding pipfile entry + :rtype: Text + """ + if not isinstance(url, six.string_types): + raise TypeError("Expected a string, got {0!r}".format(url)) + return urllib3_util.parse_url(url).host + + def init_requirement(name): + # type: (AnyStr) -> TRequirement + + if not isinstance(name, six.string_types): + raise TypeError("must supply a name to generate a requirement") from pkg_resources import Requirement + req = Requirement.parse(name) req.vcs = None req.local_file = None @@ -51,73 +141,259 @@ def init_requirement(name): def extras_to_string(extras): + # type: (Iterable[S]) -> S """Turn a list of extras into a string""" if isinstance(extras, six.string_types): if extras.startswith("["): return extras - else: extras = [extras] - return "[{0}]".format(",".join(sorted(extras))) + if not extras: + return "" + return "[{0}]".format(",".join(sorted(set(extras)))) # type: ignore def parse_extras(extras_str): - """Turn a string of extras into a parsed extras list""" + # type: (AnyStr) -> List[AnyStr] + """ + Turn a string of extras into a parsed extras list + """ + from pkg_resources import Requirement + extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras return sorted(dedup([extra.lower() for extra in extras])) def specs_to_string(specs): - """Turn a list of specifier tuples into a string""" + # type: (List[Union[STRING_TYPE, Specifier]]) -> AnyStr + """ + Turn a list of specifier tuples into a string + """ + if specs: if isinstance(specs, six.string_types): return specs try: extras = ",".join(["".join(spec) for spec in specs]) except TypeError: - extras = ",".join(["".join(spec._spec) for spec in specs]) + extras = ",".join(["".join(spec._spec) for spec in specs]) # type: ignore return extras return "" -def build_vcs_link(vcs, uri, name=None, ref=None, subdirectory=None, extras=None): +def build_vcs_uri( + vcs, # type: Optional[S] + uri, # type: S + name=None, # type: Optional[S] + ref=None, # type: Optional[S] + subdirectory=None, # type: Optional[S] + extras=None, # type: Optional[Iterable[S]] +): + # type: (...) -> STRING_TYPE if extras is None: extras = [] - vcs_start = "{0}+".format(vcs) - if not uri.startswith(vcs_start): - uri = "{0}{1}".format(vcs_start, uri) - uri = add_ssh_scheme_to_git_uri(uri) + vcs_start = "" + if vcs is not None: + vcs_start = "{0}+".format(vcs) + if not uri.startswith(vcs_start): + uri = "{0}{1}".format(vcs_start, uri) if ref: uri = "{0}@{1}".format(uri, ref) if name: uri = "{0}#egg={1}".format(uri, name) if extras: - extras = extras_to_string(extras) - uri = "{0}{1}".format(uri, extras) + extras_string = extras_to_string(extras) + uri = "{0}{1}".format(uri, extras_string) if subdirectory: uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return create_link(uri) + return uri + + +def convert_direct_url_to_url(direct_url): + # type: (AnyStr) -> AnyStr + """ + Given a direct url as defined by *PEP 508*, convert to a :class:`~pip_shims.shims.Link` + compatible URL by moving the name and extras into an **egg_fragment**. + + :param str direct_url: A pep-508 compliant direct url. + :return: A reformatted URL for use with Link objects and :class:`~pip_shims.shims.InstallRequirement` objects. + :rtype: AnyStr + """ + direct_match = DIRECT_URL_RE.match(direct_url) # type: Optional[Match] + if direct_match is None: + url_match = URL_RE.match(direct_url) + if url_match or is_valid_url(direct_url): + return direct_url + match_dict = ( + {} + ) # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] + if direct_match is not None: + match_dict = direct_match.groupdict() # type: ignore + if not match_dict: + raise ValueError( + "Failed converting value to normal URL, is it a direct URL? {0!r}".format( + direct_url + ) + ) + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + url = "" # type: STRING_TYPE + url = "".join([s for s in url_segments if s is not None]) # type: ignore + new_url = build_vcs_uri( + None, + url, + ref=match_dict.get("ref"), + name=match_dict.get("name"), + extras=match_dict.get("extras"), + subdirectory=match_dict.get("subdirectory"), + ) + return new_url + + +def convert_url_to_direct_url(url, name=None): + # type: (AnyStr, Optional[AnyStr]) -> AnyStr + """ + Given a :class:`~pip_shims.shims.Link` compatible URL, convert to a direct url as + defined by *PEP 508* by extracting the name and extras from the **egg_fragment**. + + :param AnyStr url: A :class:`~pip_shims.shims.InstallRequirement` compliant URL. + :param Optiona[AnyStr] name: A name to use in case the supplied URL doesn't provide one. + :return: A pep-508 compliant direct url. + :rtype: AnyStr + + :raises ValueError: Raised when the URL can't be parsed or a name can't be found. + :raises TypeError: When a non-string input is provided. + """ + if not isinstance(url, six.string_types): + raise TypeError( + "Expected a string to convert to a direct url, got {0!r}".format(url) + ) + direct_match = DIRECT_URL_RE.match(url) + if direct_match: + return url + url_match = URL_RE.match(url) + if url_match is None or not url_match.groupdict(): + raise ValueError("Failed parse a valid URL from {0!r}".format(url)) + match_dict = url_match.groupdict() + url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] + name = match_dict.get("name", name) + extras = match_dict.get("extras") + new_url = "" + if extras and not name: + url_segments.append(extras) + elif extras and name: + new_url = "{0}{1}@ ".format(name, extras) + else: + if name is not None: + new_url = "{0}@ ".format(name) + else: + raise ValueError( + "Failed to construct direct url: " + "No name could be parsed from {0!r}".format(url) + ) + if match_dict.get("ref"): + url_segments.append("@{0}".format(match_dict.get("ref"))) + url = "".join([s for s in url if s is not None]) + url = "{0}{1}".format(new_url, url) + return url def get_version(pipfile_entry): + # type: (Union[STRING_TYPE, Dict[STRING_TYPE, Union[STRING_TYPE, bool, Iterable[STRING_TYPE]]]]) -> STRING_TYPE if str(pipfile_entry) == "{}" or is_star(pipfile_entry): return "" elif hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: if is_star(pipfile_entry.get("version")): return "" - return pipfile_entry.get("version", "") + return pipfile_entry.get("version", "").strip().lstrip("(").rstrip(")") if isinstance(pipfile_entry, six.string_types): - return pipfile_entry + return pipfile_entry.strip().lstrip("(").rstrip(")") return "" +def strip_extras_markers_from_requirement(req): + # type: (TRequirement) -> TRequirement + """ + Given a :class:`~packaging.requirements.Requirement` instance with markers defining + *extra == 'name'*, strip out the extras from the markers and return the cleaned + requirement + + :param PackagingRequirement req: A packaging requirement to clean + :return: A cleaned requirement + :rtype: PackagingRequirement + """ + if req is None: + raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) + if getattr(req, "marker", None) is not None: + marker = req.marker # type: TMarker + marker._markers = _strip_extras_markers(marker._markers) + if not marker._markers: + req.marker = None + else: + req.marker = marker + return req + + +def _strip_extras_markers(marker): + # type: (Union[MarkerTuple, List[Union[MarkerTuple, str]]]) -> List[Union[MarkerTuple, str]] + if marker is None or not isinstance(marker, (list, tuple)): + raise TypeError("Expecting a marker type, received {0!r}".format(marker)) + markers_to_remove = [] + # iterate forwards and generate a list of indexes to remove first, then reverse the + # list so we can remove the text that normally occurs after (but we will already + # be past it in the loop) + for i, marker_list in enumerate(marker): + if isinstance(marker_list, list): + cleaned = _strip_extras_markers(marker_list) + if not cleaned: + markers_to_remove.append(i) + elif isinstance(marker_list, tuple) and marker_list[0].value == "extra": + markers_to_remove.append(i) + for i in reversed(markers_to_remove): + del marker[i] + if i > 0 and marker[i - 1] == "and": + del marker[i - 1] + return marker + + +@lru_cache() +def get_setuptools_version(): + # type: () -> Optional[STRING_TYPE] + import pkg_resources + + setuptools_dist = pkg_resources.get_distribution( + pkg_resources.Requirement("setuptools") + ) + return getattr(setuptools_dist, "version", None) + + +def get_default_pyproject_backend(): + # type: () -> STRING_TYPE + st_version = get_setuptools_version() + if st_version is not None: + parsed_st_version = parse_version(st_version) + if parsed_st_version >= parse_version("40.8.0"): + return "setuptools.build_meta:__legacy__" + return "setuptools.build_meta" + + def get_pyproject(path): - from vistir.compat import Path + # type: (Union[STRING_TYPE, Path]) -> Optional[Tuple[List[STRING_TYPE], STRING_TYPE]] + """ + Given a base path, look for the corresponding ``pyproject.toml`` file and return its + build_requires and build_backend. + + :param AnyStr path: The root path of the project, should be a directory (will be truncated) + :return: A 2 tuple of build requirements and the build backend + :rtype: Optional[Tuple[List[AnyStr], AnyStr]] + """ + if not path: return + from vistir.compat import Path + if not isinstance(path, Path): path = Path(path) if not path.is_dir(): @@ -125,8 +401,10 @@ def get_pyproject(path): pp_toml = path.joinpath("pyproject.toml") setup_py = path.joinpath("setup.py") if not pp_toml.exists(): - if setup_py.exists(): + if not setup_py.exists(): return None + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: pyproject_data = {} with io.open(pp_toml.as_posix(), encoding="utf-8") as fh: @@ -134,23 +412,21 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools", "wheel"] - backend = "setuptools.build_meta" + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() else: - requires = ["setuptools>=38.2.5", "wheel"] - backend = "setuptools.build_meta" - build_system = { - "requires": requires, - "build-backend": backend - } + requires = ["setuptools>=40.8", "wheel"] + backend = get_default_pyproject_backend() + build_system = {"requires": requires, "build-backend": backend} pyproject_data["build_system"] = build_system else: - requires = build_system.get("requires") - backend = build_system.get("build-backend") - return (requires, backend) + requires = build_system.get("requires", ["setuptools>=40.8", "wheel"]) + backend = build_system.get("build-backend", get_default_pyproject_backend()) + return requires, backend def split_markers_from_line(line): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] """Split markers from a dependency""" if not any(line.startswith(uri_prefix) for uri_prefix in SCHEME_LIST): marker_sep = ";" @@ -164,14 +440,37 @@ def split_markers_from_line(line): def split_vcs_method_from_uri(uri): + # type: (AnyStr) -> Tuple[Optional[STRING_TYPE], STRING_TYPE] """Split a vcs+uri formatted uri into (vcs, uri)""" vcs_start = "{0}+" + vcs = None # type: Optional[STRING_TYPE] vcs = first([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]) if vcs: vcs, uri = uri.split("+", 1) return vcs, uri +def split_ref_from_uri(uri): + # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] + """ + Given a path or URI, check for a ref and split it from the path if it is present, + returning a tuple of the original input and the ref or None. + + :param AnyStr uri: The path or URI to split + :returns: A 2-tuple of the path or URI and the ref + :rtype: Tuple[AnyStr, Optional[AnyStr]] + """ + if not isinstance(uri, six.string_types): + raise TypeError("Expected a string, received {0!r}".format(uri)) + parsed = urllib_parse.urlparse(uri) + path = parsed.path + ref = None + if "@" in path: + path, _, ref = path.rpartition("@") + parsed = parsed._replace(path=path) + return (urllib_parse.urlunparse(parsed), ref) + + def validate_vcs(instance, attr_, value): if value not in VCS_LIST: raise ValueError("Invalid vcs {0!r}".format(value)) @@ -208,14 +507,14 @@ def key_from_ireq(ireq): def key_from_req(req): """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, 'key'): + if hasattr(req, "key"): # from pkg_resources, such as installed dists for pip-sync key = req.key else: # from packaging, such as install requirements from requirements.txt key = req.name - key = key.replace('_', '-').lower() + key = key.replace("_", "-").lower() return key @@ -228,10 +527,11 @@ def _requirement_to_str_lowercase_name(requirement): modified to lowercase the dependency name. Previously, we were invoking the original Requirement.__str__ method and - lowercasing the entire result, which would lowercase the name, *and* other, - important stuff that should not be lowercased (such as the marker). See + lower-casing the entire result, which would lowercase the name, *and* other, + important stuff that should not be lower-cased (such as the marker). See this issue for more information: https://github.com/pypa/pipenv/issues/2113. """ + parts = [requirement.name.lower()] if requirement.extras: @@ -254,18 +554,19 @@ def format_requirement(ireq): Generic formatter for pretty printing InstallRequirements to the terminal in a less verbose way than using its `__str__` method. """ + if ireq.editable: - line = '-e {}'.format(ireq.link) + line = "-e {}".format(ireq.link) else: line = _requirement_to_str_lowercase_name(ireq.req) if str(ireq.req.marker) != str(ireq.markers): if not ireq.req.marker: - line = '{}; {}'.format(line, ireq.markers) + line = "{}; {}".format(line, ireq.markers) else: name, markers = line.split(";", 1) markers = markers.strip() - line = '{}; ({}) and ({})'.format(name, markers, ireq.markers) + line = "{}; ({}) and ({})".format(name, markers, ireq.markers) return line @@ -278,11 +579,12 @@ def format_specifier(ireq): # TODO: Ideally, this is carried over to the pip library itself specs = ireq.specifier._specs if ireq.req is not None else [] specs = sorted(specs, key=lambda x: x._spec[1]) - return ','.join(str(s) for s in specs) or '' + return ",".join(str(s) for s in specs) or "" def get_pinned_version(ireq): - """Get the pinned version of an InstallRequirement. + """ + Get the pinned version of an InstallRequirement. An InstallRequirement is considered pinned if: @@ -300,12 +602,11 @@ def get_pinned_version(ireq): Raises `TypeError` if the input is not a valid InstallRequirement, or `ValueError` if the InstallRequirement is not pinned. """ + try: specifier = ireq.specifier except AttributeError: - raise TypeError("Expected InstallRequirement, not {}".format( - type(ireq).__name__, - )) + raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) if ireq.editable: raise ValueError("InstallRequirement is editable") @@ -315,16 +616,15 @@ def get_pinned_version(ireq): raise ValueError("InstallRequirement has multiple specifications") op, version = next(iter(specifier._specs))._spec - if op not in ('==', '===') or version.endswith('.*'): - raise ValueError("InstallRequirement not pinned (is {0!r})".format( - op + version, - )) + if op not in ("==", "===") or version.endswith(".*"): + raise ValueError("InstallRequirement not pinned (is {0!r})".format(op + version)) return version def is_pinned_requirement(ireq): - """Returns whether an InstallRequirement is a "pinned" requirement. + """ + Returns whether an InstallRequirement is a "pinned" requirement. An InstallRequirement is considered pinned if: @@ -339,6 +639,7 @@ def is_pinned_requirement(ireq): django~=1.8 # NOT pinned django==1.* # NOT pinned """ + try: get_pinned_version(ireq) except (TypeError, ValueError): @@ -350,8 +651,9 @@ def as_tuple(ireq): """ Pulls out the (name: str, version:str, extras:(str)) tuple from the pinned InstallRequirement. """ + if not is_pinned_requirement(ireq): - raise TypeError('Expected a pinned InstallRequirement, got {}'.format(ireq)) + raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) name = key_from_req(ireq.req) version = first(ireq.specifier._specs)._spec[1] @@ -360,12 +662,18 @@ def as_tuple(ireq): def full_groupby(iterable, key=None): - """Like groupby(), but sorts the input on the group key first.""" + """ + Like groupby(), but sorts the input on the group key first. + """ + return groupby(sorted(iterable, key=key), key=key) def flat_map(fn, collection): - """Map a function over a collection and flatten the result by one-level""" + """ + Map a function over a collection and flatten the result by one-level + """ + return chain.from_iterable(map(fn, collection)) @@ -385,8 +693,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): For key functions that uniquely identify values, set unique=True: >>> assert lookup_table( - ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], - ... unique=True) == { + ... ['foo', 'bar', 'baz', 'qux', 'quux'], lambda s: s[0], unique=True) == { ... 'b': 'baz', ... 'f': 'foo', ... 'q': 'quux' @@ -404,13 +711,13 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): ... 'f': {'oo'}, ... 'q': {'uux', 'ux'} ... } - """ + if keyval is None: if key is None: - keyval = (lambda v: v) + keyval = lambda v: v else: - keyval = (lambda v: (key(v), v)) + keyval = lambda v: (key(v), v) if unique: return dict(keyval(v) for v in values) @@ -434,7 +741,7 @@ def lookup_table(values, key=None, keyval=None, unique=False, use_lists=False): def name_from_req(req): """Get the name of the requirement""" - if hasattr(req, 'project_name'): + if hasattr(req, "project_name"): # from pkg_resources, such as installed dists for pip-sync return req.project_name else: @@ -443,7 +750,8 @@ def name_from_req(req): def make_install_requirement(name, version, extras, markers, constraint=False): - """make_install_requirement Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. + """ + Generates an :class:`~pip._internal.req.req_install.InstallRequirement`. Create an InstallRequirement from the supplied metadata. @@ -463,6 +771,7 @@ def make_install_requirement(name, version, extras, markers, constraint=False): # If no extras are specified, the extras string is blank from pip_shims.shims import install_req_from_line + extras_string = "" if extras: # Sort extras for stability @@ -470,12 +779,13 @@ def make_install_requirement(name, version, extras, markers, constraint=False): if not markers: return install_req_from_line( - str('{}{}=={}'.format(name, extras_string, version)), - constraint=constraint) + str("{}{}=={}".format(name, extras_string, version)), constraint=constraint + ) else: return install_req_from_line( - str('{}{}=={}; {}'.format(name, extras_string, version, str(markers))), - constraint=constraint) + str("{}{}=={}; {}".format(name, extras_string, version, str(markers))), + constraint=constraint, + ) def version_from_ireq(ireq): @@ -493,9 +803,10 @@ def version_from_ireq(ireq): def clean_requires_python(candidates): """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes.""" all_candidates = [] - sys_version = '.'.join(map(str, sys.version_info[:3])) + sys_version = ".".join(map(str, sys.version_info[:3])) from packaging.version import parse as parse_version - py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', sys_version)) + + py_version = parse_version(os.environ.get("PIP_PYTHON_VERSION", sys_version)) for c in candidates: from_location = attrgetter("location.requires_python") requires_python = getattr(c, "requires_python", from_location(c)) @@ -503,7 +814,9 @@ def clean_requires_python(candidates): # Old specifications had people setting this to single digits # which is effectively the same as '>=digit,=2.6"')` - marker_key = Variable('python_version') + marker_key = Variable("python_version") for spec in specifierset: operator, val = spec._spec cleaned_val = Value(val).serialize().replace('"', "") spec_dict[Op(operator).serialize()].add(cleaned_val) - marker_str = ' and '.join([ - "{0}{1}'{2}'".format(marker_key.serialize(), op, ','.join(vals)) - for op, vals in spec_dict.items() - ]) - marker_to_add = PackagingRequirement('fakepkg; {0}'.format(marker_str)).marker + marker_str = " and ".join( + [ + "{0}{1}'{2}'".format(marker_key.serialize(), op, ",".join(vals)) + for op, vals in spec_dict.items() + ] + ) + marker_to_add = PackagingRequirement("fakepkg; {0}".format(marker_str)).marker return marker_to_add def normalize_name(pkg): + # type: (AnyStr) -> AnyStr """Given a package name, return its normalized, non-canonicalized form. - :param str pkg: The name of a package + :param AnyStr pkg: The name of a package :return: A normalized package name - :rtype: str + :rtype: AnyStr """ assert isinstance(pkg, six.string_types) return pkg.replace("_", "-").lower() + + +def get_name_variants(pkg): + # type: (STRING_TYPE) -> Set[STRING_TYPE] + """ + Given a packager name, get the variants of its name for both the canonicalized + and "safe" forms. + + :param AnyStr pkg: The package to lookup + :returns: A list of names. + :rtype: Set + """ + + if not isinstance(pkg, six.string_types): + raise TypeError("must provide a string to derive package names") + from pkg_resources import safe_name + from packaging.utils import canonicalize_name + + pkg = pkg.lower() + names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} + return names + + +SETUPTOOLS_SHIM = ( + "import setuptools, tokenize;__file__=%r;" + "f=getattr(tokenize, 'open', open)(__file__);" + "code=f.read().replace('\\r\\n', '\\n');" + "f.close();" + "exec(compile(code, __file__, 'exec'))" +) diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 6a15db3f..9296f605 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -1,11 +1,18 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + import attr +import importlib import os import pip_shims +import six +import sys -@attr.s +@attr.s(hash=True) class VCSRepository(object): + DEFAULT_RUN_ARGS = None + url = attr.ib() name = attr.ib() checkout_directory = attr.ib() @@ -14,13 +21,21 @@ class VCSRepository(object): commit_sha = attr.ib(default=None) ref = attr.ib(default=None) repo_instance = attr.ib() + clone_log = attr.ib(default=None) @repo_instance.default def get_repo_instance(self): - from pip_shims import VcsSupport + if self.DEFAULT_RUN_ARGS is None: + default_run_args = self.monkeypatch_pip() + else: + default_run_args = self.DEFAULT_RUN_ARGS + from pip_shims.shims import VcsSupport VCS_SUPPORT = VcsSupport() backend = VCS_SUPPORT._registry.get(self.vcs_type) - return backend(url=self.url) + repo = backend(url=self.url) + if repo.run_command.__func__.__defaults__ != default_run_args: + repo.run_command.__func__.__defaults__ = default_run_args + return repo @property def is_local(self): @@ -58,3 +73,22 @@ class VCSRepository(object): def get_commit_hash(self, ref=None): return self.repo_instance.get_revision(self.checkout_directory) + + @classmethod + def monkeypatch_pip(cls): + target_module = pip_shims.shims.VcsSupport.__module__ + pip_vcs = importlib.import_module(target_module) + run_command_defaults = pip_vcs.VersionControl.run_command.__defaults__ + # set the default to not write stdout, the first option sets this value + new_defaults = [False,] + list(run_command_defaults)[1:] + new_defaults = tuple(new_defaults) + if six.PY3: + try: + pip_vcs.VersionControl.run_command.__defaults__ = new_defaults + except AttributeError: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + else: + pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults + sys.modules[target_module] = pip_vcs + cls.DEFAULT_RUN_ARGS = new_defaults + return new_defaults diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/vendor/requirementslib/utils.py index f3653e32..515f9a88 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/vendor/requirementslib/utils.py @@ -1,24 +1,56 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import +from __future__ import absolute_import, print_function import contextlib import logging import os - -import six import sys -import tomlkit - -six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) -six.add_move(six.MovedAttribute("ItemsView", "collections", "collections.abc")) -from six.moves import Mapping, Sequence, Set, ItemsView -from six.moves.urllib.parse import urlparse, urlsplit import pip_shims.shims +import six +import tomlkit +import vistir +from six.moves.urllib.parse import urlparse, urlsplit, urlunparse from vistir.compat import Path -from vistir.path import is_valid_url, ensure_mkdir_p, create_tracked_tempdir +from vistir.path import create_tracked_tempdir, ensure_mkdir_p, is_valid_url + +from .environment import MYPY_RUNNING + +# fmt: off +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("Set", "collections", "collections.abc") +) # type: ignore # noqa # isort:skip +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore # noqa +from six.moves import ItemsView, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on + + +if MYPY_RUNNING: + from typing import ( + Dict, + Any, + Optional, + Union, + Tuple, + List, + Iterable, + Generator, + Text, + TypeVar, + ) + + STRING_TYPE = Union[bytes, str, Text] + S = TypeVar("S", bytes, str, Text) + PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] + PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] VCS_LIST = ("git", "svn", "hg", "bzr") @@ -68,11 +100,12 @@ VCS_SCHEMES = [ def is_installable_dir(path): + # type: (STRING_TYPE) -> bool if pip_shims.shims.is_installable_dir(path): return True - path = Path(path) - pyproject = path.joinpath("pyproject.toml") - if pyproject.exists(): + pyproject_path = os.path.join(path, "pyproject.toml") + if os.path.exists(pyproject_path): + pyproject = Path(pyproject_path) pyproject_toml = tomlkit.loads(pyproject.read_text()) build_system = pyproject_toml.get("build-system", {}).get("build-backend", "") if build_system: @@ -81,22 +114,39 @@ def is_installable_dir(path): def strip_ssh_from_git_uri(uri): + # type: (S) -> S """Return git+ssh:// formatted URI to git+git@ format""" if isinstance(uri, six.string_types): - uri = uri.replace("git+ssh://", "git+", 1) + if "git+ssh://" in uri: + parsed = urlparse(uri) + # split the path on the first separating / so we can put the first segment + # into the 'netloc' section with a : separator + path_part, _, path = parsed.path.lstrip("/").partition("/") + path = "/{0}".format(path) + parsed = parsed._replace( + netloc="{0}:{1}".format(parsed.netloc, path_part), path=path + ) + uri = urlunparse(parsed).replace("git+ssh://", "git+", 1) return uri def add_ssh_scheme_to_git_uri(uri): + # type: (S) -> S """Cleans VCS uris from pipenv.patched.notpip format""" if isinstance(uri, six.string_types): # Add scheme for parsing purposes, this is also what pip does if uri.startswith("git+") and "://" not in uri: uri = uri.replace("git+", "git+ssh://", 1) + parsed = urlparse(uri) + if ":" in parsed.netloc: + netloc, _, path_start = parsed.netloc.rpartition(":") + path = "/{0}{1}".format(path_start, parsed.path) + uri = urlunparse(parsed._replace(netloc=netloc, path=path)) return uri def is_vcs(pipfile_entry): + # type: (PipfileType) -> bool """Determine if dictionary entry from Pipfile is for a vcs dependency.""" if isinstance(pipfile_entry, Mapping): return any(key for key in pipfile_entry.keys() if key in VCS_LIST) @@ -111,12 +161,16 @@ def is_vcs(pipfile_entry): def is_editable(pipfile_entry): + # type: (PipfileType) -> bool if isinstance(pipfile_entry, Mapping): return pipfile_entry.get("editable", False) is True + if isinstance(pipfile_entry, six.string_types): + return pipfile_entry.startswith("-e ") return False def multi_split(s, split): + # type: (S, Iterable[S]) -> List[S] """Splits on multiple given separators.""" for r in split: s = s.replace(r, "|") @@ -124,21 +178,37 @@ def multi_split(s, split): def is_star(val): + # type: (PipfileType) -> bool return (isinstance(val, six.string_types) and val == "*") or ( isinstance(val, Mapping) and val.get("version", "") == "*" ) +def convert_entry_to_path(path): + # type: (Dict[S, Union[S, bool, Tuple[S], List[S]]]) -> S + """Convert a pipfile entry to a string""" + + if not isinstance(path, Mapping): + raise TypeError("expecting a mapping, received {0!r}".format(path)) + + if not any(key in path for key in ["file", "path"]): + raise ValueError("missing path-like entry in supplied mapping {0!r}".format(path)) + + if "file" in path: + path = vistir.path.url_to_path(path["file"]) + + elif "path" in path: + path = path["path"] + return path + + def is_installable_file(path): + # type: (PipfileType) -> bool """Determine if a path can potentially be installed""" from packaging import specifiers - if hasattr(path, "keys") and any( - key for key in path.keys() if key in ["file", "path"] - ): - path = urlparse(path["file"]).path if "file" in path else path["path"] - if not isinstance(path, six.string_types) or path == "*": - return False + if isinstance(path, Mapping): + path = convert_entry_to_path(path) # If the string starts with a valid specifier operator, test if it is a valid # specifier set before making a path object (to avoid breaking windows) @@ -152,41 +222,94 @@ def is_installable_file(path): return False parsed = urlparse(path) - if parsed.scheme == "file": - path = parsed.path - - if not os.path.exists(os.path.abspath(path)): + is_local = ( + not parsed.scheme + or parsed.scheme == "file" + or (len(parsed.scheme) == 1 and os.name == "nt") + ) + if parsed.scheme and parsed.scheme == "file": + path = vistir.compat.fs_decode(vistir.path.url_to_path(path)) + normalized_path = vistir.path.normalize_path(path) + if is_local and not os.path.exists(normalized_path): return False - lookup_path = Path(path) - absolute_path = "{0}".format(lookup_path.absolute()) - if lookup_path.is_dir() and is_installable_dir(absolute_path): + is_archive = pip_shims.shims.is_archive_file(normalized_path) + is_local_project = os.path.isdir(normalized_path) and is_installable_dir( + normalized_path + ) + if is_local and is_local_project or is_archive: return True - elif lookup_path.is_file() and pip_shims.shims.is_archive_file(absolute_path): + if not is_local and pip_shims.shims.is_archive_file(parsed.path): return True return False +def get_dist_metadata(dist): + import pkg_resources + from email.parser import FeedParser + + if isinstance(dist, pkg_resources.DistInfoDistribution) and dist.has_metadata( + "METADATA" + ): + metadata = dist.get_metadata("METADATA") + elif dist.has_metadata("PKG-INFO"): + metadata = dist.get_metadata("PKG-INFO") + else: + metadata = "" + + feed_parser = FeedParser() + feed_parser.feed(metadata) + return feed_parser.close() + + +def get_setup_paths(base_path, subdirectory=None): + # type: (S, Optional[S]) -> Dict[S, Optional[S]] + if base_path is None: + raise TypeError("must provide a path to derive setup paths from") + setup_py = os.path.join(base_path, "setup.py") + setup_cfg = os.path.join(base_path, "setup.cfg") + pyproject_toml = os.path.join(base_path, "pyproject.toml") + if subdirectory is not None: + base_path = os.path.join(base_path, subdirectory) + subdir_setup_py = os.path.join(subdirectory, "setup.py") + subdir_setup_cfg = os.path.join(subdirectory, "setup.cfg") + subdir_pyproject_toml = os.path.join(subdirectory, "pyproject.toml") + if subdirectory and os.path.exists(subdir_setup_py): + setup_py = subdir_setup_py + if subdirectory and os.path.exists(subdir_setup_cfg): + setup_cfg = subdir_setup_cfg + if subdirectory and os.path.exists(subdir_pyproject_toml): + pyproject_toml = subdir_pyproject_toml + return { + "setup_py": setup_py if os.path.exists(setup_py) else None, + "setup_cfg": setup_cfg if os.path.exists(setup_cfg) else None, + "pyproject_toml": pyproject_toml if os.path.exists(pyproject_toml) else None, + } + + def prepare_pip_source_args(sources, pip_args=None): + # type: (List[Dict[S, Union[S, bool]]], Optional[List[S]]) -> List[S] if pip_args is None: pip_args = [] if sources: # Add the source to pip9. - pip_args.extend(["-i", sources[0]["url"]]) + pip_args.extend(["-i", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): - pip_args.extend(["--trusted-host", urlparse(sources[0]["url"]).hostname]) + pip_args.extend( + ["--trusted-host", urlparse(sources[0]["url"]).hostname] + ) # type: ignore # Add additional sources as extra indexes. if len(sources) > 1: for source in sources[1:]: - pip_args.extend(["--extra-index-url", source["url"]]) + pip_args.extend(["--extra-index-url", source["url"]]) # type: ignore # Trust the host if it's not verified. if not source.get("verify_ssl", True): pip_args.extend( ["--trusted-host", urlparse(source["url"]).hostname] - ) + ) # type: ignore return pip_args @@ -196,10 +319,11 @@ def _ensure_dir(path): @contextlib.contextmanager -def ensure_setup_py(base_dir): - if not base_dir: - base_dir = create_tracked_tempdir(prefix="requirementslib-setup") - base_dir = Path(base_dir) +def ensure_setup_py(base): + # type: (STRING_TYPE) -> Generator[None, None, None] + if not base: + base = create_tracked_tempdir(prefix="requirementslib-setup") + base_dir = Path(base) if base_dir.exists() and base_dir.name == "setup.py": base_dir = base_dir.parent elif not (base_dir.exists() and base_dir.is_dir()): @@ -325,9 +449,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (ValueError, KeyError, IndexError, TypeError): if not getattr(cur, "__iter__", None): - exc = TypeError( - "%r object is not indexable" % type(cur).__name__ - ) + exc = TypeError("%r object is not indexable" % type(cur).__name__) raise PathAccessError(exc, seg, path) except PathAccessError: if default is _UNSET: diff --git a/pipenv/vendor/shellingham/__init__.py b/pipenv/vendor/shellingham/__init__.py index 576c4224..f879cf9d 100644 --- a/pipenv/vendor/shellingham/__init__.py +++ b/pipenv/vendor/shellingham/__init__.py @@ -4,7 +4,7 @@ import os from ._core import ShellDetectionFailure -__version__ = '1.2.7' +__version__ = '1.2.8' def detect_shell(pid=None, max_depth=6): diff --git a/pipenv/vendor/shellingham/nt.py b/pipenv/vendor/shellingham/nt.py index 7b3cc6b4..d66bc33f 100644 --- a/pipenv/vendor/shellingham/nt.py +++ b/pipenv/vendor/shellingham/nt.py @@ -75,7 +75,18 @@ def _iter_process(): # looking for. We can fix this when it actually matters. (#8) continue raise WinError() - info = {'executable': str(pe.szExeFile.decode('utf-8'))} + + # The executable name would be encoded with the current code page if + # we're in ANSI mode (usually). Try to decode it into str/unicode, + # replacing invalid characters to be safe (not thoeratically necessary, + # I think). Note that we need to use 'mbcs' instead of encoding + # settings from sys because this is from the Windows API, not Python + # internals (which those settings reflect). (pypa/pipenv#3382) + executable = pe.szExeFile + if isinstance(executable, bytes): + executable = executable.decode('mbcs', 'replace') + + info = {'executable': executable} if pe.th32ParentProcessID: info['parent_pid'] = pe.th32ParentProcessID yield pe.th32ProcessID, info diff --git a/pipenv/vendor/shellingham/posix.py b/pipenv/vendor/shellingham/posix.py index 0bbf988b..b25dd874 100644 --- a/pipenv/vendor/shellingham/posix.py +++ b/pipenv/vendor/shellingham/posix.py @@ -21,7 +21,7 @@ def _get_process_mapping(): processes = {} for line in output.split('\n'): try: - pid, ppid, args = line.strip().split(None, 2) + pid, ppid, args = line.strip().split(maxsplit=2) except ValueError: continue processes[pid] = Process( diff --git a/pipenv/vendor/shellingham/posix/_default.py b/pipenv/vendor/shellingham/posix/_default.py new file mode 100644 index 00000000..86944276 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/_default.py @@ -0,0 +1,27 @@ +import collections +import shlex +import subprocess +import sys + + +Process = collections.namedtuple('Process', 'args pid ppid') + + +def get_process_mapping(): + """Try to look up the process tree via the output of `ps`. + """ + output = subprocess.check_output([ + 'ps', '-ww', '-o', 'pid=', '-o', 'ppid=', '-o', 'args=', + ]) + if not isinstance(output, str): + output = output.decode(sys.stdout.encoding) + processes = {} + for line in output.split('\n'): + try: + pid, ppid, args = line.strip().split(None, 2) + except ValueError: + continue + processes[pid] = Process( + args=tuple(shlex.split(args)), pid=pid, ppid=ppid, + ) + return processes diff --git a/pipenv/vendor/shellingham/posix/_proc.py b/pipenv/vendor/shellingham/posix/_proc.py index e3a6e46d..921f2508 100644 --- a/pipenv/vendor/shellingham/posix/_proc.py +++ b/pipenv/vendor/shellingham/posix/_proc.py @@ -1,34 +1,40 @@ import os import re -from ._default import Process +from ._core import Process STAT_PPID = 3 STAT_TTY = 6 +STAT_PATTERN = re.compile(r'\(.+\)|\S+') + + +def _get_stat(pid): + with open(os.path.join('/proc', str(pid), 'stat')) as f: + parts = STAT_PATTERN.findall(f.read()) + return parts[STAT_TTY], parts[STAT_PPID] + + +def _get_cmdline(pid): + with open(os.path.join('/proc', str(pid), 'cmdline')) as f: + return tuple(f.read().split('\0')[:-1]) + def get_process_mapping(): """Try to look up the process tree via the /proc interface. """ - with open('/proc/{0}/stat'.format(os.getpid())) as f: - self_tty = f.read().split()[STAT_TTY] + self_tty = _get_stat(os.getpid())[0] processes = {} for pid in os.listdir('/proc'): if not pid.isdigit(): continue try: - stat = '/proc/{0}/stat'.format(pid) - cmdline = '/proc/{0}/cmdline'.format(pid) - with open(stat) as fstat, open(cmdline) as fcmdline: - stat = re.findall(r'\(.+\)|\S+', fstat.read()) - cmd = fcmdline.read().split('\x00')[:-1] - ppid = stat[STAT_PPID] - tty = stat[STAT_TTY] - if tty == self_tty: - processes[pid] = Process( - args=tuple(cmd), pid=pid, ppid=ppid, - ) + tty, ppid = _get_stat(pid) + if tty != self_tty: + continue + args = _get_cmdline(pid) + processes[pid] = Process(args=args, pid=pid, ppid=ppid) except IOError: # Process has disappeared - just ignore it. continue diff --git a/pipenv/vendor/shellingham/posix/_ps.py b/pipenv/vendor/shellingham/posix/_ps.py index 86944276..e96278cf 100644 --- a/pipenv/vendor/shellingham/posix/_ps.py +++ b/pipenv/vendor/shellingham/posix/_ps.py @@ -1,10 +1,8 @@ -import collections import shlex import subprocess import sys - -Process = collections.namedtuple('Process', 'args pid ppid') +from ._core import Process def get_process_mapping(): diff --git a/pipenv/vendor/shellingham/posix/linux.py b/pipenv/vendor/shellingham/posix/linux.py new file mode 100644 index 00000000..6db97834 --- /dev/null +++ b/pipenv/vendor/shellingham/posix/linux.py @@ -0,0 +1,35 @@ +import os +import re + +from ._default import Process + + +STAT_PPID = 3 +STAT_TTY = 6 + + +def get_process_mapping(): + """Try to look up the process tree via Linux's /proc + """ + with open('/proc/{0}/stat'.format(os.getpid())) as f: + self_tty = f.read().split()[STAT_TTY] + processes = {} + for pid in os.listdir('/proc'): + if not pid.isdigit(): + continue + try: + stat = '/proc/{0}/stat'.format(pid) + cmdline = '/proc/{0}/cmdline'.format(pid) + with open(stat) as fstat, open(cmdline) as fcmdline: + stat = re.findall(r'\(.+\)|\S+', fstat.read()) + cmd = fcmdline.read().split('\x00')[:-1] + ppid = stat[STAT_PPID] + tty = stat[STAT_TTY] + if tty == self_tty: + processes[pid] = Process( + args=tuple(cmd), pid=pid, ppid=ppid, + ) + except IOError: + # Process has disappeared - just ignore it. + continue + return processes diff --git a/pipenv/vendor/shellingham/posix/ps.py b/pipenv/vendor/shellingham/posix/ps.py index 4a155ed5..cf7e56f2 100644 --- a/pipenv/vendor/shellingham/posix/ps.py +++ b/pipenv/vendor/shellingham/posix/ps.py @@ -1,5 +1,4 @@ import errno -import shlex import subprocess import sys @@ -34,9 +33,13 @@ def get_process_mapping(): for line in output.split('\n'): try: pid, ppid, args = line.strip().split(None, 2) - processes[pid] = Process( - args=tuple(shlex.split(args)), pid=pid, ppid=ppid, - ) + # XXX: This is not right, but we are really out of options. + # ps does not offer a sane way to decode the argument display, + # and this is "Good Enough" for obtaining shell names. Hopefully + # people don't name their shell with a space, or have something + # like "/usr/bin/xonsh is uber". (sarugaku/shellingham#14) + args = tuple(a.strip() for a in args.split(' ')) except ValueError: continue + processes[pid] = Process(args=args, pid=pid, ppid=ppid) return processes diff --git a/pipenv/vendor/six.LICENSE b/pipenv/vendor/six.LICENSE index f3068bfd..365d1074 100644 --- a/pipenv/vendor/six.LICENSE +++ b/pipenv/vendor/six.LICENSE @@ -1,4 +1,4 @@ -Copyright (c) 2010-2017 Benjamin Peterson +Copyright (c) 2010-2018 Benjamin Peterson Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in diff --git a/pipenv/vendor/six.py b/pipenv/vendor/six.py index 6bf4fd38..89b2188f 100644 --- a/pipenv/vendor/six.py +++ b/pipenv/vendor/six.py @@ -1,4 +1,4 @@ -# Copyright (c) 2010-2017 Benjamin Peterson +# Copyright (c) 2010-2018 Benjamin Peterson # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -29,7 +29,7 @@ import sys import types __author__ = "Benjamin Peterson " -__version__ = "1.11.0" +__version__ = "1.12.0" # Useful for very coarse version differentiation. @@ -844,10 +844,71 @@ def add_metaclass(metaclass): orig_vars.pop(slots_var) orig_vars.pop('__dict__', None) orig_vars.pop('__weakref__', None) + if hasattr(cls, '__qualname__'): + orig_vars['__qualname__'] = cls.__qualname__ return metaclass(cls.__name__, cls.__bases__, orig_vars) return wrapper +def ensure_binary(s, encoding='utf-8', errors='strict'): + """Coerce **s** to six.binary_type. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> encoded to `bytes` + - `bytes` -> `bytes` + """ + if isinstance(s, text_type): + return s.encode(encoding, errors) + elif isinstance(s, binary_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + +def ensure_str(s, encoding='utf-8', errors='strict'): + """Coerce *s* to `str`. + + For Python 2: + - `unicode` -> encoded to `str` + - `str` -> `str` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if not isinstance(s, (text_type, binary_type)): + raise TypeError("not expecting type '%s'" % type(s)) + if PY2 and isinstance(s, text_type): + s = s.encode(encoding, errors) + elif PY3 and isinstance(s, binary_type): + s = s.decode(encoding, errors) + return s + + +def ensure_text(s, encoding='utf-8', errors='strict'): + """Coerce *s* to six.text_type. + + For Python 2: + - `unicode` -> `unicode` + - `str` -> `unicode` + + For Python 3: + - `str` -> `str` + - `bytes` -> decoded to `str` + """ + if isinstance(s, binary_type): + return s.decode(encoding, errors) + elif isinstance(s, text_type): + return s + else: + raise TypeError("not expecting type '%s'" % type(s)) + + + def python_2_unicode_compatible(klass): """ A decorator that defines __unicode__ and __str__ methods under Python 2. diff --git a/pipenv/vendor/tomlkit/__init__.py b/pipenv/vendor/tomlkit/__init__.py index 92bfa27c..9ab90e0a 100644 --- a/pipenv/vendor/tomlkit/__init__.py +++ b/pipenv/vendor/tomlkit/__init__.py @@ -22,4 +22,4 @@ from .api import value from .api import ws -__version__ = "0.5.2" +__version__ = "0.5.3" diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/container.py index 9b5db5cb..340491c1 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import copy + from ._compat import decode from .exceptions import KeyAlreadyPresent from .exceptions import NonExistentKey @@ -600,3 +602,16 @@ class Container(dict): self._map = state[0] self._body = state[1] self._parsed = state[2] + + def copy(self): # type: () -> Container + return copy.copy(self) + + def __copy__(self): # type: () -> Container + c = self.__class__(self._parsed) + for k, v in super(Container, self).copy().items(): + super(Container, c).__setitem__(k, v) + + c._body += self.body + c._map.update(self._map) + + return c diff --git a/pipenv/vendor/tomlkit/items.py b/pipenv/vendor/tomlkit/items.py index cccfd4a1..f199e8df 100644 --- a/pipenv/vendor/tomlkit/items.py +++ b/pipenv/vendor/tomlkit/items.py @@ -527,7 +527,10 @@ class DateTime(Item, datetime): def __sub__(self, other): result = super(DateTime, self).__sub__(other) - return self._new(result) + if isinstance(result, datetime): + result = self._new(result) + + return result def _new(self, result): raw = result.isoformat() diff --git a/pipenv/vendor/tomlkit/source.py b/pipenv/vendor/tomlkit/source.py index dcfdafd0..ddb580e4 100644 --- a/pipenv/vendor/tomlkit/source.py +++ b/pipenv/vendor/tomlkit/source.py @@ -45,10 +45,6 @@ class _State: if self._save_marker: self._source._marker = self._marker - # Restore exceptions are silently consumed, other exceptions need to - # propagate - return exception_type is None - class _StateHandler: """ diff --git a/pipenv/vendor/urllib3/__init__.py b/pipenv/vendor/urllib3/__init__.py index 75725167..148a9c31 100644 --- a/pipenv/vendor/urllib3/__init__.py +++ b/pipenv/vendor/urllib3/__init__.py @@ -27,7 +27,7 @@ from logging import NullHandler __author__ = 'Andrey Petrov (andrey.petrov@shazow.net)' __license__ = 'MIT' -__version__ = '1.24' +__version__ = '1.24.1' __all__ = ( 'HTTPConnectionPool', diff --git a/pipenv/vendor/urllib3/response.py b/pipenv/vendor/urllib3/response.py index f0cfbb54..c112690b 100644 --- a/pipenv/vendor/urllib3/response.py +++ b/pipenv/vendor/urllib3/response.py @@ -69,9 +69,9 @@ class GzipDecoder(object): return getattr(self._obj, name) def decompress(self, data): - ret = b'' + ret = bytearray() if self._state == GzipDecoderState.SWALLOW_DATA or not data: - return ret + return bytes(ret) while True: try: ret += self._obj.decompress(data) @@ -81,11 +81,11 @@ class GzipDecoder(object): self._state = GzipDecoderState.SWALLOW_DATA if previous_state == GzipDecoderState.OTHER_MEMBERS: # Allow trailing garbage acceptable in other gzip clients - return ret + return bytes(ret) raise data = self._obj.unused_data if not data: - return ret + return bytes(ret) self._state = GzipDecoderState.OTHER_MEMBERS self._obj = zlib.decompressobj(16 + zlib.MAX_WBITS) diff --git a/pipenv/vendor/urllib3/util/ssl_.py b/pipenv/vendor/urllib3/util/ssl_.py index 24ee26d6..64ea192a 100644 --- a/pipenv/vendor/urllib3/util/ssl_.py +++ b/pipenv/vendor/urllib3/util/ssl_.py @@ -263,6 +263,8 @@ def create_urllib3_context(ssl_version=None, cert_reqs=None, """ context = SSLContext(ssl_version or ssl.PROTOCOL_SSLv23) + context.set_ciphers(ciphers or DEFAULT_CIPHERS) + # Setting the default here, as we may have no ssl module on import cert_reqs = ssl.CERT_REQUIRED if cert_reqs is None else cert_reqs diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 1a419eef..f1fe99c0 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -5,48 +5,49 @@ blindspin==2.0.1 click==7.0 click-completion==0.5.0 click-didyoumean==0.0.3 -colorama==0.3.9 +colorama==0.4.1 delegator.py==0.1.1 pexpect==4.6.0 ptyprocess==0.6.0 -python-dotenv==0.9.1 +python-dotenv==0.10.1 first==2.0.1 iso8601==0.1.12 jinja2==2.10 -markupsafe==1.0 -parse==1.9.0 -pathlib2==2.3.2 +markupsafe==1.1.1 +parse==1.11.1 +pathlib2==2.3.3 scandir==1.9 -pipdeptree==0.13.0 +pipdeptree==0.13.2 pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder==1.1.10 -requests==2.20.1 +pythonfinder==1.2.0 +requests==2.21.0 chardet==3.0.4 - idna==2.7 - urllib3==1.24 - certifi==2018.10.15 -requirementslib==1.3.3 - attrs==18.2.0 + idna==2.8 + urllib3==1.24.1 + certifi==2018.11.29 +requirementslib==1.4.2 + attrs==19.1.0 distlib==0.2.8 - packaging==18.0 - pyparsing==2.2.2 + packaging==19.0 + pyparsing==2.3.1 git+https://github.com/sarugaku/plette.git@master#egg=plette - tomlkit==0.5.2 -shellingham==1.2.7 -six==1.11.0 + tomlkit==0.5.3 +shellingham==1.2.8 +six==1.12.0 semver==2.8.1 shutilwhich==1.1.0 toml==0.10.0 -cached-property==1.4.3 -vistir==0.2.5 +cached-property==1.5.1 +vistir==0.3.1 pip-shims==0.3.2 -ptyprocess==0.6.0 enum34==1.1.6 -yaspin==0.14.0 +yaspin==0.14.1 cerberus==1.2 -git+https://github.com/sarugaku/passa.git@master#egg=passa -cursor==1.2.0 resolvelib==0.2.2 backports.functools_lru_cache==1.5 +pep517==0.5.0 + pytoml==0.1.20 +git+https://github.com/sarugaku/passa.git@master#egg=passa +orderedmultidict==1.0 diff --git a/pipenv/vendor/vistir/__init__.py b/pipenv/vendor/vistir/__init__.py index 4c5c9061..5be3b747 100644 --- a/pipenv/vendor/vistir/__init__.py +++ b/pipenv/vendor/vistir/__init__.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, unicode_literals from .compat import ( NamedTemporaryFile, + StringIO, TemporaryDirectory, partialmethod, to_native_string, @@ -11,27 +12,30 @@ from .contextmanagers import ( atomic_open_for_write, cd, open_file, + replaced_stream, + spinner, temp_environ, temp_path, - spinner, ) +from .cursor import hide_cursor, show_cursor from .misc import ( + StreamWrapper, + chunked, + decode_for_output, + divide, + get_wrapped_stream, load_path, partialclass, run, shell_escape, - decode_for_output, - to_text, - to_bytes, take, - chunked, - divide, + to_bytes, + to_text, ) -from .path import mkdir_p, rmtree, create_tracked_tempdir, create_tracked_tempfile -from .spin import VistirSpinner, create_spinner +from .path import create_tracked_tempdir, create_tracked_tempfile, mkdir_p, rmtree +from .spin import create_spinner - -__version__ = '0.2.5' +__version__ = "0.3.1" __all__ = [ @@ -50,7 +54,6 @@ __all__ = [ "NamedTemporaryFile", "partialmethod", "spinner", - "VistirSpinner", "create_spinner", "create_tracked_tempdir", "create_tracked_tempfile", @@ -61,4 +64,10 @@ __all__ = [ "take", "chunked", "divide", + "StringIO", + "get_wrapped_stream", + "StreamWrapper", + "replaced_stream", + "show_cursor", + "hide_cursor", ] diff --git a/pipenv/vendor/vistir/backports/__init__.py b/pipenv/vendor/vistir/backports/__init__.py index 0bdac1ea..03859e4f 100644 --- a/pipenv/vendor/vistir/backports/__init__.py +++ b/pipenv/vendor/vistir/backports/__init__.py @@ -4,8 +4,4 @@ from __future__ import absolute_import, unicode_literals from .functools import partialmethod from .tempfile import NamedTemporaryFile - -__all__ = [ - "NamedTemporaryFile", - "partialmethod" -] +__all__ = ["NamedTemporaryFile", "partialmethod"] diff --git a/pipenv/vendor/vistir/backports/functools.py b/pipenv/vendor/vistir/backports/functools.py index 8060d183..5d98f41a 100644 --- a/pipenv/vendor/vistir/backports/functools.py +++ b/pipenv/vendor/vistir/backports/functools.py @@ -3,8 +3,7 @@ from __future__ import absolute_import, unicode_literals from functools import partial - -__all__ = ["partialmethod",] +__all__ = ["partialmethod"] class partialmethod(object): @@ -16,8 +15,7 @@ class partialmethod(object): def __init__(self, func, *args, **keywords): if not callable(func) and not hasattr(func, "__get__"): - raise TypeError("{!r} is not callable or a descriptor" - .format(func)) + raise TypeError("{!r} is not callable or a descriptor".format(func)) # func could be a descriptor like classmethod which isn't callable, # so we can't inherit from partial (it verifies func is callable) @@ -36,26 +34,28 @@ class partialmethod(object): def __repr__(self): args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) + keywords = ", ".join("{}={!r}".format(k, v) for k, v in self.keywords.items()) format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + return format_string.format( + module=self.__class__.__module__, + cls=self.__class__.__qualname__, + func=self.func, + args=args, + keywords=keywords, + ) def _make_unbound_method(self): def _method(*args, **keywords): call_keywords = self.keywords.copy() call_keywords.update(keywords) if len(args) > 1: - cls_or_self, rest = args[0], tuple(args[1:],) + cls_or_self, rest = args[0], tuple(args[1:]) else: cls_or_self = args[0] rest = tuple() call_args = (cls_or_self,) + self.args + tuple(rest) return self.func(*call_args, **call_keywords) + _method.__isabstractmethod__ = self.__isabstractmethod__ _method._partialmethod = self return _method diff --git a/pipenv/vendor/vistir/backports/tempfile.py b/pipenv/vendor/vistir/backports/tempfile.py index fb044acf..ce66138b 100644 --- a/pipenv/vendor/vistir/backports/tempfile.py +++ b/pipenv/vendor/vistir/backports/tempfile.py @@ -5,7 +5,6 @@ import functools import io import os import sys - from tempfile import _bin_openflags, _mkstemp_inner, gettempdir import six @@ -175,7 +174,7 @@ def NamedTemporaryFile( prefix=None, dir=None, delete=True, - wrapper_class_override=None + wrapper_class_override=None, ): """Create and return a temporary file. Arguments: @@ -203,13 +202,11 @@ def NamedTemporaryFile( else: (fd, name) = _mkstemp_inner(dir, prefix, suffix, flags, output_type) try: - file = io.open( - fd, mode, buffering=buffering, newline=newline, encoding=encoding - ) + file = io.open(fd, mode, buffering=buffering, newline=newline, encoding=encoding) if wrapper_class_override is not None: - return type( - str("_TempFileWrapper"), (wrapper_class_override, object), {} - )(file, name, delete) + return type(str("_TempFileWrapper"), (wrapper_class_override, object), {})( + file, name, delete + ) else: return _TemporaryFileWrapper(file, name, delete) diff --git a/pipenv/vendor/vistir/cmdparse.py b/pipenv/vendor/vistir/cmdparse.py index 07326c93..664ae7df 100644 --- a/pipenv/vendor/vistir/cmdparse.py +++ b/pipenv/vendor/vistir/cmdparse.py @@ -1,12 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, unicode_literals +import itertools import re import shlex import six - __all__ = ["ScriptEmptyError", "Script"] @@ -14,6 +14,12 @@ class ScriptEmptyError(ValueError): pass +def _quote_if_contains(value, pattern): + if next(re.finditer(pattern, value), None): + return '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', value)) + return value + + class Script(object): """Parse a script line (in Pipfile's [scripts] section). @@ -72,7 +78,8 @@ class Script(object): See also: https://docs.python.org/3/library/subprocess.html#converting-argument-sequence """ return " ".join( - arg if not next(re.finditer(r'\s', arg), None) - else '"{0}"'.format(re.sub(r'(\\*)"', r'\1\1\\"', arg)) - for arg in self._parts + itertools.chain( + [_quote_if_contains(self.command, r"[\s^()]")], + (_quote_if_contains(arg, r"[\s^]") for arg in self.args), + ) ) diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 27d1e75c..e8688d89 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py @@ -1,15 +1,16 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import codecs import errno import os import sys import warnings - from tempfile import mkdtemp import six +from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile __all__ = [ "Path", @@ -19,7 +20,6 @@ __all__ = [ "JSONDecodeError", "FileNotFoundError", "ResourceWarning", - "FileNotFoundError", "PermissionError", "IsADirectoryError", "fs_str", @@ -27,18 +27,28 @@ __all__ = [ "TemporaryDirectory", "NamedTemporaryFile", "to_native_string", + "Iterable", + "Mapping", + "Sequence", + "Set", + "ItemsView", + "fs_encode", + "fs_decode", + "_fs_encode_errors", + "_fs_decode_errors", ] if sys.version_info >= (3, 5): from pathlib import Path from functools import lru_cache else: - from pathlib2 import Path + from pipenv.vendor.pathlib2 import Path from pipenv.vendor.backports.functools_lru_cache import lru_cache -from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile @@ -47,20 +57,22 @@ else: try: from weakref import finalize except ImportError: - from pipenv.vendor.backports.weakref import finalize + from pipenv.vendor.backports.weakref import finalize # type: ignore try: from functools import partialmethod except Exception: - from .backports.functools import partialmethod + from .backports.functools import partialmethod # type: ignore try: from json import JSONDecodeError except ImportError: # Old Pythons. - JSONDecodeError = ValueError + JSONDecodeError = ValueError # type: ignore if six.PY2: + from io import BytesIO as StringIO + class ResourceWarning(Warning): pass @@ -78,13 +90,42 @@ if six.PY2: class IsADirectoryError(OSError): """The command does not work on directories""" + pass -else: - from builtins import ResourceWarning, FileNotFoundError, PermissionError, IsADirectoryError + class FileExistsError(OSError): + def __init__(self, *args, **kwargs): + self.errno = errno.EEXIST + super(FileExistsError, self).__init__(*args, **kwargs) -six.add_move(six.MovedAttribute("Iterable", "collections", "collections.abc")) -from six.moves import Iterable + +else: + from builtins import ( + ResourceWarning, + FileNotFoundError, + PermissionError, + IsADirectoryError, + FileExistsError, + ) + from io import StringIO + +six.add_move( + six.MovedAttribute("Iterable", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Mapping", "collections", "collections.abc") +) # type: ignore +six.add_move( + six.MovedAttribute("Sequence", "collections", "collections.abc") +) # type: ignore +six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # type: ignore +six.add_move( + six.MovedAttribute("ItemsView", "collections", "collections.abc") +) # type: ignore + +# fmt: off +from six.moves import ItemsView, Iterable, Mapping, Sequence, Set # type: ignore # noqa # isort:skip +# fmt: on if not sys.warnoptions: @@ -179,17 +220,87 @@ def fs_str(string): Borrowed from pip-tools """ + if isinstance(string, str): return string assert not isinstance(string, bytes) return string.encode(_fs_encoding) -_fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +def _get_path(path): + """ + Fetch the string value from a path-like object + + Returns **None** if there is no string value. + """ + + if isinstance(path, (six.string_types, bytes)): + return path + path_type = type(path) + try: + path_repr = path_type.__fspath__(path) + except AttributeError: + return + if isinstance(path_repr, (six.string_types, bytes)): + return path_repr + return + + +def fs_encode(path): + """ + Encode a filesystem path to the proper filesystem encoding + + :param Union[str, bytes] path: A string-like path + :returns: A bytes-encoded filesystem path representation + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to encode") + if isinstance(path, six.text_type): + path = path.encode(_fs_encoding, _fs_encode_errors) + return path + + +def fs_decode(path): + """ + Decode a filesystem path using the proper filesystem encoding + + :param path: The filesystem path to decode from bytes or string + :return: [description] + :rtype: [type] + """ + + path = _get_path(path) + if path is None: + raise TypeError("expected a valid path to decode") + if isinstance(path, six.binary_type): + path = path.decode(_fs_encoding, _fs_decode_errors) + return path + + +if sys.version_info >= (3, 3) and os.name != "nt": + _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() +else: + _fs_encoding = "utf-8" + +if six.PY3: + if os.name == "nt": + _fs_error_fn = None + alt_strategy = "surrogatepass" + else: + alt_strategy = "surrogateescape" + _fs_error_fn = getattr(sys, "getfilesystemencodeerrors", None) + _fs_encode_errors = _fs_error_fn() if _fs_error_fn is not None else alt_strategy + _fs_decode_errors = _fs_error_fn() if _fs_error_fn is not None else alt_strategy +else: + _fs_encode_errors = "backslashreplace" + _fs_decode_errors = "replace" def to_native_string(string): from .misc import to_text, to_bytes + if six.PY2: return to_bytes(string) return to_text(string) diff --git a/pipenv/vendor/vistir/contextmanagers.py b/pipenv/vendor/vistir/contextmanagers.py index 77fbb9df..d9223b66 100644 --- a/pipenv/vendor/vistir/contextmanagers.py +++ b/pipenv/vendor/vistir/contextmanagers.py @@ -1,11 +1,10 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals import io import os import stat import sys - from contextlib import contextmanager import six @@ -13,9 +12,15 @@ import six from .compat import NamedTemporaryFile, Path from .path import is_file_url, is_valid_url, path_to_url, url_to_path - __all__ = [ - "temp_environ", "temp_path", "cd", "atomic_open_for_write", "open_file", "spinner" + "temp_environ", + "temp_path", + "cd", + "atomic_open_for_write", + "open_file", + "spinner", + "dummy_spinner", + "replaced_stream", ] @@ -103,7 +108,13 @@ def dummy_spinner(spin_type, text, **kwargs): @contextmanager -def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, write_to_stdout=True): +def spinner( + spinner_name=None, + start_text=None, + handler_map=None, + nospin=False, + write_to_stdout=True, +): """Get a spinner object or a dummy spinner to wrap a context. :param str spinner_name: A spinner type e.g. "dots" or "bouncingBar" (default: {"bouncingBar"}) @@ -119,6 +130,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, """ from .spin import create_spinner + has_yaspin = None try: import yaspin @@ -145,7 +157,7 @@ def spinner(spinner_name=None, start_text=None, handler_map=None, nospin=False, handler_map=handler_map, nospin=nospin, use_yaspin=use_yaspin, - write_to_stdout=write_to_stdout + write_to_stdout=write_to_stdout, ) as _spinner: yield _spinner @@ -266,8 +278,8 @@ def open_file(link, session=None, stream=True): if os.path.isdir(local_path): raise ValueError("Cannot open directory for read: {}".format(link)) else: - with io.open(local_path, "rb") as local_file: - yield local_file + with io.open(local_path, "rb") as local_file: + yield local_file else: # Remote URL headers = {"Accept-Encoding": "identity"} @@ -286,3 +298,56 @@ def open_file(link, session=None, stream=True): if conn is not None: conn.close() result.close() + + +@contextmanager +def replaced_stream(stream_name): + """ + Context manager to temporarily swap out *stream_name* with a stream wrapper. + + :param str stream_name: The name of a sys stream to wrap + :returns: A ``StreamWrapper`` replacement, temporarily + + >>> orig_stdout = sys.stdout + >>> with replaced_stream("stdout") as stdout: + ... sys.stdout.write("hello") + ... assert stdout.getvalue() == "hello" + + >>> sys.stdout.write("hello") + 'hello' + """ + orig_stream = getattr(sys, stream_name) + new_stream = six.StringIO() + try: + setattr(sys, stream_name, new_stream) + yield getattr(sys, stream_name) + finally: + setattr(sys, stream_name, orig_stream) + + +@contextmanager +def replaced_streams(): + """ + Context manager to replace both ``sys.stdout`` and ``sys.stderr`` using + ``replaced_stream`` + + returns: *(stdout, stderr)* + + >>> import sys + >>> with vistir.contextmanagers.replaced_streams() as streams: + >>> stdout, stderr = streams + >>> sys.stderr.write("test") + >>> sys.stdout.write("hello") + >>> assert stdout.getvalue() == "hello" + >>> assert stderr.getvalue() == "test" + + >>> stdout.getvalue() + 'hello' + + >>> stderr.getvalue() + 'test' + """ + + with replaced_stream("stdout") as stdout: + with replaced_stream("stderr") as stderr: + yield (stdout, stderr) diff --git a/pipenv/vendor/vistir/cursor.py b/pipenv/vendor/vistir/cursor.py new file mode 100644 index 00000000..22d643e1 --- /dev/null +++ b/pipenv/vendor/vistir/cursor.py @@ -0,0 +1,77 @@ +# -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function + +import ctypes +import os +import sys + +__all__ = ["hide_cursor", "show_cursor"] + + +class CONSOLE_CURSOR_INFO(ctypes.Structure): + _fields_ = [("dwSize", ctypes.c_int), ("bVisible", ctypes.c_int)] + + +WIN_STDERR_HANDLE_ID = ctypes.c_ulong(-12) +WIN_STDOUT_HANDLE_ID = ctypes.c_ulong(-11) + + +def get_stream_handle(stream=sys.stdout): + """ + Get the OS appropriate handle for the corresponding output stream. + + :param str stream: The the stream to get the handle for + :return: A handle to the appropriate stream, either a ctypes buffer + or **sys.stdout** or **sys.stderr**. + """ + handle = stream + if os.name == "nt": + from ctypes import windll + + handle_id = WIN_STDOUT_HANDLE_ID + handle = windll.kernel32.GetStdHandle(handle_id) + return handle + + +def hide_cursor(stream=sys.stdout): + """ + Hide the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ctypes import windll + + cursor_info = CONSOLE_CURSOR_INFO() + windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + cursor_info.visible = False + windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + else: + handle.write("\033[?25l") + handle.flush() + + +def show_cursor(stream=sys.stdout): + """ + Show the console cursor on the given stream + + :param stream: The name of the stream to get the handle for + :return: None + :rtype: None + """ + + handle = get_stream_handle(stream=stream) + if os.name == "nt": + from ctypes import windll + + cursor_info = CONSOLE_CURSOR_INFO() + windll.kernel32.GetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + cursor_info.visible = True + windll.kernel32.SetConsoleCursorInfo(handle, ctypes.byref(cursor_info)) + else: + handle.write("\033[?25h") + handle.flush() diff --git a/pipenv/vendor/vistir/misc.py b/pipenv/vendor/vistir/misc.py index 11480e2f..42067b2d 100644 --- a/pipenv/vendor/vistir/misc.py +++ b/pipenv/vendor/vistir/misc.py @@ -1,13 +1,13 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals +from __future__ import absolute_import, print_function, unicode_literals +import io import json -import logging import locale +import logging import os import subprocess import sys - from collections import OrderedDict from functools import partial from itertools import islice, tee @@ -15,10 +15,11 @@ from itertools import islice, tee import six from .cmdparse import Script -from .compat import Path, fs_str, partialmethod, to_native_string, Iterable +from .compat import Iterable, Path, StringIO, fs_str, partialmethod, to_native_string from .contextmanagers import spinner as spinner if os.name != "nt": + class WindowsError(OSError): pass @@ -38,6 +39,9 @@ __all__ = [ "divide", "getpreferredencoding", "decode_for_output", + "get_canonical_encoding_name", + "get_wrapped_stream", + "StreamWrapper", ] @@ -140,6 +144,55 @@ def _spawn_subprocess(script, env=None, block=True, cwd=None, combine_stderr=Tru return subprocess.Popen(script.cmdify(), **options) +def _read_streams(stream_dict): + results = {} + for outstream in stream_dict.keys(): + stream = stream_dict[outstream] + if not stream: + results[outstream] = None + continue + line = to_text(stream.readline()) + if not line: + results[outstream] = None + continue + line = to_text("{0}".format(line.rstrip())) + results[outstream] = line + return results + + +def get_stream_results(cmd_instance, verbose, maxlen, spinner=None, stdout_allowed=False): + stream_results = {"stdout": [], "stderr": []} + streams = {"stderr": cmd_instance.stderr, "stdout": cmd_instance.stdout} + while True: + stream_contents = _read_streams(streams) + stdout_line = stream_contents["stdout"] + stderr_line = stream_contents["stderr"] + if not (stdout_line or stderr_line): + break + for stream_name in stream_contents.keys(): + if stream_contents[stream_name] and stream_name in stream_results: + line = stream_contents[stream_name] + stream_results[stream_name].append(line) + display_line = fs_str("{0}".format(line)) + if len(display_line) > maxlen: + display_line = "{0}...".format(display_line[:maxlen]) + if verbose: + use_stderr = not stdout_allowed or stream_name != "stdout" + if spinner: + target = spinner.stderr if use_stderr else spinner.stdout + spinner.hide_and_write(display_line, target=target) + else: + target = sys.stderr if use_stderr else sys.stdout + target.write(display_line) + target.flush() + if spinner: + spinner.text = to_native_string( + "{0} {1}".format(spinner.text, display_line) + ) + continue + return stream_results + + def _create_subprocess( cmd, env=None, @@ -151,71 +204,35 @@ def _create_subprocess( combine_stderr=False, display_limit=200, start_text="", - write_to_stdout=True + write_to_stdout=True, ): if not env: env = os.environ.copy() try: - c = _spawn_subprocess(cmd, env=env, block=block, cwd=cwd, - combine_stderr=combine_stderr) - except Exception as exc: - sys.stderr.write("Error %s while executing command %s", exc, " ".join(cmd._parts)) + c = _spawn_subprocess( + cmd, env=env, block=block, cwd=cwd, combine_stderr=combine_stderr + ) + except Exception: + import traceback + + formatted_tb = "".join(traceback.format_exception(*sys.exc_info())) + sys.stderr.write("Error while executing command %s:" % " ".join(cmd._parts)) + sys.stderr.write(formatted_tb) raise if not block: c.stdin.close() - output = [] - err = [] - spinner_orig_text = None - if spinner: - spinner_orig_text = getattr(spinner, "text", None) - if spinner_orig_text is None: - spinner_orig_text = start_text if start_text is not None else "" - streams = { - "stdout": c.stdout, - "stderr": c.stderr - } - while True: - stdout_line = None - stderr_line = None - for outstream in streams.keys(): - stream = streams[outstream] - if not stream: - continue - line = to_text(stream.readline()) - if not line: - continue - line = to_text("{0}".format(line.rstrip())) - if outstream == "stderr": - stderr_line = line - else: - stdout_line = line - if not (stdout_line or stderr_line): - break - if stderr_line is not None: - err.append(stderr_line) - err_line = fs_str("{0}".format(stderr_line)) - if verbose and err_line is not None: - if spinner: - spinner.hide_and_write(err_line, target=spinner.stderr) - else: - sys.stderr.write(err_line) - sys.stderr.flush() - if stdout_line is not None: - output.append(stdout_line) - display_line = fs_str("{0}".format(stdout_line)) - if len(stdout_line) > display_limit: - display_line = "{0}...".format(stdout_line[:display_limit]) - if verbose and display_line is not None: - if spinner: - target = spinner.stdout if write_to_stdout else spinner.stderr - spinner.hide_and_write(display_line, target=target) - else: - target = sys.stdout if write_to_stdout else sys.stderr - target.write(display_line) - target.flush() - if spinner: - spinner.text = to_native_string("{0} {1}".format(spinner_orig_text, display_line)) - continue + spinner_orig_text = "" + if spinner and getattr(spinner, "text", None) is not None: + spinner_orig_text = spinner.text + if not spinner_orig_text and start_text is not None: + spinner_orig_text = start_text + stream_results = get_stream_results( + c, + verbose=verbose, + maxlen=display_limit, + spinner=spinner, + stdout_allowed=write_to_stdout, + ) try: c.wait() finally: @@ -230,6 +247,8 @@ def _create_subprocess( spinner.ok(to_native_string("✔ Complete")) else: spinner.ok(to_native_string("Complete")) + output = stream_results["stdout"] + err = stream_results["stderr"] c.out = "\n".join(output) if output else "" c.err = "\n".join(err) if err else "" else: @@ -254,7 +273,7 @@ def run( spinner_name=None, combine_stderr=True, display_limit=200, - write_to_stdout=True + write_to_stdout=True, ): """Use `subprocess.Popen` to get the output of a command and decode it. @@ -279,14 +298,11 @@ def run( _env = os.environ.copy() if env: _env.update(env) - env = _env if six.PY2: fs_encode = partial(to_bytes, encoding=locale_encoding) - _env = {fs_encode(k): fs_encode(v) for k, v in os.environ.items()} - for key, val in env.items(): - _env[fs_encode(key)] = fs_encode(val) + _env = {fs_encode(k): fs_encode(v) for k, v in _env.items()} else: - _env = {k: fs_str(v) for k, v in os.environ.items()} + _env = {k: fs_str(v) for k, v in _env.items()} if not spinner_name: spinner_name = "bouncingBar" if six.PY2: @@ -299,8 +315,12 @@ def run( if block or not return_object: combine_stderr = False start_text = "" - with spinner(spinner_name=spinner_name, start_text=start_text, nospin=nospin, - write_to_stdout=write_to_stdout) as sp: + with spinner( + spinner_name=spinner_name, + start_text=start_text, + nospin=nospin, + write_to_stdout=write_to_stdout, + ) as sp: return _create_subprocess( cmd, env=_env, @@ -311,11 +331,10 @@ def run( spinner=sp, combine_stderr=combine_stderr, start_text=start_text, - write_to_stdout=True + write_to_stdout=True, ) - def load_path(python): """Load the :mod:`sys.path` from the given python executable's environment as json @@ -328,8 +347,9 @@ def load_path(python): """ python = Path(python).as_posix() - out, err = run([python, "-c", "import json, sys; print(json.dumps(sys.path))"], - nospin=True) + out, err = run( + [python, "-c", "import json, sys; print(json.dumps(sys.path))"], nospin=True + ) if out: return json.loads(out) else: @@ -442,7 +462,7 @@ def to_text(string, encoding="utf-8", errors=None): string = six.text_type(bytes(string), encoding, errors) else: string = string.decode(encoding, errors) - except UnicodeDecodeError as e: + except UnicodeDecodeError: string = " ".join(to_text(arg, encoding, errors) for arg in string) return string @@ -515,19 +535,187 @@ def getpreferredencoding(): PREFERRED_ENCODING = getpreferredencoding() -def decode_for_output(output): +def get_output_encoding(source_encoding): + """ + Given a source encoding, determine the preferred output encoding. + + :param str source_encoding: The encoding of the source material. + :returns: The output encoding to decode to. + :rtype: str + """ + + if source_encoding is not None: + if get_canonical_encoding_name(source_encoding) == "ascii": + return "utf-8" + return get_canonical_encoding_name(source_encoding) + return get_canonical_encoding_name(PREFERRED_ENCODING) + + +def _encode(output, encoding=None, errors=None, translation_map=None): + if encoding is None: + encoding = PREFERRED_ENCODING + try: + output = output.encode(encoding) + except (UnicodeDecodeError, UnicodeEncodeError): + if translation_map is not None: + if six.PY2: + output = unicode.translate( # noqa: F821 + to_text(output, encoding=encoding, errors=errors), translation_map + ) + else: + output = output.translate(translation_map) + else: + output = to_text(output, encoding=encoding, errors=errors) + except AttributeError: + pass + return output + + +def decode_for_output(output, target_stream=None, translation_map=None): """Given a string, decode it for output to a terminal :param str output: A string to print to a terminal + :param target_stream: A stream to write to, we will encode to target this stream if possible. + :param dict translation_map: A mapping of unicode character ordinals to replacement strings. :return: A re-encoded string using the preferred encoding :rtype: str """ if not isinstance(output, six.string_types): return output + encoding = None + if target_stream is not None: + encoding = getattr(target_stream, "encoding", None) + encoding = get_output_encoding(encoding) try: - output = output.encode(PREFERRED_ENCODING) - except AttributeError: - pass - output = output.decode(PREFERRED_ENCODING) - return output + output = _encode(output, encoding=encoding, translation_map=translation_map) + except (UnicodeDecodeError, UnicodeEncodeError): + output = _encode( + output, encoding=encoding, errors="replace", translation_map=translation_map + ) + return to_text(output, encoding=encoding, errors="replace") + + +def get_canonical_encoding_name(name): + # type: (str) -> str + """ + Given an encoding name, get the canonical name from a codec lookup. + + :param str name: The name of the codec to lookup + :return: The canonical version of the codec name + :rtype: str + """ + + import codecs + + try: + codec = codecs.lookup(name) + except LookupError: + return name + else: + return codec.name + + +def get_wrapped_stream(stream): + """ + Given a stream, wrap it in a `StreamWrapper` instance and return the wrapped stream. + + :param stream: A stream instance to wrap + :returns: A new, wrapped stream + :rtype: :class:`StreamWrapper` + """ + + if stream is None: + raise TypeError("must provide a stream to wrap") + encoding = getattr(stream, "encoding", None) + encoding = get_output_encoding(encoding) + return StreamWrapper(stream, encoding, "replace", line_buffering=True) + + +class StreamWrapper(io.TextIOWrapper): + + """ + This wrapper class will wrap a provided stream and supply an interface + for compatibility. + """ + + def __init__(self, stream, encoding, errors, line_buffering=True, **kwargs): + self._stream = stream = _StreamProvider(stream) + io.TextIOWrapper.__init__( + self, stream, encoding, errors, line_buffering=line_buffering, **kwargs + ) + + # borrowed from click's implementation of stream wrappers, see + # https://github.com/pallets/click/blob/6cafd32/click/_compat.py#L64 + if six.PY2: + + def write(self, x): + if isinstance(x, (str, buffer, bytearray)): # noqa: F821 + try: + self.flush() + except Exception: + pass + return self.buffer.write(str(x)) + return io.TextIOWrapper.write(self, x) + + def writelines(self, lines): + for line in lines: + self.write(line) + + def __del__(self): + try: + self.detach() + except Exception: + pass + + def isatty(self): + return self._stream.isatty() + + +# More things borrowed from click, this is because we are using `TextIOWrapper` instead of +# just a normal StringIO +class _StreamProvider(object): + def __init__(self, stream): + self._stream = stream + super(_StreamProvider, self).__init__() + + def __getattr__(self, name): + return getattr(self._stream, name) + + def read1(self, size): + fn = getattr(self._stream, "read1", None) + if fn is not None: + return fn(size) + if six.PY2: + return self._stream.readline(size) + return self._stream.read(size) + + def readable(self): + fn = getattr(self._stream, "readable", None) + if fn is not None: + return fn() + try: + self._stream.read(0) + except Exception: + return False + return True + + def writable(self): + fn = getattr(self._stream, "writable", None) + if fn is not None: + return fn() + try: + self._stream.write(b"") + except Exception: + return False + return True + + def seekable(self): + fn = getattr(self._stream, "seekable", None) + if fn is not None: + return fn() + try: + self._stream.seek(self._stream.tell()) + except Exception: + return False + return True diff --git a/pipenv/vendor/vistir/path.py b/pipenv/vendor/vistir/path.py index 6e9a7f65..d551aac7 100644 --- a/pipenv/vendor/vistir/path.py +++ b/pipenv/vendor/vistir/path.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- -from __future__ import absolute_import, unicode_literals, print_function +from __future__ import absolute_import, print_function, unicode_literals import atexit import errno @@ -11,21 +11,21 @@ import stat import warnings import six - from six.moves import urllib_parse from six.moves.urllib import request as urllib_request from .backports.tempfile import _TemporaryFileWrapper from .compat import ( - _NamedTemporaryFile, Path, ResourceWarning, TemporaryDirectory, _fs_encoding, + _NamedTemporaryFile, finalize, + fs_decode, + fs_encode, ) - __all__ = [ "check_for_unc_path", "get_converted_relative_path", @@ -49,7 +49,11 @@ __all__ = [ if os.name == "nt": - warnings.filterwarnings("ignore", category=DeprecationWarning, message="The Windows bytes API has been deprecated.*") + warnings.filterwarnings( + "ignore", + category=DeprecationWarning, + message="The Windows bytes API has been deprecated.*", + ) def unicode_path(path): @@ -91,9 +95,11 @@ def normalize_path(path): :rtype: str """ - return os.path.normpath(os.path.normcase( - os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) - )) + return os.path.normpath( + os.path.normcase( + os.path.abspath(os.path.expandvars(os.path.expanduser(str(path)))) + ) + ) def is_in_path(path, parent): @@ -195,9 +201,8 @@ def is_readonly_path(fn): Permissions check is `bool(path.stat & stat.S_IREAD)` or `not os.access(path, os.W_OK)` """ - from .compat import to_native_string - fn = to_native_string(fn) + fn = fs_encode(fn) if os.path.exists(fn): file_stat = os.stat(fn).st_mode return not bool(file_stat & stat.S_IWRITE) or not os.access(fn, os.W_OK) @@ -212,20 +217,19 @@ def mkdir_p(newdir, mode=0o777): :raises: OSError if a file is encountered along the way """ # http://code.activestate.com/recipes/82465-a-friendly-mkdir/ - from .misc import to_bytes, to_text - newdir = to_bytes(newdir, "utf-8") + newdir = fs_encode(newdir) if os.path.exists(newdir): if not os.path.isdir(newdir): raise OSError( "a file with the same name as the desired dir, '{0}', already exists.".format( - newdir + fs_decode(newdir) ) ) else: - head, tail = os.path.split(to_bytes(newdir, encoding="utf-8")) + head, tail = os.path.split(newdir) # Make sure the tail doesn't point to the asame place as the head - curdir = to_bytes(".", encoding="utf-8") + curdir = fs_encode(".") tail_and_head_match = ( os.path.relpath(tail, start=os.path.basename(head)) == curdir ) @@ -233,8 +237,8 @@ def mkdir_p(newdir, mode=0o777): target = os.path.join(head, tail) if os.path.exists(target) and os.path.isfile(target): raise OSError( - "A file with the same name as the desired dir, '{0}', already exists.".format( - to_text(newdir, encoding="utf-8") + "A file with the same name as the desired dir, '{0}', already exists.".format( + fs_decode(newdir) ) ) os.makedirs(os.path.join(head, tail), mode) @@ -296,9 +300,7 @@ def set_write_bit(fn): :param str fn: The target filename or path """ - from .compat import to_native_string - - fn = to_native_string(fn) + fn = fs_encode(fn) if not os.path.exists(fn): return file_stat = os.stat(fn).st_mode @@ -309,7 +311,7 @@ def set_write_bit(fn): except AttributeError: pass for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root,d) for d in dirs]: + for dir_ in [os.path.join(root, d) for d in dirs]: set_write_bit(dir_) for file_ in [os.path.join(root, f) for f in files]: set_write_bit(file_) @@ -330,20 +332,15 @@ def rmtree(directory, ignore_errors=False, onerror=None): Setting `ignore_errors=True` may cause this to silently fail to delete the path """ - from .compat import to_native_string - - directory = to_native_string(directory) + directory = fs_encode(directory) if onerror is None: onerror = handle_remove_readonly try: - shutil.rmtree( - directory, ignore_errors=ignore_errors, onerror=onerror - ) + shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) except (IOError, OSError, FileNotFoundError) as exc: # Ignore removal failures where the file doesn't exist - if exc.errno == errno.ENOENT: - pass - raise + if exc.errno != errno.ENOENT: + raise def handle_remove_readonly(func, path, exc): @@ -360,17 +357,12 @@ def handle_remove_readonly(func, path, exc): :func:`set_write_bit` on the target path and try again. """ # Check for read-only attribute - from .compat import ( - ResourceWarning, FileNotFoundError, PermissionError, to_native_string - ) + from .compat import ResourceWarning, FileNotFoundError, PermissionError PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) - default_warning_message = ( - "Unable to remove file due to permissions restriction: {!r}" - ) + default_warning_message = "Unable to remove file due to permissions restriction: {!r}" # split the initial exception out into its type, exception, and traceback exc_type, exc_exception, exc_tb = exc - path = to_native_string(path) if is_readonly_path(path): # Apply write permission and call original function set_write_bit(path) @@ -494,8 +486,8 @@ def get_converted_relative_path(path, relative_to=None): raise ValueError("The path argument does not currently accept UNC paths") relpath_s = to_text(posixpath.normpath(path.as_posix())) - if not (relpath_s == u"." or relpath_s.startswith(u"./")): - relpath_s = posixpath.join(u".", relpath_s) + if not (relpath_s == "." or relpath_s.startswith("./")): + relpath_s = posixpath.join(".", relpath_s) return relpath_s diff --git a/pipenv/vendor/vistir/spin.py b/pipenv/vendor/vistir/spin.py index a2455b7d..40dfff96 100644 --- a/pipenv/vendor/vistir/spin.py +++ b/pipenv/vendor/vistir/spin.py @@ -1,4 +1,5 @@ # -*- coding=utf-8 -*- +from __future__ import absolute_import, print_function import functools import os @@ -6,43 +7,75 @@ import signal import sys import threading import time +from io import StringIO import colorama -import cursor import six from .compat import to_native_string -from .termcolors import COLOR_MAP, COLORS, colored, DISABLE_COLORS -from io import StringIO +from .cursor import hide_cursor, show_cursor +from .misc import decode_for_output +from .termcolors import COLOR_MAP, COLORS, DISABLE_COLORS, colored try: import yaspin except ImportError: yaspin = None Spinners = None + SpinBase = None else: - from yaspin.spinners import Spinners + import yaspin.spinners + import yaspin.core + + Spinners = yaspin.spinners.Spinners + SpinBase = yaspin.core.Yaspin + +if os.name == "nt": + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.fail() + spinner.stop() + sys.exit(0) + + +else: + + def handler(signum, frame, spinner): + """Signal handler, used to gracefully shut down the ``spinner`` instance + when specified signal is received by the process running the ``spinner``. + + ``signum`` and ``frame`` are mandatory arguments. Check ``signal.signal`` + function for more details. + """ + spinner.red.fail("✘") + spinner.stop() + sys.exit(0) -handler = None -if yaspin and os.name == "nt": - handler = yaspin.signal_handlers.default_handler -elif yaspin and os.name != "nt": - handler = yaspin.signal_handlers.fancy_handler CLEAR_LINE = chr(27) + "[K" +TRANSLATION_MAP = {10004: u"OK", 10008: u"x"} + + +decode_output = functools.partial(decode_for_output, translation_map=TRANSLATION_MAP) + class DummySpinner(object): def __init__(self, text="", **kwargs): - super(DummySpinner, self).__init__() if DISABLE_COLORS: colorama.init() - from .misc import decode_for_output - self.text = to_native_string(decode_for_output(text)) if text else "" + self.text = to_native_string(decode_output(text)) if text else "" self.stdout = kwargs.get("stdout", sys.stdout) self.stderr = kwargs.get("stderr", sys.stderr) self.out_buff = StringIO() self.write_to_stdout = kwargs.get("write_to_stdout", False) + super(DummySpinner, self).__init__() def __enter__(self): if self.text and self.text != "None": @@ -50,11 +83,12 @@ class DummySpinner(object): self.write(self.text) return self - def __exit__(self, exc_type, exc_val, traceback): + def __exit__(self, exc_type, exc_val, tb): if exc_type: import traceback - from .misc import decode_for_output - self.write_err(decode_for_output(traceback.format_exception(*sys.exc_info()))) + + formatted_tb = traceback.format_exception(exc_type, exc_val, tb) + self.write_err("".join(formatted_tb)) self._close_output_buffer() return False @@ -76,56 +110,63 @@ class DummySpinner(object): pass def fail(self, exitcode=1, text="FAIL"): - from .misc import decode_for_output - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.write(decode_for_output(text)) + self.write(text) else: - self.write_err(decode_for_output(text)) + self.write_err(text) self._close_output_buffer() def ok(self, text="OK"): - if text and text != "None": + if text is not None and text != "None": if self.write_to_stdout: - self.stdout.write(self.text) + self.write(text) else: - self.stderr.write(self.text) + self.write_err(text) self._close_output_buffer() return 0 def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output("\r", target_stream=target)) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output("{0}\n".format(text), target_stream=target)) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text=None): if not self.write_to_stdout: return self.write_err(text) - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stdout.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stdout.write(line) - self.stdout.write(CLEAR_LINE) + if not self.stdout.closed: + stdout = self.stdout + else: + stdout = sys.stdout + text = decode_output(text, target_stream=stdout) + stdout.write(decode_output("\r", target_stream=stdout)) + line = decode_output("{0}\n".format(text), target_stream=stdout) + stdout.write(line) + stdout.write(CLEAR_LINE) def write_err(self, text=None): - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - text = decode_for_output(text) - self.stderr.write(decode_for_output("\r")) - line = decode_for_output("{0}\n".format(text)) - self.stderr.write(line) - self.stderr.write(CLEAR_LINE) + if not self.stderr.closed: + stderr = self.stderr + else: + if sys.stderr.closed: + print(text) + return + stderr = sys.stderr + text = decode_output(text, target_stream=stderr) + stderr.write(decode_output("\r", target_stream=stderr)) + line = decode_output("{0}\n".format(text), target_stream=stderr) + stderr.write(line) + stderr.write(CLEAR_LINE) @staticmethod def _hide_cursor(target=None): @@ -136,10 +177,11 @@ class DummySpinner(object): pass -base_obj = yaspin.core.Yaspin if yaspin is not None else DummySpinner +if SpinBase is None: + SpinBase = DummySpinner -class VistirSpinner(base_obj): +class VistirSpinner(SpinBase): "A spinner class for handling spinners on windows and posix." def __init__(self, *args, **kwargs): @@ -158,10 +200,7 @@ class VistirSpinner(base_obj): colorama.init() sigmap = {} if handler: - sigmap.update({ - signal.SIGINT: handler, - signal.SIGTERM: handler - }) + sigmap.update({signal.SIGINT: handler, signal.SIGTERM: handler}) handler_map = kwargs.pop("handler_map", {}) if os.name == "nt": sigmap[signal.SIGBREAK] = handler @@ -182,6 +221,8 @@ class VistirSpinner(base_obj): self.write_to_stdout = write_to_stdout self.is_dummy = bool(yaspin is None) super(VistirSpinner, self).__init__(*args, **kwargs) + if DISABLE_COLORS: + colorama.deinit() def ok(self, text="OK", err=False): """Set Ok (success) finalizer to a spinner.""" @@ -204,38 +245,40 @@ class VistirSpinner(base_obj): def hide_and_write(self, text, target=None): if not target: target = self.stdout - from .misc import decode_for_output if text is None or isinstance(text, six.string_types) and text == "None": pass - target.write(decode_for_output("\r")) + target.write(decode_output("\r")) self._hide_cursor(target=target) - target.write(decode_for_output("{0}\n".format(text))) + target.write(decode_output("{0}\n".format(text))) target.write(CLEAR_LINE) self._show_cursor(target=target) def write(self, text): if not self.write_to_stdout: return self.write_err(text) - from .misc import to_text - sys.stdout.write("\r") - self.stdout.write(CLEAR_LINE) + stdout = self.stdout + if self.stdout.closed: + stdout = sys.stdout + stdout.write(decode_output("\r", target_stream=stdout)) + stdout.write(decode_output(CLEAR_LINE, target_stream=stdout)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) - self.stdout.write(text) - self.out_buff.write(to_text(text)) + text = decode_output("{0}\n".format(text), target_stream=stdout) + stdout.write(text) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def write_err(self, text): """Write error text in the terminal without breaking the spinner.""" - from .misc import to_text - - self.stderr.write("\r") - self.stderr.write(CLEAR_LINE) + stderr = self.stderr + if self.stderr.closed: + stderr = sys.stderr + stderr.write(decode_output("\r", target_stream=stderr)) + stderr.write(decode_output(CLEAR_LINE, target_stream=stderr)) if text is None: text = "" - text = to_native_string("{0}\n".format(text)) + text = decode_output("{0}\n".format(text), target_stream=stderr) self.stderr.write(text) - self.out_buff.write(to_text(text)) + self.out_buff.write(decode_output(text, target_stream=self.out_buff)) def start(self): if self._sigmap: @@ -270,52 +313,45 @@ class VistirSpinner(base_obj): if target.isatty(): self._show_cursor(target=target) - if self.stderr and self.stderr != sys.stderr: - self.stderr.close() - if self.stdout and self.stdout != sys.stdout: - self.stdout.close() self.out_buff.close() def _freeze(self, final_text, err=False): """Stop spinner, compose last frame and 'freeze' it.""" if not final_text: final_text = "" - text = to_native_string(final_text) + target = self.stderr if err else self.stdout + if target.closed: + target = sys.stderr if err else sys.stdout + text = decode_output(final_text, target_stream=target) self._last_frame = self._compose_out(text, mode="last") # Should be stopped here, otherwise prints after # self._freeze call will mess up the spinner self.stop() - if err or not self.write_to_stdout: - self.stderr.write(self._last_frame) - else: - self.stdout.write(self._last_frame) + target.write(self._last_frame) def _compose_color_func(self): fn = functools.partial( - colored, - color=self._color, - on_color=self._on_color, - attrs=list(self._attrs), + colored, color=self._color, on_color=self._on_color, attrs=list(self._attrs) ) return fn def _compose_out(self, frame, mode=None): # Ensure Unicode input - frame = to_native_string(frame) + frame = decode_output(frame) if self._text is None: self._text = "" - text = to_native_string(self._text) + text = decode_output(self._text) if self._color_func is not None: frame = self._color_func(frame) if self._side == "right": frame, text = text, frame # Mode if not mode: - out = to_native_string("\r{0} {1}".format(frame, text)) + out = decode_output("\r{0} {1}".format(frame, text)) else: - out = to_native_string("{0} {1}\n".format(frame, text)) + out = decode_output("{0} {1}\n".format(frame, text)) return out def _spin(self): @@ -381,13 +417,13 @@ class VistirSpinner(base_obj): def _hide_cursor(target=None): if not target: target = sys.stdout - cursor.hide(stream=target) + hide_cursor(stream=target) @staticmethod def _show_cursor(target=None): if not target: target = sys.stdout - cursor.show(stream=target) + show_cursor(stream=target) @staticmethod def _clear_err(): diff --git a/pipenv/vendor/vistir/termcolors.py b/pipenv/vendor/vistir/termcolors.py index d72e8ebe..6aecec88 100644 --- a/pipenv/vendor/vistir/termcolors.py +++ b/pipenv/vendor/vistir/termcolors.py @@ -1,62 +1,55 @@ # -*- coding=utf-8 -*- from __future__ import absolute_import, print_function, unicode_literals -import colorama + import os + +import colorama + from .compat import to_native_string - -DISABLE_COLORS = os.getenv("CI", False) or os.getenv("ANSI_COLORS_DISABLED", - os.getenv("VISTIR_DISABLE_COLORS", False) +DISABLE_COLORS = os.getenv("CI", False) or os.getenv( + "ANSI_COLORS_DISABLED", os.getenv("VISTIR_DISABLE_COLORS", False) ) ATTRIBUTES = dict( - list(zip([ - 'bold', - 'dark', - '', - 'underline', - 'blink', - '', - 'reverse', - 'concealed' - ], - list(range(1, 9)) - )) + list( + zip( + ["bold", "dark", "", "underline", "blink", "", "reverse", "concealed"], + list(range(1, 9)), ) -del ATTRIBUTES[''] + ) +) +del ATTRIBUTES[""] HIGHLIGHTS = dict( - list(zip([ - 'on_grey', - 'on_red', - 'on_green', - 'on_yellow', - 'on_blue', - 'on_magenta', - 'on_cyan', - 'on_white' + list( + zip( + [ + "on_grey", + "on_red", + "on_green", + "on_yellow", + "on_blue", + "on_magenta", + "on_cyan", + "on_white", ], - list(range(40, 48)) - )) + list(range(40, 48)), ) + ) +) COLORS = dict( - list(zip([ - 'grey', - 'red', - 'green', - 'yellow', - 'blue', - 'magenta', - 'cyan', - 'white', - ], - list(range(30, 38)) - )) + list( + zip( + ["grey", "red", "green", "yellow", "blue", "magenta", "cyan", "white"], + list(range(30, 38)), ) + ) +) COLOR_MAP = { @@ -106,11 +99,11 @@ def colored(text, color=None, on_color=None, attrs=None): colored('Hello, World!', 'red', 'on_grey', ['blue', 'blink']) colored('Hello, World!', 'green') """ - if os.getenv('ANSI_COLORS_DISABLED') is None: + if os.getenv("ANSI_COLORS_DISABLED") is None: style = "NORMAL" - if 'bold' in attrs: + if "bold" in attrs: style = "BRIGHT" - attrs.remove('bold') + attrs.remove("bold") if color is not None: color = color.upper() text = to_native_string("%s%s%s%s%s") % ( @@ -131,10 +124,7 @@ def colored(text, color=None, on_color=None, attrs=None): ) if attrs is not None: - fmt_str = to_native_string("%s[%%dm%%s%s[9m") % ( - chr(27), - chr(27) - ) + fmt_str = to_native_string("%s[%%dm%%s%s[9m") % (chr(27), chr(27)) for attr in attrs: text = fmt_str % (ATTRIBUTES[attr], text) diff --git a/pipenv/vendor/yaspin/__version__.py b/pipenv/vendor/yaspin/__version__.py index 9e78220f..f075dd36 100644 --- a/pipenv/vendor/yaspin/__version__.py +++ b/pipenv/vendor/yaspin/__version__.py @@ -1 +1 @@ -__version__ = "0.14.0" +__version__ = "0.14.1" diff --git a/pipenv/vendor/yaspin/api.py b/pipenv/vendor/yaspin/api.py index 156630db..f59ce002 100644 --- a/pipenv/vendor/yaspin/api.py +++ b/pipenv/vendor/yaspin/api.py @@ -84,5 +84,7 @@ def kbi_safe_yaspin(*args, **kwargs): return Yaspin(*args, **kwargs) -_kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") -kbi_safe_yaspin.__doc__ = _kbi_safe_doc +# Handle PYTHONOPTIMIZE=2 case, when docstrings are set to None. +if yaspin.__doc__: + _kbi_safe_doc = yaspin.__doc__.replace("yaspin", "kbi_safe_yaspin") + kbi_safe_yaspin.__doc__ = _kbi_safe_doc diff --git a/pipenv/vendor/yaspin/core.py b/pipenv/vendor/yaspin/core.py index 06b8b621..dcaa9e42 100644 --- a/pipenv/vendor/yaspin/core.py +++ b/pipenv/vendor/yaspin/core.py @@ -17,7 +17,7 @@ import threading import time import colorama -import cursor +from pipenv.vendor.vistir import cursor from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -530,11 +530,11 @@ class Yaspin(object): @staticmethod def _hide_cursor(): - cursor.hide() + cursor.hide_cursor() @staticmethod def _show_cursor(): - cursor.show() + cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/run-tests.bat b/run-tests.bat index d20af35f..a511fa45 100644 --- a/run-tests.bat +++ b/run-tests.bat @@ -1,7 +1,7 @@ rem imdisk -a -s 964515b -m R: -p "/FS:NTFS /Y" virtualenv R:\.venv -R:\.venv\Scripts\pip install -e . --upgrade --upgrade-strategy=only-if-needed +R:\.venv\Scripts\pip install -e .[test] --upgrade --upgrade-strategy=only-if-needed R:\.venv\Scripts\pipenv install --dev git submodule sync && git submodule update --init --recursive SET RAM_DISK=R: && R:\.venv\Scripts\pipenv run pytest -n auto -v tests --tap-stream > report.tap diff --git a/run-tests.sh b/run-tests.sh index 991c0df7..a99c0461 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -53,8 +53,8 @@ fi echo "Installing dependencies…" PIPENV_PYTHON=2.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev PIPENV_PYTHON=3.7 python3 -m pipenv --venv && pipenv --rm && pipenv install --dev -PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e . -PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e . +PIPENV_PYTHON=2.7 python3 -m pipenv run pip install --upgrade -e .[test] +PIPENV_PYTHON=3.7 python3 -m pipenv run pip install --upgrade -e .[test] echo "$ git submodule sync && git submodule update --init --recursive" git submodule sync && git submodule update --init --recursive diff --git a/setup.py b/setup.py index 4c848c47..3fb48591 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,6 @@ required = [ 'enum34; python_version<"3"' ] - # https://pypi.python.org/pypi/stdeb/0.8.5#quickstart-2-just-tell-me-the-fastest-way-to-make-a-deb class DebCommand(Command): """Support for setup.py deb""" @@ -129,9 +128,12 @@ setup( ], }, python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*", - setup_requires=["invoke", "parver"], + setup_requires=["invoke", "parver", ], install_requires=required, - extras_require={}, + extras_require={ + "test": ["pytest<4.0", "pytest-tap", "pytest-xdist", "flaky", "mock"], + "dev": ["towncrier", "bs4"], + }, include_package_data=True, license="MIT", classifiers=[ diff --git a/tasks/release.py b/tasks/release.py index 7c2e7aba..375d7302 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -87,7 +87,7 @@ def release(ctx, dry_run=False): log(f'Would commit with message: "Release v{version}"') else: ctx.run('towncrier') - ctx.run("git add CHANGELOG.rst news/") + ctx.run("git add CHANGELOG.rst news/ {0}".format(get_version_file(ctx).as_posix())) ctx.run("git rm CHANGELOG.draft.rst") ctx.run(f'git commit -m "Release v{version}"') diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index c9757965..dce9e5a7 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -22,6 +22,7 @@ from urllib3.util import parse_url as urllib3_parse from pipenv.utils import mkdir_p from pipenv.vendor.vistir.compat import NamedTemporaryFile, TemporaryDirectory from pipenv.vendor.vistir.contextmanagers import open_file +import pipenv.vendor.parse as parse TASK_NAME = 'update' @@ -346,6 +347,26 @@ def post_install_cleanup(ctx, vendor_dir): remove_all(vendor_dir.glob('toml.py')) +@invoke.task +def apply_patches(ctx, patched=False, pre=False): + if patched: + vendor_dir = _get_patched_dir(ctx) + else: + vendor_dir = _get_vendor_dir(ctx) + log("Applying pre-patches...") + patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name + if pre: + if not patched: + pass + for patch in patch_dir.glob('*.patch'): + if not patch.name.startswith('_post'): + apply_patch(ctx, patch) + else: + patches = patch_dir.glob('*.patch' if not patched else '_post*.patch') + for patch in patches: + apply_patch(ctx, patch) + + def vendor(ctx, vendor_dir, package=None, rewrite=True): log('Reinstalling vendored libraries') is_patched = vendor_dir.name == 'patched' @@ -359,12 +380,8 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): # Apply pre-patches log("Applying pre-patches...") - patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name if is_patched: - for patch in patch_dir.glob('*.patch'): - if not patch.name.startswith('_post'): - apply_patch(ctx, patch) - + apply_patches(ctx, patched=is_patched, pre=True) log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) drop_dir(vendor_dir / 'setuptools') @@ -385,10 +402,7 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) if not package: - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) + apply_patches(ctx, patched=is_patched, pre=False) if is_patched: piptools_vendor = vendor_dir / 'piptools' / '_vendored' if piptools_vendor.exists(): @@ -445,19 +459,21 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx possible_pkgs.append(LIBRARY_DIRNAMES[pkg]) for pkgpath in possible_pkgs: pkgpath = vendor_dir.joinpath(pkgpath) + py_path = pkgpath.parent / "{0}.py".format(pkgpath.stem) if pkgpath.exists() and pkgpath.is_dir(): - for licensepath in LICENSES: - licensepath = pkgpath.joinpath(licensepath) - if licensepath.exists(): + for license_path in LICENSES: + license_path = pkgpath.joinpath(license_path) + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break - elif (pkgpath.exists() or pkgpath.parent.joinpath("{0}.py".format(pkgpath.stem)).exists()): - for licensepath in LICENSES: - licensepath = pkgpath.parent.joinpath("{0}.{1}".format(pkgpath.stem, licensepath)) - if licensepath.exists(): + elif pkgpath.exists() or py_path.exists(): + for license_path in LICENSES: + license_name = "{0}.{1}".format(pkgpath.stem, license_path) + license_path = pkgpath.parent / license_name + if license_path.exists(): match_found = True - # log("%s: Trying path %s... FOUND" % (pkg, licensepath)) + # log("%s: Trying path %s... FOUND" % (pkg, license_path)) break if match_found: break @@ -470,7 +486,10 @@ def packages_missing_licenses(ctx, vendor_dir=None, requirements_file='vendor.tx @invoke.task -def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, patched=False): +def download_licenses( + ctx, vendor_dir=None, requirements_file='vendor.txt', package=None, only=False, + patched=False +): log('Downloading licenses') if not vendor_dir: if patched: @@ -496,13 +515,37 @@ def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', pack requirement = package tmp_dir = vendor_dir / '__tmp__' # TODO: Fix this whenever it gets sorted out (see https://github.com/pypa/pip/issues/5739) + cmd = "pip download --no-binary :all: --only-binary requests_download --no-deps" + enum_cmd = "pip download --no-deps" ctx.run('pip install flit') # needed for the next step - ctx.run( - 'pip download --no-binary :all: --only-binary requests_download --no-build-isolation --no-deps -d {0} {1}'.format( - tmp_dir.as_posix(), - requirement, - ) - ) + for req in requirements_file.read_text().splitlines(): + if req.startswith("enum34"): + exe_cmd = "{0} -d {1} {2}".format(enum_cmd, tmp_dir.as_posix(), req) + else: + exe_cmd = "{0} --no-build-isolation --no-use-pep517 -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + try: + ctx.run(exe_cmd) + except invoke.exceptions.UnexpectedExit as e: + if "Disabling PEP 517 processing is invalid" not in e.result.stderr: + log("WARNING: Failed to download license for {0}".format(req)) + continue + parse_target = ( + "Disabling PEP 517 processing is invalid: project specifies a build " + "backend of {backend} in pyproject.toml" + ) + target = parse.parse(parse_target, e.result.stderr.strip()) + backend = target.named.get("backend") + if backend is not None: + if "." in backend: + backend, _, _ = backend.partition(".") + ctx.run("pip install {0}".format(backend)) + ctx.run( + "{0} --no-build-isolation -d {1} {2}".format( + cmd, tmp_dir.as_posix(), req + ) + ) for sdist in tmp_dir.iterdir(): extract_license(vendor_dir, sdist) new_requirements_file.unlink() @@ -514,7 +557,7 @@ def extract_license(vendor_dir, sdist): ext = sdist.suffix[1:] with tarfile.open(sdist, mode='r:{}'.format(ext)) as tar: found = find_and_extract_license(vendor_dir, tar, tar.getmembers()) - elif sdist.suffix == '.zip': + elif sdist.suffix in ('.zip', '.whl'): with zipfile.ZipFile(sdist) as zip: found = find_and_extract_license(vendor_dir, zip, zip.infolist()) else: diff --git a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch b/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch deleted file mode 100644 index c090e885..00000000 --- a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py -index 3d1bd72f..75f49c4a 100644 ---- a/pipenv/vendor/dotenv/main.py -+++ b/pipenv/vendor/dotenv/main.py -@@ -94,6 +94,13 @@ class DotEnv(): - for k, v in self.dict().items(): - if k in os.environ and not override: - continue -+ # With Python 2 on Windows, ensuree environment variables are -+ # system strings to avoid "TypeError: environment can only contain -+ # strings" in Python's subprocess module. -+ if sys.version_info.major < 3 and sys.platform == 'win32': -+ from pipenv.utils import fs_str -+ k = fs_str(k) -+ v = fs_str(v) - os.environ[k] = v - - return True diff --git a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch index d479ebfa..5d5a381f 100644 --- a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch +++ b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch @@ -7,7 +7,7 @@ index 7820aa5..2082fc8 100644 from ordereddict import OrderedDict -try: -- from pipenv.patched.notpip._internal import get_installed_distributions +- from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - from pipenv.patched.notpip._internal.operations.freeze import FrozenRequirement -except ImportError: - from pipenv.patched.notpip import get_installed_distributions, FrozenRequirement @@ -17,3 +17,4 @@ index 7820aa5..2082fc8 100644 import pkg_resources # inline: + diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch index 36e2f808..4aa6c16f 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-fix.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -14,8 +14,10 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/containe index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -1,13 +1,5 @@ +@@ -1,15 +1,7 @@ from __future__ import unicode_literals + + import copy -from typing import Any -from typing import Dict diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index 673efad8..d1dc4e3a 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -15,26 +15,28 @@ diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 9ae33fdc..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): +@@ -43,12 +43,12 @@ if sys.version_info >= (3, 5): from functools import lru_cache else: - from pathlib2 import Path +- from pathlib2 import Path - from backports.functools_lru_cache import lru_cache ++ from pipenv.vendor.pathlib2 import Path + from pipenv.vendor.backports.functools_lru_cache import lru_cache - from .backports.tempfile import NamedTemporaryFile as _NamedTemporaryFile + if sys.version_info < (3, 3): - from backports.shutil_get_terminal_size import get_terminal_size + from pipenv.vendor.backports.shutil_get_terminal_size import get_terminal_size + NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile -@@ -44,7 +44,7 @@ else: +@@ -57,7 +57,7 @@ else: try: from weakref import finalize except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize +- from backports.weakref import finalize # type: ignore ++ from pipenv.vendor.backports.weakref import finalize # type: ignore try: from functools import partialmethod diff --git a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch index a1d27cd3..511e782a 100644 --- a/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch +++ b/tasks/vendoring/patches/vendor/yaspin-signal-handling.patch @@ -7,7 +7,7 @@ index d01fb98e..06b8b621 100644 import time +import colorama -+import cursor ++from pipenv.vendor.vistir import cursor + from .base_spinner import default_spinner from .compat import PY2, basestring, builtin_str, bytes, iteritems, str @@ -48,13 +48,13 @@ index d01fb98e..06b8b621 100644 def _hide_cursor(): - sys.stdout.write("\033[?25l") - sys.stdout.flush() -+ cursor.hide() ++ cursor.hide_cursor() @staticmethod def _show_cursor(): - sys.stdout.write("\033[?25h") - sys.stdout.flush() -+ cursor.show() ++ cursor.show_cursor() @staticmethod def _clear_line(): diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b5b5bc85..e400fc06 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -1,3 +1,4 @@ +# -*- coding=utf-8 -*- import json import os import sys @@ -6,11 +7,11 @@ import warnings import pytest from vistir.compat import ResourceWarning, fs_str -from vistir.path import mkdir_p +from vistir.contextmanagers import temp_environ +from vistir.path import mkdir_p, create_tracked_tempdir from pipenv._compat import Path, TemporaryDirectory from pipenv.exceptions import VirtualenvActivationException -from pipenv.utils import temp_environ from pipenv.vendor import delegator, requests, toml, tomlkit from pytest_pypi.app import prepare_fixtures from pytest_pypi.app import prepare_packages as prepare_pypi_packages @@ -80,6 +81,10 @@ def pytest_runtest_setup(item): pytest.skip('requires github ssh') if item.get_marker('needs_hg') is not None and not WE_HAVE_HG: pytest.skip('requires mercurial') + if item.get_marker('skip_py27_win') is not None and ( + sys.version_info[:2] <= (2, 7) and os.name == "nt" + ): + pytest.skip('must use python > 2.7 on windows') @pytest.fixture @@ -91,9 +96,33 @@ def pathlib_tmpdir(request, tmpdir): pass +def _create_tracked_dir(): + tmp_location = os.environ.get("TEMP", os.environ.get("TMP")) + temp_args = {"prefix": "pipenv-", "suffix": "-test"} + if tmp_location is not None: + temp_args["dir"] = tmp_location + temp_path = create_tracked_tempdir(**temp_args) + return temp_path + + +@pytest.fixture +def vistir_tmpdir(): + temp_path = _create_tracked_dir() + yield Path(temp_path) + + +@pytest.fixture(name='create_tmpdir') +def vistir_tmpdir_factory(): + + def create_tmpdir(): + return Path(_create_tracked_dir()) + + yield create_tmpdir + + # Borrowed from pip's test runner filesystem isolation @pytest.fixture(autouse=True) -def isolate(pathlib_tmpdir): +def isolate(create_tmpdir): """ Isolate our tests so that things like global configuration files and the like do not affect our test results. @@ -102,7 +131,7 @@ def isolate(pathlib_tmpdir): """ # Create a directory to use as our home location. - home_dir = os.path.join(str(pathlib_tmpdir), "home") + home_dir = os.path.join(str(create_tmpdir()), "home") os.makedirs(home_dir) mkdir_p(os.path.join(home_dir, ".config", "git")) with open(os.path.join(home_dir, ".config", "git", "config"), "wb") as fp: @@ -112,8 +141,10 @@ def isolate(pathlib_tmpdir): os.environ["GIT_CONFIG_NOSYSTEM"] = fs_str("1") os.environ["GIT_AUTHOR_NAME"] = fs_str("pipenv") os.environ["GIT_AUTHOR_EMAIL"] = fs_str("pipenv@pipenv.org") - mkdir_p(os.path.join(home_dir, ".virtualenvs")) - os.environ["WORKON_HOME"] = fs_str(os.path.join(home_dir, ".virtualenvs")) + workon_home = create_tmpdir() + os.environ["WORKON_HOME"] = fs_str(str(workon_home)) + os.environ["HOME"] = home_dir + mkdir_p(os.path.join(home_dir, "projects")) # Ignore PIPENV_ACTIVE so that it works as under a bare environment. os.environ.pop("PIPENV_ACTIVE", None) os.environ.pop("VIRTUAL_ENV", None) @@ -187,22 +218,40 @@ class _Pipfile(object): class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): + def __init__( + self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None, + venv_root=None, ignore_virtualenvs=True, venv_in_project=True, name=None + ): self.pypi = pypi - self.original_umask = os.umask(0o007) + if ignore_virtualenvs: + os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") + if venv_root: + os.environ["VIRTUAL_ENV"] = venv_root + if venv_in_project: + os.environ["PIPENV_VENV_IN_PROJECT"] = fs_str("1") + else: + os.environ.pop("PIPENV_VENV_IN_PROJECT", None) + self.original_dir = os.path.abspath(os.curdir) - os.environ["PIPENV_NOSPIN"] = fs_str("1") - os.environ["CI"] = fs_str("1") - warnings.simplefilter("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) + if name is not None: + path = Path(os.environ["HOME"]) / "projects" / name + path.mkdir(exist_ok=True) if not path: - self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + path = TemporaryDirectory(suffix='-project', prefix='pipenv-') + if isinstance(path, TemporaryDirectory): + self._path = path path = Path(self._path.name) try: self.path = str(path.resolve()) except OSError: self.path = str(path.absolute()) + elif isinstance(path, Path): + self._path = path + try: + self.path = str(path.resolve()) + except OSError: + self.path = str(path.absolute()) else: self._path = path self.path = path @@ -224,10 +273,6 @@ class _PipenvInstance(object): self._pipfile = _Pipfile(Path(p_path)) def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') - os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') - os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -237,13 +282,12 @@ class _PipenvInstance(object): if self.chdir: os.chdir(self.original_dir) self.path = None - if self._path: + if self._path and getattr(self._path, "cleanup", None): try: self._path.cleanup() except OSError as e: _warn_msg = warn_msg.format(e) warnings.warn(_warn_msg, ResourceWarning) - os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): @@ -290,7 +334,17 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - yield _PipenvInstance + with temp_environ(): + original_umask = os.umask(0o007) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) @pytest.fixture(autouse=True) @@ -310,20 +364,47 @@ def testsroot(): return TESTS_ROOT -@pytest.fixture() -def virtualenv(pathlib_tmpdir): - virtualenv_path = pathlib_tmpdir / "venv" - with temp_environ(): - c = delegator.run("virtualenv {}".format(virtualenv_path), block=True) +class VirtualEnv(object): + def __init__(self, name="venv", base_dir=None): + if base_dir is None: + base_dir = Path(_create_tracked_dir()) + self.base_dir = base_dir + self.name = name + self.path = base_dir / name + + def __enter__(self): + self._old_environ = os.environ.copy() + self.create() + return self.activate() + + def __exit__(self, *args, **kwargs): + os.environ = self._old_environ + + def create(self): + python = Path(sys.executable).as_posix() + cmd = "{0} -m virtualenv {1}".format(python, self.path.as_posix()) + c = delegator.run(cmd, block=True) assert c.return_code == 0 - for name in ("bin", "Scripts"): - activate_this = virtualenv_path / name / "activate_this.py" - if activate_this.exists(): - with open(str(activate_this)) as f: - code = compile(f.read(), str(activate_this), "exec") - exec(code, dict(__file__=str(activate_this))) - break + + def activate(self): + script_path = "Scripts" if os.name == "nt" else "bin" + activate_this = self.path / script_path / "activate_this.py" + if activate_this.exists(): + with open(str(activate_this)) as f: + code = compile(f.read(), str(activate_this), "exec") + exec(code, dict(__file__=str(activate_this))) + os.environ["VIRTUAL_ENV"] = str(self.path) + return self.path else: raise VirtualenvActivationException("Can't find the activate_this.py script.") - os.environ["VIRTUAL_ENV"] = str(virtualenv_path) - yield virtualenv_path + + +@pytest.fixture() +def virtualenv(vistir_tmpdir): + with temp_environ(), VirtualEnv(base_dir=vistir_tmpdir) as venv: + yield venv + + +@pytest.fixture() +def raw_venv(): + yield VirtualEnv diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f858f8ac..3521ee5b 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -454,17 +454,18 @@ def test_rewrite_outline_table(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: contents = """ [packages] -six = {version = "*", editable = true} +six = {version = "*"} [packages.requests] version = "*" +extras = ["socks"] """.strip() f.write(contents) - c = p.pipenv("install -e click") + c = p.pipenv("install plette") assert c.return_code == 0 with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents - assert 'six = {version = "*", editable = true}' in contents - assert 'requests = {version = "*"}' in contents - assert 'click = {' in contents + assert 'six = {version = "*"}' in contents + assert 'requests = {version = "*"' in contents + assert 'plette = "*"' in contents diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 0879a265..98236b8a 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -23,22 +23,21 @@ def test_local_extras_install(PipenvInstance, pypi): contents = """ from setuptools import setup, find_packages setup( -name='testpipenv', -version='0.1', -description='Pipenv Test Package', -author='Pipenv Test', -author_email='test@pipenv.package', -license='MIT', -packages=find_packages(), -install_requires=[], -extras_require={'dev': ['six']}, -zip_safe=False + name='testpipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='MIT', + packages=find_packages(), + install_requires=[], + extras_require={'dev': ['six']}, + zip_safe=False ) """.strip() fh.write(contents) line = "-e .[dev]" - # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} - project = Project() + pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] @@ -54,6 +53,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "six" in p.lockfile["default"] c = p.pipenv("--rm") assert c.return_code == 0 + project = Project() project.write_toml({"packages": {}, "dev-packages": {}}) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 @@ -67,7 +67,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} @pytest.mark.local @pytest.mark.needs_internet @flaky -class TestDependencyLinks(object): +class TestDirectDependencies(object): """Ensure dependency_links are parsed and installed. This is needed for private repo dependencies. @@ -85,18 +85,15 @@ setup( version='0.1', packages=[], install_requires=[ - 'test-private-dependency' - ], - dependency_links=[ '{0}' - ] + ], ) """.strip().format(deplink) fh.write(contents) @staticmethod def helper_dependency_links_install_test(pipenv_instance, deplink): - TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink) + TestDirectDependencies.helper_dependency_links_install_make_setup(pipenv_instance, deplink) c = pipenv_instance.pipenv("install -v -e .") assert c.return_code == 0 assert "test-private-dependency" in pipenv_instance.lockfile["default"] @@ -107,19 +104,20 @@ setup( """Ensure dependency_links are parsed and installed (needed for private repo dependencies). """ with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.needs_github_ssh def test_ssh_dependency_links_install(self, PipenvInstance, pypi): with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' ) @@ -268,8 +266,8 @@ def test_local_zipfiles(PipenvInstance, pypi, testsroot): assert "file" in dep or "path" in dep assert c.return_code == 0 - key = [k for k in p.lockfile["default"].keys()][0] - dep = p.lockfile["default"][key] + # This now gets resolved to its name correctly + dep = p.lockfile["default"]["requests"] assert "file" in dep or "path" in dep diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 4dd35882..b0c1e51c 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,5 +1,4 @@ -import os - +# -*- coding=utf-8 -*- import pytest from flaky import flaky @@ -25,6 +24,7 @@ def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "https://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } assert "gitdb2" in p.lockfile["default"] @@ -42,6 +42,7 @@ def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "git://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } @@ -59,6 +60,7 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "ssh://git@github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } @@ -66,8 +68,9 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): @pytest.mark.urls @pytest.mark.needs_internet @flaky -def test_urls_work(PipenvInstance, pypi, pip_src_dir): +def test_urls_work(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: + # the library this installs is "django-cms" path = p._pipfile.get_url("django", "3.4.x.zip") c = p.pipenv( "install {0}".format(path) @@ -77,7 +80,8 @@ def test_urls_work(PipenvInstance, pypi, pip_src_dir): dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - dep = list(p.lockfile["default"].values())[0] + # now that we handle resolution with requirementslib, this will resolve to a name + dep = p.lockfile["default"]["django-cms"] assert "file" in dep, p.lockfile @@ -219,8 +223,8 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): == "5efb522b0647f7467248273ec1b893d06b984a59" ) pipfile = Path(p.pipfile_path) - new_content = pipfile.read_bytes().replace(b"1.9.0", b"1.11.0") - pipfile.write_bytes(new_content) + new_content = pipfile.read_text().replace(u"1.9.0", u"1.11.0") + pipfile.write_text(new_content) c = p.pipenv("lock") assert c.return_code == 0 assert ( @@ -234,13 +238,15 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): @pytest.mark.vcs @pytest.mark.install @pytest.mark.needs_internet +@pytest.mark.skip_py27_win def test_vcs_entry_supersedes_non_vcs(PipenvInstance, pip_src_dir): """See issue #2181 -- non-editable VCS dep was specified, but not showing up in the lockfile -- due to not running pip install before locking and not locking the resolution graph of non-editable vcs dependencies. """ with PipenvInstance(chdir=True) as p: - pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + # pyinstaller_path = p._pipfile.get_fixture_path("git/pyinstaller") + pyinstaller_uri = "https://github.com/pyinstaller/pyinstaller.git" with open(p.pipfile_path, "w") as f: f.write( """ @@ -252,7 +258,7 @@ name = "pypi" [packages] PyUpdater = "*" PyInstaller = {{ref = "develop", git = "{0}"}} - """.format(pyinstaller_path.as_uri()).strip() + """.format(pyinstaller_uri).strip() ) c = p.pipenv("install") assert c.return_code == 0 @@ -263,7 +269,7 @@ PyInstaller = {{ref = "develop", git = "{0}"}} assert p.lockfile["default"]["pyinstaller"].get("ref") is not None assert ( p.lockfile["default"]["pyinstaller"]["git"] - == pyinstaller_path.as_uri() + == pyinstaller_uri ) diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index d993b1a2..93b9d4f8 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -4,7 +4,7 @@ import sys import pytest from flaky import flaky - +from vistir.compat import Path from pipenv.utils import temp_environ @@ -585,6 +585,7 @@ six = "*" assert p.lockfile["default"]["six"]["index"] == "test" with open(p.pipfile_path, 'w') as f: f.write(contents.replace('name = "test"', 'name = "custom"')) - c = p.pipenv("lock") + c = p.pipenv("lock --clear") assert c.return_code == 0 - assert p.lockfile["default"]["six"]["index"] == "custom" + assert "index" in p.lockfile["default"]["six"] + assert p.lockfile["default"]["six"]["index"] == "custom", Path(p.lockfile_path).read_text() # p.lockfile["default"]["six"] diff --git a/tests/integration/test_pipenv.py b/tests/integration/test_pipenv.py index deacc495..ef8c23f2 100644 --- a/tests/integration/test_pipenv.py +++ b/tests/integration/test_pipenv.py @@ -5,12 +5,8 @@ XXX: Try our best to reduce tests in this file. import os -from tempfile import mkdtemp - -import mock import pytest -from pipenv._compat import Path from pipenv.project import Project from pipenv.utils import temp_environ from pipenv.vendor import delegator @@ -93,17 +89,16 @@ def test_proper_names_unamanged_virtualenv(PipenvInstance, pypi): @pytest.mark.cli -def test_directory_with_leading_dash(PipenvInstance): - def mocked_mkdtemp(suffix, prefix, dir): - if suffix == '-project': - prefix = '-dir-with-leading-dash' - return mkdtemp(suffix, prefix, dir) - - with mock.patch('pipenv.vendor.vistir.compat.mkdtemp', side_effect=mocked_mkdtemp): - with temp_environ(), PipenvInstance(chdir=True) as p: - del os.environ['PIPENV_VENV_IN_PROJECT'] - p.pipenv('--python python') - venv_path = p.pipenv('--venv').out.strip() +def test_directory_with_leading_dash(raw_venv, PipenvInstance): + with temp_environ(): + with PipenvInstance(chdir=True, venv_in_project=False, name="-project-with-dash") as p: + if "PIPENV_VENV_IN_PROJECT" in os.environ: + del os.environ['PIPENV_VENV_IN_PROJECT'] + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + c = p.pipenv('--venv') + assert c.return_code == 0 + venv_path = c.out.strip() assert os.path.isdir(venv_path) # Manually clean up environment, since PipenvInstance assumes that # the virutalenv is in the project directory. diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 42754521..d193fc7d 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -8,6 +8,7 @@ import pytest from pipenv.patched import pipfile from pipenv.project import Project from pipenv.utils import temp_environ +import pipenv.environments @pytest.mark.project @@ -168,25 +169,41 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.project @pytest.mark.virtualenv -def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): - with PipenvInstance(chdir=True, pypi=pypi) as p: - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' - c = p.pipenv('run which pip') +def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv): + with PipenvInstance(chdir=True, pypi=pypi, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p: + c = p.pipenv('run pip freeze') assert c.return_code == 0 - assert 'virtualenv' not in c.out - - os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) - c = p.pipenv('run which pip') - assert c.return_code == 0 - assert 'virtualenv' in c.out + assert 'Creating a virtualenv' not in c.err project = Project() - assert project.virtualenv_location == str(virtualenv) + assert project.virtualenv_location == virtualenv.as_posix() c = p.pipenv("run pip install click") assert c.return_code == 0 assert "Courtesy Notice" in c.err + c = p.pipenv("install six") + assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 assert c.out.strip().startswith(str(virtualenv)) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out + + +@pytest.mark.project +@pytest.mark.virtualenv +def test_run_in_virtualenv(PipenvInstance, pypi): + with PipenvInstance(chdir=True, pypi=pypi) as p: + c = p.pipenv('run pip freeze') + assert c.return_code == 0 + assert 'Creating a virtualenv' in c.err + project = Project() + c = p.pipenv("run pip install click") + assert c.return_code == 0 + c = p.pipenv("install six") + assert c.return_code == 0 + c = p.pipenv('run python -c "import click;print(click.__file__)"') + assert c.return_code == 0 + assert c.out.strip().startswith(str(project.virtualenv_location)) + c = p.pipenv("clean --dry-run") + assert c.return_code == 0 + assert "click" in c.out diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 8b8745e7..c9e5c057 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -21,11 +21,16 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): @pytest.mark.sync @pytest.mark.lock def test_mirror_lock_sync(PipenvInstance, pypi): - with temp_environ(), PipenvInstance(chdir=True) as p: + with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") assert 'pypi.org' not in mirror_url with open(p.pipfile_path, 'w') as f: f.write(""" +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + [packages] six = "*" """.strip()) diff --git a/tests/pypi/Cerberus/Cerberus-1.2.tar.gz b/tests/pypi/Cerberus/Cerberus-1.2.tar.gz new file mode 100644 index 00000000..b9f2c01d Binary files /dev/null and b/tests/pypi/Cerberus/Cerberus-1.2.tar.gz differ diff --git a/tests/pypi/cerberus/cerberus-1.2.tar.gz b/tests/pypi/cerberus/cerberus-1.2.tar.gz new file mode 100644 index 00000000..b9f2c01d Binary files /dev/null and b/tests/pypi/cerberus/cerberus-1.2.tar.gz differ diff --git a/tests/pypi/enum34/api.json b/tests/pypi/enum34/api.json new file mode 100644 index 00000000..64a47647 --- /dev/null +++ b/tests/pypi/enum34/api.json @@ -0,0 +1,190 @@ +{ + "info": { + "author": "Ethan Furman", + "author_email": "ethan@stoneleaf.us", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Programming Language :: Python", + "Programming Language :: Python :: 2.4", + "Programming Language :: Python :: 2.5", + "Programming Language :: Python :: 2.6", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Topic :: Software Development" + ], + "description": "enum --- support for enumerations\n========================================\n\nAn enumeration is a set of symbolic names (members) bound to unique, constant\nvalues. Within an enumeration, the members can be compared by identity, and\nthe enumeration itself can be iterated over.\n\n from enum import Enum\n\n class Fruit(Enum):\n apple = 1\n banana = 2\n orange = 3\n\n list(Fruit)\n # [, , ]\n\n len(Fruit)\n # 3\n\n Fruit.banana\n # \n\n Fruit['banana']\n # \n\n Fruit(2)\n # \n\n Fruit.banana is Fruit['banana'] is Fruit(2)\n # True\n\n Fruit.banana.name\n # 'banana'\n\n Fruit.banana.value\n # 2\n\nRepository and Issue Tracker at https://bitbucket.org/stoneleaf/enum34.", + "description_content_type": null, + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://bitbucket.org/stoneleaf/enum34", + "keywords": "", + "license": "BSD License", + "maintainer": "", + "maintainer_email": "", + "name": "enum34", + "package_url": "https://pypi.org/project/enum34/", + "platform": "UNKNOWN", + "project_url": "https://pypi.org/project/enum34/", + "project_urls": { + "Homepage": "https://bitbucket.org/stoneleaf/enum34" + }, + "release_url": "https://pypi.org/project/enum34/1.1.6/", + "requires_dist": null, + "requires_python": "", + "summary": "Python 3.4 Enum backported to 3.3, 3.2, 3.1, 2.7, 2.6, 2.5, and 2.4", + "version": "1.1.6" + }, + "last_serial": 2117417, + "releases": { + "1.1.6": [ + { + "comment_text": "", + "digests": { + "md5": "68f6982cc07dde78f4b500db829860bd", + "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "68f6982cc07dde78f4b500db829860bd", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 12427, + "upload_time": "2016-05-16T03:31:13", + "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a63ecb4f0b1b85fb69be64bdea999b43", + "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 12428, + "upload_time": "2016-05-16T03:31:19", + "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "5f13a0841a61f7fc295c514490d120d0", + "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + }, + "downloads": -1, + "filename": "enum34-1.1.6.tar.gz", + "has_sig": false, + "md5_digest": "5f13a0841a61f7fc295c514490d120d0", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 40048, + "upload_time": "2016-05-16T03:31:30", + "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz" + }, + { + "comment_text": "", + "digests": { + "md5": "61ad7871532d4ce2d77fac2579237a9e", + "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" + }, + "downloads": -1, + "filename": "enum34-1.1.6.zip", + "has_sig": false, + "md5_digest": "61ad7871532d4ce2d77fac2579237a9e", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 44773, + "upload_time": "2016-05-16T03:31:48", + "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "68f6982cc07dde78f4b500db829860bd", + "sha256": "6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "68f6982cc07dde78f4b500db829860bd", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 12427, + "upload_time": "2016-05-16T03:31:13", + "url": "https://files.pythonhosted.org/packages/c5/db/e56e6b4bbac7c4a06de1c50de6fe1ef3810018ae11732a50f15f62c7d050/enum34-1.1.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a63ecb4f0b1b85fb69be64bdea999b43", + "sha256": "644837f692e5f550741432dd3f223bbb9852018674981b1664e5dc339387588a" + }, + "downloads": -1, + "filename": "enum34-1.1.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "a63ecb4f0b1b85fb69be64bdea999b43", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 12428, + "upload_time": "2016-05-16T03:31:19", + "url": "https://files.pythonhosted.org/packages/af/42/cb9355df32c69b553e72a2e28daee25d1611d2c0d9c272aa1d34204205b2/enum34-1.1.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "5f13a0841a61f7fc295c514490d120d0", + "sha256": "8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" + }, + "downloads": -1, + "filename": "enum34-1.1.6.tar.gz", + "has_sig": false, + "md5_digest": "5f13a0841a61f7fc295c514490d120d0", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 40048, + "upload_time": "2016-05-16T03:31:30", + "url": "https://files.pythonhosted.org/packages/bf/3e/31d502c25302814a7c2f1d3959d2a3b3f78e509002ba91aea64993936876/enum34-1.1.6.tar.gz" + }, + { + "comment_text": "", + "digests": { + "md5": "61ad7871532d4ce2d77fac2579237a9e", + "sha256": "2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850" + }, + "downloads": -1, + "filename": "enum34-1.1.6.zip", + "has_sig": false, + "md5_digest": "61ad7871532d4ce2d77fac2579237a9e", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 44773, + "upload_time": "2016-05-16T03:31:48", + "url": "https://files.pythonhosted.org/packages/e8/26/a6101edcf724453845c850281b96b89a10dac6bd98edebc82634fccce6a5/enum34-1.1.6.zip" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl b/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl new file mode 100644 index 00000000..12be7c7e Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6-py2-none-any.whl differ diff --git a/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl b/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl new file mode 100644 index 00000000..53c1cb04 Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6-py3-none-any.whl differ diff --git a/tests/pypi/enum34/enum34-1.1.6.tar.gz b/tests/pypi/enum34/enum34-1.1.6.tar.gz new file mode 100644 index 00000000..ab548ac6 Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6.tar.gz differ diff --git a/tests/pypi/enum34/enum34-1.1.6.zip b/tests/pypi/enum34/enum34-1.1.6.zip new file mode 100644 index 00000000..caabfc64 Binary files /dev/null and b/tests/pypi/enum34/enum34-1.1.6.zip differ diff --git a/tests/pypi/functools32/functools32-3.2.3-2.tar.gz b/tests/pypi/functools32/functools32-3.2.3-2.tar.gz new file mode 100644 index 00000000..e4255eb4 Binary files /dev/null and b/tests/pypi/functools32/functools32-3.2.3-2.tar.gz differ diff --git a/tests/pypi/functools32/functools32-3.2.3-2.zip b/tests/pypi/functools32/functools32-3.2.3-2.zip new file mode 100644 index 00000000..9175388a Binary files /dev/null and b/tests/pypi/functools32/functools32-3.2.3-2.zip differ diff --git a/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl b/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl new file mode 100644 index 00000000..1ca43416 Binary files /dev/null and b/tests/pypi/plette/plette-0.2.2-py2.py3-none-any.whl differ diff --git a/tests/pypi/setuptools/setuptools-40.6.3-py2.py3-none-any.whl b/tests/pypi/setuptools/setuptools-40.6.3-py2.py3-none-any.whl new file mode 100644 index 00000000..f9fc432a Binary files /dev/null and b/tests/pypi/setuptools/setuptools-40.6.3-py2.py3-none-any.whl differ diff --git a/tests/pypi/six/six-1.12.0-py2.py3-none-any.whl b/tests/pypi/six/six-1.12.0-py2.py3-none-any.whl new file mode 100644 index 00000000..78ba9032 Binary files /dev/null and b/tests/pypi/six/six-1.12.0-py2.py3-none-any.whl differ diff --git a/tests/pypi/tomlkit/api.json b/tests/pypi/tomlkit/api.json new file mode 100644 index 00000000..c19763af --- /dev/null +++ b/tests/pypi/tomlkit/api.json @@ -0,0 +1,123 @@ +{ + "info": { + "author": "S\u00e9bastien Eustace", + "author_email": "sebastien@eustace.io", + "bugtrack_url": null, + "classifiers": [ + "License :: OSI Approved :: MIT License", + "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.4", + "Programming Language :: Python :: 3.5", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7" + ], + "description": "[github_release]: https://img.shields.io/github/release/sdispater/tomlkit.svg?logo=github&logoColor=white\n[pypi_version]: https://img.shields.io/pypi/v/tomlkit.svg?logo=python&logoColor=white\n[python_versions]: https://img.shields.io/pypi/pyversions/tomlkit.svg?logo=python&logoColor=white\n[github_license]: https://img.shields.io/github/license/sdispater/tomlkit.svg?logo=github&logoColor=white\n[travisci]: https://img.shields.io/travis/com/sdispater/tomlkit/master.svg?logo=travis&logoColor=white&label=Travis%20CI\n[appveyor]: https://img.shields.io/appveyor/ci/sdispater/tomlkit/master.svg?logo=appveyor&logoColor=white&label=AppVeyor\n\n[codecov]: https://img.shields.io/codecov/c/github/sdispater/tomlkit/master.svg?logo=data:image/svg+xml;base64,PHN2ZyB3aWR0aD0iNTAiIGhlaWdodD0iNDgiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyI+CgogPGc+CiAgPHRpdGxlPmJhY2tncm91bmQ8L3RpdGxlPgogIDxyZWN0IGZpbGw9Im5vbmUiIGlkPSJjYW52YXNfYmFja2dyb3VuZCIgaGVpZ2h0PSI0MDIiIHdpZHRoPSI1ODIiIHk9Ii0xIiB4PSItMSIvPgogPC9nPgogPGc+CiAgPHRpdGxlPkxheWVyIDE8L3RpdGxlPgogIDxwYXRoIGlkPSJzdmdfMSIgZmlsbC1ydWxlPSJldmVub2RkIiBmaWxsPSIjZmZmZmZmIiBkPSJtMjUuMDE0LDBjLTEzLjc4NCwwLjAxIC0yNS4wMDQsMTEuMTQ5IC0yNS4wMTQsMjQuODMybDAsMC4wNjJsNC4yNTQsMi40ODJsMC4wNTgsLTAuMDM5YTEyLjIzOCwxMi4yMzggMCAwIDEgOS4wNzgsLTEuOTI4YTExLjg0NCwxMS44NDQgMCAwIDEgNS45OCwyLjk3NWwwLjczLDAuNjhsMC40MTMsLTAuOTA0YzAuNCwtMC44NzQgMC44NjIsLTEuNjk2IDEuMzc0LC0yLjQ0M2MwLjIwNiwtMC4zIDAuNDMzLC0wLjYwNCAwLjY5MiwtMC45MjlsMC40MjcsLTAuNTM1bC0wLjUyNiwtMC40NGExNy40NSwxNy40NSAwIDAgMCAtOC4xLC0zLjc4MWExNy44NTMsMTcuODUzIDAgMCAwIC04LjM3NSwwLjQ5YzIuMDIzLC04Ljg2OCA5LjgyLC0xNS4wNSAxOS4wMjcsLTE1LjA1N2M1LjE5NSwwIDEwLjA3OCwyLjAwNyAxMy43NTIsNS42NTJjMi42MTksMi41OTggNC40MjIsNS44MzUgNS4yMjQsOS4zNzJhMTcuOTA4LDE3LjkwOCAwIDAgMCAtNS4yMDgsLTAuNzlsLTAuMzE4LC0wLjAwMWExOC4wOTYsMTguMDk2IDAgMCAwIC0yLjA2NywwLjE1M2wtMC4wODcsMC4wMTJjLTAuMzAzLDAuMDQgLTAuNTcsMC4wODEgLTAuODEzLDAuMTI2Yy0wLjExOSwwLjAyIC0wLjIzNywwLjA0NSAtMC4zNTUsMC4wNjhjLTAuMjgsMC4wNTcgLTAuNTU0LDAuMTE5IC0wLjgxNiwwLjE4NWwtMC4yODgsMC4wNzNjLTAuMzM2LDAuMDkgLTAuNjc1LDAuMTkxIC0xLjAwNiwwLjNsLTAuMDYxLDAuMDJjLTAuNzQsMC4yNTEgLTEuNDc4LDAuNTU4IC0yLjE5LDAuOTE0bC0wLjA1NywwLjAyOWMtMC4zMTYsMC4xNTggLTAuNjM2LDAuMzMzIC0wLjk3OCwwLjUzNGwtMC4wNzUsMC4wNDVhMTYuOTcsMTYuOTcgMCAwIDAgLTQuNDE0LDMuNzhsLTAuMTU3LDAuMTkxYy0wLjMxNywwLjM5NCAtMC41NjcsMC43MjcgLTAuNzg3LDEuMDQ4Yy0wLjE4NCwwLjI3IC0wLjM2OSwwLjU2IC0wLjYsMC45NDJsLTAuMTI2LDAuMjE3Yy0wLjE4NCwwLjMxOCAtMC4zNDgsMC42MjIgLTAuNDg3LDAuOWwtMC4wMzMsMC4wNjFjLTAuMzU0LDAuNzExIC0wLjY2MSwxLjQ1NSAtMC45MTcsMi4yMTRsLTAuMDM2LDAuMTExYTE3LjEzLDE3LjEzIDAgMCAwIC0wLjg1NSw1LjY0NGwwLjAwMywwLjIzNGEyMy41NjUsMjMuNTY1IDAgMCAwIDAuMDQzLDAuODIyYzAuMDEsMC4xMyAwLjAyMywwLjI1OSAwLjAzNiwwLjM4OGMwLjAxNSwwLjE1OCAwLjAzNCwwLjMxNiAwLjA1MywwLjQ3MWwwLjAxMSwwLjA4OGwwLjAyOCwwLjIxNGMwLjAzNywwLjI2NCAwLjA4LDAuNTI1IDAuMTMsMC43ODdjMC41MDMsMi42MzcgMS43Niw1LjI3NCAzLjYzNSw3LjYyNWwwLjA4NSwwLjEwNmwwLjA4NywtMC4xMDRjMC43NDgsLTAuODg0IDIuNjAzLC0zLjY4NyAyLjc2LC01LjM2OWwwLjAwMywtMC4wMzFsLTAuMDE1LC0wLjAyOGExMS43MzYsMTEuNzM2IDAgMCAxIC0xLjMzMywtNS40MDdjMCwtNi4yODQgNC45NCwtMTEuNTAyIDExLjI0MywtMTEuODhsMC40MTQsLTAuMDE1YzIuNTYxLC0wLjA1OCA1LjA2NCwwLjY3MyA3LjIzLDIuMTM2bDAuMDU4LDAuMDM5bDQuMTk3LC0yLjQ0bDAuMDU1LC0wLjAzM2wwLC0wLjA2MmMwLjAwNiwtNi42MzIgLTIuNTkyLC0xMi44NjUgLTcuMzE0LC0xNy41NTFjLTQuNzE2LC00LjY3OSAtMTAuOTkxLC03LjI1NSAtMTcuNjcyLC03LjI1NSIvPgogPC9nPgo8L3N2Zz4=&label=Codecov\n\n[![GitHub Release][github_release]](https://github.com/sdispater/tomlkit/releases/)\n[![PyPI Version][pypi_version]](https://pypi.python.org/pypi/tomlkit/)\n[![Python Versions][python_versions]](https://pypi.python.org/pypi/tomlkit/)\n[![License][github_license]](https://github.com/sdispater/tomlkit/blob/master/LICENSE)\n
\n[![Travis CI][travisci]](https://travis-ci.com/sdispater/tomlkit)\n[![AppVeyor][appveyor]](https://ci.appveyor.com/project/sdispater/tomlkit)\n[![Codecov][codecov]](https://codecov.io/gh/sdispater/tomlkit)\n\n# TOML Kit - Style-preserving TOML library for Python\n\nTOML Kit is a **0.5.0-compliant** [TOML](https://github.com/toml-lang/toml) library.\n\nIt includes a parser that preserves all comments, indentations, whitespace and internal element ordering,\nand makes them accessible and editable via an intuitive API.\n\nYou can also create new TOML documents from scratch using the provided helpers.\n\nPart of the implementation as been adapted, improved and fixed from [Molten](https://github.com/LeopoldArkham/Molten).\n\n## Usage\n\n### Parsing\n\nTOML Kit comes with a fast and style-preserving parser to help you access\nthe content of TOML files and strings.\n\n```python\n>>> from tomlkit import dumps\n>>> from tomlkit import parse # you can also use loads\n\n>>> content = \"\"\"[table]\n... foo = \"bar\" # String\n... \"\"\"\n>>> doc = parse(content)\n\n# doc is a TOMLDocument instance that holds all the information\n# about the TOML string.\n# It behaves like a standard dictionary.\n\n>>> assert doc[\"table\"][\"foo\"] == \"bar\"\n\n# The string generated from the document is exactly the same\n# as the original string\n>>> assert dumps(doc) == content\n```\n\n### Modifying\n\nTOML Kit provides an intuitive API to modify TOML documents.\n\n```python\n>>> from tomlkit import dumps\n>>> from tomlkit import parse\n>>> from tomlkit import table\n\n>>> doc = parse(\"\"\"[table]\n... foo = \"bar\" # String\n... \"\"\")\n\n>>> doc[\"table\"][\"baz\"] = 13\n\n>>> dumps(doc)\n\"\"\"[table]\nfoo = \"bar\" # String\nbaz = 13\n\"\"\"\n\n# Add a new table\n>>> tab = table()\n>>> tab.add(\"array\", [1, 2, 3])\n\n>>> doc[\"table2\"] = tab\n\n>>> dumps(doc)\n\"\"\"[table]\nfoo = \"bar\" # String\nbaz = 13\n\n[table2]\narray = [1, 2, 3]\n\"\"\"\n\n# Remove the newly added table\n>>> doc.remove(\"table2\")\n# del doc[\"table2] is also possible\n```\n\n### Writing\n\nYou can also write a new TOML document from scratch.\n\nLet's say we want to create this following document:\n\n```toml\n# This is a TOML document.\n\ntitle = \"TOML Example\"\n\n[owner]\nname = \"Tom Preston-Werner\"\norganization = \"GitHub\"\nbio = \"GitHub Cofounder & CEO\\nLikes tater tots and beer.\"\ndob = 1979-05-27T07:32:00Z # First class dates? Why not?\n\n[database]\nserver = \"192.168.1.1\"\nports = [ 8001, 8001, 8002 ]\nconnection_max = 5000\nenabled = true\n```\n\nIt can be created with the following code:\n\n```python\n>>> from tomlkit import comment\n>>> from tomlkit import document\n>>> from tomlkit import nl\n>>> from tomlkit import table\n\n>>> doc = document()\n>>> doc.add(comment(\"This is a TOML document.\"))\n>>> doc.add(nl())\n>>> doc.add(\"title\", \"TOML Example\")\n# Using doc[\"title\"] = \"TOML Example\" is also possible\n\n>>> owner = table()\n>>> owner.add(\"name\", \"Tom Preston-Werner\")\n>>> owner.add(\"organization\", \"GitHub\")\n>>> owner.add(\"bio\", \"GitHub Cofounder & CEO\\nLikes tater tots and beer.\")\n>>> owner.add(\"dob\", datetime(1979, 5, 27, 7, 32, tzinfo=utc))\n>>> owner[\"dob\"].comment(\"First class dates? Why not?\")\n\n# Adding the table to the document\n>>> doc.add(\"owner\", owner)\n\n>>> database = table()\n>>> database[\"server\"] = \"192.168.1.1\"\n>>> database[\"ports\"] = [8001, 8001, 8002]\n>>> database[\"connection_max\"] = 5000\n>>> database[\"enabled\"] = True\n\n>>> doc[\"database\"] = database\n```\n\n\n## Installation\n\nIf you are using [Poetry](https://poetry.eustace.io),\nadd `tomlkit` to your `pyproject.toml` file by using:\n\n```bash\npoetry add tomlkit\n```\n\nIf not, you can use `pip`:\n\n```bash\npip install tomlkit\n```\n", + "description_content_type": "text/markdown", + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://github.com/sdispater/tomlkit", + "keywords": "", + "license": "MIT", + "maintainer": "S\u00e9bastien Eustace", + "maintainer_email": "sebastien@eustace.io", + "name": "tomlkit", + "package_url": "https://pypi.org/project/tomlkit/", + "platform": "", + "project_url": "https://pypi.org/project/tomlkit/", + "project_urls": { + "Homepage": "https://github.com/sdispater/tomlkit", + "Repository": "https://github.com/sdispater/tomlkit" + }, + "release_url": "https://pypi.org/project/tomlkit/0.5.3/", + "requires_dist": [ + "enum34 (>=1.1,<2.0); python_version >= \"2.7\" and python_version < \"2.8\"", + "functools32 (>=3.2.3,<4.0.0); python_version >= \"2.7\" and python_version < \"2.8\"", + "typing (>=3.6,<4.0); python_version >= \"2.7\" and python_version < \"2.8\" or python_version >= \"3.4\" and python_version < \"3.5\"" + ], + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "summary": "Style preserving TOML library", + "version": "0.5.3" + }, + "last_serial": 4504211, + "releases": { + "0.5.3": [ + { + "comment_text": "", + "digests": { + "md5": "0a6cf417df5d0fc911f89447c9a662a9", + "sha256": "f077456d35303e7908cc233b340f71e0bec96f63429997f38ca9272b7d64029e" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3-py2.py3-none-any.whl", + "has_sig": false, + "md5_digest": "0a6cf417df5d0fc911f89447c9a662a9", + "packagetype": "bdist_wheel", + "python_version": "py2.py3", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 116796, + "upload_time": "2018-11-19T20:05:37", + "url": "https://files.pythonhosted.org/packages/71/c6/06c014b92cc48270765d6a9418d82239b158d8a9b69e031b0e2c6598740b/tomlkit-0.5.3-py2.py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a708470b53d689013f2fc9f0a7902adf", + "sha256": "d6506342615d051bc961f70bfcfa3d29b6616cc08a3ddfd4bc24196f16fd4ec2" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3.tar.gz", + "has_sig": false, + "md5_digest": "a708470b53d689013f2fc9f0a7902adf", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 29864, + "upload_time": "2018-11-19T20:05:39", + "url": "https://files.pythonhosted.org/packages/f7/f7/bbd9213bfe76cb7821c897f9ed74877fd74993b4ca2fe9513eb5a31030f9/tomlkit-0.5.3.tar.gz" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "0a6cf417df5d0fc911f89447c9a662a9", + "sha256": "f077456d35303e7908cc233b340f71e0bec96f63429997f38ca9272b7d64029e" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3-py2.py3-none-any.whl", + "has_sig": false, + "md5_digest": "0a6cf417df5d0fc911f89447c9a662a9", + "packagetype": "bdist_wheel", + "python_version": "py2.py3", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 116796, + "upload_time": "2018-11-19T20:05:37", + "url": "https://files.pythonhosted.org/packages/71/c6/06c014b92cc48270765d6a9418d82239b158d8a9b69e031b0e2c6598740b/tomlkit-0.5.3-py2.py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "a708470b53d689013f2fc9f0a7902adf", + "sha256": "d6506342615d051bc961f70bfcfa3d29b6616cc08a3ddfd4bc24196f16fd4ec2" + }, + "downloads": -1, + "filename": "tomlkit-0.5.3.tar.gz", + "has_sig": false, + "md5_digest": "a708470b53d689013f2fc9f0a7902adf", + "packagetype": "sdist", + "python_version": "source", + "requires_python": ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", + "size": 29864, + "upload_time": "2018-11-19T20:05:39", + "url": "https://files.pythonhosted.org/packages/f7/f7/bbd9213bfe76cb7821c897f9ed74877fd74993b4ca2fe9513eb5a31030f9/tomlkit-0.5.3.tar.gz" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl b/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl new file mode 100644 index 00000000..7375595e Binary files /dev/null and b/tests/pypi/tomlkit/tomlkit-0.5.3-py2.py3-none-any.whl differ diff --git a/tests/pypi/tomlkit/tomlkit-0.5.3.tar.gz b/tests/pypi/tomlkit/tomlkit-0.5.3.tar.gz new file mode 100644 index 00000000..c4a7f024 Binary files /dev/null and b/tests/pypi/tomlkit/tomlkit-0.5.3.tar.gz differ diff --git a/tests/pypi/typing/api.json b/tests/pypi/typing/api.json new file mode 100644 index 00000000..df641aa8 --- /dev/null +++ b/tests/pypi/typing/api.json @@ -0,0 +1,153 @@ +{ + "info": { + "author": "Guido van Rossum, Jukka Lehtosalo, \u0141ukasz Langa, Ivan Levkivskyi", + "author_email": "jukka.lehtosalo@iki.fi", + "bugtrack_url": null, + "classifiers": [ + "Development Status :: 5 - Production/Stable", + "Environment :: Console", + "Intended Audience :: Developers", + "License :: OSI Approved :: Python Software Foundation License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Programming Language :: Python :: 3.3", + "Programming Language :: Python :: 3.4", + "Topic :: Software Development" + ], + "description": "Typing -- Type Hints for Python\n\nThis is a backport of the standard library typing module to Python\nversions older than 3.5. (See note below for newer versions.)\n\nTyping defines a standard notation for Python function and variable\ntype annotations. The notation can be used for documenting code in a\nconcise, standard format, and it has been designed to also be used by\nstatic and runtime type checkers, static analyzers, IDEs and other\ntools.\n\nNOTE: in Python 3.5 and later, the typing module lives in the stdlib,\nand installing this package has NO EFFECT. To get a newer version of\nthe typing module in Python 3.5 or later, you have to upgrade to a\nnewer Python (bugfix) version. For example, typing in Python 3.6.0 is\nmissing the definition of 'Type' -- upgrading to 3.6.2 will fix this.\n\nAlso note that most improvements to the typing module in Python 3.7\nwill not be included in this package, since Python 3.7 has some\nbuilt-in support that is not present in older versions (See PEP 560.)\n\n\n", + "description_content_type": "", + "docs_url": null, + "download_url": "", + "downloads": { + "last_day": -1, + "last_month": -1, + "last_week": -1 + }, + "home_page": "https://docs.python.org/3/library/typing.html", + "keywords": "typing function annotations type hints hinting checking checker typehints typehinting typechecking backport", + "license": "PSF", + "maintainer": "", + "maintainer_email": "", + "name": "typing", + "package_url": "https://pypi.org/project/typing/", + "platform": "", + "project_url": "https://pypi.org/project/typing/", + "project_urls": { + "Homepage": "https://docs.python.org/3/library/typing.html" + }, + "release_url": "https://pypi.org/project/typing/3.6.6/", + "requires_dist": null, + "requires_python": "", + "summary": "Type Hints for Python", + "version": "3.6.6" + }, + "last_serial": 4208967, + "releases": { + "3.6.6": [ + { + "comment_text": "", + "digests": { + "md5": "7e50dcc98a528f47c8793c980128467c", + "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + }, + "downloads": -1, + "filename": "typing-3.6.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "7e50dcc98a528f47c8793c980128467c", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 23560, + "upload_time": "2018-08-26T18:46:05", + "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "0c84fda6fd4303fa6aee13a36ea62897", + "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4" + }, + "downloads": -1, + "filename": "typing-3.6.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 25727, + "upload_time": "2018-08-26T18:46:06", + "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "64614206b4bdc0864fc0e0bccd69efc9", + "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d" + }, + "downloads": -1, + "filename": "typing-3.6.6.tar.gz", + "has_sig": false, + "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 71799, + "upload_time": "2018-08-26T18:46:08", + "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz" + } + ] + }, + "urls": [ + { + "comment_text": "", + "digests": { + "md5": "7e50dcc98a528f47c8793c980128467c", + "sha256": "a4c8473ce11a65999c8f59cb093e70686b6c84c98df58c1dae9b3b196089858a" + }, + "downloads": -1, + "filename": "typing-3.6.6-py2-none-any.whl", + "has_sig": false, + "md5_digest": "7e50dcc98a528f47c8793c980128467c", + "packagetype": "bdist_wheel", + "python_version": "py2", + "requires_python": null, + "size": 23560, + "upload_time": "2018-08-26T18:46:05", + "url": "https://files.pythonhosted.org/packages/cc/3e/29f92b7aeda5b078c86d14f550bf85cff809042e3429ace7af6193c3bc9f/typing-3.6.6-py2-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "0c84fda6fd4303fa6aee13a36ea62897", + "sha256": "57dcf675a99b74d64dacf6fba08fb17cf7e3d5fdff53d4a30ea2a5e7e52543d4" + }, + "downloads": -1, + "filename": "typing-3.6.6-py3-none-any.whl", + "has_sig": false, + "md5_digest": "0c84fda6fd4303fa6aee13a36ea62897", + "packagetype": "bdist_wheel", + "python_version": "py3", + "requires_python": null, + "size": 25727, + "upload_time": "2018-08-26T18:46:06", + "url": "https://files.pythonhosted.org/packages/4a/bd/eee1157fc2d8514970b345d69cb9975dcd1e42cd7e61146ed841f6e68309/typing-3.6.6-py3-none-any.whl" + }, + { + "comment_text": "", + "digests": { + "md5": "64614206b4bdc0864fc0e0bccd69efc9", + "sha256": "4027c5f6127a6267a435201981ba156de91ad0d1d98e9ddc2aa173453453492d" + }, + "downloads": -1, + "filename": "typing-3.6.6.tar.gz", + "has_sig": false, + "md5_digest": "64614206b4bdc0864fc0e0bccd69efc9", + "packagetype": "sdist", + "python_version": "source", + "requires_python": null, + "size": 71799, + "upload_time": "2018-08-26T18:46:08", + "url": "https://files.pythonhosted.org/packages/bf/9b/2bf84e841575b633d8d91ad923e198a415e3901f228715524689495b4317/typing-3.6.6.tar.gz" + } + ] +} \ No newline at end of file diff --git a/tests/pypi/typing/typing-3.6.6-py2-none-any.whl b/tests/pypi/typing/typing-3.6.6-py2-none-any.whl new file mode 100644 index 00000000..a6dc8ab1 Binary files /dev/null and b/tests/pypi/typing/typing-3.6.6-py2-none-any.whl differ diff --git a/tests/pypi/typing/typing-3.6.6-py3-none-any.whl b/tests/pypi/typing/typing-3.6.6-py3-none-any.whl new file mode 100644 index 00000000..f38e8528 Binary files /dev/null and b/tests/pypi/typing/typing-3.6.6-py3-none-any.whl differ diff --git a/tests/pypi/typing/typing-3.6.6.tar.gz b/tests/pypi/typing/typing-3.6.6.tar.gz new file mode 100644 index 00000000..26d02727 Binary files /dev/null and b/tests/pypi/typing/typing-3.6.6.tar.gz differ diff --git a/tests/pypi/wheel/wheel-0.32.3-py2.py3-none-any.whl b/tests/pypi/wheel/wheel-0.32.3-py2.py3-none-any.whl new file mode 100644 index 00000000..68006a56 Binary files /dev/null and b/tests/pypi/wheel/wheel-0.32.3-py2.py3-none-any.whl differ diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 4e013ab5..fa494ea8 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -1,5 +1,6 @@ import os import json +import io import sys import requests @@ -29,7 +30,26 @@ class Package(object): with open(os.path.join(path, 'api.json')) as f: return json.load(f) except FileNotFoundError: - pass + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response def __repr__(self): return "/json') def json_for_package(package): - try: - return jsonify(packages[package].json) - except Exception: - pass - - r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) - return jsonify(r.json()) + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) if __name__ == '__main__': diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac5eb29b..17e5bab1 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -75,12 +75,20 @@ DEP_PIP_PAIRS = [ ] +def mock_unpack(link, source_dir, download_dir, only_download=False, session=None, + hashes=None, progress_bar="off"): + return + + @pytest.mark.utils @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) -def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] +def test_convert_deps_to_pip(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + if expected.startswith("Django"): + expected = expected.lower() + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] @pytest.mark.utils @@ -121,8 +129,11 @@ def test_convert_deps_to_pip(deps, expected): ), ], ) -def test_convert_deps_to_pip_one_way(deps, expected): - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] +def test_convert_deps_to_pip_one_way(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + # m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] @pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")