From 6ac1451ec86d2949855e557109c2eb92b61930fc Mon Sep 17 00:00:00 2001 From: Matt Davis Date: Sat, 19 Aug 2023 16:36:52 -0400 Subject: [PATCH] stop using requirementslib models (#5793) * Move away from requirementslib models * Revise test since PEP-440 does not support wildcard versions but does support equivalent compatible release specifiers. * simplify and remove dead code * Ensure the os_name marker is AND with the other markers. * Move what we still need from requirementslib into the pipenv utils and stop vendoring it. * Remove requirementslib. * force upgrade of virtualenv for python 3.12 * remove virtualenv-clone * Update vcs specifiers documentation; infer name from specific pip line formats where possible. * Provide helpful text and error for recently removed commands * Set the right log levels and verbosity to show users the errors generated by pip resolver when supplying -v flag * Fix the collection of all matching package hashes for non-pypi indexes. Plus lesson from testing torch which contains local identifiers. --- .github/workflows/ci.yaml | 2 +- Pipfile | 1 + Pipfile.lock | 757 ++--- docs/specifiers.md | 44 +- news/5793.bugfix.rst | 2 + pipenv/__init__.py | 20 +- pipenv/cli/command.py | 11 +- pipenv/cli/options.py | 82 + pipenv/environment.py | 53 +- pipenv/environments.py | 2 +- pipenv/pep508checker.py | 2 +- pipenv/project.py | 233 +- pipenv/resolver.py | 136 +- pipenv/routines/clear.py | 2 +- pipenv/routines/install.py | 234 +- pipenv/routines/lock.py | 11 +- pipenv/routines/outdated.py | 16 +- pipenv/routines/requirements.py | 55 +- pipenv/routines/uninstall.py | 4 +- pipenv/routines/update.py | 113 +- pipenv/shells.py | 2 +- pipenv/utils/__init__.py | 2 +- pipenv/utils/constants.py | 32 + pipenv/utils/dependencies.py | 1030 +++++- .../requirementslib => utils}/exceptions.py | 20 +- .../requirementslib => utils}/fileutils.py | 95 +- pipenv/utils/funktools.py | 297 +- pipenv/utils/internet.py | 21 +- pipenv/utils/locking.py | 411 ++- .../models => utils}/markers.py | 86 +- pipenv/utils/pip.py | 278 +- pipenv/utils/pipfile.py | 320 ++ pipenv/utils/project.py | 21 + pipenv/utils/requirements.py | 160 +- .../utils.py => utils/requirementslib.py} | 198 +- pipenv/utils/resolver.py | 688 +--- pipenv/utils/shell.py | 2 +- pipenv/utils/toml.py | 64 + pipenv/vendor/requirementslib/LICENSE | 21 - pipenv/vendor/requirementslib/__init__.py | 16 - pipenv/vendor/requirementslib/environment.py | 17 - pipenv/vendor/requirementslib/funktools.py | 77 - .../vendor/requirementslib/models/__init__.py | 0 .../vendor/requirementslib/models/common.py | 28 - .../requirementslib/models/dependencies.py | 62 - .../vendor/requirementslib/models/lockfile.py | 248 -- .../vendor/requirementslib/models/metadata.py | 1083 ------- .../requirementslib/models/old_pip_utils.py | 97 - .../vendor/requirementslib/models/pipfile.py | 324 -- .../vendor/requirementslib/models/project.py | 69 - .../requirementslib/models/requirements.py | 2885 ----------------- .../requirementslib/models/setup_info.py | 1849 ----------- pipenv/vendor/requirementslib/models/url.py | 484 --- pipenv/vendor/requirementslib/models/utils.py | 787 ----- pipenv/vendor/requirementslib/models/vcs.py | 104 - pipenv/vendor/vendor.txt | 1 - setup.py | 5 +- tasks/release.py | 2 +- tasks/vendoring/__init__.py | 3 +- tests/integration/conftest.py | 37 +- tests/integration/test_cli.py | 7 +- tests/integration/test_import_requirements.py | 40 +- tests/integration/test_install_basic.py | 11 +- tests/integration/test_install_categories.py | 5 +- tests/integration/test_install_markers.py | 6 +- tests/integration/test_install_misc.py | 11 +- tests/integration/test_install_twists.py | 15 +- tests/integration/test_install_uri.py | 32 +- tests/integration/test_lock.py | 21 +- tests/integration/test_project.py | 2 +- tests/integration/test_requirements.py | 7 +- tests/integration/test_run.py | 1 - tests/integration/test_uninstall.py | 22 +- tests/integration/test_update.py | 3 +- tests/integration/test_windows.py | 4 +- tests/pypi | 2 +- tests/unit/test_utils.py | 25 +- tests/unit/test_vendor.py | 8 +- 78 files changed, 3608 insertions(+), 10320 deletions(-) create mode 100644 news/5793.bugfix.rst rename pipenv/{vendor/requirementslib => utils}/exceptions.py (80%) rename pipenv/{vendor/requirementslib => utils}/fileutils.py (71%) rename pipenv/{vendor/requirementslib/models => utils}/markers.py (95%) rename pipenv/{vendor/requirementslib/utils.py => utils/requirementslib.py} (86%) delete mode 100644 pipenv/vendor/requirementslib/LICENSE delete mode 100644 pipenv/vendor/requirementslib/__init__.py delete mode 100644 pipenv/vendor/requirementslib/environment.py delete mode 100644 pipenv/vendor/requirementslib/funktools.py delete mode 100644 pipenv/vendor/requirementslib/models/__init__.py delete mode 100644 pipenv/vendor/requirementslib/models/common.py delete mode 100644 pipenv/vendor/requirementslib/models/dependencies.py delete mode 100644 pipenv/vendor/requirementslib/models/lockfile.py delete mode 100644 pipenv/vendor/requirementslib/models/metadata.py delete mode 100644 pipenv/vendor/requirementslib/models/old_pip_utils.py delete mode 100644 pipenv/vendor/requirementslib/models/pipfile.py delete mode 100644 pipenv/vendor/requirementslib/models/project.py delete mode 100644 pipenv/vendor/requirementslib/models/requirements.py delete mode 100644 pipenv/vendor/requirementslib/models/setup_info.py delete mode 100644 pipenv/vendor/requirementslib/models/url.py delete mode 100644 pipenv/vendor/requirementslib/models/utils.py delete mode 100644 pipenv/vendor/requirementslib/models/vcs.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 63d6a496..6095bc62 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -75,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] # "3.12-dev" Windows CI hangs indefinitely os: [MacOS, Ubuntu, Windows] steps: diff --git a/Pipfile b/Pipfile index f1c2263c..67e0637e 100644 --- a/Pipfile +++ b/Pipfile @@ -25,6 +25,7 @@ myst-parser = {extras = ["linkify"], version = "*"} invoke = "==2.0.0" exceptiongroup = "==1.1.0" tomli = "*" +pyyaml = "==6.0.1" [packages] pytz = "*" diff --git a/Pipfile.lock b/Pipfile.lock index 72e3b730..a50d34de 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "eb62088849429f204f99c914f3043a4952ab00589b6c4bd14354cff8bf95e8d9" + "sha256": "8a01aed3d68063b2b3880a1bbbb8113abeeb21fddea18c6d68dbc40f594d9c54" }, "pipfile-spec": 6, "requires": {}, @@ -16,11 +16,11 @@ "default": { "pytz": { "hashes": [ - "sha256:01a0681c4b9684a28304615eba55d1ab31ae00bf68ec157ec3708a8182dbbcd0", - "sha256:78f4f37d8198e0627c5f1143240bb0206b8691d8d7ac6d78fee88b78733f8c4a" + "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588", + "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb" ], "index": "pypi", - "version": "==2022.7.1" + "version": "==2023.3" } }, "develop": { @@ -34,26 +34,25 @@ }, "arpeggio": { "hashes": [ - "sha256:448e332deb0e9ccd04046f1c6c14529d197f41bc2fdb3931e43fc209042fbdd3", - "sha256:d6b03839019bb8a68785f9292ee6a36b1954eb84b925b84a6b8a5e1e26d3ed3d" + "sha256:c790b2b06e226d2dd468e4fbfb5b7f506cec66416031fde1441cf1de2a0ba700", + "sha256:f7c8ae4f4056a89e020c24c7202ac8df3e2bc84e416746f20b0da35bb1de0250" ], - "version": "==2.0.0" + "version": "==2.0.2" }, "atomicwrites": { "hashes": [ "sha256:81b2c9071a49367a7f770170e5eec8cb66567cfbbc8c73d20ce5ca4a8d71cf11" ], - "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==1.4.1" }, "attrs": { "hashes": [ - "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836", - "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99" + "sha256:1f28b4522cdc2fb4256ac1a020c78acf9cba2c6b461ccd2c126f3aa8e8335d04", + "sha256:6279836d581513a26f1bf235f9acd333bc9115683f14f7e8fae46c98fc50e015" ], - "markers": "python_version >= '3.6'", - "version": "==22.2.0" + "markers": "python_version >= '3.7'", + "version": "==23.1.0" }, "babel": { "hashes": [ @@ -65,50 +64,50 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:0e79446b10b3ecb499c1556f7e228a53e64a2bfcebd455f370d8927cb5b59e39", - "sha256:bc4bdda6717de5a2987436fb8d72f45dc90dd856bdfd512a1314ce90349a0106" + "sha256:492bbc69dca35d12daac71c4db1bfff0c876c00ef4a2ffacce226d4638eb72da", + "sha256:bd2520ca0d9d7d12694a53d44ac482d181b4ec1888909b035a3dbf40d0f57d4a" ], "markers": "python_full_version >= '3.6.0'", - "version": "==4.11.2" + "version": "==4.12.2" }, "black": { "hashes": [ - "sha256:0052dba51dec07ed029ed61b18183942043e00008ec65d5028814afaab9a22fd", - "sha256:0680d4380db3719ebcfb2613f34e86c8e6d15ffeabcf8ec59355c5e7b85bb555", - "sha256:121ca7f10b4a01fd99951234abdbd97728e1240be89fde18480ffac16503d481", - "sha256:162e37d49e93bd6eb6f1afc3e17a3d23a823042530c37c3c42eeeaf026f38468", - "sha256:2a951cc83ab535d248c89f300eccbd625e80ab880fbcfb5ac8afb5f01a258ac9", - "sha256:2bf649fda611c8550ca9d7592b69f0637218c2369b7744694c5e4902873b2f3a", - "sha256:382998821f58e5c8238d3166c492139573325287820963d2f7de4d518bd76958", - "sha256:49f7b39e30f326a34b5c9a4213213a6b221d7ae9d58ec70df1c4a307cf2a1580", - "sha256:57c18c5165c1dbe291d5306e53fb3988122890e57bd9b3dcb75f967f13411a26", - "sha256:7a0f701d314cfa0896b9001df70a530eb2472babb76086344e688829efd97d32", - "sha256:8178318cb74f98bc571eef19068f6ab5613b3e59d4f47771582f04e175570ed8", - "sha256:8b70eb40a78dfac24842458476135f9b99ab952dd3f2dab738c1881a9b38b753", - "sha256:9880d7d419bb7e709b37e28deb5e68a49227713b623c72b2b931028ea65f619b", - "sha256:9afd3f493666a0cd8f8df9a0200c6359ac53940cbde049dcb1a7eb6ee2dd7074", - "sha256:a29650759a6a0944e7cca036674655c2f0f63806ddecc45ed40b7b8aa314b651", - "sha256:a436e7881d33acaf2536c46a454bb964a50eff59b21b51c6ccf5a40601fbef24", - "sha256:a59db0a2094d2259c554676403fa2fac3473ccf1354c1c63eccf7ae65aac8ab6", - "sha256:a8471939da5e824b891b25751955be52ee7f8a30a916d570a5ba8e0f2eb2ecad", - "sha256:b0bd97bea8903f5a2ba7219257a44e3f1f9d00073d6cc1add68f0beec69692ac", - "sha256:b6a92a41ee34b883b359998f0c8e6eb8e99803aa8bf3123bf2b2e6fec505a221", - "sha256:bb460c8561c8c1bec7824ecbc3ce085eb50005883a6203dcfb0122e95797ee06", - "sha256:bfffba28dc52a58f04492181392ee380e95262af14ee01d4bc7bb1b1c6ca8d27", - "sha256:c1c476bc7b7d021321e7d93dc2cbd78ce103b84d5a4cf97ed535fbc0d6660648", - "sha256:c91dfc2c2a4e50df0026f88d2215e166616e0c80e86004d0003ece0488db2739", - "sha256:e6663f91b6feca5d06f2ccd49a10f254f9298cc1f7f49c46e498a0771b507104" + "sha256:064101748afa12ad2291c2b91c960be28b817c0c7eaa35bec09cc63aa56493c5", + "sha256:0945e13506be58bf7db93ee5853243eb368ace1c08a24c65ce108986eac65915", + "sha256:11c410f71b876f961d1de77b9699ad19f939094c3a677323f43d7a29855fe326", + "sha256:1c7b8d606e728a41ea1ccbd7264677e494e87cf630e399262ced92d4a8dac940", + "sha256:1d06691f1eb8de91cd1b322f21e3bfc9efe0c7ca1f0e1eb1db44ea367dff656b", + "sha256:3238f2aacf827d18d26db07524e44741233ae09a584273aa059066d644ca7b30", + "sha256:32daa9783106c28815d05b724238e30718f34155653d4d6e125dc7daec8e260c", + "sha256:35d1381d7a22cc5b2be2f72c7dfdae4072a3336060635718cc7e1ede24221d6c", + "sha256:3a150542a204124ed00683f0db1f5cf1c2aaaa9cc3495b7a3b5976fb136090ab", + "sha256:48f9d345675bb7fbc3dd85821b12487e1b9a75242028adad0333ce36ed2a6d27", + "sha256:50cb33cac881766a5cd9913e10ff75b1e8eb71babf4c7104f2e9c52da1fb7de2", + "sha256:562bd3a70495facf56814293149e51aa1be9931567474993c7942ff7d3533961", + "sha256:67de8d0c209eb5b330cce2469503de11bca4085880d62f1628bd9972cc3366b9", + "sha256:6b39abdfb402002b8a7d030ccc85cf5afff64ee90fa4c5aebc531e3ad0175ddb", + "sha256:6f3c333ea1dd6771b2d3777482429864f8e258899f6ff05826c3a4fcc5ce3f70", + "sha256:714290490c18fb0126baa0fca0a54ee795f7502b44177e1ce7624ba1c00f2331", + "sha256:7c3eb7cea23904399866c55826b31c1f55bbcd3890ce22ff70466b907b6775c2", + "sha256:92c543f6854c28a3c7f39f4d9b7694f9a6eb9d3c5e2ece488c327b6e7ea9b266", + "sha256:a6f6886c9869d4daae2d1715ce34a19bbc4b95006d20ed785ca00fa03cba312d", + "sha256:a8a968125d0a6a404842fa1bf0b349a568634f856aa08ffaff40ae0dfa52e7c6", + "sha256:c7ab5790333c448903c4b721b59c0d80b11fe5e9803d8703e84dcb8da56fec1b", + "sha256:e114420bf26b90d4b9daa597351337762b63039752bdf72bf361364c1aa05925", + "sha256:e198cf27888ad6f4ff331ca1c48ffc038848ea9f031a3b40ba36aced7e22f2c8", + "sha256:ec751418022185b0c1bb7d7736e6933d40bbb14c14a0abcf9123d1b159f98dd4", + "sha256:f0bd2f4a58d6666500542b26354978218a9babcdc972722f4bf90779524515f3" ], "markers": "python_version >= '3.7'", - "version": "==23.1.0" + "version": "==23.3.0" }, "certifi": { "hashes": [ - "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", - "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" + "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082", + "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9" ], "markers": "python_version >= '3.6'", - "version": "==2022.12.7" + "version": "==2023.7.22" }, "cfgv": { "hashes": [ @@ -120,96 +119,84 @@ }, "charset-normalizer": { "hashes": [ - "sha256:00d3ffdaafe92a5dc603cb9bd5111aaa36dfa187c8285c543be562e61b755f6b", - "sha256:024e606be3ed92216e2b6952ed859d86b4cfa52cd5bc5f050e7dc28f9b43ec42", - "sha256:0298eafff88c99982a4cf66ba2efa1128e4ddaca0b05eec4c456bbc7db691d8d", - "sha256:02a51034802cbf38db3f89c66fb5d2ec57e6fe7ef2f4a44d070a593c3688667b", - "sha256:083c8d17153ecb403e5e1eb76a7ef4babfc2c48d58899c98fcaa04833e7a2f9a", - "sha256:0a11e971ed097d24c534c037d298ad32c6ce81a45736d31e0ff0ad37ab437d59", - "sha256:0bf2dae5291758b6f84cf923bfaa285632816007db0330002fa1de38bfcb7154", - "sha256:0c0a590235ccd933d9892c627dec5bc7511ce6ad6c1011fdf5b11363022746c1", - "sha256:0f438ae3532723fb6ead77e7c604be7c8374094ef4ee2c5e03a3a17f1fca256c", - "sha256:109487860ef6a328f3eec66f2bf78b0b72400280d8f8ea05f69c51644ba6521a", - "sha256:11b53acf2411c3b09e6af37e4b9005cba376c872503c8f28218c7243582df45d", - "sha256:12db3b2c533c23ab812c2b25934f60383361f8a376ae272665f8e48b88e8e1c6", - "sha256:14e76c0f23218b8f46c4d87018ca2e441535aed3632ca134b10239dfb6dadd6b", - "sha256:16a8663d6e281208d78806dbe14ee9903715361cf81f6d4309944e4d1e59ac5b", - "sha256:292d5e8ba896bbfd6334b096e34bffb56161c81408d6d036a7dfa6929cff8783", - "sha256:2c03cc56021a4bd59be889c2b9257dae13bf55041a3372d3295416f86b295fb5", - "sha256:2e396d70bc4ef5325b72b593a72c8979999aa52fb8bcf03f701c1b03e1166918", - "sha256:2edb64ee7bf1ed524a1da60cdcd2e1f6e2b4f66ef7c077680739f1641f62f555", - "sha256:31a9ddf4718d10ae04d9b18801bd776693487cbb57d74cc3458a7673f6f34639", - "sha256:356541bf4381fa35856dafa6a965916e54bed415ad8a24ee6de6e37deccf2786", - "sha256:358a7c4cb8ba9b46c453b1dd8d9e431452d5249072e4f56cfda3149f6ab1405e", - "sha256:37f8febc8ec50c14f3ec9637505f28e58d4f66752207ea177c1d67df25da5aed", - "sha256:39049da0ffb96c8cbb65cbf5c5f3ca3168990adf3551bd1dee10c48fce8ae820", - "sha256:39cf9ed17fe3b1bc81f33c9ceb6ce67683ee7526e65fde1447c772afc54a1bb8", - "sha256:3ae1de54a77dc0d6d5fcf623290af4266412a7c4be0b1ff7444394f03f5c54e3", - "sha256:3b590df687e3c5ee0deef9fc8c547d81986d9a1b56073d82de008744452d6541", - "sha256:3e45867f1f2ab0711d60c6c71746ac53537f1684baa699f4f668d4c6f6ce8e14", - "sha256:3fc1c4a2ffd64890aebdb3f97e1278b0cc72579a08ca4de8cd2c04799a3a22be", - "sha256:4457ea6774b5611f4bed5eaa5df55f70abde42364d498c5134b7ef4c6958e20e", - "sha256:44ba614de5361b3e5278e1241fda3dc1838deed864b50a10d7ce92983797fa76", - "sha256:4a8fcf28c05c1f6d7e177a9a46a1c52798bfe2ad80681d275b10dcf317deaf0b", - "sha256:4b0d02d7102dd0f997580b51edc4cebcf2ab6397a7edf89f1c73b586c614272c", - "sha256:502218f52498a36d6bf5ea77081844017bf7982cdbe521ad85e64cabee1b608b", - "sha256:503e65837c71b875ecdd733877d852adbc465bd82c768a067badd953bf1bc5a3", - "sha256:5995f0164fa7df59db4746112fec3f49c461dd6b31b841873443bdb077c13cfc", - "sha256:59e5686dd847347e55dffcc191a96622f016bc0ad89105e24c14e0d6305acbc6", - "sha256:601f36512f9e28f029d9481bdaf8e89e5148ac5d89cffd3b05cd533eeb423b59", - "sha256:608862a7bf6957f2333fc54ab4399e405baad0163dc9f8d99cb236816db169d4", - "sha256:62595ab75873d50d57323a91dd03e6966eb79c41fa834b7a1661ed043b2d404d", - "sha256:70990b9c51340e4044cfc394a81f614f3f90d41397104d226f21e66de668730d", - "sha256:71140351489970dfe5e60fc621ada3e0f41104a5eddaca47a7acb3c1b851d6d3", - "sha256:72966d1b297c741541ca8cf1223ff262a6febe52481af742036a0b296e35fa5a", - "sha256:74292fc76c905c0ef095fe11e188a32ebd03bc38f3f3e9bcb85e4e6db177b7ea", - "sha256:761e8904c07ad053d285670f36dd94e1b6ab7f16ce62b9805c475b7aa1cffde6", - "sha256:772b87914ff1152b92a197ef4ea40efe27a378606c39446ded52c8f80f79702e", - "sha256:79909e27e8e4fcc9db4addea88aa63f6423ebb171db091fb4373e3312cb6d603", - "sha256:7e189e2e1d3ed2f4aebabd2d5b0f931e883676e51c7624826e0a4e5fe8a0bf24", - "sha256:7eb33a30d75562222b64f569c642ff3dc6689e09adda43a082208397f016c39a", - "sha256:81d6741ab457d14fdedc215516665050f3822d3e56508921cc7239f8c8e66a58", - "sha256:8499ca8f4502af841f68135133d8258f7b32a53a1d594aa98cc52013fff55678", - "sha256:84c3990934bae40ea69a82034912ffe5a62c60bbf6ec5bc9691419641d7d5c9a", - "sha256:87701167f2a5c930b403e9756fab1d31d4d4da52856143b609e30a1ce7160f3c", - "sha256:88600c72ef7587fe1708fd242b385b6ed4b8904976d5da0893e31df8b3480cb6", - "sha256:8ac7b6a045b814cf0c47f3623d21ebd88b3e8cf216a14790b455ea7ff0135d18", - "sha256:8b8af03d2e37866d023ad0ddea594edefc31e827fee64f8de5611a1dbc373174", - "sha256:8c7fe7afa480e3e82eed58e0ca89f751cd14d767638e2550c77a92a9e749c317", - "sha256:8eade758719add78ec36dc13201483f8e9b5d940329285edcd5f70c0a9edbd7f", - "sha256:911d8a40b2bef5b8bbae2e36a0b103f142ac53557ab421dc16ac4aafee6f53dc", - "sha256:93ad6d87ac18e2a90b0fe89df7c65263b9a99a0eb98f0a3d2e079f12a0735837", - "sha256:95dea361dd73757c6f1c0a1480ac499952c16ac83f7f5f4f84f0658a01b8ef41", - "sha256:9ab77acb98eba3fd2a85cd160851816bfce6871d944d885febf012713f06659c", - "sha256:9cb3032517f1627cc012dbc80a8ec976ae76d93ea2b5feaa9d2a5b8882597579", - "sha256:9cf4e8ad252f7c38dd1f676b46514f92dc0ebeb0db5552f5f403509705e24753", - "sha256:9d9153257a3f70d5f69edf2325357251ed20f772b12e593f3b3377b5f78e7ef8", - "sha256:a152f5f33d64a6be73f1d30c9cc82dfc73cec6477ec268e7c6e4c7d23c2d2291", - "sha256:a16418ecf1329f71df119e8a65f3aa68004a3f9383821edcb20f0702934d8087", - "sha256:a60332922359f920193b1d4826953c507a877b523b2395ad7bc716ddd386d866", - "sha256:a8d0fc946c784ff7f7c3742310cc8a57c5c6dc31631269876a88b809dbeff3d3", - "sha256:ab5de034a886f616a5668aa5d098af2b5385ed70142090e2a31bcbd0af0fdb3d", - "sha256:c22d3fe05ce11d3671297dc8973267daa0f938b93ec716e12e0f6dee81591dc1", - "sha256:c2ac1b08635a8cd4e0cbeaf6f5e922085908d48eb05d44c5ae9eabab148512ca", - "sha256:c512accbd6ff0270939b9ac214b84fb5ada5f0409c44298361b2f5e13f9aed9e", - "sha256:c75ffc45f25324e68ab238cb4b5c0a38cd1c3d7f1fb1f72b5541de469e2247db", - "sha256:c95a03c79bbe30eec3ec2b7f076074f4281526724c8685a42872974ef4d36b72", - "sha256:cadaeaba78750d58d3cc6ac4d1fd867da6fc73c88156b7a3212a3cd4819d679d", - "sha256:cd6056167405314a4dc3c173943f11249fa0f1b204f8b51ed4bde1a9cd1834dc", - "sha256:db72b07027db150f468fbada4d85b3b2729a3db39178abf5c543b784c1254539", - "sha256:df2c707231459e8a4028eabcd3cfc827befd635b3ef72eada84ab13b52e1574d", - "sha256:e62164b50f84e20601c1ff8eb55620d2ad25fb81b59e3cd776a1902527a788af", - "sha256:e696f0dd336161fca9adbb846875d40752e6eba585843c768935ba5c9960722b", - "sha256:eaa379fcd227ca235d04152ca6704c7cb55564116f8bc52545ff357628e10602", - "sha256:ebea339af930f8ca5d7a699b921106c6e29c617fe9606fa7baa043c1cdae326f", - "sha256:f4c39b0e3eac288fedc2b43055cfc2ca7a60362d0e5e87a637beac5d801ef478", - "sha256:f5057856d21e7586765171eac8b9fc3f7d44ef39425f85dbcccb13b3ebea806c", - "sha256:f6f45710b4459401609ebebdbcfb34515da4fc2aa886f95107f556ac69a9147e", - "sha256:f97e83fa6c25693c7a35de154681fcc257c1c41b38beb0304b9c4d2d9e164479", - "sha256:f9d0c5c045a3ca9bedfc35dca8526798eb91a07aa7a2c0fee134c6c6f321cbd7", - "sha256:ff6f3db31555657f3163b15a6b7c6938d08df7adbfc9dd13d9d19edad678f1e8" + "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96", + "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c", + "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710", + "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706", + "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020", + "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252", + "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad", + "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329", + "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a", + "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f", + "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6", + "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4", + "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a", + "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46", + "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2", + "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23", + "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace", + "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd", + "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982", + "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10", + "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2", + "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea", + "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09", + "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5", + "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149", + "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489", + "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9", + "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80", + "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592", + "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3", + "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6", + "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed", + "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c", + "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200", + "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a", + "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e", + "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d", + "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6", + "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623", + "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669", + "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3", + "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa", + "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9", + "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2", + "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f", + "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1", + "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4", + "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a", + "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8", + "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3", + "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029", + "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f", + "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959", + "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22", + "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7", + "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952", + "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346", + "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e", + "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d", + "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299", + "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd", + "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a", + "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3", + "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037", + "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94", + "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c", + "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858", + "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a", + "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449", + "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c", + "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918", + "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1", + "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c", + "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac", + "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa" ], - "version": "==3.0.1" + "markers": "python_full_version >= '3.7.0'", + "version": "==3.2.0" }, "click": { "hashes": [ @@ -217,20 +204,22 @@ "sha256:410e932b050f5eed773c4cda94de75971c89cdb3155a72a0831139a79e5ecb5b" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==8.0.3" }, "click-default-group": { "hashes": [ - "sha256:d9560e8e8dfa44b3562fbc9425042a0fd6d21956fcc2db0077f63f34253ab904" + "sha256:9b60486923720e7fc61731bdb32b617039aba820e22e1c88766b1125592eaa5f", + "sha256:eb3f3c99ec0d456ca6cd2a7f08f7d4e91771bef51b01bdd9580cc6450fe1251e" ], - "version": "==1.2.2" + "markers": "python_version >= '2.7'", + "version": "==1.2.4" }, "colorama": { "hashes": [ "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" ], - "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==0.4.6" }, @@ -239,67 +228,76 @@ "toml" ], "hashes": [ - "sha256:0339dc3237c0d31c3b574f19c57985fcbe494280153bbcad33f2cdf469f4ac3e", - "sha256:09643fb0df8e29f7417adc3f40aaf379d071ee8f0350ab290517c7004f05360b", - "sha256:0bd7e628f6c3ec4e7d2d24ec0e50aae4e5ae95ea644e849d92ae4805650b4c4e", - "sha256:0cf557827be7eca1c38a2480484d706693e7bb1929e129785fe59ec155a59de6", - "sha256:0f8318ed0f3c376cfad8d3520f496946977abde080439d6689d7799791457454", - "sha256:1b7fb13850ecb29b62a447ac3516c777b0e7a09ecb0f4bb6718a8654c87dfc80", - "sha256:22c308bc508372576ffa3d2dbc4824bb70d28eeb4fcd79d4d1aed663a06630d0", - "sha256:3004765bca3acd9e015794e5c2f0c9a05587f5e698127ff95e9cfba0d3f29339", - "sha256:3a209d512d157379cc9ab697cbdbb4cfd18daa3e7eebaa84c3d20b6af0037384", - "sha256:436313d129db7cf5b4ac355dd2bd3f7c7e5294af077b090b85de75f8458b8616", - "sha256:49567ec91fc5e0b15356da07a2feabb421d62f52a9fff4b1ec40e9e19772f5f8", - "sha256:4dd34a935de268a133e4741827ae951283a28c0125ddcdbcbba41c4b98f2dfef", - "sha256:570c21a29493b350f591a4b04c158ce1601e8d18bdcd21db136fbb135d75efa6", - "sha256:5928b85416a388dd557ddc006425b0c37e8468bd1c3dc118c1a3de42f59e2a54", - "sha256:5d2b9b5e70a21474c105a133ba227c61bc95f2ac3b66861143ce39a5ea4b3f84", - "sha256:617a94ada56bbfe547aa8d1b1a2b8299e2ec1ba14aac1d4b26a9f7d6158e1273", - "sha256:6a034480e9ebd4e83d1aa0453fd78986414b5d237aea89a8fdc35d330aa13bae", - "sha256:6fce673f79a0e017a4dc35e18dc7bb90bf6d307c67a11ad5e61ca8d42b87cbff", - "sha256:78d2c3dde4c0b9be4b02067185136b7ee4681978228ad5ec1278fa74f5ca3e99", - "sha256:7f099da6958ddfa2ed84bddea7515cb248583292e16bb9231d151cd528eab657", - "sha256:80559eaf6c15ce3da10edb7977a1548b393db36cbc6cf417633eca05d84dd1ed", - "sha256:834c2172edff5a08d78e2f53cf5e7164aacabeb66b369f76e7bb367ca4e2d993", - "sha256:861cc85dfbf55a7a768443d90a07e0ac5207704a9f97a8eb753292a7fcbdfcfc", - "sha256:8649371570551d2fd7dee22cfbf0b61f1747cdfb2b7587bb551e4beaaa44cb97", - "sha256:87dc37f16fb5e3a28429e094145bf7c1753e32bb50f662722e378c5851f7fdc6", - "sha256:8a6450da4c7afc4534305b2b7d8650131e130610cea448ff240b6ab73d7eab63", - "sha256:8d3843ca645f62c426c3d272902b9de90558e9886f15ddf5efe757b12dd376f5", - "sha256:8dca3c1706670297851bca1acff9618455122246bdae623be31eca744ade05ec", - "sha256:97a3189e019d27e914ecf5c5247ea9f13261d22c3bb0cfcfd2a9b179bb36f8b1", - "sha256:99f4dd81b2bb8fc67c3da68b1f5ee1650aca06faa585cbc6818dbf67893c6d58", - "sha256:9e872b082b32065ac2834149dc0adc2a2e6d8203080501e1e3c3c77851b466f9", - "sha256:a81dbcf6c6c877986083d00b834ac1e84b375220207a059ad45d12f6e518a4e3", - "sha256:abacd0a738e71b20e224861bc87e819ef46fedba2fb01bc1af83dfd122e9c319", - "sha256:ae82c988954722fa07ec5045c57b6d55bc1a0890defb57cf4a712ced65b26ddd", - "sha256:b0c0d46de5dd97f6c2d1b560bf0fcf0215658097b604f1840365296302a9d1fb", - "sha256:b1991a6d64231a3e5bbe3099fb0dd7c9aeaa4275ad0e0aeff4cb9ef885c62ba2", - "sha256:b2167d116309f564af56f9aa5e75ef710ef871c5f9b313a83050035097b56820", - "sha256:bd5a12239c0006252244f94863f1c518ac256160cd316ea5c47fb1a11b25889a", - "sha256:bdd3f2f285ddcf2e75174248b2406189261a79e7fedee2ceeadc76219b6faa0e", - "sha256:c77f2a9093ccf329dd523a9b2b3c854c20d2a3d968b6def3b820272ca6732242", - "sha256:cb5f152fb14857cbe7f3e8c9a5d98979c4c66319a33cad6e617f0067c9accdc4", - "sha256:cca7c0b7f5881dfe0291ef09ba7bb1582cb92ab0aeffd8afb00c700bf692415a", - "sha256:d2ef6cae70168815ed91388948b5f4fcc69681480a0061114db737f957719f03", - "sha256:d9256d4c60c4bbfec92721b51579c50f9e5062c21c12bec56b55292464873508", - "sha256:e191a63a05851f8bce77bc875e75457f9b01d42843f8bd7feed2fc26bbe60833", - "sha256:e2b50ebc2b6121edf352336d503357321b9d8738bb7a72d06fc56153fd3f4cd8", - "sha256:e3ea04b23b114572b98a88c85379e9e9ae031272ba1fb9b532aa934c621626d4", - "sha256:e4d70c853f0546855f027890b77854508bdb4d6a81242a9d804482e667fff6e6", - "sha256:f29351393eb05e6326f044a7b45ed8e38cb4dcc38570d12791f271399dc41431", - "sha256:f3d07edb912a978915576a776756069dede66d012baa503022d3a0adba1b6afa", - "sha256:fac6343bae03b176e9b58104a9810df3cdccd5cfed19f99adfa807ffbf43cf9b" + "sha256:06a9a2be0b5b576c3f18f1a241f0473575c4a26021b52b2a85263a00f034d51f", + "sha256:06fb182e69f33f6cd1d39a6c597294cff3143554b64b9825d1dc69d18cc2fff2", + "sha256:0a5f9e1dbd7fbe30196578ca36f3fba75376fb99888c395c5880b355e2875f8a", + "sha256:0e1f928eaf5469c11e886fe0885ad2bf1ec606434e79842a879277895a50942a", + "sha256:171717c7cb6b453aebac9a2ef603699da237f341b38eebfee9be75d27dc38e01", + "sha256:1e9d683426464e4a252bf70c3498756055016f99ddaec3774bf368e76bbe02b6", + "sha256:201e7389591af40950a6480bd9edfa8ed04346ff80002cec1a66cac4549c1ad7", + "sha256:245167dd26180ab4c91d5e1496a30be4cd721a5cf2abf52974f965f10f11419f", + "sha256:2aee274c46590717f38ae5e4650988d1af340fe06167546cc32fe2f58ed05b02", + "sha256:2e07b54284e381531c87f785f613b833569c14ecacdcb85d56b25c4622c16c3c", + "sha256:31563e97dae5598556600466ad9beea39fb04e0229e61c12eaa206e0aa202063", + "sha256:33d6d3ea29d5b3a1a632b3c4e4f4ecae24ef170b0b9ee493883f2df10039959a", + "sha256:3d376df58cc111dc8e21e3b6e24606b5bb5dee6024f46a5abca99124b2229ef5", + "sha256:419bfd2caae268623dd469eff96d510a920c90928b60f2073d79f8fe2bbc5959", + "sha256:48c19d2159d433ccc99e729ceae7d5293fbffa0bdb94952d3579983d1c8c9d97", + "sha256:49969a9f7ffa086d973d91cec8d2e31080436ef0fb4a359cae927e742abfaaa6", + "sha256:52edc1a60c0d34afa421c9c37078817b2e67a392cab17d97283b64c5833f427f", + "sha256:537891ae8ce59ef63d0123f7ac9e2ae0fc8b72c7ccbe5296fec45fd68967b6c9", + "sha256:54b896376ab563bd38453cecb813c295cf347cf5906e8b41d340b0321a5433e5", + "sha256:58c2ccc2f00ecb51253cbe5d8d7122a34590fac9646a960d1430d5b15321d95f", + "sha256:5b7540161790b2f28143191f5f8ec02fb132660ff175b7747b95dcb77ac26562", + "sha256:5baa06420f837184130752b7c5ea0808762083bf3487b5038d68b012e5937dbe", + "sha256:5e330fc79bd7207e46c7d7fd2bb4af2963f5f635703925543a70b99574b0fea9", + "sha256:61b9a528fb348373c433e8966535074b802c7a5d7f23c4f421e6c6e2f1697a6f", + "sha256:63426706118b7f5cf6bb6c895dc215d8a418d5952544042c8a2d9fe87fcf09cb", + "sha256:6d040ef7c9859bb11dfeb056ff5b3872436e3b5e401817d87a31e1750b9ae2fb", + "sha256:6f48351d66575f535669306aa7d6d6f71bc43372473b54a832222803eb956fd1", + "sha256:7ee7d9d4822c8acc74a5e26c50604dff824710bc8de424904c0982e25c39c6cb", + "sha256:81c13a1fc7468c40f13420732805a4c38a105d89848b7c10af65a90beff25250", + "sha256:8d13c64ee2d33eccf7437961b6ea7ad8673e2be040b4f7fd4fd4d4d28d9ccb1e", + "sha256:8de8bb0e5ad103888d65abef8bca41ab93721647590a3f740100cd65c3b00511", + "sha256:8fa03bce9bfbeeef9f3b160a8bed39a221d82308b4152b27d82d8daa7041fee5", + "sha256:924d94291ca674905fe9481f12294eb11f2d3d3fd1adb20314ba89e94f44ed59", + "sha256:975d70ab7e3c80a3fe86001d8751f6778905ec723f5b110aed1e450da9d4b7f2", + "sha256:976b9c42fb2a43ebf304fa7d4a310e5f16cc99992f33eced91ef6f908bd8f33d", + "sha256:9e31cb64d7de6b6f09702bb27c02d1904b3aebfca610c12772452c4e6c21a0d3", + "sha256:a342242fe22407f3c17f4b499276a02b01e80f861f1682ad1d95b04018e0c0d4", + "sha256:a3d33a6b3eae87ceaefa91ffdc130b5e8536182cd6dfdbfc1aa56b46ff8c86de", + "sha256:a895fcc7b15c3fc72beb43cdcbdf0ddb7d2ebc959edac9cef390b0d14f39f8a9", + "sha256:afb17f84d56068a7c29f5fa37bfd38d5aba69e3304af08ee94da8ed5b0865833", + "sha256:b1c546aca0ca4d028901d825015dc8e4d56aac4b541877690eb76490f1dc8ed0", + "sha256:b29019c76039dc3c0fd815c41392a044ce555d9bcdd38b0fb60fb4cd8e475ba9", + "sha256:b46517c02ccd08092f4fa99f24c3b83d8f92f739b4657b0f146246a0ca6a831d", + "sha256:b7aa5f8a41217360e600da646004f878250a0d6738bcdc11a0a39928d7dc2050", + "sha256:b7b4c971f05e6ae490fef852c218b0e79d4e52f79ef0c8475566584a8fb3e01d", + "sha256:ba90a9563ba44a72fda2e85302c3abc71c5589cea608ca16c22b9804262aaeb6", + "sha256:cb017fd1b2603ef59e374ba2063f593abe0fc45f2ad9abdde5b4d83bd922a353", + "sha256:d22656368f0e6189e24722214ed8d66b8022db19d182927b9a248a2a8a2f67eb", + "sha256:d2c2db7fd82e9b72937969bceac4d6ca89660db0a0967614ce2481e81a0b771e", + "sha256:d39b5b4f2a66ccae8b7263ac3c8170994b65266797fb96cbbfd3fb5b23921db8", + "sha256:d62a5c7dad11015c66fbb9d881bc4caa5b12f16292f857842d9d1871595f4495", + "sha256:e7d9405291c6928619403db1d10bd07888888ec1abcbd9748fdaa971d7d661b2", + "sha256:e84606b74eb7de6ff581a7915e2dab7a28a0517fbe1c9239eb227e1354064dcd", + "sha256:eb393e5ebc85245347950143969b241d08b52b88a3dc39479822e073a1a8eb27", + "sha256:ebba1cd308ef115925421d3e6a586e655ca5a77b5bf41e02eb0e4562a111f2d1", + "sha256:ee57190f24fba796e36bb6d3aa8a8783c643d8fa9760c89f7a98ab5455fbf818", + "sha256:f2f67fe12b22cd130d34d0ef79206061bfb5eda52feb6ce0dba0644e20a03cf4", + "sha256:f6951407391b639504e3b3be51b7ba5f3528adbf1a8ac3302b687ecababf929e", + "sha256:f75f7168ab25dd93110c8a8117a22450c19976afbc44234cbf71481094c1b850", + "sha256:fdec9e8cbf13a5bf63290fc6013d216a4c7232efb51548594ca3631a7f13c3a3" ], "markers": "python_version >= '3.7'", - "version": "==7.2.1" + "version": "==7.2.7" }, "distlib": { "hashes": [ - "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46", - "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e" + "sha256:2e24928bc811348f0feb63014e97aaae3037f2cf48712d51ae61df7fd6075057", + "sha256:9dafe54b34a028eafd95039d5e5d4851a13734540f1331060d31c9916e7147a8" ], - "version": "==0.3.6" + "version": "==0.3.7" }, "docutils": { "hashes": [ @@ -315,23 +313,24 @@ "sha256:bcb67d800a4497e1b404c2dd44fca47d3b7a5e5433dbab67f96c1a685cdfdf23" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==1.1.0" }, "execnet": { "hashes": [ - "sha256:8f694f3ba9cc92cab508b152dcfe322153975c29bda272e2fd7f3f00f36e47c5", - "sha256:a295f7cc774947aac58dde7fdc85f4aa00c42adf5d8f5468fc630c1acf30a142" + "sha256:88256416ae766bc9e8895c76a87928c0012183da3cc4fc18016e6f050e025f41", + "sha256:cc59bc4423742fd71ad227122eb0dd44db51efb3dc4095b45ac9a08c770096af" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'", - "version": "==1.9.0" + "markers": "python_version >= '3.7'", + "version": "==2.0.2" }, "filelock": { "hashes": [ - "sha256:7b319f24340b51f55a2bf7a12ac0755a9b03e718311dac567a0f4f7fabd2f5de", - "sha256:f58d535af89bb9ad5cd4df046f741f8553a418c01a7856bf0d173bbc9f6bd16d" + "sha256:002740518d8aa59a26b0c76e10fb8c6e15eae825d34b6fdf670333fd7b938d81", + "sha256:cbb791cdea2a72f23da6ac5b5269ab0a0d161e9ef0100e653b69049a7706d1ec" ], "markers": "python_version >= '3.7'", - "version": "==3.9.0" + "version": "==3.12.2" }, "flake8": { "hashes": [ @@ -359,11 +358,11 @@ }, "identify": { "hashes": [ - "sha256:89e144fa560cc4cffb6ef2ab5e9fb18ed9f9b3cb054384bab4b95c12f6c309fe", - "sha256:93aac7ecf2f6abf879b8f29a8002d3c6de7086b8c28d88e1ad15045a15ab63f9" + "sha256:7243800bce2f58404ed41b7c002e53d4d22bcf3ae1b7900c2d7aefd95394bf7f", + "sha256:c22a8ead0d4ca11f1edd6c9418c3220669b3b7533ada0a0ffa6cc0ef85cf9b54" ], - "markers": "python_version >= '3.7'", - "version": "==2.5.18" + "markers": "python_version >= '3.8'", + "version": "==2.5.26" }, "idna": { "hashes": [ @@ -410,6 +409,7 @@ "sha256:a860582bcf7a4b336fe18ef53937f0f28cec1c0053ffa767c2fcf7ba0b850f59" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==2.0.0" }, "jinja2": { @@ -437,59 +437,59 @@ }, "markupsafe": { "hashes": [ - "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed", - "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc", - "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2", - "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460", - "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7", - "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0", - "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1", - "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa", - "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03", - "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323", - "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65", - "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013", - "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036", - "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f", - "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4", - "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419", - "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2", - "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619", - "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a", - "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a", - "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd", - "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7", - "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666", - "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65", - "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859", - "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625", - "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff", - "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156", - "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd", - "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba", - "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f", - "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1", - "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094", - "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a", - "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513", - "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed", - "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d", - "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3", - "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147", - "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c", - "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603", - "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601", - "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a", - "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1", - "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d", - "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3", - "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54", - "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2", - "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6", - "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58" + "sha256:05fb21170423db021895e1ea1e1f3ab3adb85d1c2333cbc2310f2a26bc77272e", + "sha256:0a4e4a1aff6c7ac4cd55792abf96c915634c2b97e3cc1c7129578aa68ebd754e", + "sha256:10bbfe99883db80bdbaff2dcf681dfc6533a614f700da1287707e8a5d78a8431", + "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686", + "sha256:1577735524cdad32f9f694208aa75e422adba74f1baee7551620e43a3141f559", + "sha256:1b40069d487e7edb2676d3fbdb2b0829ffa2cd63a2ec26c4938b2d34391b4ecc", + "sha256:282c2cb35b5b673bbcadb33a585408104df04f14b2d9b01d4c345a3b92861c2c", + "sha256:2c1b19b3aaacc6e57b7e25710ff571c24d6c3613a45e905b1fde04d691b98ee0", + "sha256:2ef12179d3a291be237280175b542c07a36e7f60718296278d8593d21ca937d4", + "sha256:338ae27d6b8745585f87218a3f23f1512dbf52c26c28e322dbe54bcede54ccb9", + "sha256:3c0fae6c3be832a0a0473ac912810b2877c8cb9d76ca48de1ed31e1c68386575", + "sha256:3fd4abcb888d15a94f32b75d8fd18ee162ca0c064f35b11134be77050296d6ba", + "sha256:42de32b22b6b804f42c5d98be4f7e5e977ecdd9ee9b660fda1a3edf03b11792d", + "sha256:504b320cd4b7eff6f968eddf81127112db685e81f7e36e75f9f84f0df46041c3", + "sha256:525808b8019e36eb524b8c68acdd63a37e75714eac50e988180b169d64480a00", + "sha256:56d9f2ecac662ca1611d183feb03a3fa4406469dafe241673d521dd5ae92a155", + "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac", + "sha256:65c1a9bcdadc6c28eecee2c119465aebff8f7a584dd719facdd9e825ec61ab52", + "sha256:68e78619a61ecf91e76aa3e6e8e33fc4894a2bebe93410754bd28fce0a8a4f9f", + "sha256:69c0f17e9f5a7afdf2cc9fb2d1ce6aabdb3bafb7f38017c0b77862bcec2bbad8", + "sha256:6b2b56950d93e41f33b4223ead100ea0fe11f8e6ee5f641eb753ce4b77a7042b", + "sha256:787003c0ddb00500e49a10f2844fac87aa6ce977b90b0feaaf9de23c22508b24", + "sha256:7ef3cb2ebbf91e330e3bb937efada0edd9003683db6b57bb108c4001f37a02ea", + "sha256:8023faf4e01efadfa183e863fefde0046de576c6f14659e8782065bcece22198", + "sha256:8758846a7e80910096950b67071243da3e5a20ed2546e6392603c096778d48e0", + "sha256:8afafd99945ead6e075b973fefa56379c5b5c53fd8937dad92c662da5d8fd5ee", + "sha256:8c41976a29d078bb235fea9b2ecd3da465df42a562910f9022f1a03107bd02be", + "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2", + "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707", + "sha256:962f82a3086483f5e5f64dbad880d31038b698494799b097bc59c2edf392fce6", + "sha256:9dcdfd0eaf283af041973bff14a2e143b8bd64e069f4c383416ecd79a81aab58", + "sha256:aa7bd130efab1c280bed0f45501b7c8795f9fdbeb02e965371bbef3523627779", + "sha256:ab4a0df41e7c16a1392727727e7998a467472d0ad65f3ad5e6e765015df08636", + "sha256:ad9e82fb8f09ade1c3e1b996a6337afac2b8b9e365f926f5a61aacc71adc5b3c", + "sha256:af598ed32d6ae86f1b747b82783958b1a4ab8f617b06fe68795c7f026abbdcad", + "sha256:b076b6226fb84157e3f7c971a47ff3a679d837cf338547532ab866c57930dbee", + "sha256:b7ff0f54cb4ff66dd38bebd335a38e2c22c41a8ee45aa608efc890ac3e3931bc", + "sha256:bfce63a9e7834b12b87c64d6b155fdd9b3b96191b6bd334bf37db7ff1fe457f2", + "sha256:c011a4149cfbcf9f03994ec2edffcb8b1dc2d2aede7ca243746df97a5d41ce48", + "sha256:c9c804664ebe8f83a211cace637506669e7890fec1b4195b505c214e50dd4eb7", + "sha256:ca379055a47383d02a5400cb0d110cef0a776fc644cda797db0c5696cfd7e18e", + "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b", + "sha256:cd0f502fe016460680cd20aaa5a76d241d6f35a1c3350c474bac1273803893fa", + "sha256:ceb01949af7121f9fc39f7d27f91be8546f3fb112c608bc4029aef0bab86a2a5", + "sha256:d080e0a5eb2529460b30190fcfcc4199bd7f827663f858a226a81bc27beaa97e", + "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb", + "sha256:df0be2b576a7abbf737b1575f048c23fb1d769f267ec4358296f31c2479db8f9", + "sha256:e09031c87a1e51556fdcb46e5bd4f59dfb743061cf93c4d6831bf894f125eb57", + "sha256:e4dd52d80b8c83fdce44e12478ad2e85c64ea965e75d66dbeafb0a3e77308fcc", + "sha256:fec21693218efe39aa7f8599346e90c705afa52c5b31ae019b2e57e8f6542bb2" ], "markers": "python_version >= '3.7'", - "version": "==2.1.2" + "version": "==2.1.3" }, "mccabe": { "hashes": [ @@ -500,11 +500,11 @@ }, "mdit-py-plugins": { "hashes": [ - "sha256:3278aab2e2b692539082f05e1243f24742194ffd92481f48844f057b51971283", - "sha256:4f1441264ac5cb39fa40a5901921c2acf314ea098d75629750c138f80d552cdf" + "sha256:ca9a0714ea59a24b2b044a1831f48d817dd0c817e84339f20e7889f392d77c4e", + "sha256:eee0adc7195e5827e17e02d2a258a2ba159944a0748f59c5099a4a27f78fcf6a" ], "markers": "python_version >= '3.7'", - "version": "==0.3.4" + "version": "==0.3.5" }, "mdurl": { "hashes": [ @@ -516,11 +516,11 @@ }, "mock": { "hashes": [ - "sha256:c41cfb1e99ba5d341fbcc5308836e7d7c9786d302f995b2c271ce2144dece9eb", - "sha256:e3ea505c03babf7977fd21674a69ad328053d414f05e6433c30d8fa14a534a6b" + "sha256:18c694e5ae8a208cdb3d2c20a993ca1a7b0efa258c247a1e565150f477f83744", + "sha256:5e96aad5ccda4718e0a229ed94b2024df75cc2d55575ba5762d31f5767b8767d" ], "markers": "python_version >= '3.6'", - "version": "==5.0.1" + "version": "==5.1.0" }, "mypy-extensions": { "hashes": [ @@ -538,31 +538,32 @@ "sha256:61b275b85d9f58aa327f370913ae1bec26ebad372cc99f3ab85c8ec3ee8d9fb8", "sha256:79317f4bb2c13053dd6e64f9da1ba1da6cd9c40c8a430c447a7b146a594c246d" ], - "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==0.18.1" }, "nodeenv": { "hashes": [ - "sha256:27083a7b96a25f2f5e1d8cb4b6317ee8aeda3bdd121394e5ac54e498028a042e", - "sha256:e0e7f7dfb85fc5394c6fe1e8fa98131a2473e04311a45afb6508f7cf1836fa2b" + "sha256:d51e0c37e64fbf47d017feac3145cdbb58836d7eee8c6f6d3b6880c5456227d2", + "sha256:df865724bb3c3adc86b3876fa209771517b0cfe596beff01a92700e0e8be4cec" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", - "version": "==1.7.0" + "version": "==1.8.0" }, "packaging": { "hashes": [ - "sha256:714ac14496c3e68c99c29b00845f7a2b85f3bb6f1078fd9f72fd20f0570002b2", - "sha256:b6ad297f8907de0fa2fe1ccbd26fdaf387f5f47c7275fedf8cce89f99446cf97" + "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61", + "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f" ], "markers": "python_version >= '3.7'", - "version": "==23.0" + "version": "==23.1" }, "parse": { "hashes": [ - "sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b" + "sha256:371ed3800dc63983832159cc9373156613947707bc448b5215473a219dbd4362", + "sha256:cc3a47236ff05da377617ddefa867b7ba983819c664e1afe46249e5b469be464" ], "index": "pypi", - "version": "==1.19.0" + "version": "==1.19.1" }, "parver": { "hashes": [ @@ -574,11 +575,19 @@ }, "pathspec": { "hashes": [ - "sha256:3a66eb970cbac598f9e5ccb5b2cf58930cd8e3ed86d393d541eaf2d8b1705229", - "sha256:64d338d4e0914e91c1792321e6907b5a593f1ab1851de7fc269557a21b30ebbc" + "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20", + "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3" ], "markers": "python_version >= '3.7'", - "version": "==0.11.0" + "version": "==0.11.2" + }, + "pip": { + "hashes": [ + "sha256:7ccf472345f20d35bdc9d1841ff5f313260c2c33fe417f48c30ac46cccabf5be", + "sha256:fb0bd5435b3200c602b5bf61d2d43c2f13c02e29c1707567ae7fbc514eb9faf2" + ], + "markers": "python_version >= '3.7'", + "version": "==23.2.1" }, "pipenv": { "editable": true, @@ -590,19 +599,19 @@ }, "platformdirs": { "hashes": [ - "sha256:8a1228abb1ef82d788f74139988b137e78692984ec7b08eaa6c65f1723af28f9", - "sha256:b1d5eb14f221506f50d6604a561f4c5786d9e80355219694a1b244bcd96f4567" + "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d", + "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d" ], "markers": "python_version >= '3.7'", - "version": "==3.0.0" + "version": "==3.10.0" }, "pluggy": { "hashes": [ - "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", - "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" + "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849", + "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3" ], - "markers": "python_version >= '3.6'", - "version": "==1.0.0" + "markers": "python_version >= '3.7'", + "version": "==1.2.0" }, "pre-commit": { "hashes": [ @@ -610,6 +619,7 @@ "sha256:e2f91727039fc39a92f58a588a25b87f936de6567eed4f0e673e0507edc75bad" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.21.0" }, "pycodestyle": { @@ -622,13 +632,13 @@ }, "pyenchant": { "hashes": [ - "sha256:1cf830c6614362a78aab78d50eaf7c6c93831369c52e1bb64ffae1df0341e637", - "sha256:5a636832987eaf26efe971968f4d1b78e81f62bca2bde0a9da210c7de43c3bce", - "sha256:5facc821ece957208a81423af7d6ec7810dad29697cb0d77aae81e4e11c8e5a6", - "sha256:6153f521852e23a5add923dbacfbf4bebbb8d70c4e4bad609a8e0f9faeb915d1" + "sha256:0314d162b7af83adc500f5aff850c91466129363ca8c4d79a8b8d99253346204", + "sha256:377a2aaafcb41f871c573c5b74e502dcc6ddba72a62deae7d36dc601a9fcad3d", + "sha256:960cbbf4ac99cf9c662308aa9a017ef23017dbea4c8c1e8329978ee4600b3f55", + "sha256:ee514d7adf8d0fe39d3a14088f5915953e7fb3bda35092e696fc38f78aabb8b8" ], - "markers": "python_version >= '3.5'", - "version": "==3.2.2" + "markers": "python_version >= '3.7'", + "version": "==3.3.0rc1" }, "pyflakes": { "hashes": [ @@ -640,27 +650,28 @@ }, "pygments": { "hashes": [ - "sha256:b3ed06a9e8ac9a9aae5a6f5dbe78a8a58655d17b43b93c078f094ddc476ae297", - "sha256:fa7bd7bd2771287c0de303af8bfdfc731f51bd2c6a47ab69d117138893b82717" + "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c", + "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1" ], - "markers": "python_version >= '3.6'", - "version": "==2.14.0" + "markers": "python_version >= '3.7'", + "version": "==2.15.1" }, "pypiserver": { "hashes": [ - "sha256:24804a717ccd4611aa805621eead821f9d389163f6020a500c7e05f191884ba2", - "sha256:bea067627793ebcab6574549b7a7e9099cceeb5b902a73f4df09734c11fc3cdf" + "sha256:09f2f797f92b30e92287821e2dc3ca72c8011aec6a2570019254adf98318ee5c", + "sha256:70760efadc3d89b3e1b3f54f078a6520f6c6a0c3dd718b46cd0cf466c9fd01b2" ], "index": "pypi", - "version": "==1.5.1" + "markers": "python_version >= '3.6'", + "version": "==1.5.2" }, "pytest": { "hashes": [ - "sha256:c7c6ca206e93355074ae32f7403e8ea12163b1163c976fee7d4d84027c162be5", - "sha256:d45e0952f3727241918b8fd0f376f5ff6b301cc0777c6f9a556935c92d8a7d42" + "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32", + "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a" ], "markers": "python_version >= '3.7'", - "version": "==7.2.1" + "version": "==7.4.0" }, "pytest-cov": { "hashes": [ @@ -668,6 +679,7 @@ "sha256:e7f0f5b1617d2210a2cabc266dfe2f4c75a8d32fb89eafb7ad9d06f6d076d470" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==3.0.0" }, "pytest-timeout": { @@ -680,73 +692,74 @@ }, "pytest-xdist": { "hashes": [ - "sha256:336098e3bbd8193276867cc87db8b22903c3927665dff9d1ac8684c02f597b68", - "sha256:fa10f95a2564cd91652f2d132725183c3b590d9fdcdec09d3677386ecf4c1ce9" + "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93", + "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2" ], "markers": "python_version >= '3.7'", - "version": "==3.2.0" + "version": "==3.3.1" }, "pyyaml": { "hashes": [ - "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", - "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", - "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", - "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", - "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", - "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", - "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", - "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", - "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", - "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", - "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", - "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", - "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", - "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", - "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", - "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", - "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", - "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", - "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", - "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", - "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", - "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", - "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", - "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", - "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", - "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", - "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", - "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", - "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", - "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", - "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", - "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", - "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", - "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", - "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", - "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", - "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", - "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", - "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", - "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" + "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc", + "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741", + "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206", + "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27", + "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595", + "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62", + "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98", + "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696", + "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d", + "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867", + "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47", + "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486", + "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6", + "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3", + "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007", + "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938", + "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c", + "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735", + "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d", + "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba", + "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8", + "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5", + "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd", + "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3", + "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0", + "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515", + "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c", + "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c", + "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924", + "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34", + "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43", + "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859", + "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673", + "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a", + "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab", + "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa", + "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c", + "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585", + "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d", + "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f" ], + "index": "pypi", "markers": "python_version >= '3.6'", - "version": "==6.0" + "version": "==6.0.1" }, "requests": { "hashes": [ - "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa", - "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf" + "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", + "sha256:942c5a758f98d790eaed1a29cb6eefc7ffb0d1cf7af05c3d2791656dbd6ad1e1" ], - "markers": "python_version >= '3.7' and python_version < '4'", - "version": "==2.28.2" + "markers": "python_version >= '3.7'", + "version": "==2.31.0" }, "setuptools": { "hashes": [ - "sha256:e5fd0a713141a4a105412233c63dc4e17ba0090c8e8334594ac790ec97792330", - "sha256:f106dee1b506dee5102cc3f3e9e68137bbad6d47b616be7991714b0c62204251" + "sha256:11e52c67415a381d10d6b462ced9cfb97066179f0e871399e006c4ab101fc85f", + "sha256:baf1fdb41c6da4cd2eae722e135500da913332ab3f2f5c7d33af9b492acb5235" ], "markers": "python_version >= '3.7'", - "version": "==67.4.0" + "version": "==68.0.0" }, "snowballstemmer": { "hashes": [ @@ -757,11 +770,11 @@ }, "soupsieve": { "hashes": [ - "sha256:49e5368c2cda80ee7e84da9dbe3e110b70a4575f196efb74e51b94549d921955", - "sha256:e28dba9ca6c7c00173e34e4ba57448f0688bb681b7c5e8bf4971daafc093d69a" + "sha256:1c1bfee6819544a3447586c889157365a27e10d88cde3ad3da0cf0ddf646feb8", + "sha256:89d12b2d5dfcd2c9e8c22326da9d9aa9cb3dfab0a83a024f05704076ee8d35ea" ], "markers": "python_version >= '3.7'", - "version": "==2.4" + "version": "==2.4.1" }, "sphinx": { "hashes": [ @@ -769,6 +782,7 @@ "sha256:ebf612653238bcc8f4359627a9b7ce44ede6fdd75d9d30f68255c7383d3a6226" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==4.5.0" }, "sphinx-click": { @@ -777,6 +791,7 @@ "sha256:cc67692bd28f482c7f01531c61b64e9d2f069bfcf3d24cbbb51d4a84a749fa48" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==4.4.0" }, "sphinxcontrib-applehelp": { @@ -833,6 +848,7 @@ "sha256:95a0defef8ffec6526f9e83b20cc24b08c9179298729d87976891840e3aa3064" ], "index": "pypi", + "markers": "python_version >= '3.6'", "version": "==7.7.0" }, "stdeb": { @@ -848,62 +864,55 @@ "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" ], "index": "pypi", + "markers": "python_version >= '3.7'", "version": "==2.0.1" }, "towncrier": { "hashes": [ - "sha256:9767a899a4d6856950f3598acd9e8f08da2663c49fdcda5ea0f9e6ba2afc8eea", - "sha256:9c49d7e75f646a9aea02ae904c0bc1639c8fd14a01292d2b123b8d307564034d" + "sha256:da552f29192b3c2b04d630133f194c98e9f14f0558669d427708e203fea4d0a5", + "sha256:fc29bd5ab4727c8dacfbe636f7fb5dc53b99805b62da1c96b214836159ff70c1" ], "markers": "python_version >= '3.7'", - "version": "==22.12.0" + "version": "==23.6.0" }, "typing-extensions": { "hashes": [ - "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb", - "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4" + "sha256:440d5dd3af93b060174bf433bccd69b0babc3b15b1a8dca43789fd7f61514b36", + "sha256:b75ddc264f0ba5615db7ba217daeb99701ad295353c45f9e95963337ceeeffb2" ], "index": "pypi", - "version": "==4.5.0" + "markers": "python_version >= '3.7'", + "version": "==4.7.1" }, "uc-micro-py": { "hashes": [ - "sha256:316cfb8b6862a0f1d03540f0ae6e7b033ff1fa0ddbe60c12cbe0d4cec846a69f", - "sha256:b7cdf4ea79433043ddfe2c82210208f26f7962c0cfbe3bacb05ee879a7fdb596" + "sha256:30ae2ac9c49f39ac6dce743bd187fcd2b574b16ca095fa74cd9396795c954c54", + "sha256:8c9110c309db9d9e87302e2f4ad2c3152770930d88ab385cd544e7a7e75f3de0" ], - "markers": "python_version >= '3.6'", - "version": "==1.0.1" + "markers": "python_version >= '3.7'", + "version": "==1.0.2" }, "urllib3": { "hashes": [ - "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72", - "sha256:75edcdc2f7d85b137124a6c3c9fc3933cdeaa12ecb9a6a959f22797a0feca7e1" + "sha256:8d22f86aae8ef5e410d4f539fde9ce6b2113a001bb4d189e0aed70642d602b11", + "sha256:de7df1803967d2c2a98e4b11bb7d6bd9210474c46e8a0401514e3a42a75ebde4" ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", - "version": "==1.26.14" + "markers": "python_version >= '3.7'", + "version": "==2.0.4" }, "virtualenv": { "hashes": [ - "sha256:3c22fa5a7c7aa106ced59934d2c20a2ecb7f49b4130b8bf444178a16b880fa45", - "sha256:a8a4b8ca1e28f864b7514a253f98c1d62b64e31e77325ba279248c65fb4fcef4" + "sha256:43a3052be36080548bdee0b42919c88072037d50d56c28bd3f853cbe92b953ff", + "sha256:fd8a78f46f6b99a67b7ec5cf73f92357891a7b3a40fd97637c27f854aae3b9e0" ], "markers": "python_version >= '3.7'", - "version": "==20.20.0" - }, - "virtualenv-clone": { - "hashes": [ - "sha256:418ee935c36152f8f153c79824bb93eaf6f0f7984bae31d3f48f350b9183501a", - "sha256:44d5263bceed0bac3e1424d64f798095233b64def1c5689afa43dc3223caf5b0" - ], - "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==0.5.7" + "version": "==20.24.2" }, "waitress": { "hashes": [ "sha256:7500c9625927c8ec60f54377d590f67b30c8e70ef4b8894214ac6e4cad233d2a", "sha256:780a4082c5fbc0fde6a2fcfe5e26e6efc1e8f425730863c04085769781f51eba" ], - "index": "pypi", "markers": "sys_platform == 'win32'", "version": "==2.1.2" }, diff --git a/docs/specifiers.md b/docs/specifiers.md index d34bd3cc..d52741f7 100644 --- a/docs/specifiers.md +++ b/docs/specifiers.md @@ -103,30 +103,54 @@ All sub-dependencies will get added to the `Pipfile.lock` as well. Sub-dependenc ## VCS Dependencies -VCS dependencies from git and other version control systems using URLs formatted according to the following rule: +VCS dependencies from git and other version control systems using URLs formatted using preferred pip line formats: - +:////@#egg= + +:////@ -The only optional section is the `@` section. When using git over SSH, you may use the shorthand vcs and scheme alias `git+git@:/@#egg=`. Note that this is translated to `git+ssh://git@` when parsed. +Extras may be specified using the following format when issuing install command: + + @ +:////@ + +Note: that the #egg fragments should only be used for legacy pip lines which are still required in editable requirements. + + $ pipenv install -e git+https://github.com/requests/requests.git@v2.31.0#egg=requests -Note that it is **strongly recommended** that you install any version-controlled dependencies in editable mode, using `pipenv install -e`, in order to ensure that dependency resolution can be performed with an up-to-date copy of the repository each time it is performed, and that it includes all known dependencies. Below is an example usage which installs the git repository located at `https://github.com/requests/requests.git` from tag `v2.20.1` as package name `requests`: $ pipenv install -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests - Creating a Pipfile for this project... Installing -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... - [...snipped...] - Adding -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests to Pipfile's [packages]... - [...] + Resolving -e git+https://github.com/requests/requests.git@v2.20.1#egg=requests... + Added requests to Pipfile's [packages] ... + Installation Succeeded + Pipfile.lock not found, creating... + Locking [packages] dependencies... + Building requirements... + Resolving dependencies... + Success! + Locking [dev-packages] dependencies... + Updated Pipfile.lock (389441cc656bb774aaa28c7e53a35137aace7499ca01668765d528fa79f8acc8)! + Installing dependencies from Pipfile.lock (f8acc8)... + To activate this project's virtualenv, run pipenv shell. + Alternatively, run a command inside the virtualenv with pipenv run. $ cat Pipfile [packages] - requests = {git = "https://github.com/requests/requests.git", editable = true, ref = "v2.20.1"} + requests = {editable = true, ref = "v2.20.1", git = "git+https://github.com/requests/requests.git"} + + $ cat Pipfile.lock + ... + "requests": { + "editable": true, + "git": "git+https://github.com/requests/requests.git", + "markers": "python_version >= '3.7'", + "ref": "6cfbe1aedd56f8c2f9ff8b968efe65b22669795b" + }, + ... Valid values for `` include `git`, `bzr`, `svn`, and `hg`. Valid values for `` include `http`, `https`, `ssh`, and `file`. In specific cases you also have access to other schemes: `svn` may be combined with `svn` as a scheme, and `bzr` can be combined with `sftp` and `lp`. -You can read more about pip's implementation of VCS support `here `__. For more information about other options available when specifying VCS dependencies, please check the `Pipfile spec `_. +You can read more about pip's implementation of VCS support `here `__. ## Specifying Package Categories diff --git a/news/5793.bugfix.rst b/news/5793.bugfix.rst new file mode 100644 index 00000000..c9543572 --- /dev/null +++ b/news/5793.bugfix.rst @@ -0,0 +1,2 @@ +Drop requirementslib for managing pip lines and InstallRequirements, bring remaining requirementslib functionality into pipenv. +Fixes numerous reports about extras installs with vcs and file installs; format pip lines correctly to not generate deprecation warnings. diff --git a/pipenv/__init__.py b/pipenv/__init__.py index ae87ade4..3a182beb 100644 --- a/pipenv/__init__.py +++ b/pipenv/__init__.py @@ -3,8 +3,18 @@ import os import sys import warnings +# This has to come before imports of pipenv +PIPENV_ROOT = os.path.abspath(os.path.dirname(os.path.realpath(__file__))) +PIP_ROOT = os.sep.join([PIPENV_ROOT, "patched", "pip"]) +sys.path.insert(0, PIPENV_ROOT) +sys.path.insert(0, PIP_ROOT) + +# Load patched pip instead of system pip +os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" + def _ensure_modules(): + # Can be removed when we drop pydantic spec = importlib.util.spec_from_file_location( "typing_extensions", location=os.path.join( @@ -14,6 +24,14 @@ def _ensure_modules(): typing_extensions = importlib.util.module_from_spec(spec) sys.modules["typing_extensions"] = typing_extensions spec.loader.exec_module(typing_extensions) + # Ensure when pip gets invoked it uses our patched version + spec = importlib.util.spec_from_file_location( + "pip", + location=os.path.join(os.path.dirname(__file__), "patched", "pip", "__init__.py"), + ) + pip = importlib.util.module_from_spec(spec) + sys.modules["pip"] = pip + spec.loader.exec_module(pip) _ensure_modules() @@ -26,8 +44,6 @@ warnings.filterwarnings("ignore", category=DependencyWarning) warnings.filterwarnings("ignore", category=ResourceWarning) warnings.filterwarnings("ignore", category=UserWarning) -# Load patched pip instead of system pip -os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" if os.name == "nt": from pipenv.vendor import colorama diff --git a/pipenv/cli/command.py b/pipenv/cli/command.py index c296baa2..bf10693b 100644 --- a/pipenv/cli/command.py +++ b/pipenv/cli/command.py @@ -216,7 +216,7 @@ def install(state, **kwargs): requirementstxt=state.installstate.requirementstxt, pre=state.installstate.pre, deploy=state.installstate.deploy, - index_url=state.index, + index=state.index, packages=state.installstate.packages, editable_packages=state.installstate.editables, site_packages=state.site_packages, @@ -610,7 +610,7 @@ def run_open(state, module, *args, **kwargs): [ state.project._which("python"), "-c", - "import {0}; print({0}.__file__)".format(module), + f"import {module}; print({module}.__file__)", ] ) if c.returncode: @@ -693,11 +693,10 @@ def scripts(state): scripts = state.project.parsed_pipfile.get("scripts", {}) first_column_width = max(len(word) for word in ["Command"] + list(scripts)) second_column_width = max(len(word) for word in ["Script"] + list(scripts.values())) - lines = ["{0:<{width}} Script".format("Command", width=first_column_width)] - lines.append("{} {}".format("-" * first_column_width, "-" * second_column_width)) + lines = [f"{command:<{first_column_width}} Script" for command in ["Command"]] + lines.append(f"{'-' * first_column_width} {'-' * second_column_width}") lines.extend( - "{0:<{width}} {1}".format(name, script, width=first_column_width) - for name, script in scripts.items() + f"{name:<{first_column_width}} {script}" for name, script in scripts.items() ) console.print("\n".join(line for line in lines)) diff --git a/pipenv/cli/options.py b/pipenv/cli/options.py index 70b29d62..53cc33ab 100644 --- a/pipenv/cli/options.py +++ b/pipenv/cli/options.py @@ -2,6 +2,7 @@ import os import re from pipenv.project import Project +from pipenv.utils import err from pipenv.utils.internet import is_valid_url from pipenv.vendor.click import ( BadArgumentUsage, @@ -483,6 +484,83 @@ def validate_pypi_mirror(ctx, param, value): return value +# OLD REMOVED COMMANDS THAT WE STILL DISPLAY HELP TEXT FOR # +def skip_lock_option(f): + def callback(ctx, param, value): + if value: + err.print( + "The flag --skip-lock has been functionally removed. " + "Without running the lock resolver it is not possible to manage multiple package indexes. " + "Additionally it bypassed the build consistency guarantees provided by maintaining a lock file.", + style="yellow bold", + ) + raise ValueError("The flag --skip-lock flag has been removed.") + return value + + return option( + "--skip-lock", + is_flag=True, + default=False, + expose_value=False, + envvar="PIPENV_SKIP_LOCK", + callback=callback, + type=click_types.BOOL, + show_envvar=True, + hidden=True, # This hides the option from the help text. + )(f) + + +def keep_outdated_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.keep_outdated = value + if value: + err.print( + "The flag --keep-outdated has been removed. " + "The flag did not respect package resolver results and lead to inconsistent lock files. " + "Consider using the `pipenv upgrade` command to selectively upgrade packages.", + style="yellow bold", + ) + raise ValueError("The flag --keep-outdated flag has been removed.") + return value + + return option( + "--keep-outdated", + is_flag=True, + default=False, + expose_value=False, + callback=callback, + type=click_types.BOOL, + show_envvar=True, + hidden=True, # This hides the option from the help text. + )(f) + + +def selective_upgrade_option(f): + def callback(ctx, param, value): + state = ctx.ensure_object(State) + state.installstate.selective_upgrade = value + if value: + err.print( + "The flag --selective-upgrade has been removed. " + "The flag was buggy and lead to inconsistent lock files. " + "Consider using the `pipenv upgrade` command to selectively upgrade packages.", + style="yellow bold", + ) + raise ValueError("The flag --selective-upgrade flag has been removed.") + return value + + return option( + "--selective-upgrade", + is_flag=True, + default=False, + type=click_types.BOOL, + help="Update specified packages.", + callback=callback, + expose_value=False, + )(f) + + def common_options(f): f = pypi_mirror_option(f) f = verbose_option(f) @@ -496,6 +574,7 @@ def install_base_options(f): f = common_options(f) f = pre_option(f) f = extra_pip_args(f) + f = keep_outdated_option(f) # Removed, but still displayed in help text. return f @@ -505,6 +584,7 @@ def uninstall_options(f): f = uninstall_dev_option(f) f = editable_option(f) f = package_arg(f) + f = skip_lock_option(f) # Removed, but still displayed in help text. return f @@ -530,6 +610,8 @@ def install_options(f): f = ignore_pipfile_option(f) f = editable_option(f) f = package_arg(f) + f = skip_lock_option(f) # Removed, but still display help text. + f = selective_upgrade_option(f) # Removed, but still display help text. return f diff --git a/pipenv/environment.py b/pipenv/environment.py index 064fda3f..fa58d10e 100644 --- a/pipenv/environment.py +++ b/pipenv/environment.py @@ -12,21 +12,23 @@ import typing from pathlib import Path from sysconfig import get_paths, get_python_version, get_scheme_names from urllib.parse import urlparse -from urllib.request import url2pathname import pipenv from pipenv.patched.pip._internal.commands.install import InstallCommand from pipenv.patched.pip._internal.index.package_finder import PackageFinder +from pipenv.patched.pip._internal.req.req_install import InstallRequirement from pipenv.patched.pip._vendor import pkg_resources +from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name from pipenv.utils import console +from pipenv.utils.constants import VCS_LIST +from pipenv.utils.dependencies import as_pipfile +from pipenv.utils.fileutils import normalize_path, temp_path from pipenv.utils.funktools import chunked, unnest from pipenv.utils.indexes import prepare_pip_source_args from pipenv.utils.processes import subprocess_run -from pipenv.utils.shell import make_posix +from pipenv.utils.shell import make_posix, temp_environ from pipenv.vendor.pythonfinder.utils import is_in_path -from pipenv.vendor.requirementslib.fileutils import normalize_path, temp_path -from pipenv.vendor.requirementslib.utils import temp_environ try: # this is only in Python3.8 and later @@ -200,11 +202,6 @@ class Environment: .. note:: The implementation of this is borrowed from a combination of pip and virtualenv and is likely to change at some point in the future. - >>> from pipenv.core import project - >>> from pipenv.environment import Environment - >>> env = Environment(prefix=project.virtualenv_location, is_venv=True, sources=project.sources) - >>> import pprint - >>> pprint.pprint(env.base_paths) {'PATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/bin::/bin:/usr/bin', 'PYTHONPATH': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW/lib/python3.7/site-packages', 'data': '/home/hawk/.virtualenvs/pipenv-MfOPs1lW', @@ -760,38 +757,46 @@ class Environment: return any(d for d in self.get_distributions() if d.project_name == pkgname) - def is_satisfied(self, req): + def is_satisfied(self, req: InstallRequirement): match = next( iter( d for d in self.get_distributions() - if canonicalize_name(d.project_name) == req.normalized_name + if req.name + and canonicalize_name(d.project_name) == canonicalize_name(req.name) ), None, ) if match is not None: - if req.editable and req.line_instance.is_local and self.find_egg(match): - requested_path = req.line_instance.path + if req.editable and req.link and req.link.is_file: + requested_path = req.link.file_path if os.path.exists(requested_path): local_path = requested_path else: parsed_url = urlparse(requested_path) - local_path = url2pathname(parsed_url.path) + local_path = parsed_url.path return requested_path and os.path.samefile(local_path, match.location) elif match.has_metadata("direct_url.json"): direct_url_metadata = json.loads(match.get_metadata("direct_url.json")) - commit_id = direct_url_metadata.get("vcs_info", {}).get("commit_id", "") - vcs_type = direct_url_metadata.get("vcs_info", {}).get("vcs", "") - _, pipfile_part = req.as_pipfile().popitem() - return ( - vcs_type == req.vcs - and commit_id == req.commit_hash - and direct_url_metadata["url"] == pipfile_part[req.vcs] + requested_revision = direct_url_metadata.get("vcs_info", {}).get( + "requested_revision", "" ) - elif req.is_vcs or req.is_file_or_url: + vcs_type = direct_url_metadata.get("vcs_info", {}).get("vcs", "") + _, pipfile_part = as_pipfile(req).popitem() + vcs_ref = "" + for vcs in VCS_LIST: + if pipfile_part.get(vcs): + vcs_ref = pipfile_part[vcs].rsplit("@", 1)[-1] + break + return ( + vcs_type == req.link.scheme + and vcs_ref == requested_revision + and direct_url_metadata["url"] == pipfile_part[req.link.scheme] + ) + elif req.link and req.link.is_vcs: return False - elif req.line_instance.specifiers is not None: - return req.line_instance.specifiers.contains( + elif req.specifier is not None: + return SpecifierSet(str(req.specifier)).contains( match.version, prereleases=True ) return True diff --git a/pipenv/environments.py b/pipenv/environments.py index d71a499e..fea1fd7f 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -5,8 +5,8 @@ import re import sys from pipenv.patched.pip._vendor.platformdirs import user_cache_dir +from pipenv.utils.fileutils import normalize_drive from pipenv.utils.shell import env_to_bool, is_env_truthy, isatty -from pipenv.vendor.requirementslib.fileutils import normalize_drive # HACK: avoid resolver.py uses the wrong byte code files. # I hope I can remove this one day. diff --git a/pipenv/pep508checker.py b/pipenv/pep508checker.py index a771dec0..9246e7b3 100644 --- a/pipenv/pep508checker.py +++ b/pipenv/pep508checker.py @@ -5,7 +5,7 @@ import sys def format_full_version(info): - version = "{0.major}.{0.minor}.{0.micro}".format(info) + version = f"{info.major}.{info.minor}.{info.micro}" kind = info.releaselevel if kind != "final": version += kind[0] + str(info.serial) diff --git a/pipenv/project.py b/pipenv/project.py index 838507b0..3f71e374 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -11,6 +11,10 @@ import sys import urllib.parse from json.decoder import JSONDecodeError from pathlib import Path +from urllib import parse +from urllib.parse import unquote + +from pipenv.utils.constants import VCS_LIST try: import tomllib as toml @@ -23,16 +27,35 @@ from pipenv.environments import Setting, is_in_virtualenv, normalize_pipfile_pat from pipenv.patched.pip._internal.commands.install import InstallCommand from pipenv.patched.pip._internal.configuration import Configuration from pipenv.patched.pip._internal.exceptions import ConfigurationError +from pipenv.patched.pip._internal.models.link import Link +from pipenv.patched.pip._internal.req.req_install import InstallRequirement +from pipenv.patched.pip._internal.utils.hashes import FAVORITE_HASH from pipenv.patched.pip._vendor import pkg_resources +from pipenv.utils import err from pipenv.utils.constants import is_type_checking from pipenv.utils.dependencies import ( + clean_pkg_version, + determine_package_name, + determine_path_specifier, + determine_vcs_specifier, + expansive_install_req_from_line, get_canonical_names, is_editable, pep423_name, python_version, ) -from pipenv.utils.internet import get_url_name, is_pypi_url, is_valid_url, proper_case +from pipenv.utils.fileutils import open_file +from pipenv.utils.internet import ( + PackageIndexHTMLParser, + get_requests_session, + get_url_name, + is_pypi_url, + is_valid_url, + proper_case, +) from pipenv.utils.locking import atomic_open_for_write +from pipenv.utils.project import get_default_pyproject_backend +from pipenv.utils.requirements import normalize_name from pipenv.utils.shell import ( find_requirements, find_windows_executable, @@ -45,7 +68,6 @@ from pipenv.utils.shell import ( ) from pipenv.utils.toml import cleanup_toml, convert_toml_outline_tables from pipenv.vendor import click, plette, tomlkit -from pipenv.vendor.requirementslib.models.utils import get_default_pyproject_backend try: # this is only in Python3.8 and later @@ -132,6 +154,7 @@ class Project: self._environment = None self._build_system = {"requires": ["setuptools", "wheel"]} self.python_version = python_version + self.sessions = {} # pip requests sessions self.s = Setting() # Load Pip configuration and get items self.configuration = Configuration(isolated=False, load_only=None) @@ -212,6 +235,109 @@ class Project: else: return ["packages", "dev-packages"] + list(package_categories) + def get_requests_session_for_source(self, source): + if self.sessions.get(source["name"]): + session = self.sessions[source["name"]] + else: + session = get_requests_session( + self.s.PIPENV_MAX_RETRIES, source.get("verify_ssl", True) + ) + self.sessions[source["name"]] = session + return session + + @classmethod + def prepend_hash_types(cls, checksums, hash_type): + cleaned_checksums = set() + for checksum in checksums: + if not checksum: + continue + if not checksum.startswith(f"{hash_type}:"): + checksum = f"{hash_type}:{checksum}" + cleaned_checksums.add(checksum) + return sorted(cleaned_checksums) + + def get_hash_from_link(self, hash_cache, link): + if link.hash and link.hash_name == FAVORITE_HASH: + return f"{link.hash_name}:{link.hash}" + + return hash_cache.get_hash(link) + + def get_hashes_from_pypi(self, ireq, source): + pkg_url = f"https://pypi.org/pypi/{ireq.name}/json" + session = self.get_requests_session_for_source(source) + try: + collected_hashes = set() + # Grab the hashes from the new warehouse API. + r = session.get(pkg_url, timeout=10) + api_releases = r.json()["releases"] + cleaned_releases = {} + for api_version, api_info in api_releases.items(): + api_version = clean_pkg_version(api_version) + cleaned_releases[api_version] = api_info + version = "" + if ireq.specifier: + spec = next(iter(s for s in ireq.specifier), None) + if spec: + version = spec.version + for release in cleaned_releases[version]: + collected_hashes.add(release["digests"][FAVORITE_HASH]) + return self.prepend_hash_types(collected_hashes, FAVORITE_HASH) + except (ValueError, KeyError, ConnectionError): + if self.s.is_verbose(): + err.print( + f"[bold][red]Warning[/red][/bold]: Error generating hash for {ireq.name}." + ) + return None + + def get_hashes_from_remote_index_urls(self, ireq, source): + pkg_url = f"{source['url']}/{ireq.name}/" + session = self.get_requests_session_for_source(source) + try: + collected_hashes = set() + # Grab the hashes from the new warehouse API. + response = session.get(pkg_url, timeout=10) + # Create an instance of the parser + parser = PackageIndexHTMLParser() + # Feed the HTML to the parser + parser.feed(response.text) + # Extract hrefs + hrefs = parser.urls + + version = "" + if ireq.specifier: + spec = next(iter(s for s in ireq.specifier), None) + if spec: + version = spec.version + for package_url in hrefs: + if version in parse.unquote(package_url): + url_params = parse.urlparse(package_url).fragment + params_dict = parse.parse_qs(url_params) + if params_dict.get(FAVORITE_HASH): + collected_hashes.add(params_dict[FAVORITE_HASH][0]) + else: # Fallback to downloading the file to obtain hash + if source["url"] not in package_url: + package_url = f"{source['url']}{package_url}" + link = Link(package_url) + collected_hashes.add(self.get_file_hash(session, link)) + return self.prepend_hash_types(collected_hashes, FAVORITE_HASH) + except (ValueError, KeyError, ConnectionError): + if self.s.is_verbose(): + click.echo( + "{}: Error generating hash for {}".format( + click.style("Warning", bold=True, fg="red"), ireq.name + ), + err=True, + ) + return None + + def get_file_hash(self, session, link): + h = hashlib.new(FAVORITE_HASH) + err.print(f"Downloading file {link.filename} to obtain hash...") + with open_file(link.url, session) as fp: + for chunk in iter(lambda: fp.read(8096), b""): + h.update(chunk) + return f"{h.name}:{h.hexdigest()}" + @property def name(self) -> str: if self._name is None: @@ -634,7 +760,7 @@ class Project: @property def _pipfile(self): - from .vendor.requirementslib.models.pipfile import Pipfile as ReqLibPipfile + from pipenv.utils.pipfile import Pipfile as ReqLibPipfile pf = ReqLibPipfile.load(self.pipfile_location) return pf @@ -660,7 +786,7 @@ class Project: return packages def _get_vcs_packages(self, dev=False): - from pipenv.vendor.requirementslib.utils import is_vcs + from pipenv.utils.requirementslib import is_vcs section = "dev-packages" if dev else "packages" packages = { @@ -745,9 +871,7 @@ class Project: return source def get_or_create_lockfile(self, categories, from_pipfile=False): - from pipenv.vendor.requirementslib.models.lockfile import ( - Lockfile as Req_Lockfile, - ) + from pipenv.utils.locking import Lockfile as Req_Lockfile if from_pipfile and self.pipfile_exists: lockfile_dict = {} @@ -868,6 +992,14 @@ class Project: source["url"] = os.environ["PIPENV_PYPI_MIRROR"] return sources + def get_default_index(self): + return self.pipfile_sources()[0] + + def get_index_by_name(self, index_name): + for source in self.pipfile_sources(): + if source.get("name") == index_name: + return source + @property def sources(self): if self.lockfile_exists and hasattr(self.lockfile_content, "keys"): @@ -937,7 +1069,9 @@ class Project: def get_package_name_in_pipfile(self, package_name, category): """Get the equivalent package name in pipfile""" - section = self.parsed_pipfile.get(category, {}) + section = self.parsed_pipfile.get(category) + if section is None: + section = {} package_name = pep423_name(package_name) for name in section.keys(): if pep423_name(name) == package_name: @@ -968,31 +1102,90 @@ class Project: del parsed[category][pkg_name] self.write_toml(parsed) - def add_package_to_pipfile(self, package, dev=False, category=None): - from .vendor.requirementslib import Requirement + def generate_package_pipfile_entry(self, package, pip_line, category=None): + # Don't re-capitalize file URLs or VCSs. + if not isinstance(package, InstallRequirement): + package = expansive_install_req_from_line(package.strip()) + req_name = determine_package_name(package) + path_specifier = determine_path_specifier(package) + vcs_specifier = determine_vcs_specifier(package) + name = self.get_package_name_in_pipfile(req_name, category=category) + normalized_name = normalize_name(req_name) + + extras = package.extras + specifier = "*" + if package.req and package.specifier: + specifier = str(package.specifier) + + # Construct package requirement + entry = {} + if extras: + entry["extras"] = list(extras) + if path_specifier: + entry["file"] = unquote(path_specifier) + elif vcs_specifier: + for vcs in VCS_LIST: + if vcs in package.link.scheme: + if pip_line.startswith("-e"): + entry["editable"] = True + pip_line = pip_line.replace("-e ", "") + if "[" in pip_line and "]" in pip_line: + extras_section = pip_line.split("[")[1].split("]")[0] + entry["extras"] = sorted( + [extra.strip() for extra in extras_section.split(",")] + ) + if "@ " in pip_line: + vcs_part = pip_line.split("@ ", 1)[1] + else: + vcs_part = pip_line + vcs_parts = vcs_part.rsplit("@", 1) + entry["ref"] = vcs_parts[1].split("#", 1)[0].strip() + entry[vcs] = vcs_parts[0].strip() + break + else: + entry["version"] = specifier + if hasattr(package, "index"): + entry["index"] = package.index + + if len(entry) == 1 and "version" in entry: + return name, normalized_name, specifier + else: + return name, normalized_name, entry + + def add_package_to_pipfile(self, package, pip_line, dev=False, category=None): + category = category if category else "dev-packages" if dev else "packages" + + name, normalized_name, entry = self.generate_package_pipfile_entry( + package, pip_line, category=category + ) + + return self.add_pipfile_entry_to_pipfile( + name, normalized_name, entry, category=category + ) + + def add_pipfile_entry_to_pipfile(self, name, normalized_name, entry, category=None): newly_added = False + # Read and append Pipfile. p = self.parsed_pipfile - # Don't re-capitalize file URLs or VCSs. - if not isinstance(package, Requirement): - package = Requirement.from_line(package.strip(), parse_setup_info=False) - req_name, converted = package.pipfile_entry - category = category if category else "dev-packages" if dev else "packages" + # Set empty group if it doesn't exist yet. if category not in p: p[category] = {} - # Add the package to the group. - name = self.get_package_name_in_pipfile(req_name, category=category) - normalized_name = pep423_name(req_name) + if name and name != normalized_name: self.remove_package_from_pipfile(name, category=category) + + # Add the package to the group. if normalized_name not in p[category]: newly_added = True - p[category][normalized_name] = converted + + p[category][normalized_name] = entry + # Write Pipfile. self.write_toml(p) - return newly_added, category + return newly_added, category, normalized_name def src_name_from_url(self, index_url): name, _, tld_guess = urllib.parse.urlsplit(index_url).netloc.rpartition(".") diff --git a/pipenv/resolver.py b/pipenv/resolver.py index a9288e75..501e0219 100644 --- a/pipenv/resolver.py +++ b/pipenv/resolver.py @@ -1,11 +1,8 @@ import importlib.util import json -import logging import os import sys -os.environ["PIP_PYTHON_PATH"] = str(sys.executable) - def _ensure_modules(): spec = importlib.util.spec_from_file_location( @@ -74,21 +71,9 @@ def which(*args, **kwargs): def handle_parsed_args(parsed): - if "PIPENV_VERBOSITY" in os.environ: - parsed.verbose = int(os.getenv("PIPENV_VERBOSITY")) - if parsed.debug: - parsed.verbose = max(parsed.verbose, 2) - if parsed.verbose > 1: - logging.getLogger("pip").setLevel(logging.DEBUG) - elif parsed.verbose > 0: - logging.getLogger("pip").setLevel(logging.INFO) - logger = logging.getLogger( - "pipenv.patched.pip._internal.resolution.resolvelib.reporter" - ) - logger.addHandler(logging.StreamHandler()) - logger.setLevel(logging.INFO) - os.environ["PIP_RESOLVER_DEBUG"] = "" - os.environ["PIPENV_VERBOSITY"] = str(parsed.verbose) + if parsed.verbose: + os.environ["PIPENV_VERBOSITY"] = "1" + os.environ["PIP_RESOLVER_DEBUG"] = "1" if parsed.constraints_file: with open(parsed.constraints_file) as constraints: file_constraints = constraints.read().strip().split("\n") @@ -107,7 +92,7 @@ class Entry: from pipenv.utils.dependencies import ( get_lockfile_section_using_pipfile_category, ) - from pipenv.vendor.requirementslib.models.utils import tomlkit_value_to_python + from pipenv.utils.toml import tomlkit_value_to_python self.name = name if isinstance(entry_dict, dict): @@ -141,14 +126,19 @@ class Entry: @staticmethod def make_requirement(name=None, entry=None): - from pipenv.vendor.requirementslib.models.requirements import Requirement + from pipenv.utils.dependencies import from_pipfile - return Requirement.from_pipfile(name, entry) + return from_pipfile(name, entry) @classmethod def clean_initial_dict(cls, entry_dict): - if not entry_dict.get("version", "").startswith("=="): - entry_dict["version"] = cls.clean_specifier(entry_dict.get("version", "")) + from pipenv.patched.pip._vendor.packaging.requirements import Requirement + + entry_dict.get("version", "") + version = entry_dict.get("version", "") + if isinstance(version, Requirement): + version = str(version.specifier) + entry_dict["version"] = cls.clean_specifier(version) if "name" in entry_dict: del entry_dict["name"] return entry_dict @@ -176,7 +166,7 @@ class Entry: @classmethod def get_markers_from_dict(cls, entry_dict): from pipenv.patched.pip._vendor.packaging import markers as packaging_markers - from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + from pipenv.utils.markers import normalize_marker_str marker_keys = cls.parse_pyparsing_exprs(packaging_markers.VARIABLE) markers = set() @@ -217,7 +207,7 @@ class Entry: @staticmethod def marker_to_str(marker): - from pipenv.vendor.requirementslib.models.markers import normalize_marker_str + from pipenv.utils.markers import normalize_marker_str if not marker: return None @@ -242,20 +232,19 @@ class Entry: entry_extras = list(self.entry.extras) if self.lockfile_entry.extras: entry_extras.extend(list(self.lockfile_entry.extras)) - self._entry.req.extras = entry_extras - self.entry_dict["extras"] = self.entry.extras + self.entry_dict["extras"] = entry_extras if self.original_markers and not self.markers: original_markers = self.marker_to_str(self.original_markers) self.markers = original_markers self.entry_dict["markers"] = self.marker_to_str(original_markers) - entry_hashes = set(self.entry.hashes) - locked_hashes = set(self.lockfile_entry.hashes) - if entry_hashes != locked_hashes and not self.is_updated: - self.entry_dict["hashes"] = sorted(entry_hashes | locked_hashes) + entry_hashes = set(self.entry_dict.get("hashes", [])) + self.entry_dict["hashes"] = sorted(entry_hashes) self.entry_dict["name"] = self.name if "version" in self.entry_dict: self.entry_dict["version"] = self.strip_version(self.entry_dict["version"]) _, self.entry_dict = self.get_markers_from_dict(self.entry_dict) + if self.resolver.index_lookup.get(self.name): + self.entry_dict["index"] = self.resolver.index_lookup[self.name] return self.entry_dict @property @@ -383,12 +372,12 @@ class Entry: @property def updated_version(self): - version = self.entry.specifiers + version = str(self.entry.specifier) return self.strip_version(version) @property def updated_specifier(self) -> str: - return self.entry.specifiers + return str(self.entry.specifier) @property def original_specifier(self) -> str: @@ -452,15 +441,7 @@ class Entry: :return: A set of **InstallRequirement** instances representing constraints :rtype: Set """ - constraints = { - c for c in self.resolver.parsed_constraints if c and c.name == self.entry.name - } - pipfile_constraint = self.get_pipfile_constraint() - if pipfile_constraint and not ( - self.pipfile_entry.editable or pipfile_constraint.editable - ): - constraints.add(pipfile_constraint) - return constraints + return self.resolver.parsed_constraints def get_pipfile_constraint(self): """ @@ -470,7 +451,7 @@ class Entry: :return: An **InstallRequirement** instance representing a version constraint """ if self.is_in_pipfile: - return self.pipfile_entry.ireq + return self.pipfile_entry def validate_constraints(self): """ @@ -481,22 +462,24 @@ class Entry: :raises: :exc:`pipenv.exceptions.DependencyConflict` if the constraints dont exist """ from pipenv.exceptions import DependencyConflict + from pipenv.patched.pip._vendor.packaging.requirements import Requirement + from pipenv.utils import err constraints = self.get_constraints() pinned_version = self.updated_version for constraint in constraints: - if not constraint.req: + if not isinstance(constraint, Requirement): continue - if pinned_version and not constraint.req.specifier.contains( + if pinned_version and not constraint.specifier.contains( str(pinned_version), prereleases=True ): if self.project.s.is_verbose(): - print(f"Tried constraint: {constraint!r}", file=sys.stderr) + err.print(f"Tried constraint: {constraint!r}") msg = ( "Cannot resolve conflicting version {}{} while {}{} is " "locked.".format( self.name, - constraint.req.specifier, + constraint.specifier, self.name, self.updated_specifier, ) @@ -582,37 +565,6 @@ def clean_results(results, resolver, project, category): return new_results -def parse_packages(packages, pre, clear, system, requirements_dir=None): - from pipenv.utils.indexes import parse_indexes - from pipenv.vendor.requirementslib.fileutils import cd, temp_path - from pipenv.vendor.requirementslib.models.requirements import Requirement - - parsed_packages = [] - for package in packages: - *_, line = parse_indexes(package) - line = " ".join(line) - pf = {} - req = Requirement.from_line(line) - if not req.name: - with temp_path(), cd(req.req.setup_info.base_dir): - sys.path.insert(0, req.req.setup_info.base_dir) - req.req.setup_info.get_info() - req.update_name_from_path(req.req.setup_info.base_dir) - try: - name, entry = req.pipfile_entry - except Exception: - continue - else: - if name is not None and entry is not None: - pf[name] = entry - parsed_packages.append(pf) - print("RESULTS:") - if parsed_packages: - print(json.dumps(parsed_packages)) - else: - print(json.dumps([])) - - def resolve_packages( pre, clear, @@ -633,8 +585,12 @@ def resolve_packages( else None ) + if not isinstance(packages, set): + packages = set(packages) + if not isinstance(constraints, set): + constraints = set(constraints) if constraints else set() if constraints: - packages += constraints + packages |= constraints def resolve( packages, pre, project, sources, clear, system, category, requirements_dir=None @@ -692,31 +648,15 @@ def _main( parse_only=False, category=None, ): - if parse_only: - parse_packages( - packages, - pre=pre, - clear=clear, - system=system, - requirements_dir=requirements_dir, - ) - else: - resolve_packages( - pre, clear, verbose, system, write, requirements_dir, packages, category - ) + resolve_packages( + pre, clear, verbose, system, write, requirements_dir, packages, category + ) def main(argv=None): parser = get_parser() parsed, remaining = parser.parse_known_args(argv) _ensure_modules() - import warnings - - from pipenv.vendor.click.utils import get_text_stream - - warnings.simplefilter("ignore", category=ResourceWarning) - sys.stdout = get_text_stream("stdout") - sys.stderr = get_text_stream("stderr") os.environ["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" os.environ["PYTHONIOENCODING"] = "utf-8" os.environ["PYTHONUNBUFFERED"] = "1" diff --git a/pipenv/routines/clear.py b/pipenv/routines/clear.py index 86245b1c..acc421e9 100644 --- a/pipenv/routines/clear.py +++ b/pipenv/routines/clear.py @@ -1,8 +1,8 @@ import shutil from pipenv import environments +from pipenv.utils.funktools import handle_remove_readonly from pipenv.vendor import click -from pipenv.vendor.requirementslib.models.setup_info import handle_remove_readonly def do_clear(project): diff --git a/pipenv/routines/install.py b/pipenv/routines/install.py index 7d56bd8b..cf248524 100644 --- a/pipenv/routines/install.py +++ b/pipenv/routines/install.py @@ -9,6 +9,11 @@ from pipenv import environments, exceptions from pipenv.patched.pip._internal.exceptions import PipError from pipenv.patched.pip._vendor import rich from pipenv.routines.lock import do_lock +from pipenv.utils import fileutils +from pipenv.utils.dependencies import ( + expansive_install_req_from_line, + get_lockfile_section_using_pipfile_category, +) from pipenv.utils.indexes import get_source_list from pipenv.utils.internet import download_file, is_valid_url from pipenv.utils.pip import ( @@ -18,11 +23,9 @@ from pipenv.utils.pip import ( from pipenv.utils.pipfile import ensure_pipfile from pipenv.utils.project import ensure_project from pipenv.utils.requirements import add_index_to_pipfile, import_requirements +from pipenv.utils.shell import temp_environ from pipenv.utils.virtualenv import cleanup_virtualenv, do_create_virtualenv from pipenv.vendor import click -from pipenv.vendor.requirementslib import fileutils -from pipenv.vendor.requirementslib.models.requirements import Requirement -from pipenv.vendor.requirementslib.utils import temp_environ console = rich.console.Console() err = rich.console.Console(stderr=True) @@ -32,7 +35,7 @@ def do_install( project, packages=False, editable_packages=False, - index_url=False, + index=False, dev=False, python=False, pypi_mirror=None, @@ -123,7 +126,6 @@ def do_install( fd.close() # Replace the url with the temporary requirements file requirementstxt = temp_reqs - remote = True if requirementstxt: error, traceback = None, None click.secho( @@ -140,7 +142,7 @@ def do_install( ) except (UnicodeDecodeError, PipError) as e: # Don't print the temp file path if remote since it will be deleted. - req_path = requirements_url if remote else project.path_to(requirementstxt) + req_path = project.path_to(requirementstxt) error = ( "Unexpected syntax in {}. Are you sure this is a " "requirements.txt style file?".format(req_path) @@ -153,10 +155,6 @@ def do_install( ) traceback = e finally: - # If requirements file was provided by remote url delete the temporary file - if remote: - fd.close() # Close for windows to allow file cleanup. - os.remove(temp_reqs) if error and traceback: click.secho(error, fg="red") click.secho(str(traceback), fg="yellow", err=True) @@ -188,8 +186,6 @@ def do_install( # This is for if the user passed in dependencies, then we want to make sure we else: - from pipenv.vendor.requirementslib.models.requirements import Requirement - # make a tuple of (display_name, entry) pkg_list = packages + [f"-e {pkg}" for pkg in editable_packages] if not system and not project.virtualenv_exists: @@ -219,9 +215,11 @@ def do_install( os.environ["PIP_USER"] = "0" if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] - st.console.print(f"Resolving {pkg_line}...") + st.console.print(f"Resolving {pkg_line}...", markup=False) try: - pkg_requirement = Requirement.from_line(pkg_line) + pkg_requirement = expansive_install_req_from_line( + pkg_line, expand_env=True + ) except ValueError as e: err.print("{}: {}".format(click.style("WARNING", fg="red"), e)) err.print( @@ -233,7 +231,8 @@ def do_install( st.update(f"Installing {pkg_requirement.name}...") # Warn if --editable wasn't passed. if ( - pkg_requirement.is_vcs + pkg_requirement.link + and pkg_requirement.link.is_vcs and not pkg_requirement.editable and not project.s.PIPENV_RESOLVE_VCS ): @@ -254,25 +253,38 @@ def do_install( pipfile_sections = "[dev-packages]" else: pipfile_sections = "[packages]" - st.console.print( - f"[bold]Adding [green]{pkg_requirement.name}[/green][/bold] to Pipfile's [yellow]\\{pipfile_sections}[/yellow] ..." - ) # Add the package to the Pipfile. - if index_url: - index_name = add_index_to_pipfile(project, index_url) - pkg_requirement.index = index_name + if index: + source = project.get_index_by_name(index) + default_index = project.get_default_index()["name"] + if not source: + index_name = add_index_to_pipfile(project, index) + if index_name != default_index: + pkg_requirement.index = index_name + elif source["name"] != default_index: + pkg_requirement.index = source["name"] try: if categories: for category in categories: - added, cat = project.add_package_to_pipfile( - pkg_requirement, dev, category + added, cat, normalized_name = project.add_package_to_pipfile( + pkg_requirement, pkg_line, dev, category ) if added: - new_packages.append((pkg_requirement.name, cat)) + new_packages.append((normalized_name, cat)) + st.console.print( + f"[bold]Added [green]{normalized_name}[/green][/bold] to Pipfile's " + f"[yellow]\\{pipfile_sections}[/yellow] ..." + ) else: - added, cat = project.add_package_to_pipfile(pkg_requirement, dev) + added, cat, normalized_name = project.add_package_to_pipfile( + pkg_requirement, pkg_line, dev + ) if added: - new_packages.append((pkg_requirement.name, cat)) + new_packages.append((normalized_name, cat)) + st.console.print( + f"[bold]Added [green]{normalized_name}[/green][/bold] to Pipfile's " + f"[yellow]\\{pipfile_sections}[/yellow] ..." + ) except ValueError: import traceback @@ -306,11 +318,11 @@ def do_install( extra_pip_args=extra_pip_args, categories=categories, ) - except RuntimeError: + except Exception as e: # If we fail to install, remove the package from the Pipfile. for pkg_name, category in new_packages: project.remove_package_from_pipfile(pkg_name, category) - sys.exit(1) + raise e sys.exit(0) @@ -393,8 +405,6 @@ def do_install_dependencies( else: categories = ["packages"] - lockfile = None - pipfile = None for category in categories: lockfile = project.get_or_create_lockfile(categories=categories) if not bare: @@ -405,18 +415,18 @@ def do_install_dependencies( bold=True, ) dev = dev or dev_only - if lockfile: - deps_list = list( - lockfile.get_requirements(dev=dev, only=dev_only, categories=[category]) - ) - else: - deps_list = [] - for req_name, specifier in pipfile.items(): - deps_list.append(Requirement.from_pipfile(req_name, specifier)) - failed_deps_queue = queue.Queue() + deps_list = list( + lockfile.get_requirements(dev=dev, only=dev_only, categories=[category]) + ) + editable_or_vcs_deps = [ + (dep, pip_line) for dep, pip_line in deps_list if (dep.link and dep.editable) + ] + normal_deps = [ + (dep, pip_line) + for dep, pip_line in deps_list + if not (dep.link and dep.editable) + ] - editable_or_vcs_deps = [dep for dep in deps_list if (dep.editable or dep.vcs)] - normal_deps = [dep for dep in deps_list if not (dep.editable or dep.vcs)] install_kwargs = { "no_deps": True, "ignore_hashes": ignore_hashes, @@ -425,51 +435,19 @@ def do_install_dependencies( "sequential_deps": editable_or_vcs_deps, "extra_pip_args": extra_pip_args, } - + lockfile_category = get_lockfile_section_using_pipfile_category(category) + lockfile_section = lockfile[lockfile_category] batch_install( project, normal_deps, + lockfile_section, procs, - failed_deps_queue, requirements_dir, **install_kwargs, ) if not procs.empty(): - _cleanup_procs(project, procs, failed_deps_queue) - - # Iterate over the hopefully-poorly-packaged dependencies... - if not failed_deps_queue.empty(): - click.secho("Installing initially failed dependencies...", bold=True) - retry_list = [] - while not failed_deps_queue.empty(): - failed_dep = failed_deps_queue.get() - retry_list.append(failed_dep) - install_kwargs.update({"retry": False}) - batch_install( - project, - retry_list, - procs, - failed_deps_queue, - requirements_dir, - **install_kwargs, - ) - if not procs.empty(): - _cleanup_procs(project, procs, failed_deps_queue, retry=False) - if not failed_deps_queue.empty(): - failed_list = [] - while not failed_deps_queue.empty(): - failed_dep = failed_deps_queue.get() - failed_list.append(failed_dep) - click.echo( - click.style( - f"Failed to install some dependency or packages. " - f"The following have failed installation and attempted retry: {failed_list}", - fg="red", - ), - err=True, - ) - sys.exit(1) + _cleanup_procs(project, procs) def batch_install_iteration( @@ -477,33 +455,12 @@ def batch_install_iteration( deps_to_install, sources, procs, - failed_deps_queue, requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, - retry=True, extra_pip_args=None, ): - from pipenv.vendor.requirementslib.models.utils import ( - strip_extras_markers_from_requirement, - ) - - is_artifact = False - for dep in deps_to_install: - if dep.req.req: - dep.req.req = strip_extras_markers_from_requirement(dep.req.req) - if dep.markers: - dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers)) - # Install the module. - if dep.is_file_or_url and ( - dep.is_direct_url - or any(dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"]) - ): - is_artifact = True - elif dep.is_vcs: - is_artifact = True - with temp_environ(): if not allow_global: os.environ["PIP_USER"] = "0" @@ -511,10 +468,6 @@ def batch_install_iteration( del os.environ["PYTHONHOME"] if "GIT_CONFIG" in os.environ: del os.environ["GIT_CONFIG"] - use_pep517 = True - if not retry and not is_artifact: - use_pep517 = False - cmds = pip_install_deps( project, deps=deps_to_install, @@ -523,26 +476,25 @@ def batch_install_iteration( ignore_hashes=ignore_hashes, no_deps=no_deps, requirements_dir=requirements_dir, - use_pep517=use_pep517, + use_pep517=True, extra_pip_args=extra_pip_args, ) for c in cmds: procs.put(c) - _cleanup_procs(project, procs, failed_deps_queue, retry=retry) + _cleanup_procs(project, procs) def batch_install( project, deps_list, + lockfile_section, procs, - failed_deps_queue, requirements_dir, no_deps=True, ignore_hashes=False, allow_global=False, pypi_mirror=None, - retry=True, sequential_deps=None, extra_pip_args=None, ): @@ -551,7 +503,9 @@ def batch_install( deps_to_install = deps_list[:] deps_to_install.extend(sequential_deps) deps_to_install = [ - dep for dep in deps_to_install if not project.environment.is_satisfied(dep) + (dep, pip_line) + for dep, pip_line in deps_to_install + if not project.environment.is_satisfied(dep) ] search_all_sources = project.settings.get("install_search_all_sources", False) sources = get_source_list( @@ -562,27 +516,26 @@ def batch_install( pypi_mirror=pypi_mirror, ) if search_all_sources: + dependencies = [pip_line for _, pip_line in deps_to_install] batch_install_iteration( project, - deps_to_install, + dependencies, sources, procs, - failed_deps_queue, requirements_dir, no_deps=no_deps, ignore_hashes=ignore_hashes, allow_global=allow_global, - retry=retry, extra_pip_args=extra_pip_args, ) else: # Sort the dependencies out by index -- include editable/vcs in the default group deps_by_index = defaultdict(list) - for dependency in deps_to_install: - if dependency.index: - deps_by_index[dependency.index].append(dependency) - else: - deps_by_index[project.sources_default["name"]].append(dependency) + for dependency, pip_line in deps_to_install: + index = project.sources_default["name"] + if dependency.name and lockfile_section[dependency.name].get("index"): + index = lockfile_section[dependency.name]["index"] + deps_by_index[index].append(pip_line) # Treat each index as its own pip install phase for index_name, dependencies in deps_by_index.items(): try: @@ -592,12 +545,10 @@ def batch_install( dependencies, [install_source], procs, - failed_deps_queue, requirements_dir, no_deps=no_deps, ignore_hashes=ignore_hashes, allow_global=allow_global, - retry=retry, extra_pip_args=extra_pip_args, ) except StopIteration: @@ -609,7 +560,7 @@ def batch_install( sys.exit(1) -def _cleanup_procs(project, procs, failed_deps_queue, retry=True): +def _cleanup_procs(project, procs): while not procs.empty(): c = procs.get() try: @@ -621,47 +572,14 @@ def _cleanup_procs(project, procs, failed_deps_queue, retry=True): click.secho(out.strip() or err.strip(), fg="yellow") # The Installation failed... if failed: + # The Installation failed... + # We echo both c.stdout and c.stderr because pip returns error details on out. + err = err.strip().splitlines() if err else [] + out = out.strip().splitlines() if out else [] + err_lines = [line for message in [out, err] for line in message] deps = getattr(c, "deps", {}).copy() - for dep in deps: - # If there is a mismatch in installed locations or the install fails - # due to wrongful disabling of pep517, we should allow for - # additional passes at installation - if "does not match installed location" in err: - project.environment.expand_egg_links() - click.echo( - "{}".format( - click.style( - "Failed initial installation: Failed to overwrite existing " - "package, likely due to path aliasing. Expanding and trying " - "again!", - fg="yellow", - ) - ) - ) - if dep: - dep.use_pep517 = True - elif "Disabling PEP 517 processing is invalid" in err: - if dep: - dep.use_pep517 = True - elif not retry: - # The Installation failed... - # We echo both c.stdout and c.stderr because pip returns error details on out. - err = err.strip().splitlines() if err else [] - out = out.strip().splitlines() if out else [] - err_lines = [line for message in [out, err] for line in message] - # Return the subprocess' return code. - raise exceptions.InstallError(deps, extra=err_lines) - else: - # Alert the user. - click.echo( - "{} {}! Will try again.".format( - click.style("An error occurred while installing", fg="red"), - click.style(dep.as_line() if dep else "", fg="green"), - ), - err=True, - ) - # Save the Failed Dependency for later. - failed_deps_queue.put(dep) + # Return the subprocess' return code. + raise exceptions.InstallError(deps, extra=err_lines) def do_init( diff --git a/pipenv/routines/lock.py b/pipenv/routines/lock.py index fcd23b10..a8593ce4 100644 --- a/pipenv/routines/lock.py +++ b/pipenv/routines/lock.py @@ -29,10 +29,10 @@ def do_lock( lockfile_categories.insert(0, "default") # Create the lockfile. lockfile = project.lockfile(categories=lockfile_categories) - for category in lockfile_categories: - for k, v in lockfile.get(category, {}).copy().items(): - if not hasattr(v, "keys"): - del lockfile[category][k] + # for category in lockfile_categories: + # for k, v in lockfile.get(category, {}).copy().items(): + # if not hasattr(v, "keys"): + # del lockfile[category][k] # Resolve package to generate constraints before resolving other categories for category in lockfile_categories: @@ -55,7 +55,7 @@ def do_lock( # Prune old lockfile category as new one will be created. try: - del lockfile[category] + old_lock_data = lockfile.pop(category) except KeyError: pass @@ -73,6 +73,7 @@ def do_lock( pypi_mirror=pypi_mirror, pipfile=packages, lockfile=lockfile, + old_lock_data=old_lock_data, ) # Overwrite any category packages with default packages. diff --git a/pipenv/routines/outdated.py b/pipenv/routines/outdated.py index 5c2c478b..47567977 100644 --- a/pipenv/routines/outdated.py +++ b/pipenv/routines/outdated.py @@ -4,10 +4,13 @@ from collections.abc import Mapping from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name from pipenv.routines.lock import do_lock -from pipenv.utils.dependencies import pep423_name +from pipenv.utils.dependencies import ( + as_pipfile, + expansive_install_req_from_line, + get_version, + pep423_name, +) from pipenv.vendor import click -from pipenv.vendor.requirementslib.models.requirements import Requirement -from pipenv.vendor.requirementslib.models.utils import get_version def do_outdated(project, pypi_mirror=None, pre=False, clear=False): @@ -26,8 +29,11 @@ def do_outdated(project, pypi_mirror=None, pre=False, clear=False): for name, deps in project.environment.reverse_dependencies().items() } for result in installed_packages: - dep = Requirement.from_line(str(result.as_requirement())) - packages.update(dep.as_pipfile()) + dep = expansive_install_req_from_line( + str(result.as_requirement()), expand_env=True + ) + packages.update(as_pipfile(dep)) + updated_packages = {} lockfile = do_lock( project, clear=clear, pre=pre, write=False, pypi_mirror=pypi_mirror diff --git a/pipenv/routines/requirements.py b/pipenv/routines/requirements.py index 0386f38e..32c5d670 100644 --- a/pipenv/routines/requirements.py +++ b/pipenv/routines/requirements.py @@ -2,61 +2,10 @@ import re import sys from pipenv.utils.dependencies import get_lockfile_section_using_pipfile_category +from pipenv.utils.requirements import requirements_from_lockfile from pipenv.vendor import click -def requirements_from_deps(deps, include_hashes=True, include_markers=True): - pip_packages = [] - - for package_name, package_info in deps.items(): - # Handling git repositories - if "git" in package_info: - git = package_info["git"] - ref = package_info.get("ref", "") - extras = ( - "[{}]".format(",".join(package_info.get("extras", []))) - if "extras" in package_info - else "" - ) - pip_package = f"{package_name}{extras} @ git+{git}@{ref}" - # Handling file-sourced packages - elif "file" in package_info or "path" in package_info: - file = package_info.get("file") or package_info.get("path") - extras = ( - "[{}]".format(",".join(package_info.get("extras", []))) - if "extras" in package_info - else "" - ) - pip_package = f"{file}{extras}" - else: - # Handling packages from standard pypi like indexes - version = package_info.get("version", "").replace("==", "") - hashes = ( - " --hash={}".format(" --hash=".join(package_info["hashes"])) - if include_hashes and "hashes" in package_info - else "" - ) - markers = ( - "; {}".format(package_info["markers"]) - if include_markers - and "markers" in package_info - and package_info["markers"] - else "" - ) - extras = ( - "[{}]".format(",".join(package_info.get("extras", []))) - if "extras" in package_info - else "" - ) - pip_package = f"{package_name}{extras}=={version}{markers}{hashes}" - - # Append to the list - pip_packages.append(pip_package) - - # pip_packages contains the pip-installable lines - return pip_packages - - def generate_requirements( project, dev=False, @@ -84,7 +33,7 @@ def generate_requirements( if not dev_only: deps.update(lockfile["default"]) - pip_installable_lines = requirements_from_deps( + pip_installable_lines = requirements_from_lockfile( deps, include_hashes=include_hashes, include_markers=include_markers ) diff --git a/pipenv/routines/uninstall.py b/pipenv/routines/uninstall.py index 2de5e7bc..679d5d91 100644 --- a/pipenv/routines/uninstall.py +++ b/pipenv/routines/uninstall.py @@ -6,6 +6,7 @@ from pipenv.patched.pip._internal.build_env import get_runnable_pip from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name from pipenv.routines.lock import do_lock from pipenv.utils.dependencies import ( + expansive_install_req_from_line, get_canonical_names, get_lockfile_section_using_pipfile_category, get_pipfile_category_using_lockfile_section, @@ -16,7 +17,6 @@ from pipenv.utils.project import ensure_project from pipenv.utils.requirements import BAD_PACKAGES from pipenv.utils.shell import cmd_list_to_shell, project_python from pipenv.vendor import click -from pipenv.vendor.requirementslib import Requirement def do_uninstall( @@ -43,7 +43,7 @@ def do_uninstall( if not categories: categories = project.get_package_categories(for_lockfile=True) editable_pkgs = [ - Requirement.from_line(f"-e {p}").name for p in editable_packages if p + expansive_install_req_from_line(f"-e {p}").name for p in editable_packages if p ] packages += editable_pkgs package_names = {p for p in packages if p} diff --git a/pipenv/routines/update.py b/pipenv/routines/update.py index 2d8c7ad0..1ccf1348 100644 --- a/pipenv/routines/update.py +++ b/pipenv/routines/update.py @@ -1,19 +1,17 @@ import sys +from collections import defaultdict from pipenv.routines.install import do_sync from pipenv.routines.lock import do_lock from pipenv.routines.outdated import do_outdated from pipenv.utils.dependencies import ( - convert_deps_to_pip, + expansive_install_req_from_line, get_pipfile_category_using_lockfile_section, - is_star, - pep423_name, ) from pipenv.utils.project import ensure_project from pipenv.utils.requirements import add_index_to_pipfile from pipenv.utils.resolver import venv_resolve_deps from pipenv.vendor import click -from pipenv.vendor.requirementslib.models.requirements import Requirement def do_update( @@ -44,15 +42,6 @@ def do_update( site_packages=site_packages, clear=clear, ) - if not outdated: - outdated = bool(dry_run) - if outdated: - do_outdated( - project, - clear=clear, - pre=pre, - pypi_mirror=pypi_mirror, - ) packages = [p for p in packages if p] editable = [p for p in editable_packages if p] if not packages: @@ -86,6 +75,16 @@ def do_update( lock_only=lock_only, ) + if not outdated: + outdated = bool(dry_run) + if outdated: + do_outdated( + project, + clear=clear, + pre=pre, + pypi_mirror=pypi_mirror, + ) + do_sync( project, dev=dev, @@ -120,67 +119,59 @@ def upgrade( elif not categories: categories = ["default"] - package_args = [p for p in packages] + [f"-e {pkg}" for pkg in editable_packages] - index_name = None if index_url: index_name = add_index_to_pipfile(project, index_url) - reqs = {} - requested_packages = {} - for package in package_args[:]: - # section = project.packages if not dev else project.dev_packages - section = {} - package = Requirement.from_line(package) - if index_name: - package.index = index_name - package_name, package_val = package.pipfile_entry - package_name = pep423_name(package_name) - requested_packages[package_name] = package - try: - if not is_star(section[package_name]) and is_star(package_val): - # Support for VCS dependencies. - package_val = convert_deps_to_pip( - {package_name: section[package_name]}, project=project - )[0] - except KeyError: - pass - reqs[package_name] = package_val + package_args = [p for p in packages] + [f"-e {pkg}" for pkg in editable_packages] - if not reqs: - click.echo("Nothing to upgrade!") - sys.exit(0) - - # Resolve package to generate constraints of new package data - upgrade_lock_data = venv_resolve_deps( - reqs, - which=project._which, - project=project, - lockfile={}, - category="default", - pre=pre, - allow_global=system, - pypi_mirror=pypi_mirror, - ) - if not upgrade_lock_data: - click.echo("Nothing to upgrade!") - sys.exit(0) - - # Upgrade the relevant packages in the various categories specified + requested_install_reqs = defaultdict(dict) + requested_packages = defaultdict(dict) for category in categories: pipfile_category = get_pipfile_category_using_lockfile_section(category) + + for package in package_args[:]: + install_req = expansive_install_req_from_line(package, expand_env=True) + if index_name: + install_req.index = index_name + name, normalized_name, pipfile_entry = project.generate_package_pipfile_entry( + install_req, package, category=pipfile_category + ) + project.add_pipfile_entry_to_pipfile( + name, normalized_name, pipfile_entry, category=pipfile_category + ) + requested_packages[pipfile_category][normalized_name] = pipfile_entry + requested_install_reqs[pipfile_category][normalized_name] = install_req + if project.pipfile_exists: packages = project.parsed_pipfile.get(pipfile_category, {}) else: packages = project.get_pipfile_section(pipfile_category) - for package_name, requirement in requested_packages.items(): - requested_package = reqs[package_name] + + if not package_args: + click.echo("Nothing to upgrade!") + sys.exit(0) + + # Resolve package to generate constraints of new package data + upgrade_lock_data = venv_resolve_deps( + requested_packages[pipfile_category], + which=project._which, + project=project, + lockfile={}, + category="default", + pre=pre, + allow_global=system, + pypi_mirror=pypi_mirror, + ) + if not upgrade_lock_data: + click.echo("Nothing to upgrade!") + sys.exit(0) + + for package_name, pipfile_entry in requested_packages[pipfile_category].items(): if package_name not in packages: - packages.append(package_name, requested_package) + packages.append(package_name, pipfile_entry) else: - packages[package_name] = requested_package - if lock_only is False: - project.add_package_to_pipfile(requirement, category=pipfile_category) + packages[package_name] = pipfile_entry full_lock_resolution = venv_resolve_deps( packages, diff --git a/pipenv/shells.py b/pipenv/shells.py index 95200084..89467cdd 100644 --- a/pipenv/shells.py +++ b/pipenv/shells.py @@ -8,8 +8,8 @@ import sys from pathlib import Path from shutil import get_terminal_size +from pipenv.utils.shell import temp_environ from pipenv.vendor import shellingham -from pipenv.vendor.requirementslib.utils import temp_environ ShellDetectionFailure = shellingham.ShellDetectionFailure diff --git a/pipenv/utils/__init__.py b/pipenv/utils/__init__.py index 70f95c4f..3359402c 100644 --- a/pipenv/utils/__init__.py +++ b/pipenv/utils/__init__.py @@ -2,7 +2,7 @@ import logging from pipenv.patched.pip._vendor.rich.console import Console -logging.basicConfig(level=logging.ERROR) +logging.basicConfig(level=logging.INFO) console = Console() err = Console(stderr=True) diff --git a/pipenv/utils/constants.py b/pipenv/utils/constants.py index f0ad8ac4..fe66fbb0 100644 --- a/pipenv/utils/constants.py +++ b/pipenv/utils/constants.py @@ -3,6 +3,38 @@ VCS_LIST = ("git", "svn", "hg", "bzr") SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") FALSE_VALUES = ("0", "false", "no", "off") TRUE_VALUES = ("1", "true", "yes", "on") +REMOTE_FILE_SCHEMES = [ + "http", + "https", + "ftp", +] +VCS_SCHEMES = [ + "git+http", + "git+https", + "git+ssh", + "git+git", + "hg+http", + "hg+https", + "hg+ssh", + "svn+http", + "svn+https", + "svn+svn", + "bzr+http", + "bzr+https", + "bzr+ssh", + "bzr+sftp", + "bzr+ftp", + "bzr+lp", +] +REMOTE_SCHEMES = REMOTE_FILE_SCHEMES + VCS_SCHEMES + +RELEVANT_PROJECT_FILES = ( + "METADATA", + "PKG-INFO", + "setup.py", + "setup.cfg", + "pyproject.toml", +) def is_type_checking(): diff --git a/pipenv/utils/dependencies.py b/pipenv/utils/dependencies.py index 6b6db8bb..b36823bf 100644 --- a/pipenv/utils/dependencies.py +++ b/pipenv/utils/dependencies.py @@ -1,18 +1,69 @@ +import ast +import configparser import os +import re +import sys +import tarfile +import tempfile +import zipfile from contextlib import contextmanager -from tempfile import NamedTemporaryFile -from typing import Mapping, Sequence +from functools import lru_cache +from pathlib import Path +from tempfile import NamedTemporaryFile, TemporaryDirectory +from typing import Any, AnyStr, Dict, List, Mapping, Optional, Sequence, Union +from urllib.parse import urljoin, urlparse, urlsplit, urlunsplit +from pipenv.patched.pip._internal.models.link import Link +from pipenv.patched.pip._internal.network.download import Downloader +from pipenv.patched.pip._internal.req.constructors import ( + install_req_from_editable, + parse_req_from_line, +) +from pipenv.patched.pip._internal.req.req_install import InstallRequirement +from pipenv.patched.pip._internal.utils.misc import hide_url +from pipenv.patched.pip._internal.vcs.versioncontrol import VcsSupport +from pipenv.patched.pip._vendor import tomli +from pipenv.patched.pip._vendor.distlib.util import COMPARE_OP from pipenv.patched.pip._vendor.packaging.markers import Marker +from pipenv.patched.pip._vendor.packaging.requirements import Requirement +from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name from pipenv.patched.pip._vendor.packaging.version import parse -from pipenv.vendor.requirementslib.fileutils import create_tracked_tempdir -from pipenv.vendor.requirementslib.models.requirements import ( - InstallRequirement, - Requirement, +from pipenv.utils import err +from pipenv.utils.fileutils import ( + create_tracked_tempdir, +) +from pipenv.utils.requirementslib import ( + add_ssh_scheme_to_git_uri, + get_pip_command, + prepare_pip_source_args, + unpack_url, ) -from .constants import SCHEME_LIST, VCS_LIST -from .shell import temp_path +from .constants import ( + RELEVANT_PROJECT_FILES, + REMOTE_SCHEMES, + SCHEME_LIST, + VCS_LIST, + VCS_SCHEMES, +) +from .markers import PipenvMarkers + + +def get_version(pipfile_entry): + if str(pipfile_entry) == "{}" or is_star(pipfile_entry): + return "" + + if hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: + if is_star(pipfile_entry.get("version")): + return "" + version = pipfile_entry.get("version") + if version is None: + version = "" + return version.strip().lstrip("(").rstrip(")") + + if isinstance(pipfile_entry, str): + return pipfile_entry.strip().lstrip("(").rstrip(")") + return "" def python_version(path_to_python): @@ -92,58 +143,7 @@ def pep423_name(name): return name -def get_vcs_deps(project=None, dev=False, pypi_mirror=None, packages=None, reqs=None): - from pipenv.vendor.requirementslib.models.requirements import Requirement - - section = "vcs_dev_packages" if dev else "vcs_packages" - if reqs is None: - reqs = [] - lockfile = {} - if not reqs: - if not project and not packages: - raise ValueError( - "Must supply either a project or a pipfile section to lock vcs dependencies." - ) - if not packages: - try: - packages = getattr(project, section) - except AttributeError: - return [], [] - reqs = [Requirement.from_pipfile(name, entry) for name, entry in packages.items()] - result = [] - for requirement in reqs: - name = requirement.normalized_name - commit_hash = None - if requirement.is_vcs: - try: - with temp_path(), locked_repository(requirement) as repo: - from pipenv.vendor.requirementslib.models.requirements import ( - Requirement, - ) - - commit_hash = repo.commit_hash - name = requirement.normalized_name - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]["ref"] = commit_hash - result.append(requirement) - except OSError: - continue - return result, lockfile - - def translate_markers(pipfile_entry): - """Take a pipfile entry and normalize its markers - - Provide a pipfile entry which may have 'markers' as a key or it may have - any valid key from `packaging.markers.marker_context.keys()` and standardize - the format into {'markers': 'key == "some_value"'}. - - :param pipfile_entry: A dictionariy of keys and values representing a pipfile entry - :type pipfile_entry: dict - :returns: A normalized dictionary with cleaned marker entries - """ - if not isinstance(pipfile_entry, Mapping): - raise TypeError("Entry is not a pipfile formatted mapping.") from pipenv.patched.pip._vendor.packaging.markers import default_environment allowed_marker_keys = ["markers"] + list(default_environment().keys()) @@ -151,6 +151,7 @@ def translate_markers(pipfile_entry): pipfile_markers = set(provided_keys) & set(allowed_marker_keys) new_pipfile = dict(pipfile_entry).copy() marker_set = set() + os_name_marker = None if "markers" in new_pipfile: marker_str = new_pipfile.pop("markers") if marker_str: @@ -160,60 +161,53 @@ def translate_markers(pipfile_entry): for m in pipfile_markers: entry = f"{pipfile_entry[m]}" if m != "markers": - marker_set.add(str(Marker(f"{m} {entry}"))) + if m != "os_name": + marker_set.add(str(Marker(f"{m} {entry}"))) new_pipfile.pop(m) if marker_set: - new_pipfile["markers"] = str( - Marker( - " or ".join( - f"{s}" if " and " in s else s - for s in sorted(dict.fromkeys(marker_set)) - ) - ) - ).replace('"', "'") + markers_str = " and ".join( + f"{s}" if " and " in s else s for s in sorted(dict.fromkeys(marker_set)) + ) + if os_name_marker: + markers_str = f"({markers_str}) and {os_name_marker}" + new_pipfile["markers"] = str(Marker(markers_str)).replace('"', "'") return new_pipfile -def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): - from pipenv.vendor.requirementslib.utils import is_vcs +def unearth_hashes_for_dep(project, dep): + hashes = [] - name = pep423_name(dep["name"]) + index_url = "https://pypi.org/simple/" + source = "pypi" + for source in project.sources: + if source.get("name") == dep.get("index"): + index_url = source.get("url") + break + + # 1 Try to get hashes directly form index + install_req, markers = install_req_from_pipfile(dep["name"], dep) + if not install_req or not install_req.req: + return [] + if "https://pypi.org/simple/" in index_url: + hashes = project.get_hashes_from_pypi(install_req, source) + elif index_url: + hashes = project.get_hashes_from_remote_index_urls(install_req, source) + if hashes: + return hashes + + return [] + + +def clean_resolved_dep(project, dep, is_top_level=False, current_entry=None): + from pipenv.patched.pip._vendor.packaging.requirements import ( + Requirement as PipRequirement, + ) + + name = dep["name"] lockfile = {} - # We use this to determine if there are any markers on top level packages - # So we can make sure those win out during resolution if the packages reoccur - if "version" in dep and dep["version"] and not dep.get("editable", False): - version = "{}".format(dep["version"]) - if not version.startswith("=="): - version = f"=={version}" - lockfile["version"] = version - if is_vcs(dep): - ref = dep.get("ref", None) - if ref is not None: - lockfile["ref"] = ref - vcs_type = next(iter(k for k in dep.keys() if k in VCS_LIST), None) - if vcs_type: - lockfile[vcs_type] = dep[vcs_type] - if "subdirectory" in dep: - lockfile["subdirectory"] = dep["subdirectory"] - for key in ["hashes", "index", "extras", "editable"]: - if key in dep: - lockfile[key] = dep[key] - # In case we lock a uri or a file when the user supplied a path - # remove the uri or file keys from the entry and keep the path - preferred_file_keys = ["path", "file"] - dependency_file_key = next(iter(k for k in preferred_file_keys if k in dep), None) - if dependency_file_key: - lockfile[dependency_file_key] = dep[dependency_file_key] - # Pipfile entry overrides path/file from resolver - if pipfile_entry and isinstance(pipfile_entry, dict): - for k in preferred_file_keys: - if k in pipfile_entry.keys(): - lockfile[k] = pipfile_entry[k] - break - # If a package is **PRESENT** in the pipfile but has no markers, make sure we - # **NEVER** include markers in the lockfile + + # Evaluate Markers if "markers" in dep and dep.get("markers", "").strip(): - # First, handle the case where there is no top level dependency in the pipfile if not is_top_level: translated = translate_markers(dep).get("markers", "").strip() if translated: @@ -221,18 +215,127 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None): lockfile["markers"] = translated except TypeError: pass - # otherwise make sure we are prioritizing whatever the pipfile says about the markers - # If the pipfile says nothing, then we should put nothing in the lockfile else: try: - pipfile_entry = translate_markers(pipfile_entry) + pipfile_entry = translate_markers(dep) if pipfile_entry.get("markers"): lockfile["markers"] = pipfile_entry.get("markers") except TypeError: pass + + version = dep.get("version", None) + if version and not version.startswith("=="): + version = f"=={version}" + if version == "==*": + if current_entry: + version = current_entry.get("version") + dep["version"] = version + else: + version = None + + is_vcs_or_file = False + for vcs_type in VCS_LIST: + if vcs_type in dep: + if "[" in dep[vcs_type] and "]" in dep[vcs_type]: + extras_section = dep[vcs_type].split("[").pop().replace("]", "") + lockfile["extras"] = sorted( + [extra.strip() for extra in extras_section.split(",")] + ) + if has_name_with_extras(dep[vcs_type]): + lockfile[vcs_type] = dep[vcs_type].split("@ ", 1)[1] + else: + lockfile[vcs_type] = dep[vcs_type] + lockfile["ref"] = dep.get("ref") + is_vcs_or_file = True + + if "editable" in dep: + lockfile["editable"] = dep["editable"] + + preferred_file_keys = ["path", "file"] + dependency_file_key = next(iter(k for k in preferred_file_keys if k in dep), None) + if dependency_file_key: + lockfile[dependency_file_key] = dep[dependency_file_key] + is_vcs_or_file = True + if "editable" in dep: + lockfile["editable"] = dep["editable"] + + if version and not is_vcs_or_file: + if isinstance(version, PipRequirement): + if version.specifier: + lockfile["version"] = str(version.specifier) + if version.extras: + lockfile["extras"] = sorted(version.extras) + elif version: + lockfile["version"] = version + + if dep.get("hashes"): + lockfile["hashes"] = dep["hashes"] + elif is_top_level: + potential_hashes = unearth_hashes_for_dep(project, dep) + if potential_hashes: + lockfile["hashes"] = potential_hashes + + if dep.get("index"): + lockfile["index"] = dep["index"] + + if dep.get("extras"): + lockfile["extras"] = sorted(dep["extras"]) + + # In case we lock a uri or a file when the user supplied a path + # remove the uri or file keys from the entry and keep the path + if dep and isinstance(dep, dict): + for k in preferred_file_keys: + if k in dep.keys(): + lockfile[k] = dep[k] + break + + if "markers" in dep: + markers = dep["markers"] + if markers: + markers = Marker(markers) + if not markers.evaluate() and current_entry: + current_entry.update(lockfile) + return {name: current_entry} + return {name: lockfile} +def as_pipfile(dep: InstallRequirement) -> Dict[str, Any]: + """Create a pipfile entry for the given InstallRequirement.""" + pipfile_dict = {} + name = dep.name + version = dep.req.specifier + + # Construct the pipfile entry + pipfile_dict[name] = { + "version": str(version), + "editable": dep.editable, + "extras": list(dep.extras), + } + + if dep.link: + # If it's a VCS link + if dep.link.is_vcs: + vcs = dep.link.scheme.split("+")[0] + pipfile_dict[name][vcs] = dep.link.url_without_fragment + # If it's a URL link + elif dep.link.scheme.startswith("http"): + pipfile_dict[name]["file"] = dep.link.url_without_fragment + # If it's a local file + elif dep.link.is_file: + pipfile_dict[name]["path"] = dep.link.file_path + + # Convert any markers to their string representation + if dep.markers: + pipfile_dict[name]["markers"] = str(dep.markers) + + # If a hash is available, add it to the pipfile entry + if dep.hash_options: + pipfile_dict[name]["hashes"] = dep.hash_options + + return pipfile_dict + + def is_star(val): return isinstance(val, str) and val == "*" @@ -257,46 +360,699 @@ def is_pinned_requirement(ireq): return spec.operator in {"==", "==="} and not spec.version.endswith(".*") +def is_editable_path(path): + not_editable = [".whl", ".zip", ".tar", ".tar.gz", ".tgz"] + if os.path.isfile(path): + return False + if os.path.isdir(path): + return True + if os.path.splitext(path)[1] in not_editable: + return False + for ext in not_editable: + if path.endswith(ext): + return False + return False + + +def dependency_as_pip_install_line( + dep_name: str, + dep: Union[str, Mapping], + include_hashes: bool, + include_markers: bool, + include_index: bool, + indexes: list, + constraint: bool = False, +): + if isinstance(dep, str): + if is_star(dep): + return dep_name + elif not COMPARE_OP.match(dep): + return f"{dep_name}=={dep}" + return f"{dep_name}{dep}" + line = [] + is_constraint = False + vcs = next(iter([vcs for vcs in VCS_LIST if vcs in dep]), None) + if not vcs: + for k in ["file", "path"]: + if k in dep: + if is_editable_path(dep[k]): + line.append("-e") + extras = "" + if "extras" in dep: + extras = f"[{','.join(dep['extras'])}]" + location = dep["file"] if "file" in dep else dep["path"] + if location.startswith(("http:", "https:")): + line.append(f"{dep_name}{extras} @ {location}") + else: + line.append(f"{location}{extras}") + break + else: + # Normal/Named Requirements + is_constraint = True + line.append(dep_name) + if "extras" in dep: + line[-1] += f"[{','.join(dep['extras'])}]" + if "version" in dep: + version = dep["version"] + if version and not is_star(version): + if not COMPARE_OP.match(version): + version = f"=={version}" + line[-1] += version + if include_markers and dep.get("markers"): + line[-1] = f'{line[-1]}; {dep["markers"]}' + + if include_hashes and dep.get("hashes"): + line.extend([f" --hash={hash}" for hash in dep["hashes"]]) + + if include_index: + if dep.get("index"): + indexes = [s for s in indexes if s.get("name") == dep["index"]] + else: + indexes = [indexes[0]] if indexes else [] + index_list = prepare_pip_source_args(indexes) + line.extend(index_list) + elif vcs and vcs in dep: # VCS Requirements + extras = "" + ref = "" + if dep.get("ref"): + ref = f"@{dep['ref']}" + if "extras" in dep: + extras = f"[{','.join(dep['extras'])}]" + include_vcs = "" if f"{vcs}+" in dep[vcs] else f"{vcs}+" + vcs_url = dep[vcs] + # legacy format is the only format supported for editable installs https://github.com/pypa/pip/issues/9106 + if is_editable_path(dep[vcs]) or "file://" in dep[vcs]: + if "#egg=" not in dep[vcs]: + git_req = f"-e {include_vcs}{dep[vcs]}{ref}#egg={dep_name}{extras}" + else: + git_req = f"-e {include_vcs}{dep[vcs]}{ref}" + if "subdirectory" in dep: + git_req += f"&subdirectory={dep['subdirectory']}" + else: + if "#egg=" in vcs_url: + vcs_url = vcs_url.split("#egg=")[0] + git_req = f"{dep_name}{extras}@ {include_vcs}{vcs_url}{ref}" + if "subdirectory" in dep: + git_req += f"#subdirectory={dep['subdirectory']}" + + line.append(git_req) + + if constraint and not is_constraint: + pip_line = "" + else: + pip_line = " ".join(line) + return pip_line + + def convert_deps_to_pip( deps, - project=None, - include_index=True, + indexes=None, include_hashes=True, include_markers=True, + include_index=False, ): """ "Converts a Pipfile-formatted dependency to a pip-formatted one.""" dependencies = [] - for dep_name, dep in deps.items(): - if project: - project.clear_pipfile_cache() + if indexes is None: indexes = [] - if project: - indexes = project.pipfile_sources() - new_dep = Requirement.from_pipfile(dep_name, dep) - if new_dep.index: - include_index = True - sources = indexes if include_index else None - req = new_dep.as_line( - sources=sources, - include_hashes=include_hashes, - include_markers=include_markers, - ).strip() + for dep_name, dep in deps.items(): + req = dependency_as_pip_install_line( + dep_name, dep, include_hashes, include_markers, include_index, indexes + ) dependencies.append(req) return dependencies +def parse_metadata_file(content: str): + """ + Parse a METADATA file to get the package name. + + Parameters: + content (str): Contents of the METADATA file. + + Returns: + str: Name of the package or None if not found. + """ + + for line in content.splitlines(): + if line.startswith("Name:"): + return line.split("Name: ")[1].strip() + + return None + + +def parse_pkginfo_file(content: str): + """ + Parse a PKG-INFO file to get the package name. + + Parameters: + content (str): Contents of the PKG-INFO file. + + Returns: + str: Name of the package or None if not found. + """ + for line in content.splitlines(): + if line.startswith("Name:"): + return line.split("Name: ")[1].strip() + + return None + + +def parse_setup_file(content): + try: + tree = ast.parse(content) + for node in ast.walk(tree): + if isinstance(node, ast.Call) and getattr(node.func, "id", "") == "setup": + for keyword in node.keywords: + if keyword.arg == "name": + if isinstance(keyword.value, ast.Str): + return keyword.value.s + elif sys.version_info < (3, 9) and isinstance( + keyword.value, ast.Subscript + ): + if ( + isinstance(keyword.value.value, ast.Name) + and keyword.value.value.id == "about" + ): + if isinstance( + keyword.value.slice, ast.Index + ) and isinstance(keyword.value.slice.value, ast.Str): + return keyword.value.slice.value.s + return keyword.value.s + elif sys.version_info >= (3, 9) and isinstance( + keyword.value, ast.Subscript + ): + # If the name is a lookup in a dictionary, only handle the case where it's a static lookup + if ( + isinstance(keyword.value.value, ast.Name) + and isinstance(keyword.value.slice, ast.Str) + and keyword.value.value.id == "about" + ): + return keyword.value.slice.s + + except ValueError: + pass # We will not exec unsafe code to determine the name pre-resolver + + return None + + +def parse_cfg_file(content): + config = configparser.ConfigParser() + config.read_string(content) + try: + return config["metadata"]["name"] + except configparser.NoSectionError: + return None + except KeyError: + return None + + +def parse_toml_file(content): + toml_dict = tomli.loads(content) + if "project" in toml_dict and "name" in toml_dict["project"]: + return toml_dict["project"]["name"] + if "tool" in toml_dict and "poetry" in toml_dict["tool"]: + return toml_dict["tool"]["poetry"]["name"] + + return None + + +def find_package_name_from_tarball(tarball_filepath): + if tarball_filepath.startswith("file://") and os.name != "nt": + tarball_filepath = tarball_filepath[7:] + with tarfile.open(tarball_filepath, "r") as tar_ref: + for filename in tar_ref.getnames(): + if filename.endswith(RELEVANT_PROJECT_FILES): + with tar_ref.extractfile(filename) as file: + possible_name = find_package_name_from_filename(filename, file) + if possible_name: + return possible_name + + +def find_package_name_from_zipfile(zip_filepath): + if zip_filepath.startswith("file://") and os.name != "nt": + zip_filepath = zip_filepath[7:] + with zipfile.ZipFile(zip_filepath, "r") as zip_ref: + for filename in zip_ref.namelist(): + if filename.endswith(RELEVANT_PROJECT_FILES): + with zip_ref.open(filename) as file: + possible_name = find_package_name_from_filename(file.name, file) + if possible_name: + return possible_name + + +def find_package_name_from_directory(directory): + parsed_url = urlparse(directory) + directory = ( + os.path.normpath(parsed_url.path) + if parsed_url.scheme + else os.path.normpath(directory) + ) + if "#egg=" in directory: # parse includes the fragment in py3.7 and py3.8 + expected_name = directory.split("#egg=")[1] + return expected_name + if os.name == "nt": + if directory.startswith("\\") and (":\\" in directory or ":/" in directory): + directory = directory[1:] + if directory.startswith("\\\\"): + directory = directory[1:] + directory_contents = sorted( + os.listdir(directory), + key=lambda x: (os.path.isdir(os.path.join(directory, x)), x), + ) + for filename in directory_contents: + filepath = os.path.join(directory, filename) + if os.path.isfile(filepath): + if filename.endswith(RELEVANT_PROJECT_FILES): + with open(filepath, "rb") as file: + possible_name = find_package_name_from_filename(filename, file) + if possible_name: + return possible_name + elif os.path.isdir(filepath): + possible_name = find_package_name_from_directory(filepath) + if possible_name: + return possible_name + + return None + + +def determine_path_specifier(package: InstallRequirement): + if package.link: + if package.link.scheme in ["http", "https"]: + path_specifier = package.link.url_without_fragment + return path_specifier + if package.link.scheme == "file": + try: + path_specifier = os.path.relpath(package.link.file_path) + except ValueError: + # If os.path.relpath() fails, use the absolute path instead + path_specifier = os.path.abspath(package.link.file_path) + return path_specifier + + +def determine_vcs_specifier(package: InstallRequirement): + if package.link and package.link.scheme in VCS_SCHEMES: + vcs_specifier = package.link.url_without_fragment + return vcs_specifier + + +def get_vcs_backend(vcs_type): + backend = VcsSupport().get_backend(vcs_type) + return backend + + +def generate_temp_dir_path(): + # Create a temporary directory using mkdtemp + temp_dir = tempfile.mkdtemp() + # Remove the created directory + os.rmdir(temp_dir) + return temp_dir + + +def determine_vcs_revision_hash( + package: InstallRequirement, vcs_type: str, revision: str +): + try: # Windows python 3.7 will sometimes raise PermissionError cleaning up + checkout_directory = generate_temp_dir_path() + repo_backend = get_vcs_backend(vcs_type) + repo_backend.obtain(checkout_directory, hide_url(package.link.url), verbosity=1) + return repo_backend.get_revision(checkout_directory) + except Exception as e: + err.print( + f"Error {e} obtaining {vcs_type} revision hash for {package}; falling back to {revision}." + ) + return revision + + +@lru_cache(maxsize=None) +def determine_package_name(package: InstallRequirement): + req_name = None + if package.name: + req_name = package.name + elif "#egg=" in str(package): + req_name = str(package).split("#egg=")[1] + req_name = req_name.split("[")[0] + elif "@ " in str(package): + req_name = str(package).split("@ ")[0] + req_name = req_name.split("[")[0] + elif package.link and package.link.scheme in REMOTE_SCHEMES: + try: # Windows python 3.7 will sometimes raise PermissionError cleaning up + with TemporaryDirectory() as td: + cmd = get_pip_command() + options, _ = cmd.parser.parse_args([]) + session = cmd._build_session(options) + local_file = unpack_url( + link=package.link, + location=td, + download=Downloader(session, "off"), + verbosity=1, + ) + if local_file.path.endswith(".whl") or local_file.path.endswith(".zip"): + req_name = find_package_name_from_zipfile(local_file.path) + elif local_file.path.endswith(".tar.gz") or local_file.path.endswith( + ".tar.bz2" + ): + req_name = find_package_name_from_tarball(local_file.path) + else: + req_name = find_package_name_from_directory(local_file.path) + except PermissionError: + pass + elif package.link and package.link.scheme in [ + "bzr+file", + "git+file", + "hg+file", + "svn+file", + ]: + parsed_url = urlparse(package.link.url) + repository_path = parsed_url.path + repository_path = repository_path.rsplit("@", 1)[ + 0 + ] # extract the actual directory path + repository_path = repository_path.split("#egg=")[0] + req_name = find_package_name_from_directory(repository_path) + elif package.link and package.link.scheme == "file": + if package.link.file_path.endswith(".whl") or package.link.file_path.endswith( + ".zip" + ): + req_name = find_package_name_from_zipfile(package.link.file_path) + elif package.link.file_path.endswith(".tar.gz"): + req_name = find_package_name_from_tarball(package.link.file_path) + else: + req_name = find_package_name_from_directory(package.link.file_path) + if req_name: + return req_name + else: + raise ValueError(f"Could not determine package name from {package}") + + +def find_package_name_from_filename(filename, file): + if filename.endswith("METADATA"): + content = file.read().decode() + possible_name = parse_metadata_file(content) + if possible_name: + return possible_name + + if filename.endswith("PKG-INFO"): + content = file.read().decode() + possible_name = parse_pkginfo_file(content) + if possible_name: + return possible_name + + if filename.endswith("setup.py"): + content = file.read().decode() + possible_name = parse_setup_file(content) + if possible_name: + return possible_name + + if filename.endswith("setup.cfg"): + content = file.read().decode() + possible_name = parse_cfg_file(content) + if possible_name: + return possible_name + + if filename.endswith("pyproject.toml"): + content = file.read().decode() + possible_name = parse_toml_file(content) + if possible_name: + return possible_name + return None + + +def create_link(link): + # type: (AnyStr) -> Link + + if not isinstance(link, str): + raise TypeError("must provide a string to instantiate a new link") + + return Link(link) + + +def get_link_from_line(line): + """Parse link information from given requirement line. Return a + 6-tuple: + + - `vcs_type` indicates the VCS to use (e.g. "git"), or None. + - `prefer` is either "file", "path" or "uri", indicating how the + information should be used in later stages. + - `relpath` is the relative path to use when recording the dependency, + instead of the absolute path/URI used to perform installation. + This can be None (to prefer the absolute path or URI). + - `path` is the absolute file path to the package. This will always use + forward slashes. Can be None if the line is a remote URI. + - `uri` is the absolute URI to the package. Can be None if the line is + not a URI. + - `link` is an instance of :class:`pipenv.patched.pip._internal.index.Link`, + representing a URI parse result based on the value of `uri`. + This function is provided to deal with edge cases concerning URIs + without a valid netloc. Those URIs are problematic to a straight + ``urlsplit` call because they cannot be reliably reconstructed with + ``urlunsplit`` due to a bug in the standard library: + >>> from urllib.parse import urlsplit, urlunsplit + >>> urlunsplit(urlsplit('git+file:///this/breaks')) + 'git+file:/this/breaks' + >>> urlunsplit(urlsplit('file:///this/works')) + 'file:///this/works' + See `https://bugs.python.org/issue23505#msg277350`. + """ + + # Git allows `git@github.com...` lines that are not really URIs. + # Add "ssh://" so we can parse correctly, and restore afterward. + fixed_line = add_ssh_scheme_to_git_uri(line) # type: str + + # We can assume a lot of things if this is a local filesystem path. + if "://" not in fixed_line: + p = Path(fixed_line).absolute() # type: Path + p.as_posix() # type: Optional[str] + uri = p.as_uri() # type: str + link = create_link(uri) # type: Link + return link + + # This is an URI. We'll need to perform some elaborated parsing. + parsed_url = urlsplit(fixed_line) # type: SplitResult + + # Split the VCS part out if needed. + original_scheme = parsed_url.scheme # type: str + if "+" in original_scheme: + vcs_type, _, scheme = original_scheme.partition("+") + parsed_url = parsed_url._replace(scheme=scheme) # type: ignore + else: + pass + + # Re-attach VCS prefix to build a Link. + link = create_link( + urlunsplit(parsed_url._replace(scheme=original_scheme)) # type: ignore + ) + + return link + + +def has_name_with_extras(requirement): + pattern = r"^([a-zA-Z0-9_-]+(\[[a-zA-Z0-9_-]+\])?) @ .*" + match = re.match(pattern, requirement) + return match is not None + + +def expand_env_variables(line) -> AnyStr: + """Expand the env vars in a line following pip's standard. + https://pip.pypa.io/en/stable/reference/pip_install/#id10. + + Matches environment variable-style values in '${MY_VARIABLE_1}' with + the variable name consisting of only uppercase letters, digits or + the '_' + """ + + def replace_with_env(match): + value = os.getenv(match.group(1)) + return value if value else match.group() + + return re.sub(r"\$\{([A-Z0-9_]+)\}", replace_with_env, line) + + +def expansive_install_req_from_line( + name: str, + comes_from: Optional[Union[str, InstallRequirement]] = None, + *, + use_pep517: Optional[bool] = None, + isolated: bool = False, + global_options: Optional[List[str]] = None, + hash_options: Optional[Dict[str, List[str]]] = None, + constraint: bool = False, + line_source: Optional[str] = None, + user_supplied: bool = False, + config_settings: Optional[Dict[str, Union[str, List[str]]]] = None, + expand_env: bool = False, +) -> InstallRequirement: + """Creates an InstallRequirement from a name, which might be a + requirement, directory containing 'setup.py', filename, or URL. + + :param line_source: An optional string describing where the line is from, + for logging purposes in case of an error. + """ + name = name.strip("'") + editable = False + if name.startswith("-e "): + # Editable requirement + editable = True + name = name.split("-e ")[1] + if has_name_with_extras(name): + name = name.split(" @ ", 1)[1] + + if expand_env: + name = expand_env_variables(name) + + if os.path.isfile(name) or os.path.isdir(name): + if not name.startswith("file:"): + # Make sure the path is absolute and properly formatted as a file: URL + absolute_path = os.path.abspath(name) + name = urljoin("file:", absolute_path) + name = "file:" + name + + return install_req_from_editable(name, line_source) + + vcs_part = name + if "@ " in name: # Check for new style vcs lines + vcs_part = name.split("@ ", 1)[1] + for vcs in VCS_LIST: + if vcs_part.startswith(f"{vcs}+"): + link = get_link_from_line(vcs_part) + return InstallRequirement( + None, + comes_from, + link=link, + use_pep517=use_pep517, + isolated=isolated, + global_options=global_options, + hash_options=hash_options, + constraint=constraint, + user_supplied=user_supplied, + ) + if editable: + return install_req_from_editable(name, line_source) + if urlparse(name).scheme in ("http", "https", "file"): + parts = parse_req_from_line(name, line_source) + else: + # It's a requirement + if "--index" in name: + name = name.split("--index")[0] + if " -i " in name: + name = name.split(" -i ")[0] + # handle local version identifiers (like the ones torch uses in their public index) + if "+" in name: + name = name.split("+")[0] + parts = parse_req_from_line(name, line_source) + + return InstallRequirement( + parts.requirement, + comes_from, + link=parts.link, + markers=parts.markers, + use_pep517=use_pep517, + isolated=isolated, + global_options=global_options, + hash_options=hash_options, + config_settings=config_settings, + constraint=constraint, + extras=parts.extras, + user_supplied=user_supplied, + ) + + +def install_req_from_pipfile(name, pipfile): + _pipfile = {} + if hasattr(pipfile, "keys"): + _pipfile = dict(pipfile).copy() + + extras = _pipfile.get("extras", []) + extras_str = "" + if extras: + extras_str = f"[{','.join(extras)}]" + vcs = next(iter([vcs for vcs in VCS_LIST if vcs in _pipfile]), None) + + if vcs: + _pipfile["vcs"] = vcs + req_str = f"{_pipfile[vcs]}{_pipfile.get('ref', '')}{extras_str}" + if not req_str.startswith(f"{vcs}+"): + req_str = f"{vcs}+{req_str}" + if f"{vcs}+file://" in req_str: + req_str = f"-e {req_str}#egg={name}{extras_str}" + else: + req_str = f"{name}{extras_str}@ {req_str}" + elif "path" in _pipfile: + req_str = f"{_pipfile['path']}{extras_str}" + elif "file" in _pipfile: + req_str = f"{_pipfile['file']}{extras_str}" + else: + # We ensure version contains an operator. Default to equals (==) + _pipfile["version"] = version = get_version(pipfile) + if version and not is_star(version) and COMPARE_OP.match(version) is None: + _pipfile["version"] = f"=={version}" + if is_star(version) or version == "==*": + version = "" + req_str = f"{name}{extras_str}{version}" + + install_req = expansive_install_req_from_line( + req_str, + comes_from=None, + use_pep517=False, + isolated=False, + hash_options={"hashes": _pipfile.get("hashes", [])}, + constraint=False, + expand_env=True, + ) + markers = PipenvMarkers.from_pipfile(name, _pipfile) + return install_req, markers + + +def from_pipfile(name, pipfile): + install_req, markers = install_req_from_pipfile(name, pipfile) + + if markers: + markers = str(markers) + install_req.markers = Marker(markers) + + # Construct the requirement string for your Requirement class + extras_str = "" + if install_req.req and install_req.req.extras: + extras_str = f"[{','.join(install_req.req.extras)}]" + specifier = install_req.req.specifier if install_req.req else "" + req_str = f"{install_req.name}{extras_str}{specifier}" + if install_req.markers: + req_str += f"; {install_req.markers}" + + # Create the Requirement instance + cls_inst = Requirement(req_str) + + return cls_inst + + def get_constraints_from_deps(deps): - """Get constraints from Pipfile-formatted dependency""" - - def is_constraints(dep: InstallRequirement) -> bool: - return dep.name and not dep.editable and not dep.extras - - constraints = [] - for dep_name, dep in deps.items(): - new_dep = Requirement.from_pipfile(dep_name, dep) - if new_dep.is_named and is_constraints(new_dep.ireq): - c = new_dep.as_line().strip() - constraints.append(c) + """Get constraints from dictionary-formatted dependency""" + constraints = set() + for dep_name, dep_version in deps.items(): + c = None + # Constraints cannot contain extras + dep_name = dep_name.split("[", 1)[0] + # Creating a constraint as a canonical name plus a version specifier + if isinstance(dep_version, str): + if dep_version and not is_star(dep_version): + if COMPARE_OP.match(dep_version) is None: + dep_version = f"=={dep_version}" + c = f"{canonicalize_name(dep_name)}{dep_version}" + else: + c = canonicalize_name(dep_name) + else: + if not any([k in dep_version for k in ["path", "file", "uri"]]): + if dep_version.get("skip_resolver") is True: + continue + version = dep_version.get("version", None) + if version and not is_star(version): + if COMPARE_OP.match(version) is None: + version = f"=={dep_version}" + c = f"{canonicalize_name(dep_name)}{version}" + else: + c = canonicalize_name(dep_name) + if c: + constraints.add(c) return constraints @@ -309,6 +1065,7 @@ def prepare_constraint_file( if not directory: directory = create_tracked_tempdir(suffix="-requirements", prefix="pipenv-") + constraints = set(constraints) constraints_file = NamedTemporaryFile( mode="w", prefix="pipenv-", @@ -326,7 +1083,8 @@ def prepare_constraint_file( requirementstxt_sources = requirementstxt_sources.replace(" --", "\n--") constraints_file.write(f"{requirementstxt_sources}\n") - constraints_file.write("\n".join([c for c in constraints])) + if constraints: + constraints_file.write("\n".join([c for c in constraints])) constraints_file.close() return constraints_file.name diff --git a/pipenv/vendor/requirementslib/exceptions.py b/pipenv/utils/exceptions.py similarity index 80% rename from pipenv/vendor/requirementslib/exceptions.py rename to pipenv/utils/exceptions.py index eacef437..121399da 100644 --- a/pipenv/vendor/requirementslib/exceptions.py +++ b/pipenv/utils/exceptions.py @@ -1,6 +1,3 @@ -# -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function - import errno import os import sys @@ -13,7 +10,7 @@ class RequirementError(Exception): class MissingParameter(Exception): def __init__(self, param): self.message = self.get_message(param) - super(MissingParameter, self).__init__(self.message) + super().__init__(self.message) @classmethod def get_message(cls, param): @@ -25,7 +22,6 @@ class MissingParameter(Exception): class FileCorruptException(OSError): def __init__(self, path, *args, **kwargs): - path = path backup_path = kwargs.pop("backup_path", None) if not backup_path and args: args = reversed(args) @@ -38,7 +34,7 @@ class FileCorruptException(OSError): if args: args = reversed(args) self.message = self.get_message(path, backup_path=backup_path) - super(FileCorruptException, self).__init__(self.message) + super().__init__(self.message) def get_message(self, path, backup_path=None): message = "ERROR: Failed to load file at %s" % path @@ -46,7 +42,7 @@ class FileCorruptException(OSError): msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + message = f"{message}\nYour lockfile is corrupt, {msg}" return message def show(self): @@ -56,7 +52,7 @@ class FileCorruptException(OSError): class LockfileCorruptException(FileCorruptException): def __init__(self, path, backup_path=None): self.message = self.get_message(path, backup_path=backup_path) - super(LockfileCorruptException, self).__init__(self.message) + super().__init__(self.message) def get_message(self, path, backup_path=None): message = "ERROR: Failed to load lockfile at %s" % path @@ -64,7 +60,7 @@ class LockfileCorruptException(FileCorruptException): msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - message = "{0}\nYour lockfile is corrupt, {1}".format(message, msg) + message = f"{message}\nYour lockfile is corrupt, {msg}" return message def show(self, path, backup_path=None): @@ -74,7 +70,7 @@ class LockfileCorruptException(FileCorruptException): class PipfileCorruptException(FileCorruptException): def __init__(self, path, backup_path=None): self.message = self.get_message(path, backup_path=backup_path) - super(PipfileCorruptException, self).__init__(self.message) + super().__init__(self.message) def get_message(self, path, backup_path=None): message = "ERROR: Failed to load Pipfile at %s" % path @@ -82,7 +78,7 @@ class PipfileCorruptException(FileCorruptException): msg = "it will be backed up to %s and removed" % backup_path else: msg = "it will be removed and replaced on the next lock." - message = "{0}\nYour Pipfile is corrupt, {1}".format(message, msg) + message = f"{message}\nYour Pipfile is corrupt, {msg}" return message def show(self, path, backup_path=None): @@ -93,4 +89,4 @@ class PipfileNotFound(FileNotFoundError): def __init__(self, path, *args, **kwargs): self.errno = errno.ENOENT self.filename = path - super(PipfileNotFound, self).__init__(self.filename) + super().__init__(self.filename) diff --git a/pipenv/vendor/requirementslib/fileutils.py b/pipenv/utils/fileutils.py similarity index 71% rename from pipenv/vendor/requirementslib/fileutils.py rename to pipenv/utils/fileutils.py index 1abdda66..e5bb1c61 100644 --- a/pipenv/vendor/requirementslib/fileutils.py +++ b/pipenv/utils/fileutils.py @@ -1,51 +1,25 @@ """A collection for utilities for working with files and paths.""" import atexit -import io import os -import posixpath import sys import warnings from contextlib import closing, contextmanager from http.client import HTTPResponse as Urllib_HTTPResponse from pathlib import Path from tempfile import TemporaryDirectory -from typing import IO, Any, ContextManager, Iterator, Optional, Text, TypeVar, Union +from typing import IO, Any, ContextManager, Optional, TypeVar, Union from urllib import parse as urllib_parse from urllib import request as urllib_request from urllib.parse import quote, urlparse from pipenv.patched.pip._vendor.requests import Session -from pipenv.patched.pip._vendor.urllib3.response import HTTPResponse as Urllib3_HTTPResponse +from pipenv.patched.pip._vendor.urllib3.response import ( + HTTPResponse as Urllib3_HTTPResponse, +) _T = TypeVar("_T") -@contextmanager -def cd(path): - # type: () -> Iterator[None] - """Context manager to temporarily change working directories. - - :param str path: The directory to move into - >>> print(os.path.abspath(os.curdir)) - '/home/user/code/myrepo' - >>> with cd("/home/user/code/otherdir/subdir"): - ... print("Changed directory: %s" % os.path.abspath(os.curdir)) - Changed directory: /home/user/code/otherdir/subdir - >>> print(os.path.abspath(os.curdir)) - '/home/user/code/myrepo' - """ - if not path: - return - prev_cwd = Path.cwd().as_posix() - if isinstance(path, Path): - path = path.as_posix() - os.chdir(str(path)) - try: - yield - finally: - os.chdir(prev_cwd) - - def is_file_url(url: Any) -> bool: """Returns true if the given url is a file url.""" if not url: @@ -54,7 +28,7 @@ def is_file_url(url: Any) -> bool: try: url = url.url except AttributeError: - raise ValueError("Cannot parse url from unknown type: {!r}".format(url)) + raise ValueError(f"Cannot parse url from unknown type: {url!r}") return urllib_parse.urlparse(url.lower()).scheme == "file" @@ -83,7 +57,7 @@ if os.name == "nt": # from click _winconsole.py from ctypes import create_unicode_buffer, windll - def get_long_path(short_path: Text) -> Text: + def get_long_path(short_path: str) -> str: BUFFER_SIZE = 500 buffer = create_unicode_buffer(BUFFER_SIZE) get_long_path_name = windll.kernel32.GetLongPathNameW @@ -135,7 +109,7 @@ def path_to_url(path): # XXX: actually part of a surrogate pair, but were just incidentally # XXX: passed in as a piece of a filename quoted_path = quote(path, errors="backslashreplace") - return "file:///{}:{}".format(drive, quoted_path) + return f"file:///{drive}:{quoted_path}" # XXX: This is also here to help deal with incidental dangling surrogates # XXX: on linux, by making sure they are preserved during encoding so that # XXX: we can urlencode the backslash correctly @@ -160,7 +134,7 @@ def open_file( try: link = link.url_without_fragment except AttributeError: - raise ValueError("Cannot parse url from unknown type: {0!r}".format(link)) + raise ValueError(f"Cannot parse url from unknown type: {link!r}") if not is_valid_url(link) and os.path.exists(link): link = path_to_url(link) @@ -169,9 +143,9 @@ def open_file( # Local URL local_path = url_to_path(link) if os.path.isdir(local_path): - raise ValueError("Cannot open directory for read: {}".format(link)) + raise ValueError(f"Cannot open directory for read: {link}") else: - with io.open(local_path, "rb") as local_file: + with open(local_path, "rb") as local_file: yield local_file else: # Remote URL @@ -264,52 +238,3 @@ def check_for_unc_path(path): return True else: return False - - -def get_converted_relative_path(path, relative_to=None): - """Convert `path` to be relative. - - Given a vague relative path, return the path relative to the given - location. - - :param str path: The location of a target path - :param str relative_to: The starting path to build against, optional - :returns: A relative posix-style path with a leading `./` - - This performs additional conversion to ensure the result is of POSIX form, - and starts with `./`, or is precisely `.`. - - >>> os.chdir('/home/user/code/myrepo/myfolder') - >>> vistir.path.get_converted_relative_path('/home/user/code/file.zip') - './../../file.zip' - >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder/mysubfolder') - './mysubfolder' - >>> vistir.path.get_converted_relative_path('/home/user/code/myrepo/myfolder') - '.' - """ - if not relative_to: - relative_to = os.getcwd() - - start_path = Path(str(relative_to)) - try: - start = start_path.resolve() - except OSError: - start = start_path.absolute() - - # check if there is a drive letter or mount point - # if it is a mountpoint use the original absolute path - # instead of the unc path - if check_for_unc_path(start): - start = start_path.absolute() - - path = start.joinpath(str(path)).relative_to(start) - - # check and see if the path that was passed into the function is a UNC path - # and raise value error if it is not. - if check_for_unc_path(path): - raise ValueError("The path argument does not currently accept UNC paths") - - relpath_s = posixpath.normpath(path.as_posix()) - if not (relpath_s == "." or relpath_s.startswith("./")): - relpath_s = posixpath.join(".", relpath_s) - return relpath_s diff --git a/pipenv/utils/funktools.py b/pipenv/utils/funktools.py index f1a682cc..be40778e 100644 --- a/pipenv/utils/funktools.py +++ b/pipenv/utils/funktools.py @@ -1,13 +1,19 @@ """ A small collection of useful functional tools for working with iterables. - -This module should be in requirementslib. Once we release a new version of requirementslib -we can remove this file and use the one in requirementslib. """ +import errno +import locale +import os +import stat +import subprocess +import time +import warnings from functools import partial -from itertools import islice, tee +from itertools import count, islice, tee from typing import Any, Iterable +DIRECTORY_CLEANUP_TIMEOUT = 1.0 + def _is_iterable(elem: Any) -> bool: if getattr(elem, "__iter__", False) or isinstance(elem, Iterable): @@ -35,7 +41,6 @@ def chunked(n: int, iterable: Iterable) -> Iterable: def unnest(elem: Iterable) -> Any: - # type: (Iterable) -> Any """Flatten an arbitrarily nested iterable. :param elem: An iterable to flatten @@ -73,8 +78,288 @@ def unnest(elem: Iterable) -> Any: def dedup(iterable: Iterable) -> Iterable: - # type: (Iterable) -> Iterable """Deduplicate an iterable object like iter(set(iterable)) but order- preserved.""" return iter(dict.fromkeys(iterable)) + + +def is_readonly_path(fn: os.PathLike) -> bool: + """check if a provided path exists and is readonly. + + permissions check is `bool(path.stat & stat.s_iread)` or `not + os.access(path, os.w_ok)` + """ + if os.path.exists(fn): + file_stat = os.stat(fn).st_mode + return not bool(file_stat & stat.s_iwrite) or not os.access(fn, os.w_ok) + return False + + +def _wait_for_files(path): # pragma: no cover + """Retry with backoff up to 1 second to delete files from a directory. + + :param str path: The path to crawl to delete files from + :return: A list of remaining paths or None + :rtype: Optional[List[str]] + """ + timeout = 0.001 # noqa:S101 + remaining = [] + while timeout < DIRECTORY_CLEANUP_TIMEOUT: + remaining = [] + if os.path.isdir(path): + L = os.listdir(path) + for target in L: + _remaining = _wait_for_files(target) + if _remaining: + remaining.extend(_remaining) + continue + try: + os.unlink(path) + except FileNotFoundError as e: + if e.errno == errno.ENOENT: + return + except (OSError, PermissionError): # noqa:B014 + time.sleep(timeout) + timeout *= 2 + remaining.append(path) + else: + return + return remaining + + +def _walk_for_powershell(directory): + for _, dirs, files in os.walk(directory): + powershell = next( + iter(fn for fn in files if fn.lower() == "powershell.exe"), None + ) + if powershell is not None: + return os.path.join(directory, powershell) + for subdir in dirs: + powershell = _walk_for_powershell(os.path.join(directory, subdir)) + if powershell: + return powershell + return None + + +def _get_powershell_path(): + paths = [ + os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir) + for subdir in ("SysWOW64", "system32") + ] + powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None) + if not powershell_path: + powershell_path = subprocess.run(["where", "powershell"]) + if powershell_path.stdout: + return powershell_path.stdout.strip() + + +def _get_sid_with_powershell(): + powershell_path = _get_powershell_path() + if not powershell_path: + return None + args = [ + powershell_path, + "-ExecutionPolicy", + "Bypass", + "-Command", + "Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'", + ] + sid = subprocess.run(args, capture_output=True) + return sid.stdout.strip() + + +def get_value_from_tuple(value, value_type): + try: + import winreg + except ImportError: + import _winreg as winreg + if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ): + if "\0" in value: + return value[: value.index("\0")] + return value + return None + + +def query_registry_value(root, key_name, value): + try: + import winreg + except ImportError: + import _winreg as winreg + try: + with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key: + return get_value_from_tuple(*winreg.QueryValueEx(key, value)) + except OSError: + return None + + +def _get_sid_from_registry(): + try: + import winreg + except ImportError: + import _winreg as winreg + var_names = ("%USERPROFILE%", "%HOME%") + current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None) + root, subkey = ( + winreg.HKEY_LOCAL_MACHINE, + r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList", + ) + subkey_names = [] + value = None + matching_key = None + try: + with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key: + for i in count(): + key_name = winreg.EnumKey(key, i) + subkey_names.append(key_name) + value = query_registry_value( + root, rf"{subkey}\{key_name}", "ProfileImagePath" + ) + if value and value.lower() == current_user_home.lower(): + matching_key = key_name + break + except OSError: + pass + if matching_key is not None: + return matching_key + + +def _get_current_user(): + fns = (_get_sid_from_registry, _get_sid_with_powershell) + for fn in fns: + result = fn() + if result: + return result + return None + + +def _find_icacls_exe(): + if os.name == "nt": + paths = [ + os.path.expandvars(r"%windir%\{0}").format(subdir) + for subdir in ("system32", "SysWOW64") + ] + for path in paths: + icacls_path = next( + iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None + ) + if icacls_path is not None: + icacls_path = os.path.join(path, icacls_path) + return icacls_path + return None + + +def set_write_bit(fn: str) -> None: + """Set read-write permissions for the current user on the target path. Fail + silently if the path doesn't exist. + + :param str fn: The target filename or path + :return: None + """ + if not os.path.exists(fn): + return + file_stat = os.stat(fn).st_mode + os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) + if os.name == "nt": + user_sid = _get_current_user() + icacls_exe = _find_icacls_exe() or "icacls" + + if user_sid: + c = subprocess.run( + [ + icacls_exe, + f"''{fn}''", + "/grant", + f"{user_sid}:WD", + "/T", + "/C", + "/Q", + ], + capture_output=True, + # 2020-06-12 Yukihiko Shinoda + # There are 3 way to get system default encoding in Stack Overflow. + # see: https://stackoverflow.com/questions/37506535/how-to-get-the-system-default-encoding-in-python-2-x + # I investigated these way by using Shift-JIS Windows. + # >>> import locale + # >>> locale.getpreferredencoding() + # "cp932" (Shift-JIS) + # >>> import sys + # >>> sys.getdefaultencoding() + # "utf-8" + # >>> sys.stdout.encoding + # "UTF8" + encoding=locale.getpreferredencoding(), + ) + if not c.err and c.returncode == 0: + return + + if not os.path.isdir(fn): + for path in [fn, os.path.dirname(fn)]: + try: + os.chflags(path, 0) + except AttributeError: + pass + return None + for root, dirs, files in os.walk(fn, topdown=False): + for dir_ in [os.path.join(root, d) for d in dirs]: + set_write_bit(dir_) + for file_ in [os.path.join(root, f) for f in files]: + set_write_bit(file_) + + +def handle_remove_readonly(func, path, exc): + """Error handler for shutil.rmtree. + + Windows source repo folders are read-only by default, so this error handler + attempts to set them as writeable and then proceed with deletion. + + :param function func: The caller function + :param str path: The target path for removal + :param Exception exc: The raised exception + + This function will call check :func:`is_readonly_path` before attempting to call + :func:`set_write_bit` on the target path and try again. + """ + + PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) + default_warning_message = "Unable to remove file due to permissions restriction: {!r}" + # split the initial exception out into its type, exception, and traceback + exc_type, exc_exception, exc_tb = exc + if is_readonly_path(path): + # Apply write permission and call original function + set_write_bit(path) + try: + func(path) + except (OSError, FileNotFoundError, PermissionError) as e: # pragma: no cover + if e.errno in PERM_ERRORS: + if e.errno == errno.ENOENT: + return + remaining = None + if os.path.isdir(path): + remaining = _wait_for_files(path) + if remaining: + warnings.warn( + default_warning_message.format(path), + ResourceWarning, + stacklevel=2, + ) + else: + func(path, ignore_errors=True) + return + + if exc_exception.errno in PERM_ERRORS: + set_write_bit(path) + remaining = _wait_for_files(path) + try: + func(path) + except (OSError, FileNotFoundError, PermissionError) as e: # noqa:B014 + if e.errno in PERM_ERRORS: + if e.errno != errno.ENOENT: # File still exists + warnings.warn( + default_warning_message.format(path), + ResourceWarning, + stacklevel=2, + ) + return + else: + raise exc_exception diff --git a/pipenv/utils/internet.py b/pipenv/utils/internet.py index 5fde7237..f4d57b38 100644 --- a/pipenv/utils/internet.py +++ b/pipenv/utils/internet.py @@ -1,5 +1,6 @@ import os import re +from html.parser import HTMLParser from urllib.parse import urlparse from pipenv.patched.pip._vendor import requests @@ -7,7 +8,7 @@ from pipenv.patched.pip._vendor.requests.adapters import HTTPAdapter from pipenv.patched.pip._vendor.urllib3 import util as urllib3_util -def _get_requests_session(max_retries=1, verify_ssl=True): +def get_requests_session(max_retries=1, verify_ssl=True): """Load requests lazily.""" pip_client_cert = os.environ.get("PIP_CLIENT_CERT") requests_session = requests.Session() @@ -46,7 +47,7 @@ def create_mirror_source(url, name): def download_file(url, filename, max_retries=1): """Downloads file from url to a path with filename""" - r = _get_requests_session(max_retries).get(url, stream=True) + r = get_requests_session(max_retries).get(url, stream=True) r.close() if not r.ok: raise OSError("Unable to download file") @@ -116,7 +117,7 @@ def is_url_equal(url: str, other_url: str) -> bool: def proper_case(package_name): """Properly case project name from pypi.org.""" # Hit the simple API. - r = _get_requests_session().get( + r = get_requests_session().get( f"https://pypi.org/pypi/{package_name}/json", timeout=0.3, stream=True ) r.close() @@ -128,3 +129,17 @@ def proper_case(package_name): good_name = match.group(1) return good_name + + +class PackageIndexHTMLParser(HTMLParser): + def __init__(self): + super().__init__() + self.urls = [] + + def handle_starttag(self, tag, attrs): + # If tag is an anchor + if tag == "a": + # find href attribute + for attr in attrs: + if attr[0] == "href": + self.urls.append(attr[1]) diff --git a/pipenv/utils/locking.py b/pipenv/utils/locking.py index 4803fb48..c8bc8ad0 100644 --- a/pipenv/utils/locking.py +++ b/pipenv/utils/locking.py @@ -1,93 +1,157 @@ +import copy +import itertools import os import stat from contextlib import contextmanager +from json import JSONDecodeError +from pathlib import Path from tempfile import NamedTemporaryFile -from typing import Mapping +from typing import Dict, Iterator, List, Optional -from .dependencies import clean_resolved_dep, pep423_name, translate_markers +from pipenv.patched.pip._internal.req.req_install import InstallRequirement +from pipenv.utils.dependencies import ( + clean_resolved_dep, + determine_vcs_revision_hash, + expansive_install_req_from_line, + pep423_name, + translate_markers, +) +from pipenv.utils.exceptions import ( + LockfileCorruptException, + MissingParameter, + PipfileNotFound, +) +from pipenv.utils.pipfile import DEFAULT_NEWLINES, ProjectFile +from pipenv.utils.requirements import normalize_name +from pipenv.utils.requirementslib import is_editable, is_vcs, merge_items +from pipenv.vendor.plette import lockfiles +from pipenv.vendor.pydantic import BaseModel, Field -def format_requirement_for_lockfile(req, markers_lookup, index_lookup, hashes=None): - if req.specifiers: - version = str(req.get_version) +def merge_markers(entry, markers): + if not isinstance(markers, list): + markers = [markers] + for marker in markers: + if not isinstance(marker, str): + marker = str(marker) + if "markers" not in entry: + entry["markers"] = marker + elif marker not in entry["markers"]: + entry["markers"] = f"({entry['markers']}) and ({marker})" + + +def format_requirement_for_lockfile( + req: InstallRequirement, + markers_lookup, + index_lookup, + original_deps, + pipfile_entries, + hashes=None, +): + if req.specifier: + version = str(req.specifier) else: version = None - index = index_lookup.get(req.normalized_name) - markers = markers_lookup.get(req.normalized_name) + name = normalize_name(req.name) + index = index_lookup.get(name) + markers = req.markers req.index = index - name, pf_entry = req.pipfile_entry - name = pep423_name(req.name) + pipfile_entry = pipfile_entries[name] if name in pipfile_entries else {} entry = {} - if isinstance(pf_entry, str): - entry["version"] = pf_entry.lstrip("=") - else: - entry.update(pf_entry) - if version is not None and not req.is_vcs: - entry["version"] = version - if req.line_instance.is_direct_url and not req.is_vcs: - entry["file"] = req.req.uri + if req.link and req.link.is_vcs: + vcs = req.link.scheme.split("+", 1)[0] + entry["ref"] = determine_vcs_revision_hash(req, vcs, pipfile_entry.get("ref")) + if name in original_deps: + entry[vcs] = original_deps[name] + else: + entry[vcs] = req.link.url + if req.req: + entry["version"] = str(req.specifier) + elif version: + entry["version"] = version + elif req.link and req.link.is_file: + entry["file"] = req.link.url if hashes: entry["hashes"] = sorted(set(hashes)) entry["name"] = name if index: entry.update({"index": index}) if markers: - entry.update({"markers": markers}) + entry.update({"markers": str(markers)}) + if name in markers_lookup: + merge_markers(entry, markers_lookup[name]) + if isinstance(pipfile_entry, dict) and "markers" in pipfile_entry: + merge_markers(entry, pipfile_entry["markers"]) + if isinstance(pipfile_entry, dict) and "os_name" in pipfile_entry: + merge_markers(entry, f"os_name {pipfile_entry['os_name']}") entry = translate_markers(entry) - if req.vcs or req.editable: - for key in ("index", "version", "file"): - try: - del entry[key] - except KeyError: - pass + if req.extras: + entry["extras"] = sorted(req.extras) + if isinstance(pipfile_entry, dict) and pipfile_entry.get("file"): + entry["file"] = pipfile_entry["file"] + entry["editable"] = True + entry.pop("version", None) + entry.pop("index", None) + elif isinstance(pipfile_entry, dict) and pipfile_entry.get("path"): + entry["path"] = pipfile_entry["path"] + entry["editable"] = True + entry.pop("version", None) + entry.pop("index", None) return name, entry -def get_locked_dep(dep, pipfile_section): - entry = None - cleaner_kwargs = {"is_top_level": False, "pipfile_entry": None} - if isinstance(dep, Mapping) and dep.get("name"): +def get_locked_dep(project, dep, pipfile_section, current_entry=None): + # initialize default values + is_top_level = False + + # if the dependency has a name, find corresponding entry in pipfile + if isinstance(dep, dict) and dep.get("name"): dep_name = pep423_name(dep["name"]) for pipfile_key, pipfile_entry in pipfile_section.items(): - if pep423_name(pipfile_key) == dep_name: - entry = pipfile_entry + if pep423_name(pipfile_key) == dep_name or pipfile_key == dep_name: + is_top_level = True + if isinstance(pipfile_entry, dict): + if pipfile_entry.get("version"): + pipfile_entry.pop("version") + if pipfile_entry.get("ref"): + pipfile_entry.pop("ref") + dep.update(pipfile_entry) + break - if entry: - cleaner_kwargs.update({"is_top_level": True, "pipfile_entry": entry}) - lockfile_entry = clean_resolved_dep(dep, **cleaner_kwargs) - if entry and isinstance(entry, Mapping): - version = entry.get("version", "") if entry else "" - else: - version = entry if entry else "" + # clean the dependency + lockfile_entry = clean_resolved_dep(project, dep, is_top_level, current_entry) + + # get the lockfile version and compare with pipfile version lockfile_name, lockfile_dict = lockfile_entry.copy().popitem() - lockfile_version = lockfile_dict.get("version", "") - # Keep pins from the lockfile - if lockfile_version != version and version.startswith("==") and "*" not in version: - lockfile_dict["version"] = version lockfile_entry[lockfile_name] = lockfile_dict + return lockfile_entry -def prepare_lockfile(results, pipfile, lockfile): - # from .vendor.requirementslib.utils import is_vcs +def prepare_lockfile(project, results, pipfile, lockfile_section, old_lock_data=None): for dep in results: if not dep: continue - # Merge in any relevant information from the pipfile entry, including - # markers, normalized names, URL info, etc that we may have dropped during lock - # if not is_vcs(dep): - lockfile_entry = get_locked_dep(dep, pipfile) - name = next(iter(k for k in lockfile_entry.keys())) - current_entry = lockfile.get(name) - if current_entry: - if not isinstance(current_entry, Mapping): - lockfile[name] = lockfile_entry[name] - else: - lockfile[name].update(lockfile_entry[name]) - lockfile[name] = translate_markers(lockfile[name]) + dep_name = dep["name"] + current_entry = None + if dep_name in old_lock_data: + current_entry = old_lock_data[dep_name] + lockfile_entry = get_locked_dep(project, dep, pipfile, current_entry) + + # If the current dependency doesn't exist in the lockfile, add it + if dep_name not in lockfile_section: + lockfile_section[dep_name] = lockfile_entry[dep_name] else: - lockfile[name] = lockfile_entry[name] - return lockfile + # If the dependency exists, update the details + current_entry = lockfile_section[dep_name] + if not isinstance(current_entry, dict): + lockfile_section[dep_name] = lockfile_entry[dep_name] + else: + # If the current entry is a dict, merge the new details + lockfile_section[dep_name].update(lockfile_entry[dep_name]) + lockfile_section[dep_name] = translate_markers(lockfile_section[dep_name]) + + return lockfile_section @contextmanager @@ -170,3 +234,236 @@ def atomic_open_for_write(target, binary=False, newline=None, encoding=None) -> except OSError: pass os.rename(f.name, target) # No os.replace() on Python 2. + + +class Lockfile(BaseModel): + path: Path = Field( + default_factory=lambda: Path(os.curdir).joinpath("Pipfile.lock").absolute() + ) + _requirements: Optional[list] = Field(default_factory=list) + _dev_requirements: Optional[list] = Field(default_factory=list) + projectfile: ProjectFile = None + lockfile: lockfiles.Lockfile + newlines: str = DEFAULT_NEWLINES + + class Config: + validate_assignment = True + arbitrary_types_allowed = True + allow_mutation = True + include_private_attributes = True + # keep_untouched = (cached_property,) + + @property + def section_keys(self): + return set(self.lockfile.keys()) - {"_meta"} + + @property + def extended_keys(self): + return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) + if check_lockfile: + return True + return super().__contains__(k) + + def __setitem__(self, k, v): + lockfile = self.lockfile + lockfile.__setitem__(k, v) + + def __getitem__(self, k, *args, **kwargs): + retval = None + lockfile = self.lockfile + try: + retval = lockfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(lockfile.get(section, {}), "_data", {}) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + lockfile = self.lockfile + try: + return super().__getattribute__(k) + except AttributeError: + retval = getattr(lockfile, k, None) + if retval is not None: + return retval + return super().__getattribute__(k, *args, **kwargs) + + def get_deps(self, dev=False, only=True): + deps = {} + if dev: + deps.update(self.develop._data) + if only: + return deps + deps = merge_items([deps, self.default._data]) + return deps + + @classmethod + def read_projectfile(cls, path): + pf = ProjectFile.read(path, lockfiles.Lockfile, invalid_ok=True) + return pf + + @classmethod + def lockfile_from_pipfile(cls, pipfile_path): + from pipenv.utils.pipfile import Pipfile + + if os.path.isfile(pipfile_path): + if not os.path.isabs(pipfile_path): + pipfile_path = os.path.abspath(pipfile_path) + pipfile = Pipfile.load(os.path.dirname(pipfile_path)) + return lockfiles.Lockfile.with_meta_from(pipfile.pipfile) + raise PipfileNotFound(pipfile_path) + + @classmethod + def load_projectfile( + cls, path: Optional[str] = None, create: bool = True, data: Optional[Dict] = None + ) -> "ProjectFile": + if not path: + path = os.curdir + path = Path(path).absolute() + project_path = path if path.is_dir() else path.parent + lockfile_path = path if path.is_file() else project_path / "Pipfile.lock" + if not project_path.exists(): + raise OSError(f"Project does not exist: {project_path.as_posix()}") + elif not lockfile_path.exists() and not create: + raise FileNotFoundError( + f"Lockfile does not exist: {lockfile_path.as_posix()}" + ) + projectfile = cls.read_projectfile(lockfile_path.as_posix()) + if not lockfile_path.exists(): + if not data: + pipfile = project_path.joinpath("Pipfile") + lf = cls.lockfile_from_pipfile(pipfile) + else: + lf = lockfiles.Lockfile(data) + projectfile.model = lf + else: + if data: + raise ValueError("Cannot pass data when loading existing lockfile") + with open(lockfile_path.as_posix()) as f: + projectfile.model = lockfiles.Lockfile.load(f) + return projectfile + + @classmethod + def from_data( + cls, path: Optional[str], data: Optional[Dict], meta_from_project: bool = True + ) -> "Lockfile": + if path is None: + raise MissingParameter("path") + if data is None: + raise MissingParameter("data") + if not isinstance(data, dict): + raise TypeError("Expecting a dictionary for parameter 'data'") + path = os.path.abspath(str(path)) + if os.path.isdir(path): + project_path = path + elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): + project_path = os.path.dirname(path) + pipfile_path = os.path.join(project_path, "Pipfile") + lockfile_path = os.path.join(project_path, "Pipfile.lock") + if meta_from_project: + lockfile = cls.lockfile_from_pipfile(pipfile_path) + lockfile.update(data) + else: + lockfile = lockfiles.Lockfile(data) + projectfile = ProjectFile( + line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile + ) + return cls( + projectfile=projectfile, + lockfile=lockfile, + newlines=projectfile.line_ending, + path=Path(projectfile.location), + ) + + @classmethod + def load(cls, path: Optional[str], create: bool = True) -> "Lockfile": + try: + projectfile = cls.load_projectfile(path, create=create) + except JSONDecodeError: + path = os.path.abspath(path) + path = Path( + os.path.join(path, "Pipfile.lock") if os.path.isdir(path) else path + ) + formatted_path = path.as_posix() + backup_path = f"{formatted_path}.bak" + LockfileCorruptException.show(formatted_path, backup_path=backup_path) + path.rename(backup_path) + cls.load(formatted_path, create=True) + lockfile_path = Path(projectfile.location) + creation_args = { + "projectfile": projectfile, + "lockfile": projectfile.model, + "newlines": projectfile.line_ending, + "path": lockfile_path, + } + return cls(**creation_args) + + @classmethod + def create(cls, path: Optional[str], create: bool = True) -> "Lockfile": + return cls.load(path, create=create) + + def get_section(self, name: str) -> Optional[Dict]: + return self.lockfile.get(name) + + @property + def develop(self) -> Dict: + return self.lockfile.develop + + @property + def default(self) -> Dict: + return self.lockfile.default + + def get_requirements( + self, dev: bool = True, only: bool = False, categories: Optional[List[str]] = None + ) -> Iterator[InstallRequirement]: + from pipenv.utils.requirements import requirement_from_lockfile + + if categories: + deps = {} + for category in categories: + if category == "packages": + category = "default" + elif category == "dev-packages": + category = "develop" + try: + category_deps = self[category] + except KeyError: + category_deps = {} + self.lockfile[category] = category_deps + deps = merge_items([deps, category_deps]) + else: + deps = self.get_deps(dev=dev, only=only) + for package_name, package_info in deps.items(): + pip_line = requirement_from_lockfile( + package_name, package_info, include_hashes=False, include_markers=False + ) + pip_line_specified = requirement_from_lockfile( + package_name, package_info, include_hashes=True, include_markers=True + ) + yield expansive_install_req_from_line(pip_line), pip_line_specified + + def requirements_list(self, category: str) -> List[Dict]: + if self.lockfile.get(category): + return [ + {name: entry._data} for name, entry in self.lockfile[category].items() + ] + return [] + + def write(self) -> None: + self.projectfile.model = copy.deepcopy(self.lockfile) + self.projectfile.write() diff --git a/pipenv/vendor/requirementslib/models/markers.py b/pipenv/utils/markers.py similarity index 95% rename from pipenv/vendor/requirementslib/models/markers.py rename to pipenv/utils/markers.py index fd5451d8..5b162877 100644 --- a/pipenv/vendor/requirementslib/models/markers.py +++ b/pipenv/utils/markers.py @@ -3,24 +3,23 @@ import operator import re from collections.abc import Mapping, Set from functools import reduce -from typing import Any, AnyStr, Iterator, List, Optional, Tuple, Type, Union +from typing import Optional from pipenv.patched.pip._vendor.distlib import markers from pipenv.patched.pip._vendor.packaging.markers import InvalidMarker, Marker -from pipenv.patched.pip._vendor.packaging.specifiers import LegacySpecifier, Specifier, SpecifierSet +from pipenv.patched.pip._vendor.packaging.specifiers import ( + LegacySpecifier, + Specifier, + SpecifierSet, +) from pipenv.vendor.pydantic import BaseModel -from ..exceptions import RequirementError - MAX_VERSIONS = {1: 7, 2: 7, 3: 11, 4: 0} DEPRECATED_VERSIONS = ["3.0", "3.1", "3.2", "3.3"] -def is_instance(item, cls): - # type: (Any, Type) -> bool - if isinstance(item, cls) or item.__class__.__name__ == cls.__name__: - return True - return False +class RequirementError(Exception): + pass class PipenvMarkers(BaseModel): @@ -38,9 +37,7 @@ class PipenvMarkers(BaseModel): @property def line_part(self): - return " and ".join( - ["{0} {1}".format(k, v) for k, v in self.dict().items() if v is not None] - ) + return " and ".join([f"{k} {v}" for k, v in self.dict().items() if v is not None]) @property def pipfile_part(self): @@ -67,7 +64,7 @@ class PipenvMarkers(BaseModel): def from_pipfile(cls, name, pipfile): attr_fields = [field_name for field_name in cls.__fields__] found_keys = [k for k in pipfile.keys() if k in attr_fields] - marker_strings = ["{0} {1}".format(k, pipfile[k]) for k in found_keys] + marker_strings = [f"{k} {pipfile[k]}" for k in found_keys] if pipfile.get("markers"): marker_strings.append(pipfile.get("markers")) markers = set() @@ -82,6 +79,13 @@ class PipenvMarkers(BaseModel): return combined_marker +def is_instance(item, cls): + # type: (Any, Type) -> bool + if isinstance(item, cls) or item.__class__.__name__ == cls.__name__: + return True + return False + + def _tuplize_version(version): # type: (str) -> Union[Tuple[()], Tuple[int, ...], Tuple[int, int, str]] output = [] @@ -110,7 +114,7 @@ def _format_pyspec(specifier): # type: (Union[str, Specifier]) -> Specifier if isinstance(specifier, str): if not specifier.startswith(tuple(Specifier._operators.keys())): - specifier = "=={0}".format(specifier) + specifier = f"=={specifier}" specifier = Specifier(specifier) version = getattr(specifier, "version", specifier).rstrip() if version: @@ -121,7 +125,7 @@ def _format_pyspec(specifier): if version.endswith(".*"): version = version[:-2] version = version.rstrip("*") - specifier = Specifier("{0}{1}".format(specifier.operator, version)) + specifier = Specifier(f"{specifier.operator}{version}") try: op = REPLACE_RANGES[specifier.operator] except KeyError: @@ -137,7 +141,7 @@ def _format_pyspec(specifier): next_tuple = (next_tuple[0], curr_tuple[1]) else: return specifier - specifier = Specifier("{0}{1}".format(op, _format_version(next_tuple))) + specifier = Specifier(f"{op}{_format_version(next_tuple)}") return specifier @@ -214,9 +218,7 @@ def normalize_specifier_set(specs): # And rename it to something meaningful def get_sorted_version_string(version_set): # type: (Set[AnyStr]) -> AnyStr - version_list = sorted( - "{0}".format(_format_version(version)) for version in version_set - ) + version_list = sorted(f"{_format_version(version)}" for version in version_set) version = ", ".join(version_list) return version @@ -495,7 +497,7 @@ def _split_specifierset_str(specset_str, prefix="=="): if prefix == "!=" and any(v in values for v in DEPRECATED_VERSIONS): values += DEPRECATED_VERSIONS[:] for value in sorted(values): - specifiers.add(Specifier("{0}{1}".format(prefix, value))) + specifiers.add(Specifier(f"{prefix}{value}")) return specifiers @@ -516,7 +518,7 @@ def _get_specifiers_from_markers(marker_item): elif op.value == "not in": specifiers.update(_split_specifierset_str(value.value, prefix="!=")) else: - specifiers.add(Specifier("{0}{1}".format(op.value, value.value))) + specifiers.add(Specifier(f"{op.value}{value.value}")) elif isinstance(marker_item, list): parts = get_specset(marker_item) if parts: @@ -611,12 +613,17 @@ def _contains_micro_version(version_string): return re.search(r"\d+\.\d+\.\d+", version_string) is not None -def format_pyversion(parts): - op, val = parts - version_marker = ( - "python_full_version" if _contains_micro_version(val) else "python_version" - ) - return "{0} {1} '{2}'".format(version_marker, op, val) +def merge_markers(m1, m2): + # type: (Marker, Marker) -> Optional[Marker] + if not all((m1, m2)): + return next(iter(v for v in (m1, m2) if v), None) + m1 = _ensure_marker(m1) + m2 = _ensure_marker(m2) + _markers = [] # type: List[Marker] + for marker in (m1, m2): + _markers.append(str(marker)) + marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) + return _ensure_marker(normalize_marker_str(marker_str)) def normalize_marker_str(marker) -> str: @@ -632,9 +639,9 @@ def normalize_marker_str(marker) -> str: marker_str = " and ".join([format_pyversion(pv) for pv in parts]) if marker: if marker_str: - marker_str = "{0!s} and {1!s}".format(marker_str, marker) + marker_str = f"{marker_str!s} and {marker!s}" else: - marker_str = "{0!s}".format(marker) + marker_str = f"{marker!s}" return marker_str.replace('"', "'") @@ -642,9 +649,9 @@ def marker_from_specifier(spec) -> Marker: if not any(spec.startswith(k) for k in Specifier._operators.keys()): if spec.strip().lower() in ["any", "", "*"]: return None - spec = "=={0}".format(spec) + spec = f"=={spec}" elif spec.startswith("==") and spec.count("=") > 3: - spec = "=={0}".format(spec.lstrip("=")) + spec = "=={}".format(spec.lstrip("=")) if not spec: return None marker_segments = [] @@ -654,14 +661,9 @@ def marker_from_specifier(spec) -> Marker: return Marker(marker_str) -def merge_markers(m1, m2): - # type: (Marker, Marker) -> Optional[Marker] - if not all((m1, m2)): - return next(iter(v for v in (m1, m2) if v), None) - m1 = _ensure_marker(m1) - m2 = _ensure_marker(m2) - _markers = [] # type: List[Marker] - for marker in (m1, m2): - _markers.append(str(marker)) - marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) - return _ensure_marker(normalize_marker_str(marker_str)) +def format_pyversion(parts): + op, val = parts + version_marker = ( + "python_full_version" if _contains_micro_version(val) else "python_version" + ) + return f"{version_marker} {op} '{val}'" diff --git a/pipenv/utils/pip.py b/pipenv/utils/pip.py index 35805ba8..1f5d56d1 100644 --- a/pipenv/utils/pip.py +++ b/pipenv/utils/pip.py @@ -1,72 +1,14 @@ -import logging import os import tempfile from pathlib import Path from typing import List, Optional from pipenv.patched.pip._internal.build_env import get_runnable_pip -from pipenv.project import Project from pipenv.utils import err -from pipenv.utils.dependencies import get_constraints_from_deps, prepare_constraint_file -from pipenv.utils.indexes import get_source_list, prepare_pip_source_args +from pipenv.utils.fileutils import create_tracked_tempdir, normalize_path +from pipenv.utils.indexes import prepare_pip_source_args from pipenv.utils.processes import subprocess_run from pipenv.utils.shell import cmd_list_to_shell, project_python -from pipenv.vendor.requirementslib import Requirement -from pipenv.vendor.requirementslib.fileutils import create_tracked_tempdir, normalize_path - - -def format_pip_output(out, r=None): - def gen(out): - for line in out.split("\n"): - # Remove requirements file information from pip9 output. - if "(from -r" in line: - yield line[: line.index("(from -r")] - - else: - yield line - - out = "\n".join([line for line in gen(out)]) - return out - - -def format_pip_error(error): - error = error.replace("Expected", "[bold green]Expected[/bold green]") - error = error.replace("Got", "[bold red]Got[/red bold]") - error = error.replace( - "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE", - "[bold red]THESE PACKAGES DO NOT MATCH THE HASHES FROM Pipfile.lock![/bold red]", - ) - error = error.replace( - "someone may have tampered with them", - "[red]someone may have tampered with them[/red]", - ) - error = error.replace("option to pip install", "option to 'pipenv install'") - return error - - -def pip_download(project, package_name): - cache_dir = Path(project.s.PIPENV_CACHE_DIR) - pip_config = { - "PIP_CACHE_DIR": cache_dir.as_posix(), - "PIP_WHEEL_DIR": cache_dir.joinpath("wheels").as_posix(), - "PIP_DESTINATION_DIR": cache_dir.joinpath("pkgs").as_posix(), - } - for source in project.sources: - cmd = [ - project_python(project), - get_runnable_pip(), - "download", - package_name, - "-i", - source["url"], - "-d", - project.download_location, - ] - c = subprocess_run(cmd, env=pip_config) - if c.returncode == 0: - break - - return c def pip_install_deps( @@ -95,56 +37,29 @@ def pip_install_deps( editable_requirements = tempfile.NamedTemporaryFile( prefix="pipenv-", suffix="-reqs.txt", dir=requirements_dir, delete=False ) - for requirement in deps: - ignore_hash = ignore_hashes - vcs_or_editable = ( - requirement.is_vcs - or requirement.vcs - or requirement.editable - or (requirement.is_file_or_url and not requirement.hashes) - ) - if vcs_or_editable: - ignore_hash = True - if requirement and vcs_or_editable: - requirement.index = None - line = requirement.line_instance.get_line( - with_prefix=True, - with_hashes=not ignore_hash, - with_markers=True, - as_list=False, - ) + for pip_line in deps: + ignore_hash = ignore_hashes or "--hash" not in pip_line + if project.s.is_verbose(): - err.print(f"Writing supplied requirement line to temporary file: {line!r}") - target = editable_requirements if vcs_or_editable else standard_requirements - target.write(line.encode()) + err.print( + f"Writing supplied requirement line to temporary file: {pip_line!r}" + ) + target = editable_requirements if ignore_hash else standard_requirements + target.write(pip_line.encode()) target.write(b"\n") + standard_requirements.close() editable_requirements.close() cmds = [] files = [] - standard_deps = list( - filter( - lambda d: not ( - d.is_vcs or d.vcs or d.editable or (d.is_file_or_url and not d.hashes) - ), - deps, - ) - ) - if standard_deps: - files.append(standard_requirements) - editable_deps = list( - filter( - lambda d: d.is_vcs - or d.vcs - or d.editable - or (d.is_file_or_url and not d.hashes), - deps, - ) - ) - if editable_deps: - files.append(editable_requirements) + for pip_line in deps: + if "--hash" in pip_line and standard_requirements not in files: + files.append(standard_requirements) + elif editable_requirements not in files: + files.append(editable_requirements) + for file in files: pip_command = [ project_python(project, system=allow_global), @@ -166,10 +81,8 @@ def pip_install_deps( if project.s.is_verbose(): msg = f"Install Phase: {'Standard Requirements' if file == standard_requirements else 'Editable Requirements'}" err.print(msg, style="bold") - for requirement in ( - standard_deps if file == standard_requirements else editable_deps - ): - err.print(f"Preparing Installation of {requirement.name!r}", style="bold") + for pip_line in deps: + err.print(f"Preparing Installation of {pip_line!r}", style="bold") err.print(f"$ {cmd_list_to_shell(pip_command)}", style="cyan") cache_dir = Path(project.s.PIPENV_CACHE_DIR) default_exists_action = "w" @@ -186,10 +99,6 @@ def pip_install_deps( err.print(f"Using source directory: {src_dir!r}") pip_config.update({"PIP_SRC": src_dir}) c = subprocess_run(pip_command, block=False, capture_output=True, env=pip_config) - if file == standard_requirements: - c.deps = standard_deps - else: - c.deps = editable_deps c.env = pip_config cmds.append(c) if project.s.is_verbose(): @@ -204,132 +113,6 @@ def pip_install_deps( return cmds -def pip_install( - project, - requirement=None, - r=None, - allow_global=False, - ignore_hashes=False, - no_deps=False, - block=True, - index=None, - pre=False, - dev=False, - requirements_dir=None, - extra_indexes=None, - pypi_mirror=None, - use_pep517=True, - use_constraint=False, - extra_pip_args: Optional[List] = None, -): - piplogger = logging.getLogger("pipenv.patched.pip._internal.commands.install") - trusted_hosts = get_trusted_hosts() - if not allow_global: - src_dir = os.getenv( - "PIP_SRC", os.getenv("PIP_SRC_DIR", project.virtualenv_src_location) - ) - else: - src_dir = os.getenv("PIP_SRC", os.getenv("PIP_SRC_DIR")) - if requirement: - if requirement.editable or not requirement.hashes: - ignore_hashes = True - elif not (requirement.is_vcs or requirement.editable or requirement.vcs): - ignore_hashes = False - line = None - # Try installing for each source in project.sources. - search_all_sources = project.settings.get("install_search_all_sources", False) - if not index and requirement.index: - index = requirement.index - if index and not extra_indexes: - if search_all_sources: - extra_indexes = list(project.sources) - else: # Default: index restrictions apply during installation - extra_indexes = [] - if requirement.index: - extra_indexes = list( - filter(lambda d: d.get("name") == requirement.index, project.sources) - ) - if not extra_indexes: - extra_indexes = list(project.sources) - if requirement and requirement.vcs or requirement.editable: - requirement.index = None - - r = write_requirement_to_file( - project, - requirement, - requirements_dir=requirements_dir, - include_hashes=not ignore_hashes, - ) - sources = get_source_list( - project, - index, - extra_indexes=extra_indexes, - trusted_hosts=trusted_hosts, - pypi_mirror=pypi_mirror, - ) - source_names = {src.get("name") for src in sources} - if not search_all_sources and requirement.index in source_names: - sources = list(filter(lambda d: d.get("name") == requirement.index, sources)) - if r: - with open(r) as fh: - if "--hash" not in fh.read(): - ignore_hashes = True - if project.s.is_verbose(): - piplogger.setLevel(logging.WARN) - if requirement: - err.print(f"Installing {requirement.name!r}", style="bold") - - pip_command = [ - project_python(project, system=allow_global), - get_runnable_pip(), - "install", - ] - pip_args = get_pip_args( - project, - pre=pre, - verbose=project.s.is_verbose(), - upgrade=True, - no_use_pep517=not use_pep517, - no_deps=no_deps, - require_hashes=not ignore_hashes, - extra_pip_args=extra_pip_args, - ) - pip_command.extend(pip_args) - if r: - pip_command.extend(["-r", normalize_path(r)]) - elif line: - pip_command.extend(line) - if dev and use_constraint: - default_constraints = get_constraints_from_deps(project.packages) - constraint_filename = prepare_constraint_file( - default_constraints, - directory=requirements_dir, - sources=None, - pip_args=None, - ) - pip_command.extend(["-c", normalize_path(constraint_filename)]) - pip_command.extend(prepare_pip_source_args(sources)) - if project.s.is_verbose(): - err.print(f"$ {cmd_list_to_shell(pip_command)}") - cache_dir = Path(project.s.PIPENV_CACHE_DIR) - default_exists_action = "w" - exists_action = project.s.PIP_EXISTS_ACTION or default_exists_action - pip_config = { - "PIP_CACHE_DIR": cache_dir.as_posix(), - "PIP_WHEEL_DIR": cache_dir.joinpath("wheels").as_posix(), - "PIP_DESTINATION_DIR": cache_dir.joinpath("pkgs").as_posix(), - "PIP_EXISTS_ACTION": exists_action, - "PATH": os.environ.get("PATH"), - } - if src_dir: - if project.s.is_verbose(): - err(f"Using source directory: {src_dir!r}") - pip_config.update({"PIP_SRC": src_dir}) - c = subprocess_run(pip_command, block=block, env=pip_config) - c.env = pip_config - return c - - def get_pip_args( project, pre: bool = False, @@ -366,26 +149,3 @@ def get_trusted_hosts(): return os.environ.get("PIP_TRUSTED_HOSTS", []).split(" ") except AttributeError: return [] - - -def write_requirement_to_file( - project: Project, - requirement: Requirement, - requirements_dir: Optional[str] = None, - include_hashes: bool = True, -) -> str: - if not requirements_dir: - requirements_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") - line = requirement.line_instance.get_line( - with_prefix=True, with_hashes=include_hashes, with_markers=True, as_list=False - ) - - f = tempfile.NamedTemporaryFile( - prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, delete=False - ) - if project.s.is_verbose(): - err.print(f"Writing supplied requirement line to temporary file: {line!r}") - f.write(line.encode()) - r = f.name - f.close() - return r diff --git a/pipenv/utils/pipfile.py b/pipenv/utils/pipfile.py index 79b6af96..32b7b473 100644 --- a/pipenv/utils/pipfile.py +++ b/pipenv/utils/pipfile.py @@ -1,8 +1,21 @@ +import io +import itertools import os +from pathlib import Path +from typing import Any, Dict, List, Optional from pipenv import environments, exceptions from pipenv.utils import console, err +from pipenv.utils.internet import get_url_name +from pipenv.utils.markers import RequirementError from pipenv.utils.requirements import import_requirements +from pipenv.utils.requirementslib import is_editable, is_vcs, merge_items +from pipenv.utils.toml import tomlkit_value_to_python +from pipenv.vendor import tomlkit +from pipenv.vendor.plette import pipfiles +from pipenv.vendor.pydantic import BaseModel, Field, validator + +DEFAULT_NEWLINES = "\n" def walk_up(bottom): @@ -110,3 +123,310 @@ def ensure_pipfile( if changed: err.print("Fixing package names in Pipfile...", style="bold") project.write_toml(p) + + +def reorder_source_keys(data): + # type: (tomlkit.toml_document.TOMLDocument) -> tomlkit.toml_document.TOMLDocument + sources = [] # type: sources_type + for source_key in ["source", "sources"]: + sources.extend(data.get(source_key, tomlkit.aot()).value) + new_source_aot = tomlkit.aot() + for entry in sources: + table = tomlkit.table() # type: tomlkit.items.Table + source_entry = PipfileLoader.populate_source(entry.copy()) + for key in ["name", "url", "verify_ssl"]: + table.update({key: source_entry[key]}) + new_source_aot.append(table) + data["source"] = new_source_aot + if data.get("sources", None): + del data["sources"] + return data + + +def preferred_newlines(f): + if isinstance(f.newlines, str): + return f.newlines + return DEFAULT_NEWLINES + + +class ProjectFile(BaseModel): + location: str + line_ending: str + model: Optional[Any] = Field(default_factory=lambda: {}) + + @classmethod + def read(cls, location: str, model_cls, invalid_ok: bool = False) -> "ProjectFile": + if not os.path.exists(location) and not invalid_ok: + raise FileNotFoundError(location) + try: + with open(location, encoding="utf-8") as f: + model = model_cls.load(f) + line_ending = preferred_newlines(f) + except Exception: + if not invalid_ok: + raise + model = {} + line_ending = DEFAULT_NEWLINES + return cls(location=location, line_ending=line_ending, model=model) + + def write(self) -> None: + kwargs = {"encoding": "utf-8", "newline": self.line_ending} + with open(self.location, "w", **kwargs) as f: + if self.model: + self.model.dump(f) + + def dumps(self) -> str: + if self.model: + strio = io.StringIO() + self.model.dump(strio) + return strio.getvalue() + return "" + + +class PipfileLoader(pipfiles.Pipfile): + @classmethod + def validate(cls, data): + # type: (tomlkit.toml_document.TOMLDocument) -> None + for key, klass in pipfiles.PIPFILE_SECTIONS.items(): + if key not in data or key == "sources": + continue + try: + klass.validate(data[key]) + except Exception: + pass + + @classmethod + def ensure_package_sections(cls, data): + # type: (tomlkit.toml_document.TOMLDocument[Text, Any]) -> tomlkit.toml_document.TOMLDocument[Text, Any] + """Ensure that all pipfile package sections are present in the given + toml document. + + :param :class:`~tomlkit.toml_document.TOMLDocument` data: The toml document to + ensure package sections are present on + :return: The updated toml document, ensuring ``packages`` and ``dev-packages`` + sections are present + :rtype: :class:`~tomlkit.toml_document.TOMLDocument` + """ + package_keys = ( + k for k in pipfiles.PIPFILE_SECTIONS.keys() if k.endswith("packages") + ) + for key in package_keys: + if key not in data: + data.update({key: tomlkit.table()}) + return data + + @classmethod + def populate_source(cls, source): + """Derive missing values of source from the existing fields.""" + # Only URL pararemter is mandatory, let the KeyError be thrown. + if "name" not in source: + source["name"] = get_url_name(source["url"]) + if "verify_ssl" not in source: + source["verify_ssl"] = "https://" in source["url"] + if not isinstance(source["verify_ssl"], bool): + source["verify_ssl"] = str(source["verify_ssl"]).lower() == "true" + return source + + @classmethod + def load(cls, f, encoding=None): + # type: (Any, Text) -> PipfileLoader + content = f.read() + if encoding is not None: + content = content.decode(encoding) + _data = tomlkit.loads(content) + should_reload = "source" not in _data + _data = reorder_source_keys(_data) + if should_reload: + if "sources" in _data: + content = tomlkit.dumps(_data) + else: + # HACK: There is no good way to prepend a section to an existing + # TOML document, but there's no good way to copy non-structural + # content from one TOML document to another either. Modify the + # TOML content directly, and load the new in-memory document. + sep = "" if content.startswith("\n") else "\n" + content = pipfiles.DEFAULT_SOURCE_TOML + sep + content + data = tomlkit.loads(content) + data = cls.ensure_package_sections(data) + instance = cls(data) + instance._data = dict(instance._data) + return instance + + def __contains__(self, key): + # type: (Text) -> bool + if key not in self._data: + package_keys = self._data.get("packages", {}).keys() + dev_package_keys = self._data.get("dev-packages", {}).keys() + return any(key in pkg_list for pkg_list in (package_keys, dev_package_keys)) + return True + + def __getattribute__(self, key): + # type: (Text) -> Any + if key == "source": + return self._data[key] + return super().__getattribute__(key) + + +class Pipfile(BaseModel): + path: Path + projectfile: ProjectFile + pipfile: Optional[PipfileLoader] + _pyproject: Optional[tomlkit.TOMLDocument] = tomlkit.document() + build_system: Optional[Dict] = {} + _requirements: Optional[List] = [] + _dev_requirements: Optional[List] = [] + + class Config: + validate_assignment = True + arbitrary_types_allowed = True + allow_mutation = True + include_private_attributes = True + # keep_untouched = (cached_property,) + + @validator("path", pre=True, always=True) + def _get_path(cls, v): + return v or Path(os.curdir).absolute() + + @validator("projectfile", pre=True, always=True) + def _get_projectfile(cls, v, values): + return v or cls.load_projectfile(os.curdir, create=False) + + @validator("pipfile", pre=True, always=True) + def _get_pipfile(cls, v, values): + return v or values["projectfile"].model + + @property + def root(self): + return self.path.parent + + @property + def extended_keys(self): + return [ + k + for k in itertools.product( + ("packages", "dev-packages"), ("", "vcs", "editable") + ) + ] + + def get_deps(self, dev=False, only=True): + deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] + if dev: + deps.update(dict(self.pipfile._data.get("dev-packages", {}))) + if only: + return deps + return tomlkit_value_to_python( + merge_items([deps, dict(self.pipfile._data.get("packages", {}))]) + ) + + def get(self, k): + return self.__getitem__(k) + + def __contains__(self, k): + check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) + if check_pipfile: + return True + return False + + def __getitem__(self, k, *args, **kwargs): + retval = None + pipfile = self.pipfile + section = None + pkg_type = None + try: + retval = pipfile[k] + except KeyError: + if "-" in k: + section, _, pkg_type = k.rpartition("-") + vals = getattr(pipfile.get(section, {}), "_data", {}) + vals = tomlkit_value_to_python(vals) + if pkg_type == "vcs": + retval = {k: v for k, v in vals.items() if is_vcs(v)} + elif pkg_type == "editable": + retval = {k: v for k, v in vals.items() if is_editable(v)} + if retval is None: + raise + else: + retval = getattr(retval, "_data", retval) + return retval + + def __getattr__(self, k, *args, **kwargs): + pipfile = self.pipfile + try: + retval = super(Pipfile).__getattribute__(k) + except AttributeError: + retval = getattr(pipfile, k, None) + return retval + + @property + def requires_python(self): + # type: () -> bool + return getattr( + self.pipfile.requires, + "python_version", + getattr(self.pipfile.requires, "python_full_version", None), + ) + + @property + def allow_prereleases(self): + # type: () -> bool + return self.pipfile.get("pipenv", {}).get("allow_prereleases", False) + + @classmethod + def read_projectfile(cls, path): + # type: (Text) -> ProjectFile + """Read the specified project file and provide an interface for + writing/updating. + + :param Text path: Path to the target file. + :return: A project file with the model and location for interaction + :rtype: :class:`~project.ProjectFile` + """ + pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True) + return pf + + @classmethod + def load_projectfile(cls, path, create=False): + # type: (Text, bool) -> ProjectFile + """Given a path, load or create the necessary pipfile. + + :param Text path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A project file instance for the supplied project + :rtype: :class:`~project.ProjectFile` + """ + if not path: + raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") + if not isinstance(path, Path): + path = Path(path).absolute() + pipfile_path = path if path.is_file() else path.joinpath("Pipfile") + project_path = pipfile_path.parent + if not project_path.exists(): + raise FileNotFoundError("%s is not a valid project path!" % path) + elif not pipfile_path.exists() or not pipfile_path.is_file(): + if not create: + raise RequirementError("%s is not a valid Pipfile" % pipfile_path) + return cls.read_projectfile(pipfile_path.as_posix()) + + @classmethod + def load(cls, path, create=False): + # type: (Text, bool) -> Pipfile + """Given a path, load or create the necessary pipfile. + + :param Text path: Path to the project root or pipfile + :param bool create: Whether to create the pipfile if not found, defaults to True + :raises OSError: Thrown if the project root directory doesn't exist + :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` + :return: A pipfile instance pointing at the supplied project + :rtype:: class:`~pipfile.Pipfile` + """ + + projectfile = cls.load_projectfile(path, create=create) + pipfile = projectfile.model + creation_args = { + "projectfile": projectfile, + "pipfile": pipfile, + "path": Path(projectfile.location), + } + return cls(**creation_args) diff --git a/pipenv/utils/project.py b/pipenv/utils/project.py index 44fa1c95..84166961 100644 --- a/pipenv/utils/project.py +++ b/pipenv/utils/project.py @@ -1,6 +1,9 @@ import os +from functools import lru_cache from pipenv import exceptions +from pipenv.patched.pip._vendor.packaging.version import parse as parse_version +from pipenv.patched.pip._vendor.pkg_resources import Requirement, get_distribution from pipenv.utils.dependencies import python_version from pipenv.utils.pipfile import ensure_pipfile from pipenv.utils.shell import shorten_path @@ -82,3 +85,21 @@ def ensure_project( categories=categories, ) os.environ["PIP_PYTHON_PATH"] = project.python(system=system) + + +@lru_cache() +def get_setuptools_version(): + # type: () -> Optional[STRING_TYPE] + + setuptools_dist = get_distribution(Requirement("setuptools")) + return getattr(setuptools_dist, "version", None) + + +def get_default_pyproject_backend(): + # type: () -> STRING_TYPE + st_version = get_setuptools_version() + if st_version is not None: + parsed_st_version = parse_version(st_version) + if parsed_st_version >= parse_version("40.8.0"): + return "setuptools.build_meta:__legacy__" + return "setuptools.build_meta" diff --git a/pipenv/utils/requirements.py b/pipenv/utils/requirements.py index 2efe2c1c..23c1003c 100644 --- a/pipenv/utils/requirements.py +++ b/pipenv/utils/requirements.py @@ -1,17 +1,60 @@ import os import re +import urllib.parse +from typing import Tuple from pipenv.patched.pip._internal.network.session import PipSession from pipenv.patched.pip._internal.req import parse_requirements from pipenv.patched.pip._internal.req.constructors import ( install_req_from_parsed_requirement, ) -from pipenv.patched.pip._internal.utils.misc import split_auth_from_netloc +from pipenv.patched.pip._internal.utils.misc import _transform_url, split_auth_from_netloc +from pipenv.utils.constants import VCS_LIST from pipenv.utils.indexes import parse_indexes from pipenv.utils.internet import get_host_and_port from pipenv.utils.pip import get_trusted_hosts +def redact_netloc(netloc: str) -> Tuple[str]: + """ + Replace the sensitive data in a netloc with "****", if it exists, unless it's an environment variable. + + For example: + - "user:pass@example.com" returns "user:****@example.com" + - "accesstoken@example.com" returns "****@example.com" + - "${ENV_VAR}:pass@example.com" returns "${ENV_VAR}:****@example.com" if ${ENV_VAR} is an environment variable + """ + netloc, (user, password) = split_auth_from_netloc(netloc) + if user is None: + return (netloc,) + if password is None: + # Check if user is an environment variable + if not re.match(r"\$\{\w+\}", user): + # If not, redact the user + user = "****" + password = "" + else: + # Check if password is an environment variable + if not re.match(r"\$\{\w+\}", password): + # If not, redact the password + password = ":****" + else: + # If it is, leave it as is + password = ":" + password + user = urllib.parse.quote(user) + return (f"{user}{password}@{netloc}",) + + +def redact_auth_from_url(url: str) -> str: + """Replace the password in a given url with ****.""" + return _transform_url(url, redact_netloc)[0] + + +def normalize_name(pkg) -> str: + """Given a package name, return its normalized, non-canonicalized form.""" + return pkg.replace("_", "-").lower() + + def import_requirements(project, r=None, dev=False, categories=None): # Parse requirements.txt file with Pip's parser. # Pip requires a `PipSession` which is a subclass of requests.Session. @@ -36,44 +79,34 @@ def import_requirements(project, r=None, dev=False, categories=None): indexes.append(extra_index) if trusted_host: trusted_hosts.append(get_host_and_port(trusted_host)) - indexes = sorted(set(indexes)) - trusted_hosts = sorted(set(trusted_hosts)) - reqs = [ - install_req_from_parsed_requirement(f) - for f in parse_requirements(r, session=PipSession()) - ] - for package in reqs: + for f in parse_requirements(r, session=PipSession()): + package = install_req_from_parsed_requirement(f) if package.name not in BAD_PACKAGES: if package.link is not None: if package.editable: package_string = f"-e {package.link}" else: - netloc, (user, pw) = split_auth_from_netloc(package.link.netloc) - safe = True - if user and not re.match(r"\${[\W\w]+}", user): - safe = False - if pw and not re.match(r"\${[\W\w]+}", pw): - safe = False - if safe: - package_string = str(package.link._url) - else: - package_string = str(package.link) + package_string = urllib.parse.unquote( + redact_auth_from_url(package.original_link.url) + ) + if categories: for category in categories: project.add_package_to_pipfile( - package_string, dev=dev, category=category + package, package_string, dev=dev, category=category ) else: - project.add_package_to_pipfile(package_string, dev=dev) + project.add_package_to_pipfile(package, package_string, dev=dev) else: if categories: for category in categories: project.add_package_to_pipfile( - str(package.req), dev=dev, category=category + package, str(package.req), dev=dev, category=category ) else: - project.add_package_to_pipfile(str(package.req), dev=dev) - + project.add_package_to_pipfile(package, str(package.req), dev=dev) + indexes = sorted(set(indexes)) + trusted_hosts = sorted(set(trusted_hosts)) for index in indexes: add_index_to_pipfile(project, index, trusted_hosts) project.recase_pipfile() @@ -105,3 +138,84 @@ BAD_PACKAGES = ( "setuptools", "wheel", ) + + +def requirement_from_lockfile( + package_name, package_info, include_hashes=True, include_markers=True +): + from pipenv.utils.dependencies import is_editable_path, is_star + + # Handle string requirements + if isinstance(package_info, str): + if package_info and not is_star(package_info): + return f"{package_name}=={package_info}" + else: + return package_name + # Handling vcs repositories + for vcs in VCS_LIST: + if vcs in package_info: + url = package_info[vcs] + ref = package_info.get("ref", "") + extras = ( + "[{}]".format(",".join(package_info.get("extras", []))) + if "extras" in package_info + else "" + ) + include_vcs = "" if f"{vcs}+" in url else f"{vcs}+" + egg_fragment = "" if "#egg=" in url else f"#egg={package_name}" + ref_str = "" if f"@{ref}" in url else f"@{ref}" + if is_editable_path(url) or "file://" in url: + pip_line = f"-e {include_vcs}{url}{ref_str}{egg_fragment}{extras}" + else: + pip_line = f"{package_name}{extras}@ {include_vcs}{url}{ref_str}" + return pip_line + # Handling file-sourced packages + for k in ["file", "path"]: + line = [] + if k in package_info: + path = package_info[k] + if is_editable_path(path): + line.append("-e") + line.append(f"{package_info[k]}") + pip_line = " ".join(line) + return pip_line + + # Handling packages from standard pypi like indexes + version = package_info.get("version", "").replace("==", "") + hashes = ( + f" --hash={' --hash='.join(package_info['hashes'])}" + if include_hashes and "hashes" in package_info + else "" + ) + markers = ( + "; {}".format(package_info["markers"]) + if include_markers and "markers" in package_info and package_info["markers"] + else "" + ) + os_markers = ( + "; {}".format(package_info["os_markers"]) + if include_markers and "os_markers" in package_info and package_info["os_markers"] + else "" + ) + extras = ( + "[{}]".format(",".join(package_info.get("extras", []))) + if "extras" in package_info + else "" + ) + pip_line = f"{package_name}{extras}=={version}{os_markers}{markers}{hashes}" + return pip_line + + +def requirements_from_lockfile(deps, include_hashes=True, include_markers=True): + pip_packages = [] + + for package_name, package_info in deps.items(): + pip_package = requirement_from_lockfile( + package_name, package_info, include_hashes, include_markers + ) + + # Append to the list + pip_packages.append(pip_package) + + # pip_packages contains the pip-installable lines + return pip_packages diff --git a/pipenv/vendor/requirementslib/utils.py b/pipenv/utils/requirementslib.py similarity index 86% rename from pipenv/vendor/requirementslib/utils.py rename to pipenv/utils/requirementslib.py index f5800466..3cb3e6f1 100644 --- a/pipenv/vendor/requirementslib/utils.py +++ b/pipenv/utils/requirementslib.py @@ -1,77 +1,35 @@ -# This Module is taken in part from the click project and expanded -# see https://github.com/pallets/click/blob/6cafd32/click/_winconsole.py -# Copyright © 2014 by the Pallets team. - -# Some rights reserved. - -# Redistribution and use in source and binary forms of the software as well as -# documentation, with or without modification, are permitted provided that the -# following conditions are met: -# Redistributions of source code must retain the above copyright notice, -# this list of conditions and the following disclaimer. -# Redistributions in binary form must reproduce the above copyright notice, -# this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# Neither the name of the copyright holder nor the names of its contributors -# may be used to endorse or promote products derived from this -# software without specific prior written permission. - -# THIS SOFTWARE AND DOCUMENTATION IS PROVIDED BY THE COPYRIGHT HOLDERS AND -# CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT -# NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -# PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; -# OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -# WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR -# OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE AND -# DOCUMENTATION, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import logging import os -import sys from collections.abc import ItemsView, Mapping, Sequence, Set -from contextlib import contextmanager from pathlib import Path +from typing import Dict, List, Optional, Tuple, TypeVar, Union from urllib.parse import urlparse, urlsplit, urlunparse -import pipenv.vendor.tomlkit as tomlkit from pipenv.patched.pip._internal.commands.install import InstallCommand +from pipenv.patched.pip._internal.models.link import Link from pipenv.patched.pip._internal.models.target_python import TargetPython +from pipenv.patched.pip._internal.network.download import Downloader +from pipenv.patched.pip._internal.operations.prepare import ( + File, + _check_download_dir, + get_file_url, + unpack_vcs_link, +) from pipenv.patched.pip._internal.utils.filetypes import is_archive_file +from pipenv.patched.pip._internal.utils.hashes import Hashes from pipenv.patched.pip._internal.utils.misc import is_installable_dir +from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory +from pipenv.patched.pip._internal.utils.unpacking import unpack_file from pipenv.patched.pip._vendor.packaging import specifiers +from pipenv.utils.fileutils import is_valid_url, normalize_path, url_to_path +from pipenv.vendor import tomlkit -from .environment import MYPY_RUNNING -from .fileutils import is_valid_url, normalize_path, url_to_path - -if MYPY_RUNNING: - from typing import Dict, Iterator, List, Optional, Text, Tuple, TypeVar, Union - - STRING_TYPE = Union[bytes, str, Text] - S = TypeVar("S", bytes, str, Text) - PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] - PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] +STRING_TYPE = Union[bytes, str, str] +S = TypeVar("S", bytes, str, str) +PipfileEntryType = Union[STRING_TYPE, bool, Tuple[STRING_TYPE], List[STRING_TYPE]] +PipfileType = Union[STRING_TYPE, Dict[STRING_TYPE, PipfileEntryType]] VCS_LIST = ("git", "svn", "hg", "bzr") - - -def setup_logger(): - logger = logging.getLogger("requirementslib") - loglevel = logging.DEBUG - handler = logging.StreamHandler(stream=sys.stderr) - handler.setLevel(loglevel) - logger.addHandler(handler) - logger.setLevel(loglevel) - return logger - - -log = setup_logger() - - SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") @@ -111,10 +69,8 @@ def strip_ssh_from_git_uri(uri): # split the path on the first separating / so we can put the first segment # into the 'netloc' section with a : separator path_part, _, path = parsed.path.lstrip("/").partition("/") - path = "/{0}".format(path) - parsed = parsed._replace( - netloc="{0}:{1}".format(parsed.netloc, path_part), path=path - ) + path = f"/{path}" + parsed = parsed._replace(netloc=f"{parsed.netloc}:{path_part}", path=path) uri = urlunparse(parsed).replace("git+ssh://", "git+", 1) return uri @@ -129,7 +85,7 @@ def add_ssh_scheme_to_git_uri(uri): parsed = urlparse(uri) if ":" in parsed.netloc: netloc, _, path_start = parsed.netloc.rpartition(":") - path = "/{0}{1}".format(path_start, parsed.path) + path = f"/{path_start}{parsed.path}" uri = urlunparse(parsed._replace(netloc=netloc, path=path)) return uri @@ -170,10 +126,10 @@ def convert_entry_to_path(path): """Convert a pipfile entry to a string.""" if not isinstance(path, Mapping): - raise TypeError("expecting a mapping, received {0!r}".format(path)) + raise TypeError(f"expecting a mapping, received {path!r}") if not any(key in path for key in ["file", "path"]): - raise ValueError("missing path-like entry in supplied mapping {0!r}".format(path)) + raise ValueError(f"missing path-like entry in supplied mapping {path!r}") if "file" in path: path = url_to_path(path["file"]) @@ -276,7 +232,7 @@ def prepare_pip_source_args(sources, pip_args=None): pip_args = [] if sources: # Add the source to pip9. - pip_args.extend(["-i", sources[0]["url"]]) # type: ignore + pip_args.extend(["-i ", sources[0]["url"]]) # type: ignore # Trust the host if it's not verified. if not sources[0].get("verify_ssl", True): pip_args.extend( @@ -375,14 +331,10 @@ class PathAccessError(KeyError, IndexError, TypeError): def __repr__(self): cn = self.__class__.__name__ - return "%s(%r, %r, %r)" % (cn, self.exc, self.seg, self.path) + return f"{cn}({self.exc!r}, {self.seg!r}, {self.path!r})" def __str__(self): - return "could not access %r from path %r, got error: %r" % ( - self.seg, - self.path, - self.exc, - ) + return f"could not access {self.seg} from path {self.path}, got error: {self.exc}" def get_path(root, path, default=_UNSET): @@ -421,7 +373,7 @@ def get_path(root, path, default=_UNSET): cur = cur[seg] except (KeyError, IndexError) as exc: raise PathAccessError(exc, seg, path) - except TypeError as exc: + except TypeError: # either string index in a list, or a parent that # doesn't support indexing try: @@ -701,20 +653,90 @@ def get_pip_command() -> InstallCommand: # General options (find_links, index_url, extra_index_url, trusted_host, # and pre) are deferred to pip. pip_command = InstallCommand( - name="InstallCommand", summary="requirementslib pip Install command." + name="InstallCommand", summary="pipenv pip Install command." ) return pip_command -# Borrowed from Pew. -# See https://github.com/berdario/pew/blob/master/pew/_utils.py#L82 -@contextmanager -def temp_environ(): - # type: () -> Iterator[None] - """Allow the ability to set os.environ temporarily.""" - environ = dict(os.environ) - try: - yield - finally: - os.environ.clear() - os.environ.update(environ) +def unpack_url( + link: Link, + location: str, + download: Downloader, + verbosity: int, + download_dir: Optional[str] = None, + hashes: Optional[Hashes] = None, +) -> Optional[File]: + """Unpack link into location, downloading if required. + + :param hashes: A Hashes object, one of whose embedded hashes must match, + or HashMismatch will be raised. If the Hashes is empty, no matches are + required, and unhashable types of requirements (like VCS ones, which + would ordinarily raise HashUnsupported) are allowed. + """ + # non-editable vcs urls + if link.scheme in [ + "git+http", + "git+https", + "git+ssh", + "git+git", + "hg+http", + "hg+https", + "hg+ssh", + "svn+http", + "svn+https", + "svn+svn", + "bzr+http", + "bzr+https", + "bzr+ssh", + "bzr+sftp", + "bzr+ftp", + "bzr+lp", + ]: + unpack_vcs_link(link, location, verbosity=verbosity) + return File(location, content_type=None) + + assert not link.is_existing_dir() + + # file urls + if link.is_file: + file = get_file_url(link, download_dir, hashes=hashes) + + # http urls + else: + file = get_http_url( + link, + download, + download_dir, + hashes=hashes, + ) + + # unpack the archive to the build dir location. even when only downloading + # archives, they have to be unpacked to parse dependencies, except wheels + if not link.is_wheel: + unpack_file(file.path, location, file.content_type) + + return file + + +def get_http_url( + link: Link, + download: Downloader, + download_dir: Optional[str] = None, + hashes: Optional[Hashes] = None, +) -> File: + temp_dir = TempDirectory(kind="unpack", globally_managed=False) + # If a download dir is specified, is the file already downloaded there? + already_downloaded_path = None + if download_dir: + already_downloaded_path = _check_download_dir(link, download_dir, hashes) + + if already_downloaded_path: + from_path = already_downloaded_path + content_type = None + else: + # let's download to a tmp dir + from_path, content_type = download(link, temp_dir.path) + if hashes: + hashes.check_against_path(from_path) + + return File(from_path, content_type) diff --git a/pipenv/utils/resolver.py b/pipenv/utils/resolver.py index 25b24f03..97fa7883 100644 --- a/pipenv/utils/resolver.py +++ b/pipenv/utils/resolver.py @@ -1,5 +1,4 @@ import contextlib -import hashlib import json import os import subprocess @@ -7,18 +6,15 @@ import sys import tempfile import warnings from functools import lru_cache -from html.parser import HTMLParser from pathlib import Path -from typing import Dict, List, Optional, Set, Tuple, Union -from urllib import parse +from typing import Dict, List, Optional, Set from pipenv import environments, resolver -from pipenv.exceptions import RequirementError, ResolutionFailure +from pipenv.exceptions import ResolutionFailure from pipenv.patched.pip._internal.cache import WheelCache from pipenv.patched.pip._internal.commands.install import InstallCommand from pipenv.patched.pip._internal.exceptions import InstallationError from pipenv.patched.pip._internal.models.target_python import TargetPython -from pipenv.patched.pip._internal.network.cache import SafeFileCache from pipenv.patched.pip._internal.operations.build.build_tracker import ( get_build_tracker, ) @@ -26,14 +22,13 @@ from pipenv.patched.pip._internal.req.constructors import ( install_req_from_parsed_requirement, ) from pipenv.patched.pip._internal.req.req_file import parse_requirements -from pipenv.patched.pip._internal.utils.hashes import FAVORITE_HASH +from pipenv.patched.pip._internal.req.req_install import InstallRequirement from pipenv.patched.pip._internal.utils.temp_dir import global_tempdir_manager from pipenv.patched.pip._vendor import pkg_resources, rich from pipenv.project import Project +from pipenv.utils.fileutils import create_tracked_tempdir +from pipenv.utils.requirements import normalize_name from pipenv.vendor import click -from pipenv.vendor.requirementslib.fileutils import create_tracked_tempdir, open_file -from pipenv.vendor.requirementslib.models.requirements import Line, Requirement -from pipenv.vendor.requirementslib.models.utils import DIRECT_URL_RE try: # this is only in Python3.8 and later @@ -44,18 +39,17 @@ except ImportError: from .dependencies import ( HackedPythonVersion, - clean_pkg_version, convert_deps_to_pip, + determine_package_name, + expansive_install_req_from_line, get_constraints_from_deps, get_lockfile_section_using_pipfile_category, - get_vcs_deps, is_pinned_requirement, - pep423_name, prepare_constraint_file, translate_markers, ) from .indexes import parse_indexes, prepare_pip_source_args -from .internet import _get_requests_session, is_pypi_url +from .internet import is_pypi_url from .locking import format_requirement_for_lockfile, prepare_lockfile from .shell import make_posix, subprocess_run, temp_environ @@ -101,42 +95,14 @@ class HashCacheMixin: avoid issues where the location on the server changes. """ - def __init__(self, directory, session): + def __init__(self, project, session): + self.project = project self.session = session - if not os.path.isdir(directory): - os.makedirs(directory, exist_ok=True) - super().__init__(directory=directory) def get_hash(self, link): - # If there is no link hash (i.e., md5, sha256, etc.), we don't want - # to store it. - hash_value = self.get(link.url) - if not hash_value: - hash_value = self._get_file_hash(link).encode() - self.set(link.url, hash_value) + hash_value = self.project.get_file_hash(self.session, link).encode() return hash_value.decode("utf8") - def _get_file_hash(self, link): - h = hashlib.new(FAVORITE_HASH) - with open_file(link.url, self.session) as fp: - for chunk in iter(lambda: fp.read(8096), b""): - h.update(chunk) - return f"{h.name}:{h.hexdigest()}" - - -class PackageIndexHTMLParser(HTMLParser): - def __init__(self): - super().__init__() - self.urls = [] - - def handle_starttag(self, tag, attrs): - # If tag is an anchor - if tag == "a": - # find href attribute - for attr in attrs: - if attr[0] == "href": - self.urls.append(attr[1]) - class Resolver: def __init__( @@ -151,6 +117,9 @@ class Resolver: clear=False, pre=False, category=None, + original_deps=None, + install_reqs=None, + pipfile_entries=None, ): self.initial_constraints = constraints self.req_dir = req_dir @@ -167,18 +136,11 @@ class Resolver: self.skipped = skipped if skipped is not None else {} self.markers = {} self.requires_python_markers = {} - self._pip_args = None - self._constraints = None - self._parsed_constraints = None - self._resolver = None - self._finder = None - self._session = None - self._constraint_file = None - self._pip_options = None - self._pip_command = None + self.original_deps = original_deps if original_deps is not None else {} + self.install_reqs = install_reqs if install_reqs is not None else {} + self.pipfile_entries = pipfile_entries if pipfile_entries is not None else {} self._retry_attempts = 0 self._hash_cache = None - self._sessions = {} def __repr__(self): return ( @@ -194,260 +156,25 @@ class Resolver: @property def hash_cache(self): if not self._hash_cache: - self._hash_cache = type("HashCache", (HashCacheMixin, SafeFileCache), {})( - os.path.join(self.project.s.PIPENV_CACHE_DIR, "hashes"), self.session - ) + self._hash_cache = HashCacheMixin(self.project, self.session) return self._hash_cache - def get_metadata( + def check_if_package_req_skipped( self, - deps: List[str], - index_lookup: Dict[str, str], - markers_lookup: Dict[str, str], - project: Project, - sources: Dict[str, str], - req_dir: Optional[str] = None, - pre: bool = False, - clear: bool = False, - category: str = None, - ) -> Tuple[ - Set[str], - Dict[str, Dict[str, Union[str, bool, List[str]]]], - Dict[str, str], - Dict[str, str], - ]: - constraints: Set[str] = set() - skipped: Dict[str, Dict[str, Union[str, bool, List[str]]]] = {} - if index_lookup is None: - index_lookup = {} - if markers_lookup is None: - markers_lookup = {} - if not req_dir: - req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-reqdir") - for dep in deps: - if not dep: - continue - req, req_idx, markers_idx = self.parse_line( - dep, - index_lookup=index_lookup, - markers_lookup=markers_lookup, - project=project, + req: InstallRequirement, + ) -> bool: + if req.markers and not req.markers.evaluate(): + err.print( + f"Could not find a matching version of {req}; {req.markers} for your environment, " + "its dependencies will be skipped.", ) - index_lookup.update(req_idx) - markers_lookup.update(markers_idx) - # Add dependencies of any file (e.g. wheels/tarballs), source, or local - # directories into the initial constraint pool to be resolved with the - # rest of the dependencies, while adding the files/vcs deps/paths themselves - # to the lockfile directly - use_sources = None - if req.name in index_lookup: - use_sources = list( - filter(lambda s: s.get("name") == index_lookup[req.name], sources) - ) - if not use_sources: - use_sources = sources - transient_resolver = Resolver( - [], - req_dir, - project, - use_sources, - index_lookup=index_lookup, - markers_lookup=markers_lookup, - clear=clear, - pre=pre, - category=category, - ) - constraint_update, lockfile_update = self.get_deps_from_req( - req, resolver=transient_resolver, resolve_vcs=project.s.PIPENV_RESOLVE_VCS - ) - constraints |= constraint_update - skipped.update(lockfile_update) - return constraints, skipped, index_lookup, markers_lookup - - def parse_line( - self, - line: str, - index_lookup: Dict[str, str] = None, - markers_lookup: Dict[str, str] = None, - project: Optional[Project] = None, - ) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]: - if index_lookup is None: - index_lookup = {} - if markers_lookup is None: - markers_lookup = {} - if project is None: - from pipenv.project import Project - - project = Project() - index, extra_index, trust_host, remainder = parse_indexes(line) - line = " ".join(remainder) - try: - req = Requirement.from_line(line) - except ValueError: - direct_url = DIRECT_URL_RE.match(line) - if direct_url: - name = direct_url.groupdict()["name"] - line = f"{name}@ {line}" - try: - req = Requirement.from_line(line) - except ValueError: - raise ResolutionFailure( - f"Failed to resolve requirement from line: {line!s}" - ) - else: - raise ResolutionFailure( - f"Failed to resolve requirement from line: {line!s}" - ) - if index: - try: - index_lookup[req.normalized_name] = project.get_source( - url=index, refresh=True - ).get("name") - except TypeError: - pass - try: - req.normalized_name - except TypeError: - raise RequirementError(req=req) - # strip the marker and re-add it later after resolution - # but we will need a fallback in case resolution fails - # eg pypiwin32 - if req.markers: - markers_lookup[req.normalized_name] = req.markers.replace('"', "'") - return req, index_lookup, markers_lookup - - def get_deps_from_req( - self, - req: Requirement, - resolver: Optional["Resolver"] = None, - resolve_vcs: bool = True, - ) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]: - from pipenv.vendor.requirementslib.models.requirements import Requirement - from pipenv.vendor.requirementslib.models.utils import ( - _requirement_to_str_lowercase_name, - ) - from pipenv.vendor.requirementslib.utils import is_installable_dir - - # TODO: this is way too complex, refactor this - constraints: Set[str] = set() - locked_deps: Dict[str, Dict[str, Union[str, bool, List[str]]]] = {} - editable_packages = self.project.get_editable_packages(category=self.category) - if (req.is_file_or_url or req.is_vcs) and not req.is_wheel: - # for local packages with setup.py files and potential direct url deps: - if req.is_vcs: - req_list, lockfile = get_vcs_deps(reqs=[req]) - req = next(iter(req for req in req_list if req is not None), req_list) - entry = lockfile[pep423_name(req.normalized_name)] - else: - _, entry = req.pipfile_entry - parsed_line: Line = req.req.parsed_line - try: - name = req.normalized_name - except TypeError: - raise RequirementError(req=req) - if parsed_line.setup_info: - setup_info = parsed_line.setup_info - else: - setup_info = req.req.parse_setup_info() - locked_deps[pep423_name(name)] = entry - requirements = [] - # Allow users to toggle resolution off for non-editable VCS packages - # but leave it on for local, installable folders on the filesystem - if resolve_vcs or ( - req.editable - or parsed_line.is_wheel - or ( - req.is_file_or_url - and parsed_line.is_local - and is_installable_dir(parsed_line.path) - ) - ): - setup_info.run_pyproject() - setup_info.run_setup() - requirements = [v for v in getattr(setup_info, "requires", {}).values()] - if req.extras: - for extra in req.extras: - requirements.extend( - v - for v in getattr(setup_info, "extras", {}).get(extra, []) - if v not in requirements - ) - for r in requirements: - if getattr(r, "url", None) and not getattr(r, "editable", False): - if r is not None: - if not r.url: - continue - line = _requirement_to_str_lowercase_name(r) - new_req, _, _ = self.parse_line(line) - if r.marker and not r.marker.evaluate(): - new_constraints = {} - _, new_entry = req.pipfile_entry - new_lock = {pep423_name(new_req.normalized_name): new_entry} - else: - new_constraints, new_lock = self.get_deps_from_req( - new_req, resolver - ) - locked_deps.update(new_lock) - constraints |= new_constraints - # if there is no marker or there is a valid marker, add the constraint line - elif r and (not r.marker or (r.marker and r.marker.evaluate())): - if r.name not in editable_packages: - line = _requirement_to_str_lowercase_name(r) - constraints.add(line) - # ensure the top level entry remains as provided - # note that we shouldn't pin versions for editable vcs deps - if not req.is_vcs: - if req.specifiers: - locked_deps[name]["version"] = req.specifiers - elif parsed_line.setup_info and parsed_line.setup_info.version: - locked_deps[name]["version"] = f"=={parsed_line.setup_info.version}" - # if not req.is_vcs: - locked_deps.update({name: entry}) - else: - # if the dependency isn't installable, don't add it to constraints - # and instead add it directly to the lock - if ( - req - and req.requirement - and (req.requirement.marker and not req.requirement.marker.evaluate()) - ): - pypi = resolver.finder if resolver else None - ireq = req.ireq - best_match = ( - pypi.find_best_candidate(ireq.name, ireq.specifier).best_candidate - if pypi - else None - ) - if best_match: - ireq.req.specifier = ireq.specifier.__class__( - f"=={best_match.version}" - ) - hashes = resolver.collect_hashes(ireq) if resolver else [] - new_req = Requirement.from_ireq(ireq) - new_req.add_hashes(hashes) - new_req.merge_markers(req.markers) - name, entry = new_req.pipfile_entry - locked_deps[pep423_name(name)] = translate_markers(entry) - click.echo( - "{} doesn't match your environment, " - "its dependencies won't be resolved.".format(req.as_line()), - err=True, - ) - else: - click.echo( - "Could not find a version of {} that matches your environment, " - "it will be skipped.".format(req.as_line()), - err=True, - ) - return constraints, locked_deps - constraints.add(req.constraint_line) - return constraints, locked_deps - return constraints, locked_deps + return True + return False @classmethod def create( cls, - deps: List[str], + deps: Set[str], project: Project, index_lookup: Dict[str, str] = None, markers_lookup: Dict[str, str] = None, @@ -463,41 +190,68 @@ class Resolver: index_lookup = {} if markers_lookup is None: markers_lookup = {} + original_deps = {} + install_reqs = {} + pipfile_entries = {} + skipped = {} if sources is None: sources = project.sources + packages = project.get_pipfile_section(category) + constraints = set() + for dep in deps: # Build up the index and markers lookups + if not dep: + continue + is_constraint = True + install_req = expansive_install_req_from_line(dep, expand_env=True) + package_name = determine_package_name(install_req) + original_deps[package_name] = dep + install_reqs[package_name] = install_req + index, extra_index, trust_host, remainder = parse_indexes(dep) + if package_name in packages: + pipfile_entry = packages[package_name] + pipfile_entries[package_name] = pipfile_entry + if isinstance(pipfile_entry, dict): + if packages[package_name].get("index"): + index_lookup[package_name] = packages[package_name].get("index") + if packages[package_name].get("skip_resolver"): + is_constraint = False + skipped[package_name] = dep + elif index: + index_lookup[package_name] = index + else: + index_lookup[package_name] = project.get_default_index()["name"] + if install_req.markers: + markers_lookup[package_name] = install_req.markers + if is_constraint: + constraints.add(dep) resolver = Resolver( - [], + set(), req_dir, project, sources, index_lookup=index_lookup, markers_lookup=markers_lookup, + skipped=skipped, clear=clear, pre=pre, category=category, + original_deps=original_deps, + install_reqs=install_reqs, + pipfile_entries=pipfile_entries, ) - constraints, skipped, index_lookup, markers_lookup = resolver.get_metadata( - deps, - index_lookup, - markers_lookup, - project, - sources, - req_dir=req_dir, - pre=pre, - clear=clear, - category=category, - ) # Workaround to the fact `get_metadata` instantiates a transient Resolver + for package_name, dep in original_deps.items(): + install_req = install_reqs[package_name] + if resolver.check_if_package_req_skipped(install_req): + resolver.skipped[package_name] = dep resolver.initial_constraints = constraints - resolver.skipped = skipped resolver.index_lookup = index_lookup + resolver.finder.index_lookup = index_lookup resolver.markers_lookup = markers_lookup return resolver @property def pip_command(self): - if self._pip_command is None: - self._pip_command = self._get_pip_command() - return self._pip_command + return self._get_pip_command() def prepare_pip_args(self, use_pep517=None, build_isolation=True): pip_args = [] @@ -516,11 +270,9 @@ class Resolver: def pip_args(self): use_pep517 = environments.get_from_env("USE_PEP517", prefix="PIP") build_isolation = environments.get_from_env("BUILD_ISOLATION", prefix="PIP") - if self._pip_args is None: - self._pip_args = self.prepare_pip_args( - use_pep517=use_pep517, build_isolation=build_isolation - ) - return self._pip_args + return self.prepare_pip_args( + use_pep517=use_pep517, build_isolation=build_isolation + ) def prepare_constraint_file(self): constraint_filename = prepare_constraint_file( @@ -533,9 +285,7 @@ class Resolver: @property def constraint_file(self): - if self._constraint_file is None: - self._constraint_file = self.prepare_constraint_file() - return self._constraint_file + return self.prepare_constraint_file() @cached_property def default_constraint_file(self): @@ -550,24 +300,20 @@ class Resolver: @property def pip_options(self): - if self._pip_options is None: - pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) - pip_options.cache_dir = self.project.s.PIPENV_CACHE_DIR - pip_options.no_python_version_warning = True - pip_options.no_input = self.project.settings.get("disable_pip_input", True) - pip_options.progress_bar = "off" - pip_options.ignore_requires_python = True - pip_options.pre = self.pre or self.project.settings.get( - "allow_prereleases", False - ) - self._pip_options = pip_options - return self._pip_options + pip_options, _ = self.pip_command.parser.parse_args(self.pip_args) + pip_options.cache_dir = self.project.s.PIPENV_CACHE_DIR + pip_options.no_python_version_warning = True + pip_options.no_input = self.project.settings.get("disable_pip_input", True) + pip_options.progress_bar = "off" + pip_options.ignore_requires_python = True + pip_options.pre = self.pre or self.project.settings.get( + "allow_prereleases", False + ) + return pip_options @property def session(self): - if self._session is None: - self._session = self.pip_command._build_session(self.pip_options) - return self._session + return self.pip_command._build_session(self.pip_options) def prepare_index_lookup(self): index_mapping = {} @@ -582,29 +328,26 @@ class Resolver: @property def finder(self): - if self._finder is None: - self._finder = get_package_finder( - install_cmd=self.pip_command, - options=self.pip_options, - session=self.session, - ) + finder = get_package_finder( + install_cmd=self.pip_command, + options=self.pip_options, + session=self.session, + ) index_lookup = self.prepare_index_lookup() - self._finder._link_collector.index_lookup = index_lookup - self._finder._link_collector.search_scope.index_lookup = index_lookup - return self._finder + finder._link_collector.index_lookup = index_lookup + finder._link_collector.search_scope.index_lookup = index_lookup + return finder @property def parsed_constraints(self): pip_options = self.pip_options pip_options.extra_index_urls = [] - if self._parsed_constraints is None: - self._parsed_constraints = parse_requirements( - self.constraint_file, - finder=self.finder, - session=self.session, - options=pip_options, - ) - return self._parsed_constraints + return parse_requirements( + self.constraint_file, + finder=self.finder, + session=self.session, + options=pip_options, + ) @cached_property def parsed_default_constraints(self): @@ -617,11 +360,11 @@ class Resolver: session=self.session, options=pip_options, ) - return parsed_default_constraints + return set(parsed_default_constraints) @cached_property def default_constraints(self): - default_constraints = [ + possible_default_constraints = [ install_req_from_parsed_requirement( c, isolated=self.pip_options.build_isolation, @@ -629,25 +372,34 @@ class Resolver: ) for c in self.parsed_default_constraints ] - return default_constraints + default_constraints = [] + for c in possible_default_constraints: + default_constraints.append(c) + return set(default_constraints) + + @property + def possible_constraints(self): + possible_constraints_list = [ + install_req_from_parsed_requirement( + c, + isolated=self.pip_options.build_isolation, + use_pep517=self.pip_options.use_pep517, + user_supplied=True, + ) + for c in self.parsed_constraints + ] + return possible_constraints_list @property def constraints(self): - if self._constraints is None: - self._constraints = [ - install_req_from_parsed_requirement( - c, - isolated=self.pip_options.build_isolation, - use_pep517=self.pip_options.use_pep517, - user_supplied=True, - ) - for c in self.parsed_constraints - ] - # Only use default_constraints when installing dev-packages - if self.category != "packages": - self._constraints += self.default_constraints - self._constraints.sort(key=lambda ireq: ireq.name) - return self._constraints + possible_constraints_list = self.possible_constraints + constraints_list = set() + for c in possible_constraints_list: + constraints_list.add(c) + # Only use default_constraints when installing dev-packages + if self.category != "packages": + constraints_list |= self.default_constraints + return set(constraints_list) @contextlib.contextmanager def get_resolver(self, clear=False): @@ -682,7 +434,6 @@ class Resolver: yield resolver def resolve(self): - self.constraints # For some reason it is important to evaluate constraints before resolver context with temp_environ(), self.get_resolver() as resolver: try: results = resolver.resolve(self.constraints, check_supported_wheels=False) @@ -694,7 +445,7 @@ class Resolver: return self.resolved_tree def resolve_constraints(self): - from pipenv.vendor.requirementslib.models.markers import marker_from_specifier + from .markers import marker_from_specifier new_tree = set() for result in self.resolved_tree: @@ -723,100 +474,10 @@ class Resolver: new_tree.add(result) self.resolved_tree = new_tree - @classmethod - def prepend_hash_types(cls, checksums, hash_type): - cleaned_checksums = set() - for checksum in checksums: - if not checksum: - continue - if not checksum.startswith(f"{hash_type}:"): - checksum = f"{hash_type}:{checksum}" - cleaned_checksums.add(checksum) - return cleaned_checksums - - def _get_requests_session_for_source(self, source): - if self._sessions.get(source["name"]): - session = self._sessions[source["name"]] - else: - session = _get_requests_session( - self.project.s.PIPENV_MAX_RETRIES, source.get("verify_ssl", True) - ) - self._sessions[source["name"]] = session - return session - - def _get_hashes_from_pypi(self, ireq, source): - pkg_url = f"https://pypi.org/pypi/{ireq.name}/json" - session = self._get_requests_session_for_source(source) - try: - collected_hashes = set() - # Grab the hashes from the new warehouse API. - r = session.get(pkg_url, timeout=10) - api_releases = r.json()["releases"] - cleaned_releases = {} - for api_version, api_info in api_releases.items(): - api_version = clean_pkg_version(api_version) - cleaned_releases[api_version] = api_info - version = "" - if ireq.specifier: - spec = next(iter(s for s in ireq.specifier), None) - if spec: - version = spec.version - for release in cleaned_releases[version]: - collected_hashes.add(release["digests"][FAVORITE_HASH]) - return self.prepend_hash_types(collected_hashes, FAVORITE_HASH) - except (ValueError, KeyError, ConnectionError): - if self.project.s.is_verbose(): - click.echo( - "{}: Error generating hash for {}".format( - click.style("Warning", bold=True, fg="red"), ireq.name - ), - err=True, - ) - return None - - def _get_hashes_from_remote_index_urls(self, ireq, source): - pkg_url = f"{source['url']}/{ireq.name}/" - session = self._get_requests_session_for_source(source) - try: - collected_hashes = set() - # Grab the hashes from the new warehouse API. - response = session.get(pkg_url, timeout=10) - # Create an instance of the parser - parser = PackageIndexHTMLParser() - # Feed the HTML to the parser - parser.feed(response.text) - # Extract hrefs - hrefs = parser.urls - - version = "" - if ireq.specifier: - spec = next(iter(s for s in ireq.specifier), None) - if spec: - version = spec.version - for package_url in hrefs: - if version in package_url: - url_params = parse.urlparse(package_url).fragment - params_dict = parse.parse_qs(url_params) - if params_dict.get(FAVORITE_HASH): - collected_hashes.add(params_dict[FAVORITE_HASH][0]) - return self.prepend_hash_types(collected_hashes, FAVORITE_HASH) - except (ValueError, KeyError, ConnectionError): - if self.project.s.is_verbose(): - click.echo( - "{}: Error generating hash for {}".format( - click.style("Warning", bold=True, fg="red"), ireq.name - ), - err=True, - ) - return None - def collect_hashes(self, ireq): link = ireq.link # Handle VCS and file links first if link and (link.is_vcs or (link.is_file and link.is_existing_dir())): return set() - if link and ireq.original_link: - return {self._get_hash_from_link(ireq.original_link)} - if not is_pinned_requirement(ireq): return set() @@ -828,11 +489,11 @@ class Resolver: source = sources[0] if len(sources) else None if source: if is_pypi_url(source["url"]): - hashes = self._get_hashes_from_pypi(ireq, source) + hashes = self.project.get_hashes_from_pypi(ireq, source) if hashes: return hashes else: - hashes = self._get_hashes_from_remote_index_urls(ireq, source) + hashes = self.project.get_hashes_from_remote_index_urls(ireq, source) if hashes: return hashes @@ -841,12 +502,14 @@ class Resolver: ).iter_applicable() applicable_candidates = list(applicable_candidates) if applicable_candidates: - return { - self._get_hash_from_link(candidate.link) - for candidate in applicable_candidates - } + return sorted( + { + self.project.get_hash_from_link(self.hash_cache, candidate.link) + for candidate in applicable_candidates + } + ) if link: - return {self._get_hash_from_link(link)} + return {self.project.get_hash_from_link(self.hash_cache, link)} return set() def resolve_hashes(self): @@ -855,19 +518,18 @@ class Resolver: self.hashes[ireq] = self.collect_hashes(ireq) return self.hashes - def _get_hash_from_link(self, link): - if link.hash and link.hash_name == FAVORITE_HASH: - return f"{link.hash_name}:{link.hash}" - - return self.hash_cache.get_hash(link) - - def _clean_skipped_result(self, req, value): + def clean_skipped_result( + self, req_name: str, ireq: InstallRequirement, pipfile_entry + ): ref = None - if req.is_vcs: - ref = req.commit_hash - ireq = req.ireq - entry = value.copy() - entry["name"] = req.name + if ireq.link and ireq.link.is_vcs: + ref = ireq.link.egg_fragment + + if isinstance(pipfile_entry, dict): + entry = pipfile_entry.copy() + else: + entry = {} + entry["name"] = req_name if entry.get("editable", False) and entry.get("version"): del entry["version"] ref = ref if ref is not None else entry.get("ref") @@ -876,31 +538,35 @@ class Resolver: collected_hashes = self.collect_hashes(ireq) if collected_hashes: entry["hashes"] = sorted(set(collected_hashes)) - return req.name, entry + return req_name, entry def clean_results(self): - reqs = [(Requirement.from_ireq(ireq), ireq) for ireq in self.resolved_tree] + reqs = [(ireq,) for ireq in self.resolved_tree] results = {} - for req, ireq in reqs: - if req.vcs and req.editable and not req.is_direct_url: - continue - elif req.normalized_name in self.skipped.keys(): + for (ireq,) in reqs: + if normalize_name(ireq.name) in self.skipped.keys(): continue collected_hashes = self.hashes.get(ireq, set()) - req.add_hashes(collected_hashes) if collected_hashes: collected_hashes = sorted(collected_hashes) name, entry = format_requirement_for_lockfile( - req, self.markers_lookup, self.index_lookup, collected_hashes + ireq, + self.markers_lookup, + self.index_lookup, + self.original_deps, + self.pipfile_entries, + collected_hashes, ) entry = translate_markers(entry) if name in results: results[name].update(entry) else: results[name] = entry - for k in list(self.skipped.keys()): - req = Requirement.from_pipfile(k, self.skipped[k]) - name, entry = self._clean_skipped_result(req, self.skipped[k]) + for req_name in self.skipped: + install_req = self.install_reqs[req_name] + name, entry = self.clean_skipped_result( + req_name, install_req, self.pipfile_entries[req_name] + ) entry = translate_markers(entry) if name in results: results[name].update(entry) @@ -957,7 +623,7 @@ def actually_resolve_deps( warning.lineno, warning.line, ) - return (results, hashes, resolver.markers_lookup, resolver, resolver.skipped) + return (results, hashes, resolver) def resolve(cmd, st, project): @@ -999,6 +665,7 @@ def venv_resolve_deps( pypi_mirror=None, pipfile=None, lockfile=None, + old_lock_data=None, ): """ Resolve dependencies for a pipenv project, acts as a portal to the target environment. @@ -1008,7 +675,7 @@ def venv_resolve_deps( dependency resolution. This function reads the output of that call and mutates the provided lockfile accordingly, returning nothing. - :param List[:class:`~requirementslib.Requirement`] deps: A list of dependencies to resolve. + :param List[:class:`~pip.InstallRequirement`] deps: A list of dependencies to resolve. :param Callable which: [description] :param project: The pipenv Project instance to use during resolution :param Optional[bool] pre: Whether to resolve pre-release candidates, defaults to False @@ -1036,6 +703,8 @@ def venv_resolve_deps( pipfile = getattr(project, category, {}) if lockfile is None: lockfile = project.lockfile(categories=[category]) + if old_lock_data is None: + old_lock_data = lockfile.get(lockfile_section, {}) req_dir = create_tracked_tempdir(prefix="pipenv", suffix="requirements") results = [] with temp_environ(): @@ -1054,9 +723,10 @@ def venv_resolve_deps( # dependency resolution on them, so we are including this step inside the # spinner context manager for the UX improvement st.console.print("Building requirements...") - deps = convert_deps_to_pip(deps, project, include_index=True) + deps = convert_deps_to_pip( + deps, project.pipfile_sources(), include_index=True + ) constraints = set(deps) - st.console.print("Resolving dependencies...") # Useful for debugging and hitting breakpoints in the resolver if project.s.PIPENV_RESOLVER_PARENT_PYTHON: try: @@ -1079,7 +749,7 @@ def venv_resolve_deps( st.console.print( environments.PIPENV_SPINNER_FAIL_TEXT.format("Locking Failed!") ) - raise + raise # maybe sys.exit(1) here? else: # Default/Production behavior is to use project python's resolver cmd = [ which("python", allow_global=allow_global), @@ -1094,6 +764,8 @@ def venv_resolve_deps( if category: cmd.append("--category") cmd.append(category) + if project.s.is_verbose(): + cmd.append("--verbose") target_file = tempfile.NamedTemporaryFile( prefix="resolver", suffix=".json", delete=False ) @@ -1133,7 +805,9 @@ def venv_resolve_deps( click.echo(f"Error: {c.stderr.strip()}", err=True) if lockfile_section not in lockfile: lockfile[lockfile_section] = {} - return prepare_lockfile(results, pipfile, lockfile[lockfile_section]) + return prepare_lockfile( + project, results, pipfile, lockfile[lockfile_section], old_lock_data + ) def resolve_deps( @@ -1164,7 +838,7 @@ def resolve_deps( req_dir = create_tracked_tempdir(prefix="pipenv-", suffix="-requirements") with HackedPythonVersion(python_path=project.python(system=allow_global)): try: - results, hashes, markers_lookup, resolver, skipped = actually_resolve_deps( + results, hashes, internal_resolver = actually_resolve_deps( deps, index_lookup, markers_lookup, @@ -1189,9 +863,7 @@ def resolve_deps( ( results, hashes, - markers_lookup, - resolver, - skipped, + internal_resolver, ) = actually_resolve_deps( deps, index_lookup, @@ -1205,7 +877,7 @@ def resolve_deps( ) except RuntimeError: sys.exit(1) - return results, resolver + return results, internal_resolver @lru_cache() diff --git a/pipenv/utils/shell.py b/pipenv/utils/shell.py index 89c6ef9b..36881cfa 100644 --- a/pipenv/utils/shell.py +++ b/pipenv/utils/shell.py @@ -11,9 +11,9 @@ from contextlib import contextmanager from functools import lru_cache from pathlib import Path +from pipenv.utils.fileutils import normalize_drive, normalize_path from pipenv.vendor import click from pipenv.vendor.pythonfinder.utils import ensure_path -from pipenv.vendor.requirementslib.fileutils import normalize_drive, normalize_path from .constants import FALSE_VALUES, SCHEME_LIST, TRUE_VALUES from .processes import subprocess_run diff --git a/pipenv/utils/toml.py b/pipenv/utils/toml.py index 583332a0..e1261469 100644 --- a/pipenv/utils/toml.py +++ b/pipenv/utils/toml.py @@ -1,3 +1,9 @@ +from typing import Union + +from pipenv.vendor.plette.models import Package, PackageCollection +from pipenv.vendor.tomlkit.container import Container +from pipenv.vendor.tomlkit.items import AoT, Array, Bool, InlineTable, Item, String, Table + try: import tomllib as toml except ImportError: @@ -5,6 +11,10 @@ except ImportError: from pipenv.vendor import tomlkit +TOML_DICT_TYPES = Union[Container, Package, PackageCollection, Table, InlineTable] +TOML_DICT_OBJECTS = (Container, Package, Table, InlineTable, PackageCollection) +TOML_DICT_NAMES = [o.__class__.__name__ for o in TOML_DICT_OBJECTS] + def cleanup_toml(tml): toml = tml.split("\n") @@ -74,3 +84,57 @@ def convert_toml_outline_tables(parsed, project): parsed[section] = result return parsed + + +def tomlkit_value_to_python(toml_value): + # type: (Union[Array, AoT, TOML_DICT_TYPES, Item]) -> Union[List, Dict] + value_type = type(toml_value).__name__ + if ( + isinstance(toml_value, TOML_DICT_OBJECTS + (dict,)) + or value_type in TOML_DICT_NAMES + ): + return tomlkit_dict_to_python(toml_value) + elif isinstance(toml_value, AoT) or value_type == "AoT": + return [tomlkit_value_to_python(val) for val in toml_value._body] + elif isinstance(toml_value, Array) or value_type == "Array": + return [tomlkit_value_to_python(val) for val in list(toml_value)] + elif isinstance(toml_value, String) or value_type == "String": + return f"{toml_value!s}" + elif isinstance(toml_value, Bool) or value_type == "Bool": + return toml_value.value + elif isinstance(toml_value, Item): + return toml_value.value + return toml_value + + +def tomlkit_dict_to_python(toml_dict): + # type: (TOML_DICT_TYPES) -> Dict + value_type = type(toml_dict).__name__ + if toml_dict is None: + raise TypeError("Invalid type NoneType when converting toml dict to python") + converted = None # type: Optional[Dict] + if isinstance(toml_dict, (InlineTable, Table)) or value_type in ( + "InlineTable", + "Table", + ): + converted = toml_dict.value + elif isinstance(toml_dict, (Package, PackageCollection)) or value_type in ( + "Package, PackageCollection" + ): + converted = toml_dict._data + if isinstance(converted, Container) or type(converted).__name__ == "Container": + converted = converted.value + elif isinstance(toml_dict, Container) or value_type == "Container": + converted = toml_dict.value + elif isinstance(toml_dict, dict): + converted = toml_dict.copy() + else: + raise TypeError( + "Invalid type for conversion: expected Container, Dict, or Table, " + "got {!r}".format(toml_dict) + ) + if isinstance(converted, dict): + return {k: tomlkit_value_to_python(v) for k, v in converted.items()} + elif isinstance(converted, (TOML_DICT_OBJECTS)) or value_type in TOML_DICT_NAMES: + return tomlkit_dict_to_python(converted) + return converted diff --git a/pipenv/vendor/requirementslib/LICENSE b/pipenv/vendor/requirementslib/LICENSE deleted file mode 100644 index 21aaa475..00000000 --- a/pipenv/vendor/requirementslib/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright 2019-2021 Dan Ryan and Frost Ming. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/pipenv/vendor/requirementslib/__init__.py b/pipenv/vendor/requirementslib/__init__.py deleted file mode 100644 index 2127bc7e..00000000 --- a/pipenv/vendor/requirementslib/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -import logging -import warnings - -from .models.lockfile import Lockfile -from .models.pipfile import Pipfile -from .models.requirements import Requirement - -__version__ = "3.0.0" - - -logger = logging.getLogger(__name__) -logger.addHandler(logging.NullHandler()) -warnings.filterwarnings("ignore", category=ResourceWarning) - - -__all__ = ["Lockfile", "Pipfile", "Requirement"] diff --git a/pipenv/vendor/requirementslib/environment.py b/pipenv/vendor/requirementslib/environment.py deleted file mode 100644 index 5ad3b974..00000000 --- a/pipenv/vendor/requirementslib/environment.py +++ /dev/null @@ -1,17 +0,0 @@ -import os - -from pipenv.patched.pip._vendor.platformdirs import user_cache_dir - - -def is_type_checking(): - try: - from typing import TYPE_CHECKING - except ImportError: - return False - return TYPE_CHECKING - - -REQUIREMENTSLIB_CACHE_DIR = os.getenv( - "REQUIREMENTSLIB_CACHE_DIR", user_cache_dir("pipenv") -) -MYPY_RUNNING = os.environ.get("MYPY_RUNNING", is_type_checking()) diff --git a/pipenv/vendor/requirementslib/funktools.py b/pipenv/vendor/requirementslib/funktools.py deleted file mode 100644 index 4c4cdcb2..00000000 --- a/pipenv/vendor/requirementslib/funktools.py +++ /dev/null @@ -1,77 +0,0 @@ -"""A small collection of useful functional tools for working with iterables. - -unnest, chunked and take are used by pipenv. However, since pipenv -relies on requirementslib, we can them here. -""" -from functools import partial -from itertools import islice, tee -from typing import Any, Iterable - - -def _is_iterable(elem: Any) -> bool: - if getattr(elem, "__iter__", False) or isinstance(elem, Iterable): - return True - return False - - -def take(n: int, iterable: Iterable) -> Iterable: - """Take n elements from the supplied iterable without consuming it. - - :param int n: Number of unique groups - :param iter iterable: An iterable to split up - """ - return list(islice(iterable, n)) - - -def chunked(n: int, iterable: Iterable) -> Iterable: - """Split an iterable into lists of length *n*. - - :param int n: Number of unique groups - :param iter iterable: An iterable to split up - """ - return iter(partial(take, n, iter(iterable)), []) - - -def unnest(elem: Iterable) -> Any: - # type: (Iterable) -> Any - """Flatten an arbitrarily nested iterable. - - :param elem: An iterable to flatten - :type elem: :class:`~collections.Iterable` - >>> nested_iterable = ( - 1234, (3456, 4398345, (234234)), ( - 2396, ( - 928379, 29384, ( - 293759, 2347, ( - 2098, 7987, 27599 - ) - ) - ) - ) - ) - >>> list(unnest(nested_iterable)) - [1234, 3456, 4398345, 234234, 2396, 928379, 29384, 293759, - 2347, 2098, 7987, 27599] - """ - - if isinstance(elem, Iterable) and not isinstance(elem, str): - elem, target = tee(elem, 2) - else: - target = elem - if not target or not _is_iterable(target): - yield target - else: - for el in target: - if isinstance(el, Iterable) and not isinstance(el, str): - el, el_copy = tee(el, 2) - for sub in unnest(el_copy): - yield sub - else: - yield el - - -def dedup(iterable: Iterable) -> Iterable: - # type: (Iterable) -> Iterable - """Deduplicate an iterable object like iter(set(iterable)) but order- - preserved.""" - return iter(dict.fromkeys(iterable)) diff --git a/pipenv/vendor/requirementslib/models/__init__.py b/pipenv/vendor/requirementslib/models/__init__.py deleted file mode 100644 index e69de29b..00000000 diff --git a/pipenv/vendor/requirementslib/models/common.py b/pipenv/vendor/requirementslib/models/common.py deleted file mode 100644 index 0f06bcef..00000000 --- a/pipenv/vendor/requirementslib/models/common.py +++ /dev/null @@ -1,28 +0,0 @@ -from typing import Any, Dict - -from pipenv.vendor.pydantic import BaseModel, Extra - - -class ReqLibBaseModel(BaseModel): - def __setattr__(self, name, value): # noqa: C901 (ignore complexity) - private_attributes = { - field_name - for field_name in self.__annotations__ - if field_name.startswith("_") - } - - if name in private_attributes or name in self.__fields__: - return object.__setattr__(self, name, value) - - if self.__config__.extra is not Extra.allow and name not in self.__fields__: - raise ValueError(f'"{self.__class__.__name__}" object has no field "{name}"') - - object.__setattr__(self, name, value) - - def dict(self, *args, **kwargs) -> Dict[str, Any]: - """The requirementslib classes make use of a lot of private attributes - which do not get serialized out to the dict by default in pydantic.""" - model_dict = super().dict(*args, **kwargs) - private_attrs = {k: v for k, v in self.__dict__.items() if k.startswith("_")} - model_dict.update(private_attrs) - return model_dict diff --git a/pipenv/vendor/requirementslib/models/dependencies.py b/pipenv/vendor/requirementslib/models/dependencies.py deleted file mode 100644 index ab509b48..00000000 --- a/pipenv/vendor/requirementslib/models/dependencies.py +++ /dev/null @@ -1,62 +0,0 @@ -import atexit -import os - -from pipenv.patched.pip._internal.index.package_finder import PackageFinder -from pipenv.patched.pip._vendor.platformdirs import user_cache_dir - -from ..utils import get_package_finder, get_pip_command, prepare_pip_source_args - -CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) - - -def is_python(section): - return section.startswith("[") and ":" in section - - -def get_pip_options(args=None, sources=None, pip_command=None): - """Build a pip command from a list of sources. - - :param args: positional arguments passed through to the pip parser - :param sources: A list of pipfile-formatted sources, defaults to None - :param sources: list[dict], optional - :param pip_command: A pre-built pip command instance - :type pip_command: :class:`~pipenv.patched.pip._internal.cli.base_command.Command` - :return: An instance of pip_options using the supplied arguments plus sane defaults - :rtype: :class:`~pipenv.patched.pip._internal.cli.cmdoptions` - """ - - if not pip_command: - pip_command = get_pip_command() - if not sources: - sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] - os.makedirs(CACHE_DIR, mode=0o777, exist_ok=True) - pip_args = args or [] - pip_args = prepare_pip_source_args(sources, pip_args) - pip_options, _ = pip_command.parser.parse_args(pip_args) - pip_options.cache_dir = CACHE_DIR - return pip_options - - -def get_finder(sources=None, pip_command=None, pip_options=None) -> PackageFinder: - """Get a package finder for looking up candidates to install. - - :param sources: A list of pipfile-formatted sources, defaults to None - :param sources: list[dict], optional - :param pip_command: A pip command instance, defaults to None - :type pip_command: :class:`~pipenv.patched.pip._internal.cli.base_command.Command` - :param pip_options: A pip options, defaults to None - :type pip_options: :class:`~pipenv.patched.pip._internal.cli.cmdoptions` - :return: A package finder - :rtype: :class:`~pipenv.patched.pip._internal.index.PackageFinder` - """ - - if not pip_command: - pip_command = get_pip_command() - if not sources: - sources = [{"url": "https://pypi.org/simple", "name": "pypi", "verify_ssl": True}] - if not pip_options: - pip_options = get_pip_options(sources=sources, pip_command=pip_command) - session = pip_command._build_session(pip_options) - atexit.register(session.close) - finder = get_package_finder(get_pip_command(), options=pip_options, session=session) - return session, finder diff --git a/pipenv/vendor/requirementslib/models/lockfile.py b/pipenv/vendor/requirementslib/models/lockfile.py deleted file mode 100644 index 88917006..00000000 --- a/pipenv/vendor/requirementslib/models/lockfile.py +++ /dev/null @@ -1,248 +0,0 @@ -import copy -import itertools -import os -from json import JSONDecodeError -from pathlib import Path -from typing import Dict, Iterator, List, Optional - -from pipenv.vendor.plette import lockfiles -from pipenv.vendor.pydantic import Field - -from ..exceptions import LockfileCorruptException, MissingParameter, PipfileNotFound -from ..utils import is_editable, is_vcs, merge_items -from .common import ReqLibBaseModel -from .project import ProjectFile -from .requirements import Requirement - -DEFAULT_NEWLINES = "\n" - - -def preferred_newlines(f): - if isinstance(f.newlines, str): - return f.newlines - return DEFAULT_NEWLINES - - -class Lockfile(ReqLibBaseModel): - path: Path = Field( - default_factory=lambda: Path(os.curdir).joinpath("Pipfile.lock").absolute() - ) - _requirements: Optional[list] = Field(default_factory=list) - _dev_requirements: Optional[list] = Field(default_factory=list) - projectfile: ProjectFile = None - lockfile: lockfiles.Lockfile - newlines: str = DEFAULT_NEWLINES - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - # keep_untouched = (cached_property,) - - @property - def section_keys(self): - return set(self.lockfile.keys()) - {"_meta"} - - @property - def extended_keys(self): - return [k for k in itertools.product(self.section_keys, ["", "vcs", "editable"])] - - def get(self, k): - return self.__getitem__(k) - - def __contains__(self, k): - check_lockfile = k in self.extended_keys or self.lockfile.__contains__(k) - if check_lockfile: - return True - return super(Lockfile, self).__contains__(k) - - def __setitem__(self, k, v): - lockfile = self.lockfile - lockfile.__setitem__(k, v) - - def __getitem__(self, k, *args, **kwargs): - retval = None - lockfile = self.lockfile - try: - retval = lockfile[k] - except KeyError: - if "-" in k: - section, _, pkg_type = k.rpartition("-") - vals = getattr(lockfile.get(section, {}), "_data", {}) - if pkg_type == "vcs": - retval = {k: v for k, v in vals.items() if is_vcs(v)} - elif pkg_type == "editable": - retval = {k: v for k, v in vals.items() if is_editable(v)} - if retval is None: - raise - else: - retval = getattr(retval, "_data", retval) - return retval - - def __getattr__(self, k, *args, **kwargs): - lockfile = self.lockfile - try: - return super(Lockfile, self).__getattribute__(k) - except AttributeError: - retval = getattr(lockfile, k, None) - if retval is not None: - return retval - return super(Lockfile, self).__getattribute__(k, *args, **kwargs) - - def get_deps(self, dev=False, only=True): - deps = {} - if dev: - deps.update(self.develop._data) - if only: - return deps - deps = merge_items([deps, self.default._data]) - return deps - - @classmethod - def read_projectfile(cls, path): - pf = ProjectFile.read(path, lockfiles.Lockfile, invalid_ok=True) - return pf - - @classmethod - def lockfile_from_pipfile(cls, pipfile_path): - from .pipfile import Pipfile - - if os.path.isfile(pipfile_path): - if not os.path.isabs(pipfile_path): - pipfile_path = os.path.abspath(pipfile_path) - pipfile = Pipfile.load(os.path.dirname(pipfile_path)) - return lockfiles.Lockfile.with_meta_from(pipfile.pipfile) - raise PipfileNotFound(pipfile_path) - - @classmethod - def load_projectfile( - cls, path: Optional[str] = None, create: bool = True, data: Optional[Dict] = None - ) -> "ProjectFile": - if not path: - path = os.curdir - path = Path(path).absolute() - project_path = path if path.is_dir() else path.parent - lockfile_path = path if path.is_file() else project_path / "Pipfile.lock" - if not project_path.exists(): - raise OSError(f"Project does not exist: {project_path.as_posix()}") - elif not lockfile_path.exists() and not create: - raise FileNotFoundError( - f"Lockfile does not exist: {lockfile_path.as_posix()}" - ) - projectfile = cls.read_projectfile(lockfile_path.as_posix()) - if not lockfile_path.exists(): - if not data: - pipfile = project_path.joinpath("Pipfile") - lf = cls.lockfile_from_pipfile(pipfile) - else: - lf = lockfiles.Lockfile(data) - projectfile.model = lf - else: - if data: - raise ValueError("Cannot pass data when loading existing lockfile") - with open(lockfile_path.as_posix(), "r") as f: - projectfile.model = lockfiles.Lockfile.load(f) - return projectfile - - @classmethod - def from_data( - cls, path: Optional[str], data: Optional[Dict], meta_from_project: bool = True - ) -> "Lockfile": - if path is None: - raise MissingParameter("path") - if data is None: - raise MissingParameter("data") - if not isinstance(data, dict): - raise TypeError("Expecting a dictionary for parameter 'data'") - path = os.path.abspath(str(path)) - if os.path.isdir(path): - project_path = path - elif not os.path.isdir(path) and os.path.isdir(os.path.dirname(path)): - project_path = os.path.dirname(path) - pipfile_path = os.path.join(project_path, "Pipfile") - lockfile_path = os.path.join(project_path, "Pipfile.lock") - if meta_from_project: - lockfile = cls.lockfile_from_pipfile(pipfile_path) - lockfile.update(data) - else: - lockfile = lockfiles.Lockfile(data) - projectfile = ProjectFile( - line_ending=DEFAULT_NEWLINES, location=lockfile_path, model=lockfile - ) - return cls( - projectfile=projectfile, - lockfile=lockfile, - newlines=projectfile.line_ending, - path=Path(projectfile.location), - ) - - @classmethod - def load(cls, path: Optional[str], create: bool = True) -> "Lockfile": - try: - projectfile = cls.load_projectfile(path, create=create) - except JSONDecodeError: - path = os.path.abspath(path) - path = Path( - os.path.join(path, "Pipfile.lock") if os.path.isdir(path) else path - ) - formatted_path = path.as_posix() - backup_path = f"{formatted_path}.bak" - LockfileCorruptException.show(formatted_path, backup_path=backup_path) - path.rename(backup_path) - cls.load(formatted_path, create=True) - lockfile_path = Path(projectfile.location) - creation_args = { - "projectfile": projectfile, - "lockfile": projectfile.model, - "newlines": projectfile.line_ending, - "path": lockfile_path, - } - return cls(**creation_args) - - @classmethod - def create(cls, path: Optional[str], create: bool = True) -> "Lockfile": - return cls.load(path, create=create) - - def get_section(self, name: str) -> Optional[Dict]: - return self.lockfile.get(name) - - @property - def develop(self) -> Dict: - return self.lockfile.develop - - @property - def default(self) -> Dict: - return self.lockfile.default - - def get_requirements( - self, dev: bool = True, only: bool = False, categories: Optional[List[str]] = None - ) -> Iterator[Requirement]: - if categories: - deps = {} - for category in categories: - if category == "packages": - category = "default" - elif category == "dev-packages": - category = "develop" - try: - category_deps = self[category] - except KeyError: - category_deps = {} - self.lockfile[category] = category_deps - deps = merge_items([deps, category_deps]) - else: - deps = self.get_deps(dev=dev, only=only) - for k, v in deps.items(): - yield Requirement.from_pipfile(k, v) - - def requirements_list(self, category: str) -> List[Dict]: - if self.lockfile.get(category): - return [ - {name: entry._data} for name, entry in self.lockfile[category].items() - ] - return [] - - def write(self) -> None: - self.projectfile.model = copy.deepcopy(self.lockfile) - self.projectfile.write() diff --git a/pipenv/vendor/requirementslib/models/metadata.py b/pipenv/vendor/requirementslib/models/metadata.py deleted file mode 100644 index a3906360..00000000 --- a/pipenv/vendor/requirementslib/models/metadata.py +++ /dev/null @@ -1,1083 +0,0 @@ -import io -import json -import logging -import operator -import os -import zipfile -from collections import defaultdict -from datetime import datetime -from functools import reduce -from typing import Any, Dict, Iterator, List, Optional, Sequence, Set, Tuple, Union - -import pipenv.patched.pip._vendor.requests as requests -from pipenv.patched.pip._vendor.distlib import wheel -from pipenv.patched.pip._vendor.distlib.metadata import Metadata -from pipenv.patched.pip._vendor.packaging.markers import Marker -from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement -from pipenv.patched.pip._vendor.packaging.specifiers import Specifier, SpecifierSet -from pipenv.patched.pip._vendor.packaging.tags import Tag -from pipenv.patched.pip._vendor.packaging.version import _BaseVersion, parse -from pipenv.vendor.pydantic import BaseModel, Field, validator -from pipenv.vendor.pydantic.json import pydantic_encoder - -from ..fileutils import open_file -from .common import ReqLibBaseModel -from .markers import ( - get_contained_extras, - get_contained_pyversions, - get_without_extra, - get_without_pyversion, - marker_from_specifier, - merge_markers, - normalize_specifier_set, -) -from .requirements import Requirement -from .setup_info import SetupInfo -from .utils import filter_dict, get_pinned_version, is_pinned_requirement - -ch = logging.StreamHandler() -formatter = logging.Formatter("%(asctime)s %(levelname)s: %(message)s") -ch.setFormatter(formatter) -logger = logging.getLogger(__name__) -logger.setLevel(logging.INFO) - -VALID_ALGORITHMS = { - "sha1": 40, - "sha3_224": 56, - "sha512": 128, - "blake2b": 128, - "sha256": 64, - "sha384": 96, - "blake2s": 64, - "sha3_256": 64, - "sha3_512": 128, - "md5": 32, - "sha3_384": 96, - "sha224": 56, -} - -PACKAGE_TYPES = { - "sdist", - "bdist_wheel", - "bdist_egg", - "bdist_dumb", - "bdist_wininst", - "bdist_rpm", - "bdist_msi", - "bdist_dmg", -} - - -class PackageEncoder(json.JSONEncoder): - def default(self, obj): # noqa:E0202 # noqa:W0221 - if isinstance(obj, datetime): - return obj.isoformat() - elif isinstance(obj, PackagingRequirement): - return obj.__dict__ - elif isinstance(obj, set): - return tuple(obj) - elif isinstance(obj, (Specifier, SpecifierSet, Marker)): - return str(obj) - else: - return json.JSONEncoder.default(self, obj) - - -def validate_extras(inst, attrib, value) -> None: - duplicates = [k for k in value if value.count(k) > 1] - if duplicates: - raise ValueError("Found duplicate keys: {0}".format(", ".join(duplicates))) - return None - - -def validate_digest(algorithm, value) -> None: - expected_length = VALID_ALGORITHMS[algorithm.lower()] - if len(value) != expected_length: - raise ValueError( - "Expected a digest of length {0!s}, got one of length {1!s}".format( - expected_length, len(value) - ) - ) - return None - - -def get_local_wheel_metadata(wheel_file) -> Optional[Metadata]: - parsed_metadata = None - with io.open(wheel_file, "rb") as fh: - with zipfile.ZipFile(fh, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: - metadata = None - for fn in zf.namelist(): - if os.path.basename(fn) == "METADATA": - metadata = fn - break - if metadata is None: - raise RuntimeError("No metadata found in wheel: {0}".format(wheel_file)) - with zf.open(metadata, "r") as metadata_fh: - parsed_metadata = Metadata(fileobj=metadata_fh) - return parsed_metadata - - -def get_remote_sdist_metadata(line) -> SetupInfo: - req = Requirement.from_line(line) - try: - _ = req.run_requires() - except SystemExit: - raise RuntimeError("Failed to compute metadata for dependency {0}".format(line)) - else: - return req.line_instance.setup_info - - -def get_remote_wheel_metadata(whl_file) -> Optional[Metadata]: - parsed_metadata = None - data = io.BytesIO() - with open_file(whl_file) as fp: - for chunk in iter(lambda: fp.read(8096), b""): - data.write(chunk) - with zipfile.ZipFile(data, mode="r", compression=zipfile.ZIP_DEFLATED) as zf: - metadata = None - for fn in zf.namelist(): - if os.path.basename(fn) == "METADATA": - metadata = fn - break - if metadata is None: - raise RuntimeError("No metadata found in wheel: {0}".format(whl_file)) - with zf.open(metadata, "r") as metadata_fh: - parsed_metadata = Metadata(fileobj=metadata_fh) - return parsed_metadata - - -def create_specifierset(spec=None): - # type: (Optional[str]) -> SpecifierSet - if isinstance(spec, SpecifierSet): - return spec - elif isinstance(spec, (set, list, tuple)): - spec = " and ".join(spec) - if spec is None: - spec = "" - return SpecifierSet(spec) - - -class Dependency(ReqLibBaseModel): - name: str - requirement: PackagingRequirement - specifier: SpecifierSet - extras: Tuple[str, ...] = tuple() - from_extras: Optional[str] = None - python_version: Any = "" - parent: Optional["Dependency"] = None - markers: Optional[Marker] = None - _specset_str: str = "" - _python_version_str: str = "" - _marker_str: str = "" - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - # keep_untouched = (cached_property,) - - def __hash__(self): - # type: () -> int - return hash(self.name) - - def __str__(self): - # type: () -> str - return str(self.requirement) - - def as_line(self): - # type: () -> str - line_str = "{0}".format(self.name) - if self.extras: - line_str = "{0}[{1}]".format(line_str, ",".join(self.extras)) - if self.specifier: - line_str = "{0}{1!s}".format(line_str, self.specifier) - py_version_part = "" - if self.python_version: - specifiers = normalize_specifier_set(self.python_version) - markers = [] - if specifiers is not None: - markers = [marker_from_specifier(str(s)) for s in specifiers] - py_version_part = reduce(merge_markers, markers) - if self.markers: - line_str = "{0} ; {1}".format(line_str, str(self.markers)) - if py_version_part: - line_str = "{0} and {1}".format(line_str, py_version_part) - elif py_version_part and not self.markers: - line_str = "{0} ; {1}".format(line_str, py_version_part) - return line_str - - def pin(self): - # type: () -> "Package" - base_package = get_package(self.name) - sorted_releases = sorted( - base_package.releases.non_yanked_releases, - key=operator.attrgetter("parsed_version"), - reverse=True, - ) - version = next( - iter(self.specifier.filter((r.version for r in sorted_releases))), None - ) - if not version: - version = next( - iter( - self.specifier.filter( - (r.version for r in sorted_releases), prereleases=True - ) - ), - None, - ) - if not version: - raise RuntimeError( - "Failed to resolve {0} ({1!s})".format(self.name, self.specifier) - ) - match = get_package_version(self.name, str(version)) - return match - - @classmethod - def from_requirement(cls, req, parent=None): - # type: (PackagingRequirement, Optional["Dependency"]) -> "Dependency" - from_extras, marker, python_version = None, None, None - specset_str, py_version_str, marker_str = "", "", "" - if req.marker: - marker = Marker(str(req.marker)) - from_extras = next(iter(list(get_contained_extras(marker))), None) - python_version = get_contained_pyversions(marker) - marker = get_without_extra(get_without_pyversion(marker)) - if not str(marker) or not marker or not marker._markers: - marker = None - req.marker = marker - if marker is not None: - marker_str = str(marker) - if req.specifier: - specset_str = str(req.specifier) - if python_version: - py_version_str = str(python_version) - return cls( - name=req.name, - specifier=req.specifier, - extras=tuple(sorted(set(req.extras))) - if req.extras is not None - else req.extras, - requirement=req, - from_extras=from_extras, - python_version=python_version, - markers=marker, - parent=parent, - specset_str=specset_str, - python_version_str=py_version_str, - marker_str=marker_str, - ) - - @classmethod - def from_info(cls, info): - # type: ("PackageInfo") -> "Dependency" - marker_str = "" - specset_str, py_version_str = "", "" - if info.requires_python: - # Some markers are improperly formatted -- we already handle most cases - # but learned about new broken formats, such as - # python_version in "2.6 2.7 3.2 3.3" (note the lack of commas) - # as a marker on a dependency of a library called 'pickleshare' - # Some packages also have invalid markers with stray characters, such as 'algoliasearch' - try: - marker = marker_from_specifier(info.requires_python) - except Exception: - marker_str = "" - else: - if not marker or not marker._markers: - marker_str = "" - else: - marker_str = "{0!s}".format(marker) - req_str = "{0}=={1}".format(info.name, info.version) - if marker_str: - req_str = "{0} ; {1}".format(req_str, marker_str) - req = PackagingRequirement(req_str) - requires_python_str = ( - info.requires_python if info.requires_python is not None else "" - ) - if req.specifier: - specset_str = str(req.specifier) - if requires_python_str: - py_version_str = requires_python_str - return cls( - name=info.name, - specifier=req.specifier, - extras=tuple(sorted(set(req.extras))) - if req.extras is not None - else req.extras, - requirement=req, - from_extras=None, - python_version=SpecifierSet(requires_python_str), - markers=None, - parent=None, - specset_str=specset_str, - python_version_str=py_version_str, - marker_str=marker_str, - ) - - @classmethod - def from_str(cls, depstr, parent=None): - # type: (str, Optional["Dependency"]) -> "Dependency" - try: - req = PackagingRequirement(depstr) - except Exception: - raise - return cls.from_requirement(req, parent=parent) - - -class Digest(BaseModel): - algorithm: str - value: str - - class Config: - frozen = True - - def __str__(self): - return f"{self.algorithm}:{self.value}" - - @classmethod - def create(cls, algorithm: str, value: str) -> "Digest": - if algorithm not in VALID_ALGORITHMS.keys(): - raise ValueError("Invalid algorithm") - validate_digest(algorithm, value) - return cls(algorithm=algorithm, value=value) - - @classmethod - def collection_from_dict(cls, digest_dict: Dict[str, str]) -> List["Digest"]: - return [cls.create(k, v) for k, v in digest_dict.items()] - - def check_algorithm(self, algorithm): - if algorithm not in VALID_ALGORITHMS.keys(): - raise ValueError("Invalid algorithm") - return algorithm - - def check_value(self, algorithm, value): - self.check_algorithm(algorithm) - validate_digest(algorithm, value) - return value - - -def create_digest_collection(digest_dict) -> List["Digest"]: - return Digest.collection_from_dict(digest_dict) - - -def instance_check_converter(expected_type=None, converter=None): - def _converter(val): - if expected_type is not None and isinstance(val, expected_type): - return val - return converter(val) - - return _converter - - -class ParsedTag(BaseModel): - marker_string: Optional[str] = None - python_version: Optional[str] = None - platform_system: Optional[str] = None - abi: Optional[str] = None - - class Config: - frozen = True - - -def parse_tag(tag) -> ParsedTag: - """Parse a :class:`~packaging.tags.Tag` instance. - - :param :class:`~packaging.tags.Tag` tag: A tag to parse - :return: A parsed tag with combined markers, supported platform and python version - """ - platform_system = None - python_version = None - version = None - marker_str = "" - if tag.platform.startswith("macos"): - platform_system = "Darwin" - elif tag.platform.startswith("manylinux") or tag.platform.startswith("linux"): - platform_system = "Linux" - elif tag.platform.startswith("win32"): - platform_system = "Windows" - if platform_system: - marker_str = 'platform_system == "{}"'.format(platform_system) - if tag.interpreter: - version = tag.interpreter[2:] - py_version_str = "" - if len(version) == 1: - py_version_str = ">={}.0,<{}".format(version, str(int(version) + 1)) - elif len(version) > 1 and len(version) <= 3: - # reverse the existing version so we can add 1 to the first element - # and re-reverse, generating the new version, e.g. [3, 2, 8] => - # [8, 2, 3] => [9, 2, 3] => [3, 2, 9] - next_version_list = list(reversed(version[:])) - next_version_list[0] = str(int(next_version_list[0]) + 1) - next_version = ".".join(list(reversed(next_version_list))) - version = ".".join(version) - py_version_str = ">={},<{}".format(version, next_version) - else: - py_version_str = "{0}".format(version) - python_version = marker_from_specifier(py_version_str) - if python_version: - if marker_str: - marker_str = "{0} and {1!s}".format(marker_str, python_version) - else: - marker_str = str(python_version) - return ParsedTag( - marker_string=marker_str, - python_version=version, - platform_system=platform_system, - abi=tag.abi, - ) - - -class ReleaseUrl(BaseModel): - md5_digest: Digest - packagetype: str - upload_time: datetime - upload_time_iso_8601: datetime - size: int - url: str - digests: Any - name: Optional[str] = None - comment_text: str = "" - yanked: bool = False - downloads: int = -1 - filename: str = "" - has_sig: bool = False - python_version: Optional[str] = "source" - requires_python: Optional[str] = None - tags: List[ParsedTag] = [] - - @validator("packagetype", pre=True) - def validate_package_type(cls, packagetype): - if packagetype not in PACKAGE_TYPES: - raise ValueError( - "Invalid package type: {0}. Expected one of {1}".format( - packagetype, " ".join(PACKAGE_TYPES) - ) - ) - return packagetype - - @property - def is_wheel(self) -> bool: - return os.path.splitext(self.filename)[-1].lower() == ".whl" - - @property - def is_sdist(self) -> bool: - return self.python_version == "source" - - @property - def markers(self): - if self.requires_python: - return marker_from_specifier(self.requires_python) - return None - - @property - def pep508_url(self): - # type: () -> str - markers = self.markers - req_str = "{0} @ {1}#egg={0}".format(self.name, self.url) - if markers: - req_str = "{0} ; {1}".format(req_str, markers) - return req_str - - def get_markers_from_wheel(self) -> str: - supported_platforms = [] - supported_pyversions = [] - supported_abis = [] - markers = [] - for parsed_tag in self.tags: - if parsed_tag.marker_string: - markers.append(Marker(parsed_tag.marker_string)) - if parsed_tag.python_version: - supported_pyversions.append(parsed_tag.python_version) - if parsed_tag.abi: - supported_abis.append(parsed_tag.abi) - if not (markers or supported_platforms): - return "" - if ( - all(pyversion in supported_pyversions for pyversion in ["2", "3"]) - and not supported_platforms - ): - marker_line = "" - else: - marker_line = " or ".join(["{}".format(str(marker)) for marker in markers]) - return marker_line - - def get_dependencies(self) -> Tuple["ReleaseUrl", Dict[str, Union[List[str], str]]]: - results = {"requires_python": None} - requires_dist = [] # type: List[str] - if self.is_wheel: - metadata = get_remote_wheel_metadata(self.url) - if metadata is not None: - requires_dist = metadata.run_requires - if not self.requires_python: - results["requires_python"] = metadata._legacy.get("Requires-Python") - else: - requires_dist = [] - try: - metadata = get_remote_sdist_metadata(self.pep508_url) - except Exception: - pass - else: - if metadata.requires: - requires_dist = [str(v) for v in metadata.requires.values()] - results["requires_dist"] = requires_dist - requires_python = getattr(self, "requires_python", results["requires_python"]) - self.requires_python = requires_python - return self, results - - @property - def sha256(self) -> str: - return next( - iter(digest for digest in self.digests if digest.algorithm == "sha256") - ).value - - @classmethod - def create(cls, release_dict: Dict, name: Optional[str] = None) -> "ReleaseUrl": - valid_digest_keys = set("{0}_digest".format(k) for k in VALID_ALGORITHMS.keys()) - digest_keys = set(release_dict.keys()) & valid_digest_keys - creation_kwargs = {k: v for k, v in release_dict.items() if k not in digest_keys} - if name is not None: - creation_kwargs["name"] = name - for k in digest_keys: - digest = release_dict[k] - if not isinstance(digest, str): - raise TypeError("Digests must be strings, got {!r}".format(digest)) - creation_kwargs[k] = Digest.create(k.replace("_digest", ""), digest) - release_url = cls(**filter_dict(creation_kwargs)) - if release_url.is_wheel: - supported_tags = [ - parse_tag(Tag(*tag)) for tag in wheel.Wheel(release_url.url).tags - ] - release_url.tags = supported_tags - return release_url - - -def create_release_urls_from_list(urls, name=None) -> List[ReleaseUrl]: - url_list = [] - for release_dict in urls: - if isinstance(release_dict, ReleaseUrl): - if name and not release_dict.name: - release_dict.name = name - url_list.append(release_dict) - continue - url_list.append(ReleaseUrl.create(release_dict, name=name)) - return url_list - - -class ReleaseUrlCollection(BaseModel, Sequence): - urls: List[ReleaseUrl] = [] - name: Optional[str] = None - - class Config: - frozen = True - - @classmethod - def create(cls, urls, name: Optional[str] = None) -> "ReleaseUrlCollection": - return cls(urls=create_release_urls_from_list(urls), name=name) - - @property - def wheels(self) -> Iterator[ReleaseUrl]: - for url in self.urls: - if not url.is_wheel: - continue - yield url - - @property - def sdists(self) -> Iterator[ReleaseUrl]: - for url in self.urls: - if not url.is_sdist: - continue - yield url - - def __iter__(self) -> Iterator[ReleaseUrl]: - return iter(self.urls) - - def __getitem__(self, key: int) -> ReleaseUrl: - return self.urls.__getitem__(key) - - def __len__(self) -> int: - return len(self.urls) - - @property - def latest(self) -> Optional[ReleaseUrl]: - if not self.urls: - return None - return next( - iter(sorted(self.urls, key=operator.attrgetter("upload_time"), reverse=True)) - ) - - @property - def latest_timestamp(self) -> Optional[datetime]: - latest = self.latest - if latest is not None: - return latest.upload_time - return None - - def find_package_type(self, type_: str) -> Optional[ReleaseUrl]: - if type_ not in PACKAGE_TYPES: - raise ValueError( - "Invalid package type: {0}. Expected one of {1}".format( - type_, " ".join(PACKAGE_TYPES) - ) - ) - return next(iter(url for url in self.urls if url.packagetype == type_), None) - - -def convert_release_urls_to_collection(urls=None, name=None) -> ReleaseUrlCollection: - if urls is None: - urls = [] - urls = create_release_urls_from_list(urls, name=name) - return ReleaseUrlCollection.create(urls, name=name) - - -class Release(BaseModel, Sequence): - version: str - urls: ReleaseUrlCollection - name: Optional[str] = None - - class Config: - frozen = True - - def __iter__(self) -> Iterator[ReleaseUrlCollection]: - return iter(self.urls) - - def __getitem__(self, key: int) -> ReleaseUrl: - return self.urls[key] - - def __len__(self) -> int: - return len(self.urls) - - @property - def yanked(self) -> bool: - if not self.urls: - return True - return False - - @property - def parsed_version(self) -> _BaseVersion: - return parse(self.version) - - @property - def wheels(self) -> Iterator[ReleaseUrl]: - return self.urls.wheels - - @property - def sdists(self) -> Iterator[ReleaseUrl]: - return self.urls.sdists - - @property - def latest(self) -> ReleaseUrl: - return self.urls.latest - - @property - def latest_timestamp(self) -> datetime: - return self.urls.latest_timestamp - - def to_lockfile(self) -> Dict[str, Union[List[str], str]]: - return { - "hashes": [str(url.sha256) for url in self.urls if url.sha256 is not None], - "version": "=={0}".format(self.version), - } - - -def get_release(version, urls, name=None) -> Release: - release_kwargs = {"version": version, "name": name} - if not isinstance(urls, ReleaseUrlCollection): - release_kwargs["urls"] = convert_release_urls_to_collection(urls, name=name) - else: - release_kwargs["urls"] = urls - return Release(**release_kwargs) # type: ignore - - -def get_releases_from_package(releases, name=None) -> List[Release]: - release_list = [] - for version, urls in releases.items(): - release_list.append(get_release(version, urls, name=name)) - return release_list - - -class ReleaseCollection(BaseModel): - releases: List[Release] = [] - - def __iter__(self) -> Iterator[Release]: - return iter(self.releases) - - def __getitem__(self, key: str) -> Release: - result = next(iter(r for r in self.releases if r.version == key), None) - if result is None: - raise KeyError(key) - return result - - def __len__(self) -> int: - return len(self.releases) - - def get_latest_lockfile(self) -> Dict[str, Union[str, List[str]]]: - return self.latest.to_lockfile() - - def wheels(self) -> Iterator[ReleaseUrl]: - for release in self.sort_releases(): - for w in release.wheels: - yield w - - def sdists(self) -> Iterator[ReleaseUrl]: - for release in self.sort_releases(): - for sdist in release.sdists: - yield sdist - - @property - def non_yanked_releases(self) -> List[Release]: - return list(r for r in self.releases if not r.yanked) - - def sort_releases(self) -> List[Release]: - return sorted( - self.non_yanked_releases, - key=operator.attrgetter("latest_timestamp"), - reverse=True, - ) - - @property - def latest(self) -> Optional[Release]: - return next(iter(r for r in self.sort_releases() if not r.yanked)) - - @classmethod - def load(cls, releases, name: Optional[str] = None) -> "ReleaseCollection": - if not isinstance(releases, list): - releases = get_releases_from_package(releases, name=name) - return cls(releases=releases) - - class Config: - frozen = True - - -def convert_releases_to_collection(releases, name=None) -> ReleaseCollection: - return ReleaseCollection.load(releases, name=name) - - -def split_keywords(value): - # type: (Union[str, List]) -> List[str] - if value and isinstance(value, str): - return value.split(",") - elif isinstance(value, list): - return value - return [] - - -def create_dependencies( - requires_dist, # type: Optional[List[Dependency]] - parent=None, # type: Optional[Dependency] -): - # type: (...) -> Optional[Set[Dependency]] - if requires_dist is None: - return None - dependencies = set() - for req in requires_dist: - if not isinstance(req, Dependency): - dependencies.add(Dependency.from_str(req, parent=parent)) - else: - dependencies.add(req) - return dependencies - - -class PackageInfo(ReqLibBaseModel): - name: str - version: str - package_url: str - summary: Optional[str] = None - author: Optional[str] = None - keywords: Optional[Union[str, List[str]]] = [] - description: Optional[str] = "" - download_url: Optional[str] = "" - home_page: Optional[str] = "" - license: Optional[str] = "" - maintainer: Optional[str] = "" - maintainer_email: Optional[str] = "" - downloads: Optional[Dict[str, int]] = {} - docs_url: Optional[str] = None - platform: Optional[str] = "" - project_url: Optional[str] = "" - project_urls: Optional[Dict[str, str]] = {} - requires_python: Optional[str] = None - requires_dist: Optional[Any] = [] - release_url: Optional[str] = None - description_content_type: Optional[str] = "text/md" - bugtrack_url: Optional[str] = None - classifiers: Optional[List[str]] = [] - author_email: Optional[str] = None - markers: Optional[str] = None - dependencies: Optional[Any] = None - - @classmethod - def from_json(cls, info_json) -> "PackageInfo": - return cls(**filter_dict(info_json)) # type: ignore - - def to_dependency(self) -> Dependency: - return Dependency.from_info(self) - - def create_dependencies(self, force=False) -> "PackageInfo": - """Create values for **self.dependencies**. - - :param bool force: Sets **self.dependencies** to an empty tuple if it would be - None, defaults to False. - :return: An updated instance of the current object with **self.dependencies** - updated accordingly. - :rtype: :class:`PackageInfo` - """ - if not self.dependencies and not self.requires_dist: - if force: - self.dependencies = tuple() - return self - self_dependency = self.to_dependency() - deps = set() - self_dependencies = tuple() if not self.dependencies else self.dependencies - for dep in self_dependencies: - if dep is None: - continue - new_dep = dep.parent = self_dependency - deps.add(new_dep) - created_deps = create_dependencies(self.requires_dist, parent=self_dependency) - if created_deps is not None: - for dep in created_deps: - if dep is None: - continue - deps.add(dep) - self.dependencies = deps - return self - - -def convert_package_info(info_json) -> PackageInfo: - if isinstance(info_json, PackageInfo): - return info_json - return PackageInfo.from_json(info_json) - - -def add_markers_to_dep(d, marker_str): - # type: (str, Union[str, Marker]) -> str - req = PackagingRequirement(d) - existing_marker = getattr(req, "marker", None) - if isinstance(marker_str, Marker): - marker_str = str(marker_str) - if existing_marker is not None: - marker_str = str(merge_markers(existing_marker, marker_str)) - if marker_str: - marker_str = marker_str.replace("'", '"') - req.marker = Marker(marker_str) - return str(req) - - -class Package(BaseModel): - info: PackageInfo = Field(converter=convert_package_info) - last_serial: int - releases: ReleaseCollection = Field( - converter=instance_check_converter( - ReleaseCollection, convert_releases_to_collection - ) - ) - urls: ReleaseUrlCollection = Field( - default_factory=lambda: [], - converter=instance_check_converter( - ReleaseUrlCollection, convert_release_urls_to_collection - ), - ) - - @property - def name(self) -> str: - return self.info.name - - @property - def version(self) -> str: - return self.info.version - - @property - def requirement(self) -> PackagingRequirement: - return self.info.to_dependency().requirement - - @property - def latest_sdist(self) -> ReleaseUrl: - return next(iter(self.urls.sdists)) - - @property - def latest_wheels(self) -> Iterator[ReleaseUrl]: - for w in self.urls.wheels: - yield w - - @property - def dependencies(self) -> Set[Dependency]: - if self.info.dependencies is None and list(self.urls): - rval = self.get_dependencies() - return set(rval.dependencies) - return set(self.info.dependencies) - - def get_dependencies(self) -> "Package": - urls = [] - deps = set() - info = self.info - if info.dependencies is None: - for url in self.urls: - try: - url, dep_dict = url.get_dependencies() - except (RuntimeError, TypeError): - # This happens if we are parsing `setup.py` and we fail - if url.is_sdist: - continue - else: - raise - markers = url.markers - dep_list = dep_dict.get("requires_dist", []) - for dep in dep_list: - # XXX: We need to parse these as requirements and "and" the markers - # XXX: together because they may contain "extra" markers which we - # XXX: will need to parse and remove - deps.add(add_markers_to_dep(dep, markers)) - urls.append(url) - if None in deps: - deps.remove(None) - info.requires_dist = tuple(sorted(deps)) - info = info.create_dependencies(force=True) - self.info = info - self.urls = urls - return self - - @classmethod - def from_json(cls, package_json) -> "Package": - info = convert_package_info(package_json["info"]).create_dependencies() - releases = convert_releases_to_collection( - package_json["releases"], name=info.name - ) - urls = convert_release_urls_to_collection(package_json["urls"], name=info.name) - return cls( - info=info, - releases=releases, - urls=urls, - last_serial=package_json["last_serial"], - ) - - def pin_dependencies(self, include_extras=None): - # type: (Optional[List[str]]) -> Tuple[List["Package"], Dict[str, List[SpecifierSet]]] - deps = [] - if include_extras: - include_extras = list(sorted(set(include_extras))) - else: - include_extras = [] - constraints = defaultdict(list) - for dep in self.dependencies: - if dep.from_extras and dep.from_extras not in include_extras: - continue - if dep.specifier: - constraints[dep.name].append(dep.specifier) - try: - pinned = dep.pin() - except requests.exceptions.HTTPError: - continue - deps.append(pinned) - return deps, constraints - - def get_latest_lockfile(self): - # type: () -> Dict[str, Dict[str, Union[List[str], str]]] - lockfile = {} - constraints = {dep.name: dep.specifier for dep in self.dependencies} - deps, _ = self.pin_dependencies() - for dep in deps: - dep = dep.get_dependencies() - for sub_dep in dep.dependencies: - if sub_dep.name not in constraints: - logger.info( - "Adding {0} (from {1}) {2!s}".format( - sub_dep.name, dep.name, sub_dep.specifier - ) - ) - constraints[sub_dep.name] = sub_dep.specifier - else: - existing = "{0} (from {1}): {2!s} + ".format( - sub_dep.name, dep.name, constraints[sub_dep.name] - ) - new_specifier = sub_dep.specifier - merged = constraints[sub_dep.name] & new_specifier - logger.info( - "Updating: {0}{1!s} = {2!s}".format( - existing, new_specifier, merged - ) - ) - constraints[sub_dep.name] = merged - - lockfile.update({dep.info.name: dep.releases.get_latest_lockfile()}) - for sub_dep_name, specset in constraints.items(): - try: - sub_dep_pkg = get_package(sub_dep_name) - except requests.exceptions.HTTPError: - continue - logger.info("Getting package: {0} ({1!s})".format(sub_dep, specset)) - sorted_releases = list( - sorted( - sub_dep_pkg.releases, - key=operator.attrgetter("parsed_version"), - reverse=True, - ) - ) - try: - version = next(iter(specset.filter((r.version for r in sorted_releases)))) - except StopIteration: - logger.info( - "No version of {0} matches specifier: {1}".format(sub_dep, specset) - ) - logger.info( - "Available versions: {0}".format( - " ".join([r.version for r in sorted_releases]) - ) - ) - raise - sub_dep_instance = get_package_version(sub_dep_name, version=str(version)) - if sub_dep_instance is None: - continue - lockfile.update( - { - sub_dep_instance.info.name: sub_dep_instance.releases.get_latest_lockfile() - } - ) - # lockfile.update(dep.get_latest_lockfile()) - lockfile.update({self.info.name: self.releases.get_latest_lockfile()}) - return lockfile - - def as_dict(self) -> Dict[str, Any]: - return self.dict() - - def serialize(self) -> str: - return json.dumps(self.dict(), default=pydantic_encoder, indent=4) - - -def get_package(name): - # type: (str) -> Package - url = "https://pypi.org/pypi/{}/json".format(name) - with requests.get(url) as r: - r.raise_for_status() - result = r.json() - package = Package.from_json(result) - return package - - -def get_package_version(name, version): - # type: (str, str) -> Package - url = "https://pypi.org/pypi/{0}/{1}/json".format(name, version) - with requests.get(url) as r: - r.raise_for_status() - result = r.json() - package = Package.from_json(result) - return package - - -def get_package_from_requirement(req): - # type: (PackagingRequirement) -> Tuple[Package, Set[str]] - versions = set() - if is_pinned_requirement(req): - version = get_pinned_version(req) - versions.add(version) - pkg = get_package_version(req.name, version) - else: - pkg = get_package(req.name) - sorted_releases = list( - sorted(pkg.releases, key=operator.attrgetter("parsed_version"), reverse=True) - ) - versions = set(req.specifier.filter((r.version for r in sorted_releases))) - version = next(iter(req.specifier.filter((r.version for r in sorted_releases)))) - if pkg.version not in versions: - pkg = get_package_version(pkg.name, version) - return pkg, versions diff --git a/pipenv/vendor/requirementslib/models/old_pip_utils.py b/pipenv/vendor/requirementslib/models/old_pip_utils.py deleted file mode 100644 index 8f51e6af..00000000 --- a/pipenv/vendor/requirementslib/models/old_pip_utils.py +++ /dev/null @@ -1,97 +0,0 @@ -"""These were old pip utils that were dropped starting in pip 22.1 but -`requirementslib` still deeply depends upon. - -In the interest of getting the build working with the latest version of -pip again, this workaround to copy these dependent utils in was -provided. Ideally the code that depends on this behavior would be -modernized and refactored to not require it. -""" -import logging -import os -import shutil -import stat - -logger = logging.getLogger(__name__) - - -from typing import Dict, Iterable, List - - -# This can be removed once this pr is merged -# https://github.com/python/cpython/pull/16575 -def is_socket(path: str) -> bool: - return stat.S_ISSOCK(os.lstat(path).st_mode) - - -def copy2_fixed(src: str, dest: str) -> None: - """Wrap shutil.copy2() but map errors copying socket files to - SpecialFileError as expected. - - See also https://bugs.python.org/issue37700. - """ - try: - shutil.copy2(src, dest) - except OSError: - for f in [src, dest]: - try: - is_socket_file = is_socket(f) - except OSError: - # An error has already occurred. Another error here is not - # a problem and we can ignore it. - pass - else: - if is_socket_file: - raise shutil.SpecialFileError("`{f}` is a socket".format(**locals())) - - raise - - -def _copy2_ignoring_special_files(src: str, dest: str) -> None: - """Copying special files is not supported, but as a convenience to users we - skip errors copying them. - - This supports tools that may create e.g. socket files in the project - source directory. - """ - try: - copy2_fixed(src, dest) - except shutil.SpecialFileError as e: - # SpecialFileError may be raised due to either the source or - # destination. If the destination was the cause then we would actually - # care, but since the destination directory is deleted prior to - # copy we ignore all of them assuming it is caused by the source. - logger.warning( - "Ignoring special file error '%s' encountered copying %s to %s.", - str(e), - src, - dest, - ) - - -def _copy_source_tree(source: str, target: str) -> None: - target_abspath = os.path.abspath(target) - target_basename = os.path.basename(target_abspath) - target_dirname = os.path.dirname(target_abspath) - - def ignore(d: str, names: List[str]) -> List[str]: - skipped: List[str] = [] - if d == source: - # Pulling in those directories can potentially be very slow, - # exclude the following directories if they appear in the top - # level dir (and only it). - # See discussion at https://github.com/pypa/pip/pull/6770 - skipped += [".tox", ".nox"] - if os.path.abspath(d) == target_dirname: - # Prevent an infinite recursion if the target is in source. - # This can happen when TMPDIR is set to ${PWD}/... - # and we copy PWD to TMPDIR. - skipped += [target_basename] - return skipped - - shutil.copytree( - source, - target, - ignore=ignore, - symlinks=True, - copy_function=_copy2_ignoring_special_files, - ) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py deleted file mode 100644 index ab273fbf..00000000 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ /dev/null @@ -1,324 +0,0 @@ -import itertools -import os -from pathlib import Path -from typing import Any, Dict, Iterable, List, Optional, Text, Union - -import pipenv.vendor.tomlkit as tomlkit -from pipenv.vendor.plette import pipfiles -from pipenv.vendor.pydantic import BaseModel, validator - -from ..environment import MYPY_RUNNING -from ..exceptions import RequirementError -from ..utils import is_editable, is_vcs, merge_items -from .common import ReqLibBaseModel -from .project import ProjectFile -from .requirements import Requirement -from .utils import get_url_name, tomlkit_value_to_python - -if MYPY_RUNNING: - package_type = Dict[Text, Dict[Text, Union[List[Text], Text]]] - source_type = Dict[Text, Union[Text, bool]] - sources_type = Iterable[source_type] - meta_type = Dict[Text, Union[int, Dict[Text, Text], sources_type]] - lockfile_type = Dict[Text, Union[package_type, meta_type]] - - -def reorder_source_keys(data): - # type: (tomlkit.toml_document.TOMLDocument) -> tomlkit.toml_document.TOMLDocument - sources = [] # type: sources_type - for source_key in ["source", "sources"]: - sources.extend(data.get(source_key, tomlkit.aot()).value) - new_source_aot = tomlkit.aot() - for entry in sources: - table = tomlkit.table() # type: tomlkit.items.Table - source_entry = PipfileLoader.populate_source(entry.copy()) - for key in ["name", "url", "verify_ssl"]: - table.update({key: source_entry[key]}) - new_source_aot.append(table) - data["source"] = new_source_aot - if data.get("sources", None): - del data["sources"] - return data - - -class PipfileLoader(pipfiles.Pipfile): - @classmethod - def validate(cls, data): - # type: (tomlkit.toml_document.TOMLDocument) -> None - for key, klass in pipfiles.PIPFILE_SECTIONS.items(): - if key not in data or key == "sources": - continue - try: - klass.validate(data[key]) - except Exception: - pass - - @classmethod - def ensure_package_sections(cls, data): - # type: (tomlkit.toml_document.TOMLDocument[Text, Any]) -> tomlkit.toml_document.TOMLDocument[Text, Any] - """Ensure that all pipfile package sections are present in the given - toml document. - - :param :class:`~tomlkit.toml_document.TOMLDocument` data: The toml document to - ensure package sections are present on - :return: The updated toml document, ensuring ``packages`` and ``dev-packages`` - sections are present - :rtype: :class:`~tomlkit.toml_document.TOMLDocument` - """ - package_keys = ( - k for k in pipfiles.PIPFILE_SECTIONS.keys() if k.endswith("packages") - ) - for key in package_keys: - if key not in data: - data.update({key: tomlkit.table()}) - return data - - @classmethod - def populate_source(cls, source): - """Derive missing values of source from the existing fields.""" - # Only URL pararemter is mandatory, let the KeyError be thrown. - if "name" not in source: - source["name"] = get_url_name(source["url"]) - if "verify_ssl" not in source: - source["verify_ssl"] = "https://" in source["url"] - if not isinstance(source["verify_ssl"], bool): - source["verify_ssl"] = str(source["verify_ssl"]).lower() == "true" - return source - - @classmethod - def load(cls, f, encoding=None): - # type: (Any, Text) -> PipfileLoader - content = f.read() - if encoding is not None: - content = content.decode(encoding) - _data = tomlkit.loads(content) - should_reload = "source" not in _data - _data = reorder_source_keys(_data) - if should_reload: - if "sources" in _data: - content = tomlkit.dumps(_data) - else: - # HACK: There is no good way to prepend a section to an existing - # TOML document, but there's no good way to copy non-structural - # content from one TOML document to another either. Modify the - # TOML content directly, and load the new in-memory document. - sep = "" if content.startswith("\n") else "\n" - content = pipfiles.DEFAULT_SOURCE_TOML + sep + content - data = tomlkit.loads(content) - data = cls.ensure_package_sections(data) - instance = cls(data) - instance._data = dict(instance._data) - return instance - - def __contains__(self, key): - # type: (Text) -> bool - if key not in self._data: - package_keys = self._data.get("packages", {}).keys() - dev_package_keys = self._data.get("dev-packages", {}).keys() - return any(key in pkg_list for pkg_list in (package_keys, dev_package_keys)) - return True - - def __getattribute__(self, key): - # type: (Text) -> Any - if key == "source": - return self._data[key] - return super(PipfileLoader, self).__getattribute__(key) - - -class Pipfile(ReqLibBaseModel): - path: Path - projectfile: ProjectFile - pipfile: Optional[PipfileLoader] - _pyproject: Optional[tomlkit.TOMLDocument] = tomlkit.document() - build_system: Optional[Dict] = dict() - _requirements: Optional[List] = list() - _dev_requirements: Optional[List] = list() - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - # keep_untouched = (cached_property,) - - @validator("path", pre=True, always=True) - def _get_path(cls, v): - return v or Path(os.curdir).absolute() - - @validator("projectfile", pre=True, always=True) - def _get_projectfile(cls, v, values): - return v or cls.load_projectfile(os.curdir, create=False) - - @validator("pipfile", pre=True, always=True) - def _get_pipfile(cls, v, values): - return v or values["projectfile"].model - - @property - def root(self): - return self.path.parent - - @property - def extended_keys(self): - return [ - k - for k in itertools.product( - ("packages", "dev-packages"), ("", "vcs", "editable") - ) - ] - - def get_deps(self, dev=False, only=True): - deps = {} # type: Dict[Text, Dict[Text, Union[List[Text], Text]]] - if dev: - deps.update(dict(self.pipfile._data.get("dev-packages", {}))) - if only: - return deps - return tomlkit_value_to_python( - merge_items([deps, dict(self.pipfile._data.get("packages", {}))]) - ) - - def get(self, k): - return self.__getitem__(k) - - def __contains__(self, k): - check_pipfile = k in self.extended_keys or self.pipfile.__contains__(k) - if check_pipfile: - return True - return False - - def __getitem__(self, k, *args, **kwargs): - retval = None - pipfile = self.pipfile - section = None - pkg_type = None - try: - retval = pipfile[k] - except KeyError: - if "-" in k: - section, _, pkg_type = k.rpartition("-") - vals = getattr(pipfile.get(section, {}), "_data", {}) - vals = tomlkit_value_to_python(vals) - if pkg_type == "vcs": - retval = {k: v for k, v in vals.items() if is_vcs(v)} - elif pkg_type == "editable": - retval = {k: v for k, v in vals.items() if is_editable(v)} - if retval is None: - raise - else: - retval = getattr(retval, "_data", retval) - return retval - - def __getattr__(self, k, *args, **kwargs): - pipfile = self.pipfile - try: - retval = super(Pipfile).__getattribute__(k) - except AttributeError: - retval = getattr(pipfile, k, None) - return retval - - @property - def requires_python(self): - # type: () -> bool - return getattr( - self.pipfile.requires, - "python_version", - getattr(self.pipfile.requires, "python_full_version", None), - ) - - @property - def allow_prereleases(self): - # type: () -> bool - return self.pipfile.get("pipenv", {}).get("allow_prereleases", False) - - @classmethod - def read_projectfile(cls, path): - # type: (Text) -> ProjectFile - """Read the specified project file and provide an interface for - writing/updating. - - :param Text path: Path to the target file. - :return: A project file with the model and location for interaction - :rtype: :class:`~requirementslib.models.project.ProjectFile` - """ - pf = ProjectFile.read(path, PipfileLoader, invalid_ok=True) - return pf - - @classmethod - def load_projectfile(cls, path, create=False): - # type: (Text, bool) -> ProjectFile - """Given a path, load or create the necessary pipfile. - - :param Text path: Path to the project root or pipfile - :param bool create: Whether to create the pipfile if not found, defaults to True - :raises OSError: Thrown if the project root directory doesn't exist - :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` - :return: A project file instance for the supplied project - :rtype: :class:`~requirementslib.models.project.ProjectFile` - """ - if not path: - raise RuntimeError("Must pass a path to classmethod 'Pipfile.load'") - if not isinstance(path, Path): - path = Path(path).absolute() - pipfile_path = path if path.is_file() else path.joinpath("Pipfile") - project_path = pipfile_path.parent - if not project_path.exists(): - raise FileNotFoundError("%s is not a valid project path!" % path) - elif not pipfile_path.exists() or not pipfile_path.is_file(): - if not create: - raise RequirementError("%s is not a valid Pipfile" % pipfile_path) - return cls.read_projectfile(pipfile_path.as_posix()) - - @classmethod - def load(cls, path, create=False): - # type: (Text, bool) -> Pipfile - """Given a path, load or create the necessary pipfile. - - :param Text path: Path to the project root or pipfile - :param bool create: Whether to create the pipfile if not found, defaults to True - :raises OSError: Thrown if the project root directory doesn't exist - :raises FileNotFoundError: Thrown if the pipfile doesn't exist and ``create=False`` - :return: A pipfile instance pointing at the supplied project - :rtype:: class:`~requirementslib.models.pipfile.Pipfile` - """ - - projectfile = cls.load_projectfile(path, create=create) - pipfile = projectfile.model - creation_args = { - "projectfile": projectfile, - "pipfile": pipfile, - "path": Path(projectfile.location), - } - return cls(**creation_args) - - @property - def dev_packages(self): - # type: () -> List[Requirement] - return self.dev_requirements - - @property - def packages(self): - # type: () -> List[Requirement] - return self.requirements - - @property - def dev_requirements(self): - # type: () -> List[Requirement] - if not self._dev_requirements: - packages = tomlkit_value_to_python(self.pipfile.get("dev-packages", {})) - self._dev_requirements = [ - Requirement.from_pipfile(k, v) - for k, v in packages.items() - if v is not None - ] - return self._dev_requirements - - @property - def requirements(self): - # type: () -> List[Requirement] - if not self._requirements: - packages = tomlkit_value_to_python(self.pipfile.get("packages", {})) - self._requirements = [ - Requirement.from_pipfile(k, v) - for k, v in packages.items() - if v is not None - ] - return self._requirements diff --git a/pipenv/vendor/requirementslib/models/project.py b/pipenv/vendor/requirementslib/models/project.py deleted file mode 100644 index c7829b04..00000000 --- a/pipenv/vendor/requirementslib/models/project.py +++ /dev/null @@ -1,69 +0,0 @@ -import collections -import io -import os -from typing import Any, Optional - -from pipenv.patched.pip._vendor.packaging.markers import Marker -from pipenv.vendor.pydantic import BaseModel, Field - -SectionDifference = collections.namedtuple("SectionDifference", ["inthis", "inthat"]) -FileDifference = collections.namedtuple("FileDifference", ["default", "develop"]) - - -def _are_pipfile_entries_equal(a, b): - a = {k: v for k, v in a.items() if k not in ("markers", "hashes", "hash")} - b = {k: v for k, v in b.items() if k not in ("markers", "hashes", "hash")} - if a != b: - return False - try: - marker_eval_a = Marker(a["markers"]).evaluate() - except (AttributeError, KeyError, TypeError, ValueError): - marker_eval_a = True - try: - marker_eval_b = Marker(b["markers"]).evaluate() - except (AttributeError, KeyError, TypeError, ValueError): - marker_eval_b = True - return marker_eval_a == marker_eval_b - - -DEFAULT_NEWLINES = "\n" - - -def preferred_newlines(f): - if isinstance(f.newlines, str): - return f.newlines - return DEFAULT_NEWLINES - - -class ProjectFile(BaseModel): - location: str - line_ending: str - model: Optional[Any] = Field(default_factory=lambda: dict()) - - @classmethod - def read(cls, location: str, model_cls, invalid_ok: bool = False) -> "ProjectFile": - if not os.path.exists(location) and not invalid_ok: - raise FileNotFoundError(location) - try: - with io.open(location, encoding="utf-8") as f: - model = model_cls.load(f) - line_ending = preferred_newlines(f) - except Exception: - if not invalid_ok: - raise - model = {} - line_ending = DEFAULT_NEWLINES - return cls(location=location, line_ending=line_ending, model=model) - - def write(self) -> None: - kwargs = {"encoding": "utf-8", "newline": self.line_ending} - with io.open(self.location, "w", **kwargs) as f: - if self.model: - self.model.dump(f) - - def dumps(self) -> str: - if self.model: - strio = io.StringIO() - self.model.dump(strio) - return strio.getvalue() - return "" diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py deleted file mode 100644 index aaea6f6b..00000000 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ /dev/null @@ -1,2885 +0,0 @@ -import collections -import copy -import os -import sys -from contextlib import contextmanager -from pathlib import Path -from sysconfig import get_path -from typing import ( - Any, - AnyStr, - Dict, - Generator, - List, - Optional, - Set, - Text, - Tuple, - TypeVar, - Union, -) -from urllib import parse as urllib_parse -from urllib.parse import SplitResult, unquote, urlparse -from urllib.request import url2pathname - -from pipenv.patched.pip._internal.index.package_finder import PackageFinder -from pipenv.patched.pip._internal.models.link import Link -from pipenv.patched.pip._internal.models.wheel import Wheel -from pipenv.patched.pip._internal.req.constructors import ( - _strip_extras, - install_req_from_editable, - install_req_from_line, -) -from pipenv.patched.pip._internal.req.req_install import InstallRequirement -from pipenv.patched.pip._internal.utils.urls import path_to_url, url_to_path -from pipenv.patched.pip._vendor.distlib.util import cached_property, COMPARE_OP -from pipenv.patched.pip._vendor.packaging.markers import Marker -from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement -from pipenv.patched.pip._vendor.packaging.specifiers import ( - InvalidSpecifier, - LegacySpecifier, - Specifier, - SpecifierSet, -) -from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.pip._vendor.packaging.version import parse -from pipenv.vendor.pydantic import Field, validator - -from ..environment import MYPY_RUNNING -from ..exceptions import RequirementError -from ..fileutils import ( - create_tracked_tempdir, - get_converted_relative_path, - is_file_url, - is_valid_url, - normalize_path, - temp_path, -) -from ..funktools import dedup -from ..utils import ( - VCS_LIST, - add_ssh_scheme_to_git_uri, - get_setup_paths, - is_installable_dir, - is_installable_file, - is_vcs, - strip_ssh_from_git_uri, -) -from .common import ReqLibBaseModel -from .markers import normalize_marker_str -from .setup_info import ( - SetupInfo, - _prepare_wheel_building_kwargs, - ast_parse_setup_py, - get_metadata, -) -from .url import URI -from .utils import ( - DIRECT_URL_RE, - HASH_STRING, - build_vcs_uri, - convert_direct_url_to_url, - create_link, - expand_env_variables, - extras_to_string, - format_requirement, - get_default_pyproject_backend, - get_pyproject, - get_version, - init_requirement, - make_install_requirement, - normalize_name, - parse_extras_str, - specs_to_string, - split_markers_from_line, - split_ref_from_uri, - split_vcs_method_from_uri, - validate_path, - validate_vcs, -) -from .vcs import VCSRepository - -if MYPY_RUNNING: - S = TypeVar("S", bytes, str, Text) - - -SPECIFIERS_BY_LENGTH = sorted(list(Specifier._operators.keys()), key=len, reverse=True) - - -class Line(ReqLibBaseModel): - line: str - extras: Optional[Union[List[str], Set[str], Tuple[str, ...]]] = None - editable: bool = False - hashes: List[str] = [] - markers: Optional[str] = None - vcs: Optional[str] = None - path: Optional[str] = None - relpath: Optional[str] = None - uri: Optional[str] = None - _link: Optional[Any] = None - is_local: bool = False - _name: Optional[str] = None - _specifier: Optional[str] = None - parsed_marker: Optional[Any] = None - preferred_scheme: Optional[str] = None - _requirement: Optional[Any] = None - _parsed_url: Optional[Any] = None - _setup_cfg: Optional[str] = None - _setup_py: Optional[str] = None - _pyproject_toml: Optional[str] = None - _pyproject_requires: Optional[Tuple[str, ...]] = None - _pyproject_backend: Optional[str] = None - _wheel_kwargs: Optional[Dict[str, str]] = None - _vcsrepo: Optional[Any] = None - _stack: Optional[Any] = None - setup_info: Optional[Any] = None - _ref: Optional[str] = None - _ireq: Optional[Any] = None - _src_root: Optional[str] = None - dist: Optional[Any] = None - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __init__( - self, - line: str, - extras: Optional[Union[List[str], Set[str], Tuple[str, ...]]] = None, - name: Optional[str] = None, - editable: Optional[bool] = False, - ) -> None: - super().__init__(line=line, extras=extras) - if line.startswith("-e "): - line = line[len("-e ") :] - self.editable = True - if extras is not None: - self.extras = tuple(sorted(set(extras))) - self.line = line - self.parse() - - def __hash__(self): - # Convert the _requirement attribute to a hashable type if it's a dict - requirement_hash = ( - hash(tuple(self._requirement.items())) - if isinstance(self._requirement, dict) - else self._requirement - ) - - # Convert the extras attribute to a hashable type if it's a dict - extras_hash = ( - hash(tuple(self.extras.items())) - if isinstance(self.extras, dict) - else tuple(self.extras) - ) - - return hash( - ( - self.editable, - self.line, - self.markers, - extras_hash, - tuple(self.hashes), - self.vcs, - self.uri, - self.path, - self.name, - requirement_hash, - ) - ) - - def __repr__(self): - try: - return ( - "".format( - self=self - ) - ) - except Exception: - return "".format(self.__dict__.values()) - - def __str__(self) -> str: - if self.markers: - return "{0} ; {1}".format(self.get_line(), self.markers) - return self.get_line() - - def get_line( - self, - with_prefix: bool = False, - with_markers: bool = False, - with_hashes: bool = True, - as_list: bool = False, - ) -> Union[str, List[str]]: - line = self.line - extras_str = extras_to_string(self.extras) - with_hashes = False if self.editable or self.is_vcs else with_hashes - hash_list = ["--hash={0}".format(h) for h in sorted(self.hashes)] - if self.is_named: - line = self.name_and_specifier - elif self.is_direct_url: - line = self.link.url - elif extras_str: - if self.is_vcs: - line = self.link.url - if "git+file:/" in line and "git+file:///" not in line: - line = line.replace("git+file:/", "git+file:///") - elif extras_str not in line: - line = "{0}{1}".format(line, extras_str) - # XXX: For using markers on vcs or url requirements, they can be used - # as normal (i.e. no space between the requirement and the semicolon) - # and no additional quoting as long as they are not editable requirements - # HOWEVER, for editable requirements, the requirement+marker must be quoted - # We do this here for the line-formatted versions, but leave it up to the - # `Script.parse()` functionality in pipenv, for instance, to handle that - # in a cross-platform manner for the `as_list` approach since that is how - # we anticipate this will be used if passing directly to the command line - # for pip. - if with_markers and self.markers: - line = "{0} ; {1}".format(line, self.markers) - if with_prefix and self.editable and not as_list: - line = '"{0}"'.format(line) - if as_list: - result_list = [] - if with_prefix and self.editable: - result_list.append("-e") - result_list.append(line) - if with_hashes: - result_list.extend(self.hashes) - return result_list - if with_prefix and self.editable: - line = "-e {0}".format(line) - if with_hashes and hash_list: - line = "{0} {1}".format(line, " ".join(hash_list)) - return line - - @cached_property - def name_and_specifier(self) -> str: - name_str, spec_str = "", "" - if self.name: - name_str = "{0}".format(self.name.lower()) - extras_str = extras_to_string(self.extras) - if extras_str: - name_str = "{0}{1}".format(name_str, extras_str) - if self.specifier: - spec_str = "{0}".format(self.specifier) - return "{0}{1}".format(name_str, spec_str) - - @classmethod - def split_hashes(cls, line: str) -> Tuple[str, List[str]]: - if "--hash" not in line: - return line, [] - split_line = line.split() - line_parts = [] # type: List[str] - hashes = [] # type: List[str] - for part in split_line: - if part.startswith("--hash"): - param, _, value = part.partition("=") - hashes.append(value) - else: - line_parts.append(part) - line = " ".join(line_parts) - return line, hashes - - @property - def line_with_prefix(self) -> str: - return self.get_line(with_prefix=True, with_hashes=False) - - def line_for_ireq(self) -> str: - line = "" - if self.is_file or self.is_remote_url and not self.is_vcs: - scheme = self.preferred_scheme if self.preferred_scheme is not None else "uri" - local_line = next( - iter( - [ - os.path.dirname(os.path.abspath(f)) - for f in [self.setup_py, self.setup_cfg, self.pyproject_toml] - if f is not None - ] - ), - None, - ) - if local_line and self.extras: - local_line = "{0}{1}".format(local_line, extras_to_string(self.extras)) - line = local_line if local_line is not None else self.line - if scheme == "path": - if not line and self.base_path is not None: - line = os.path.abspath(self.base_path) - else: - if DIRECT_URL_RE.match(self.line): - uri = URI.parse(self.line) - line = uri.full_url - self._requirement = init_requirement(self.line) - line = convert_direct_url_to_url(self.line) - else: - if self.link: - line = self.link.url - else: - try: - uri = URI.parse(line) - except ValueError: - line = line - else: - line = uri.base_url - self._link = uri.as_link - - if self.editable: - if not line: - if self.is_path or self.is_file: - if not self.path and self.url is not None: - line = url_to_path(self.url) - else: - line = self.path - if self.extras: - line = "{0}{1}".format(line, extras_to_string(self.extras)) - else: - line = self.link.url - elif self.is_vcs and not self.editable: - line = add_ssh_scheme_to_git_uri(self.line) - if not line: - line = self.line - return line - - @cached_property - def base_path(self): - # type: () -> Optional[S] - self.parse_link() - if not self.path: - return None - path = normalize_path(self.path) - if os.path.exists(path) and os.path.isdir(path): - path = path - elif os.path.exists(path) and os.path.isfile(path): - path = os.path.dirname(path) - else: - path = None - return path - - @property - def setup_py(self): - # type: () -> Optional[str] - if self._setup_py is None: - self.populate_setup_paths() - return self._setup_py - - @property - def setup_cfg(self): - # type: () -> Optional[str] - if self._setup_cfg is None: - self.populate_setup_paths() - return self._setup_cfg - - @property - def pyproject_toml(self): - # type: () -> Optional[str] - if self._pyproject_toml is None: - self.populate_setup_paths() - return self._pyproject_toml - - @cached_property - def specifier(self) -> Optional[str]: - options = [self._specifier] - for req in (self.ireq, self.requirement): - if req is not None and getattr(req, "specifier", None): - options.append(req.specifier) - specifier = next( - iter(spec for spec in options if spec is not None), None - ) # type: Optional[Union[Specifier, SpecifierSet]] - spec_string = None # type: Optional[str] - if specifier is not None: - spec_string = specs_to_string(specifier) - self.set_specifiers(specifier) - elif ( - specifier is None - and not self.is_named - and (self.setup_info is not None and self.setup_info.version) - ): - spec_string = "=={0}".format(self.setup_info.version) - if spec_string: - self._specifier = spec_string - return self._specifier - - def set_specifier(self, spec: str) -> None: - if not spec.startswith("=="): - spec = "=={0}".format(spec) - self._specifier = spec - self.set_specifiers(SpecifierSet(spec)) - - @property - def specifiers(self) -> Optional[SpecifierSet]: - ireq_needs_specifier = False - req_needs_specifier = False - if self.ireq is None or self.ireq.req is None or not self.ireq.req.specifier: - ireq_needs_specifier = True - if self.requirement is None or not self.requirement.specifier: - req_needs_specifier = True - if any([ireq_needs_specifier, req_needs_specifier]): - # TODO: Should we include versions for VCS dependencies? IS there a reason not - # to? For now we are using hashes as the equivalent to pin - # note: we need versions for direct dependencies at the very least - if ( - self.is_file - or self.is_remote_url - or self.is_path - or (self.is_vcs and not self.editable) - ): - if self.specifier is not None: - specifier = self.specifier - if not isinstance(specifier, SpecifierSet): - specifier = SpecifierSet(specifier) - return specifier - if self._ireq is not None and self._ireq.req is not None: - return self._ireq.req.specifier - elif self.requirement is not None: - return self.requirement.specifier - return None - - def set_specifiers(self, specifiers): - if not isinstance(specifiers, SpecifierSet): - if isinstance(specifiers, str): - specifiers = SpecifierSet(specifiers) - else: - raise TypeError("Must pass a string or a SpecifierSet") - specs = self.get_requirement_specs(specifiers) - if self._ireq and self._ireq.req is not None: - self._ireq.req.specifier = specifiers - self._ireq.req.specs = specs - if self.requirement is not None: - self.requirement.specifier = specifiers - self.requirement.specs = specs - - @classmethod - def get_requirement_specs(cls, specifierset): - # type: (SpecifierSet) -> List[Tuple[AnyStr, AnyStr]] - specs = [] - spec = next(iter(specifierset._specs), None) - if spec: - specs.append(spec._spec) - return specs - - @property - def requirement(self): - if self._requirement is None: - self.parse_requirement() - if self._requirement is None and self._name is not None: - self._requirement = init_requirement(canonicalize_name(self.name)) - if self.is_file or self.is_remote_url and self._requirement is not None: - self._requirement.url = self.url - if ( - self._requirement - and self._requirement.specifier - and not self._requirement.specs - ): - specs = self.get_requirement_specs(self._requirement.specifier) - self._requirement.specs = specs - return self._requirement - - def populate_setup_paths(self) -> None: - if not self.link and not self.path: - self.parse_link() - if not self.path: - return - base_path = self.base_path - if base_path is None: - return - setup_paths = get_setup_paths(base_path, subdirectory=self.subdirectory) - self._setup_py = setup_paths.get("setup_py") - self._setup_cfg = setup_paths.get("setup_cfg") - self._pyproject_toml = setup_paths.get("pyproject_toml") - - def get_setup_paths(self) -> None: - if not self.link and not self.path: - self.parse_link() - base_path = self.base_path - if base_path is None: - return - setup_paths = get_setup_paths(base_path, subdirectory=self.subdirectory) - self._setup_py = setup_paths.get("setup_py") - self._setup_cfg = setup_paths.get("setup_cfg") - self._pyproject_toml = setup_paths.get("pyproject_toml") - - @property - def pyproject_requires(self): - # type: () -> Optional[Tuple[str, ...]] - if self._pyproject_requires is None and self.pyproject_toml is not None: - if self.path is not None: - results = get_pyproject(self.path) # type: ignore - if results and results.get("build_requires"): - self._pyproject_requires = results.get("build_requires") - return self._pyproject_requires - - @property - def pyproject_backend(self): - # type: () -> Optional[str] - if self._pyproject_requires is None and self.pyproject_toml is not None: - results = get_pyproject(self.path) # type: ignore - if results: - if not results.get("build_backend") and self.setup_cfg is not None: - setup_dict = SetupInfo.get_setup_cfg(self.setup_cfg) - self._pyproject_backend = get_default_pyproject_backend() - else: - self._pyproject_backend = results.get("build_backend") - if results.get("build_requires"): - self._pyproject_requires = tuple(results.get("build_requires")) - else: - self._pyproject_requires = setup_dict.get( - "build_requires", ["setuptools", "wheel"] - ) # type: ignore - - return self._pyproject_backend - - def parse_hashes(self): - # type: () -> "Line" - """Parse hashes from *self.line* and set them on the current object. - - :returns: Self - :rtype: `:class:~Line` - """ - line, hashes = self.split_hashes(self.line) - self.hashes = hashes - self.line = line - return self - - def parse_extras(self): - # type: () -> "Line" - """ - Parse extras from *self.line* and set them on the current object - :returns: self - :rtype: :class:`~Line` - """ - extras = None - line = "{0}".format(self.line) - if any([self.is_vcs, self.is_url]): - try: - if self.parsed_url.name: - self._name = self.parsed_url.name - if ( - self.parsed_url.host - and self.parsed_url.path - and self.parsed_url.scheme - ): - self.line = self.parsed_url.to_string( - escape_password=False, - direct=False, - strip_ssh=self.parsed_url.is_implicit_ssh, - ) - except ValueError: - self.line, extras = _strip_extras(self.line) - else: - self.line, extras = _strip_extras(self.line) - extras_set = set() # type: Set[str] - if extras is not None: - extras_set = set(parse_extras_str(extras)) - if self._name: - self._name, name_extras = _strip_extras(self._name) - if name_extras: - name_extras = set(parse_extras_str(name_extras)) - extras_set |= name_extras - if extras_set is not None: - self.extras = tuple(sorted(extras_set)) - return self - - def get_url(self): - # type: () -> str - """Sets ``self.name`` if given a **PEP-508** style URL.""" - return self.parsed_url.to_string( - escape_password=False, direct=False, strip_ref=True - ) - - @property - def name(self) -> Optional[str]: - if self._name is None: - self.parse_name() - if self._name is None and not self.is_named and not self.is_wheel: - if self.setup_info: - self._name = self.setup_info.name - elif self.is_wheel: - self._name = self._parse_wheel() - if not self._name: - self._name = self.ireq.name - return self._name - - @name.setter - def name(self, name): - # type: (str) -> None - self._name = name - if self.setup_info: - self.setup_info.name = name - if self.requirement and self._requirement: - self._requirement.name = name - if self.ireq and self._ireq and self._ireq.req: - self._ireq.req.name = name - - @property - def url(self): - # type: () -> Optional[str] - try: - return self.parsed_url.to_string( - escape_password=False, - strip_ref=True, - strip_name=True, - strip_subdir=True, - strip_ssh=False, - ) - except ValueError: - return None - - @property - def link(self): - # type: () -> Link - if self._link is None: - self.parse_link() - return self._link - - @property - def subdirectory(self): - # type: () -> Optional[str] - if self.link is not None: - return self.link.subdirectory_fragment - return "" - - @property - def is_wheel(self): - # type: () -> bool - if self.link is None: - return False - return self.link.is_wheel - - @property - def is_artifact(self): - # type: () -> bool - - if self.link is None: - return False - return getattr(self.link, "is_vcs", False) - - @property - def is_vcs(self): - # type: () -> bool - # Installable local files and installable non-vcs urls are handled - # as files, generally speaking - try: - if is_vcs(self.line) or is_vcs(self.get_url()): - return True - except ValueError: - return False - return False - - @property - def is_url(self): - # type: () -> bool - try: - url = self.get_url() - except ValueError: - return False - if is_valid_url(url) or is_file_url(url): - return True - return False - - @property - def is_remote_url(self): - # type: () -> bool - return self.is_url and self.parsed_url.host is not None - - @property - def is_path(self): - # type: () -> bool - try: - line_url = self.get_url() - except ValueError: - line_url = None - if ( - self.path - and ( - self.path.startswith(".") - or os.path.isabs(self.path) - or os.path.exists(self.path) - ) - and is_installable_file(self.path) - ): - return True - elif (os.path.exists(self.line) and is_installable_file(self.line)) or ( - line_url and os.path.exists(line_url) and is_installable_file(line_url) - ): - return True - return False - - @property - def is_file_url(self): - # type: () -> bool - try: - url = self.get_url() - except ValueError: - return False - try: - parsed_url_scheme = self.parsed_url.scheme - except ValueError: - return False - if url and is_file_url(url) or parsed_url_scheme == "file": - return True - return False - - @property - def is_file(self): - # type: () -> bool - try: - url = self.get_url() - except ValueError: - return False - if ( - self.is_path - or (is_file_url(url) and is_installable_file(url)) - or ( - self._parsed_url - and self._parsed_url.is_file_url - and is_installable_file(self._parsed_url.url_without_fragment_or_ref) - ) - ): - return True - return False - - @property - def is_named(self): - # type: () -> bool - return not ( - self.is_file_url - or self.is_url - or self.is_file - or self.is_vcs - or self.is_direct_url - ) - - @property - def ref(self): - # type: () -> Optional[str] - if self._ref is None and self.relpath is not None: - self.relpath, self._ref = split_ref_from_uri(self.relpath) - return self._ref - - @property - def ireq(self): - # type: () -> Optional[InstallRequirement] - if self._ireq is None: - self.parse_ireq() - return self._ireq - - @property - def is_installable(self): - # type: () -> bool - try: - url = self.get_url() - except ValueError: - url = None - possible_paths = (self.line, url, self.path, self.base_path) - return any(is_installable_file(p) for p in possible_paths if p is not None) - - @property - def wheel_kwargs(self): - if not self._wheel_kwargs: - self._wheel_kwargs = _prepare_wheel_building_kwargs(self.ireq) - return self._wheel_kwargs - - def get_setup_info(self) -> SetupInfo: - setup_info = self.setup_info - if setup_info is None: - setup_info = SetupInfo.from_ireq(self.ireq, subdir=self.subdirectory) - if not setup_info.name: - setup_info.get_info() - return setup_info - - def set_setup_info(self, setup_info) -> None: - self.setup_info = setup_info - if setup_info.version: - self.set_specifiers(f"=={setup_info.version}") - if setup_info.name and not self.name: - self.name = setup_info.name - - def _get_vcsrepo(self): - # type: () -> Optional[VCSRepository] - from .vcs import VCSRepository - - checkout_directory = self.wheel_kwargs["src_dir"] # type: ignore - if self.name is not None: - checkout_directory = os.path.join( - checkout_directory, self.name - ) # type: ignore - vcsrepo = VCSRepository( - url=self.link.url, - name=self.name, - ref=self.ref if self.ref else None, - checkout_directory=checkout_directory, - vcs_type=self.vcs, - subdirectory=self.subdirectory, - ) - if not (self.link.scheme.startswith("file") and self.editable): - vcsrepo.obtain() - return vcsrepo - - @property - def vcsrepo(self): - # type: () -> Optional[VCSRepository] - if self._vcsrepo is None and self.is_vcs: - self._vcsrepo = self._get_vcsrepo() - return self._vcsrepo - - @property - def parsed_url(self) -> URI: - if self._parsed_url is None: - self._parsed_url = URI.parse(self.line) - return self._parsed_url - - @property - def is_direct_url(self): - # type: () -> bool - try: - return self.is_url and self._parsed_url.is_direct_url - except ValueError: - return self.is_url and bool(DIRECT_URL_RE.match(self.line)) - - @cached_property - def metadata(self): - # type: () -> Dict[Any, Any] - if self.is_local and self.path and is_installable_dir(self.path): - return get_metadata(self.path) - return {} - - @cached_property - def parsed_setup_cfg(self): - # type: () -> Dict[Any, Any] - if not ( - self.is_local - and self.path - and is_installable_dir(self.path) - and self.setup_cfg - ): - return {} - return self.setup_info.parse_setup_cfg() - - @cached_property - def parsed_setup_py(self): - # type: () -> Dict[Any, Any] - if self.is_local and self.path and is_installable_dir(self.path): - if self.setup_py: - return ast_parse_setup_py(self.setup_py, raising=False) - return {} - - def set_vcsrepo(self, repo): - # type (VCSRepository) -> None - self._vcsrepo = repo - ireq = self.ireq - wheel_kwargs = self.wheel_kwargs.copy() - wheel_kwargs["src_dir"] = repo.checkout_directory - setupinfo = SetupInfo.create( - repo.checkout_directory, - ireq=ireq, - subdirectory=self.subdirectory, - kwargs=wheel_kwargs, - ) - self.setup_info = setupinfo - - def get_ireq(self) -> InstallRequirement: - line = self.line_for_ireq() - if self.editable: - ireq = install_req_from_editable(line) - else: - ireq = install_req_from_line(line) - if self.is_named: - ireq = install_req_from_line(self.line) - if self.is_file or self.is_remote_url: - ireq.link = Link(expand_env_variables(self.link.url)) - if self.extras and not ireq.extras: - ireq.extras = set(self.extras) - if self.parsed_marker is not None and not ireq.markers: - ireq.markers = self.parsed_marker - if not ireq.req and self._requirement is not None: - ireq.req = self._requirement - return ireq - - def parse_ireq(self): - # type: () -> None - if self._ireq is None: - self._ireq = self.get_ireq() - if self._ireq is not None: - if self.requirement is not None and self._ireq.req is None: - self._ireq.req = self.requirement - - def _parse_wheel(self): - # type: () -> Optional[str] - if not self.is_wheel: - return - _wheel = Wheel(self.link.filename) - name = _wheel.name - version = _wheel.version - self._specifier = "=={0}".format(version) - return name - - def _parse_name_from_link(self): - # type: () -> Optional[str] - if self.link is None: - return None - if getattr(self.link, "egg_fragment", None): - return self.link.egg_fragment - elif self.is_wheel: - return self._parse_wheel() - return None - - def _parse_name_from_line(self) -> Optional[str]: - if not self.is_named: - pass - try: - self._requirement = init_requirement(self.line) - except Exception: - raise RequirementError( - "Failed parsing requirement from {0!r}".format(self.line) - ) - name = self._requirement.name - if not self._specifier and self._requirement and self._requirement.specifier: - self._specifier = specs_to_string(self._requirement.specifier) - if self._requirement.extras and not self.extras: - self.extras = self._requirement.extras - if not name: - name = self.line - specifier_match = next( - iter(spec for spec in SPECIFIERS_BY_LENGTH if spec in self.line), None - ) - specifier = None # type: Optional[str] - if specifier_match: - specifier = "{0!s}".format(specifier_match) - if specifier is not None and specifier in name: - name, specifier, version = name.partition(specifier) - self._specifier = "{0}{1}".format(specifier, version) - return name - - def _parse_name_from_path(self): - # type: () -> Optional[S] - path = self.path - if path and path.startswith("file:"): - parsed_url = urlparse(path) - path = url2pathname(parsed_url.path) - if path and self.is_local and is_installable_dir(path): - metadata = get_metadata(path) - if metadata: - name = metadata.get("name", "") - if name and name != "wheel": - return name - parsed_setup_cfg = self.parsed_setup_cfg - if parsed_setup_cfg: - name = parsed_setup_cfg.get("name", "") - if name: - return name - - parsed_setup_py = self.parsed_setup_py - if parsed_setup_py: - name = parsed_setup_py.get("name", "") - if name and isinstance(name, str): - return name - return None - - def parse_name(self) -> None: - name = None - if self.link is not None and self.line_is_installable: - name = self._parse_name_from_link() - if name is None and ( - (self.is_remote_url or self.is_artifact or self.is_vcs) and self.parsed_url - ): - if self._parsed_url.fragment: - _, _, name = self._parsed_url.fragment.partition("egg=") - if "&" in name: - # subdirectory fragments might also be in here - name, _, _ = name.partition("&") - if name is None and self.is_named: - name = self._parse_name_from_line() - elif name is None and (self.is_file or self.is_remote_url or self.is_path): - if self.is_local: - name = self._parse_name_from_path() - if name is not None: - name, extras = _strip_extras(name) - if extras is not None and not self.extras: - self.extras = tuple(sorted(set(parse_extras_str(extras)))) - self._name = name - - def _parse_requirement_from_vcs(self): - # type: () -> Optional[PackagingRequirement] - url = self.url if self.url else self.link.url - if url: - url = unquote(url) - if ( - url - and self.uri != url - and "git+ssh://" in url - and (self.uri is not None and "git+git@" in self.uri) - and self._requirement is not None - ): - self._requirement.line = self.uri - self._requirement.url = self.url - vcs_uri = build_vcs_uri( # type: ignore - vcs=self.vcs, - uri=self.url, - ref=self.ref, - subdirectory=self.subdirectory, - extras=self.extras, - name=self.name, - ) - if vcs_uri: - self._requirement.link = create_link(vcs_uri) - elif self.link: - self._requirement.link = self.link - # else: - # req.link = self.link - if self.ref and self._requirement is not None: - self._requirement.revision = self.ref - if self._vcsrepo is not None: - self._requirement.revision = self._vcsrepo.commit_hash - return self._requirement - - def parse_requirement(self) -> "Line": - if self._name is None: - self.parse_name() - if not any([self._name, self.is_vcs, self.is_named]): - if self.setup_info and self.setup_info.name: - self._name = self.setup_info.name - name, extras, url = self.requirement_info - if name: - self._requirement = init_requirement(name) # type: PackagingRequirement - if extras: - self._requirement.extras = set(extras) - if url: - self._requirement.url = url - if self.is_direct_url: - url = self.link.url - if self.link: - self._requirement.link = self.link - self._requirement.editable = self.editable - if self.path and self.link and self.link.scheme.startswith("file"): - self._requirement.local_file = True - self._requirement.path = self.path - if self.is_vcs: - self._requirement.vcs = self.vcs - self._requirement.line = self.link.url - self._parse_requirement_from_vcs() - else: - self._requirement.line = self.line - if self.parsed_marker is not None: - self._requirement.marker = self.parsed_marker - if self.specifiers: - self._requirement.specifier = self.specifiers - specs = [] - spec = next(iter(s for s in self.specifiers._specs), None) - if spec: - specs.append(spec._spec) - self._requirement.spec = spec - else: - if self.is_vcs: - raise ValueError( - "pipenv requires an #egg fragment for version controlled " - "dependencies. Please install remote dependency " - "in the form {0}#egg=.".format(url) - ) - return self - - def parse_link(self): - # type: () -> "Line" - parsed_url = None # type: Optional[URI] - if ( - not is_valid_url(self.line) - and is_installable_file(os.path.abspath(self.line)) - and ( - self.line.startswith("./") - or (os.path.exists(self.line) or os.path.isabs(self.line)) - ) - ): - url = path_to_url(os.path.abspath(self.line)) - self._parsed_url = parsed_url = URI.parse(url) - elif any( - [ - is_valid_url(self.line), - is_vcs(self.line), - is_file_url(self.line), - self.is_direct_url, - ] - ): - parsed_url = self.parsed_url - if parsed_url is None or ( - parsed_url.is_file_url and not parsed_url.is_installable - ): - return None - if parsed_url.is_vcs: - self.vcs, _ = parsed_url.scheme.split("+") - if parsed_url.is_file_url: - self.is_local = True - parsed_link = parsed_url.as_link - self._ref = parsed_url.ref - self.uri = parsed_url.bare_url - if parsed_url.name: - self._name = parsed_url.name - if parsed_url.extras: - self.extras = tuple(sorted(set(parsed_url.extras))) - self._link = parsed_link - vcs, prefer, relpath, path, uri, link = FileRequirement.get_link_from_line( - self.line - ) - ref = None - if link is not None and "@" in unquote(link.path) and uri is not None: - uri, _, ref = unquote(uri).rpartition("@") - if relpath is not None and "@" in relpath: - relpath, _, ref = relpath.rpartition("@") - if path is not None and "@" in path: - path, prefix = split_ref_from_uri(path) - link_url = link.url_without_fragment - if "@" in link_url: - link_url, _ = split_ref_from_uri(link_url) - self.preferred_scheme = prefer - self.relpath = relpath - self.path = path - self.uri = uri - if prefer in ("path", "relpath") or uri.startswith("file"): - self.is_local = True - if uri.startswith("file"): - self.path = uri - if parsed_url.is_vcs or parsed_url.is_direct_url and parsed_link: - self._link = parsed_link - else: - self._link = link - return self - - def parse_markers(self): - # type: () -> None - if self.markers: - pkg_name, markers = split_markers_from_line(self.line) - self.parsed_marker = markers - - @property - def requirement_info(self): - # type: () -> Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] - """ - Generates a 3-tuple of the requisite *name*, *extras* and *url* to generate a - :class:`~packaging.requirements.Requirement` out of. - :return: A Tuple of an optional name, a Tuple of extras, and an optional URL. - :rtype: Tuple[Optional[S], Tuple[Optional[S], ...], Optional[S]] - """ - - # Direct URLs can be converted to packaging requirements directly, but - # only if they are `file://` (with only two slashes) - name = None # type: Optional[S] - extras = () # type: Tuple[Optional[S], ...] - url = None # type: Optional[str] - # if self.is_direct_url: - if self._name: - name = canonicalize_name(self._name) - if self.is_file or self.is_url or self.is_path or self.is_file_url or self.is_vcs: - url = "" - if self.is_vcs: - url = self.url if self.url else self.uri - if self.is_direct_url: - url = self.link.url_without_fragment - else: - if self.link: - url = self.link.url_without_fragment - elif self.url: - url = self.url - if self.ref: - url = "{0}@{1}".format(url, self.ref) - else: - url = self.uri - if self.link and name is None: - self._name = self.link.egg_fragment - if self._name: - name = canonicalize_name(self._name) - return name, extras, url # type: ignore - - @property - def line_is_installable(self): - # type: () -> bool - """This is a safeguard against decoy requirements when a user installs - a package whose name coincides with the name of a folder in the cwd, - e.g. install *alembic* when there is a folder called *alembic* in the - working directory. - - In this case we first need to check that the given requirement - is a valid URL, VCS requirement, or installable filesystem path - before deciding to treat it as a file requirement over a named - requirement. - """ - line = self.line - direct_url_match = DIRECT_URL_RE.match(line) - if direct_url_match: - match_dict = direct_url_match.groupdict() - auth = "" - username = match_dict.get("username", None) - password = match_dict.get("password", None) - port = match_dict.get("port", None) - path = match_dict.get("path", None) - ref = match_dict.get("ref", None) - if username is not None: - auth = "{0}".format(username) - if password: - auth = "{0}:{1}".format(auth, password) if auth else password - line = match_dict.get("host", "") - if auth: - line = "{auth}@{line}".format(auth=auth, line=line) - if port: - line = "{line}:{port}".format(line=line, port=port) - if path: - line = "{line}{pathsep}{path}".format( - line=line, pathsep=match_dict["pathsep"], path=path - ) - if ref: - line = "{line}@{ref}".format(line=line, ref=ref) - line = "{scheme}{line}".format(scheme=match_dict["scheme"], line=line) - if is_file_url(line): - link = create_link(line) - line = link.url_without_fragment - line, _ = split_ref_from_uri(line) - if ( - is_vcs(line) - or (not is_file_url(line) and is_valid_url(line)) - or (is_file_url(line) and is_installable_file(line)) - or is_installable_file(line) - ): - return True - return False - - def parse(self): - # type: () -> None - self.line = self.line.strip() - if self.line.startswith('"'): - self.line = self.line.strip('"') - self.line, self.markers = split_markers_from_line(self.parse_hashes().line) - if self.markers: - self.markers = self.markers.replace('"', "'") - self.parse_extras() - if self.line.startswith("git+file:/") and not self.line.startswith( - "git+file:///" - ): - self.line = self.line.replace("git+file:/", "git+file:///") - self.parse_markers() - if self.is_file_url: - if self.line_is_installable: - self.populate_setup_paths() - else: - raise RequirementError( - "Supplied requirement is not installable: {0!r}".format(self.line) - ) - elif self.is_named and self._name is None: - self.parse_name() - self.parse_link() - - -class NamedRequirement(ReqLibBaseModel): - name: str - version: Optional[str] - req: PackagingRequirement - extras: Optional[Tuple[str, ...]] = Field(default_factory=list) - editable: bool = False - parsed_line: Optional[Line] = None - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __init__(self, **data): - super().__init__(**data) - - self.parsed_line = Line(line=self.line_part) - - @classmethod - def from_line(cls, line, parsed_line=None) -> "NamedRequirement": - req = init_requirement(line) - specifiers = None # type: Optional[str] - if req.specifier: - specifiers = specs_to_string(req.specifier) - req.line = line - name = getattr(req, "name", None) - if not name: - name = getattr(req, "project_name", None) - req.name = name - if not name: - name = getattr(req, "key", line) - req.name = name - creation_kwargs = { - "name": name, - "version": specifiers, - "req": req, - "parsed_line": parsed_line, - "extras": None, - } - extras = None # type: Optional[Tuple[str, ...]] - if req.extras: - extras = tuple(req.extras) - creation_kwargs["extras"] = extras - return cls(**creation_kwargs) - - @classmethod - def from_pipfile(cls, name: str, pipfile: Dict[str, Any]) -> "NamedRequirement": - creation_args = {} - if hasattr(pipfile, "keys"): - # Get field names from the Pydantic model - pydantic_fields = cls.__fields__.keys() - creation_args = {k: v for k, v in pipfile.items() if k in pydantic_fields} - creation_args["name"] = name - version = get_version(pipfile) - extras = creation_args.get("extras", None) - creation_args["version"] = version - req = init_requirement("{0}{1}".format(name, version)) - if req and extras and req.extras and isinstance(req.extras, tuple): - if isinstance(extras, str): - req.extras = (extras,) + tuple( - ["{0}".format(xtra) for xtra in req.extras] - ) - elif isinstance(extras, (tuple, list)): - req.extras += tuple(extras) - creation_args["req"] = req - return cls(**creation_args) - - @cached_property - def line_part(self) -> str: - return normalize_name(self.name) - - @cached_property - def pipfile_part(self) -> Dict[str, Any]: - pipfile_dict = self.dict() - if "version" not in pipfile_dict or pipfile_dict["version"] is None: - pipfile_dict["version"] = "*" - if "parsed_line" in pipfile_dict: - pipfile_dict.pop("parsed_line") - if pipfile_dict.get("editable") is False: - pipfile_dict.pop("editable") - if pipfile_dict.get("line_part"): - pipfile_dict.pop("line_part") - if not pipfile_dict.get("extras", True): - pipfile_dict.pop("extras") - name = pipfile_dict.pop("name") - return {name: pipfile_dict} - - -LinkInfo = collections.namedtuple( - "LinkInfo", ["vcs_type", "prefer", "relpath", "path", "uri", "link"] -) - - -class FileRequirement(ReqLibBaseModel): - """File requirements for tar.gz installable files or wheels or setup.py - containing directories.""" - - setup_path: Optional[str] = None - path: Optional[str] = None - editable: bool = False - extras: Optional[Tuple[str, ...]] = () - uri_scheme: Optional[str] = None - uri: Optional[str] = Field(default_factory=lambda: "") - name: Optional[str] = Field(default_factory=lambda: "") - link: Optional[Link] = Field(default_factory=lambda: None) - pyproject_requires: Optional[Tuple[str, ...]] = () - pyproject_backend: Optional[str] = None - pyproject_path: Optional[Any] = None - subdirectory: Optional[str] = None - setup_info: Optional[SetupInfo] = None - _has_hashed_name: bool = False - parsed_line: Optional[Line] = None - req: Optional[PackagingRequirement] = Field(default_factory=lambda: None) - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __init__(self, **data): - super().__init__(**data) - # Set default values using the methods - if not self.name: - self.name = self.get_name() - if not self.link: - self.link = self.get_link() - if not self.req: - self.req = self.get_requirement() - if not self.uri: - self.uri = self.get_uri() - if not self.path and self.uri: - self.path = self.uri - self.uri_scheme = "file" - elif self.path and self.path.startswith("file:"): - self.uri_scheme = "file" - elif self.path: - self.uri_scheme = "path" - # self.parse_setup_info() - if self.parsed_line is None: - self.parsed_line = Line(line=self.line_part) - if self.name is None and self.parsed_line: - if self.parsed_line.setup_info: - self.setup_info = self.parsed_line.setup_info - if self.parsed_line.setup_info.name: - self.name = self.parsed_line.setup_info.name - if self.req is None and ( - self.parsed_line is not None and self.parsed_line.requirement is not None - ): - self.req = self.parsed_line.requirement - if self.parsed_line and self.parsed_line.ireq and not self.parsed_line.ireq.req: - if self.req is not None and self.parsed_line._ireq is not None: - self.parsed_line._ireq.req = self.req - - @classmethod - def get_link_from_line(cls, line): - # type: (str) -> LinkInfo - """Parse link information from given requirement line. Return a - 6-tuple: - - - `vcs_type` indicates the VCS to use (e.g. "git"), or None. - - `prefer` is either "file", "path" or "uri", indicating how the - information should be used in later stages. - - `relpath` is the relative path to use when recording the dependency, - instead of the absolute path/URI used to perform installation. - This can be None (to prefer the absolute path or URI). - - `path` is the absolute file path to the package. This will always use - forward slashes. Can be None if the line is a remote URI. - - `uri` is the absolute URI to the package. Can be None if the line is - not a URI. - - `link` is an instance of :class:`pipenv.patched.pip._internal.index.Link`, - representing a URI parse result based on the value of `uri`. - This function is provided to deal with edge cases concerning URIs - without a valid netloc. Those URIs are problematic to a straight - ``urlsplit` call because they cannot be reliably reconstructed with - ``urlunsplit`` due to a bug in the standard library: - >>> from urllib.parse import urlsplit, urlunsplit - >>> urlunsplit(urlsplit('git+file:///this/breaks')) - 'git+file:/this/breaks' - >>> urlunsplit(urlsplit('file:///this/works')) - 'file:///this/works' - See `https://bugs.python.org/issue23505#msg277350`. - """ - - # Git allows `git@github.com...` lines that are not really URIs. - # Add "ssh://" so we can parse correctly, and restore afterward. - fixed_line = add_ssh_scheme_to_git_uri(line) # type: str - added_ssh_scheme = fixed_line != line # type: bool - - # We can assume a lot of things if this is a local filesystem path. - if "://" not in fixed_line: - p = Path(fixed_line).absolute() # type: Path - path = p.as_posix() # type: Optional[str] - uri = p.as_uri() # type: str - link = create_link(uri) # type: Link - relpath = None # type: Optional[str] - try: - relpath = get_converted_relative_path(path) - except ValueError: - relpath = None - return LinkInfo(None, "path", relpath, path, uri, link) - - # This is an URI. We'll need to perform some elaborated parsing. - - parsed_url = urllib_parse.urlsplit(fixed_line) # type: SplitResult - original_url = parsed_url._replace() # type: SplitResult - - # Split the VCS part out if needed. - original_scheme = parsed_url.scheme # type: str - vcs_type = None # type: Optional[str] - if "+" in original_scheme: - scheme = None # type: Optional[str] - vcs_type, _, scheme = original_scheme.partition("+") - parsed_url = parsed_url._replace(scheme=scheme) # type: ignore - prefer = "uri" # type: str - else: - vcs_type = None - prefer = "file" - - if parsed_url.scheme == "file" and parsed_url.path: - # This is a "file://" URI. Use url_to_path and path_to_url to - # ensure the path is absolute. Also we need to build relpath. - path = Path(url_to_path(urllib_parse.urlunsplit(parsed_url))).as_posix() - try: - relpath = get_converted_relative_path(path) - except ValueError: - relpath = None - uri = path_to_url(path) - else: - # This is a remote URI. Simply use it. - path = None - relpath = None - # Cut the fragment, but otherwise this is fixed_line. - uri = urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme, fragment="") # type: ignore - ) - - if added_ssh_scheme: - original_uri = urllib_parse.urlunsplit( - original_url._replace(scheme=original_scheme, fragment="") # type: ignore - ) - uri = strip_ssh_from_git_uri(original_uri) - - # Re-attach VCS prefix to build a Link. - link = create_link( - urllib_parse.urlunsplit( - parsed_url._replace(scheme=original_scheme) - ) # type: ignore - ) - - return LinkInfo(vcs_type, prefer, relpath, path, uri, link) - - @property - def setup_py_dir(self) -> Optional[str]: - if self.setup_path: - return os.path.dirname(os.path.abspath(self.setup_path)) - return None - - @property - def dependencies(self): - # type: () -> Tuple[Dict[S, PackagingRequirement], List[Union[S, PackagingRequirement]], List[S]] - build_deps = [] # type: List[Union[S, PackagingRequirement]] - setup_deps = [] # type: List[S] - deps = {} # type: Dict[S, PackagingRequirement] - if self.pyproject_requires: - build_deps.extend(list(self.pyproject_requires)) - else: - if not self.setup_info: - self.parse_setup_info() - if self.setup_info: - setup_info = self.setup_info.as_dict() - deps.update(setup_info.get("requires", {})) - setup_deps.extend(setup_info.get("setup_requires", [])) - build_deps.extend(setup_info.get("build_requires", [])) - if self.extras and self.setup_info.extras: - for dep in self.extras: - if dep not in self.setup_info.extras: - continue - extras_list = self.setup_info.extras.get(dep, []) # type: ignore - for req_instance in extras_list: # type: ignore - deps[req_instance.key] = req_instance - setup_deps = list(set(setup_deps)) - build_deps = list(set(build_deps)) - return deps, setup_deps, build_deps - - def parse_setup_info(self) -> Optional[SetupInfo]: - if self.setup_info is None and self.parsed_line: - if self.parsed_line and self.parsed_line and self.parsed_line.setup_info: - if self.parsed_line.setup_info and not self.parsed_line.setup_info.name: - self.parsed_line.setup_info.get_info() - self.setup_info = self.parsed_line.setup_info - elif self.parsed_line and ( - self.parsed_line.ireq and not self.parsed_line.is_wheel - ): - self.setup_info = SetupInfo.from_ireq( - self.parsed_line.ireq, subdir=self.subdirectory - ) - else: - if self.link and not self.link.is_wheel: - self.setup_info = Line(self.line_part).setup_info - self.setup_info.get_info() - return self.setup_info - - def set_setup_info(self, setup_info) -> None: - self.setup_info = setup_info - if self.parsed_line: - self.parsed_line.setup_info = setup_info - - def get_uri(self) -> str: - if self.path and not self.uri: - self.uri_scheme = "path" - return path_to_url(os.path.abspath(self.path)) - elif ( - getattr(self, "req", None) - and self.req is not None - and getattr(self.req, "url", None) - ): - return self.req.url - elif self.link is not None: - return self.link.url_without_fragment - return "" - - def get_name(self) -> str: - if self.parsed_line and self.parsed_line.name: - return self.parsed_line.name - elif self.link and self.link.egg_fragment: - return self.link.egg_fragment - elif self.setup_info and self.setup_info.name: - return self.setup_info.name - - def get_link(self) -> Link: - target = "{0}".format(self.uri) - if hasattr(self, "name") and not self._has_hashed_name: - target = "{0}#egg={1}".format(target, self.name) - link = create_link(target) - return link - - def get_requirement(self) -> PackagingRequirement: - if self.name is None: - if self.parsed_line is not None and self.parsed_line.name is not None: - self.name = self.parsed_line.name - else: - raise ValueError( - "Failed to generate a requirement: missing name for {0!r}".format( - self - ) - ) - if self.parsed_line: - try: - # initialize specifiers to make sure we capture them - self.parsed_line.specifiers - except Exception: - pass - req = copy.deepcopy(self.parsed_line.requirement) - if req: - return req - - @property - def is_local(self): - # type: () -> bool - uri = getattr(self, "uri", None) - if uri is None: - if getattr(self, "path", None) and self.path is not None: - uri = path_to_url(os.path.abspath(self.path)) - elif ( - getattr(self, "req", None) - and self.req is not None - and (getattr(self.req, "url", None) and self.req.url is not None) - ): - uri = self.req.url - if uri and is_file_url(uri): - return True - return False - - @cached_property - def is_remote_artifact(self): - # type: () -> bool - if self.link is None: - return False - return ( - self.parsed_line - and not self.parsed_line.is_local - and (self.parsed_line.is_artifact or self.parsed_line.is_wheel) - and not self.editable - ) - - @property - def is_direct_url(self): - # type: () -> bool - if self.parsed_line is not None and self.parsed_line.is_direct_url: - return True - return self.is_remote_artifact - - @property - def formatted_path(self): - # type: () -> Optional[str] - if self.path: - path = self.path - if not isinstance(path, Path): - path = Path(path) - return path.as_posix() - return None - - @classmethod - def from_line( - cls, line, editable=None, extras=None, parsed_line=None - ) -> Union["FileRequirement"]: - parsed_line = Line(line=line) - file_req_from_parsed_line(parsed_line) - - @classmethod - def from_pipfile(cls, name, pipfile) -> Union["FileRequirement", "VCSRequirement"]: - # Parse the values out. After this dance we should have two variables: - # path - Local filesystem path. - # uri - Absolute URI that is parsable with urlsplit. - # One of these will be a string; the other would be None. - uri = pipfile.get("uri") - fil = pipfile.get("file") - path = pipfile.get("path") - if path and isinstance(path, str): - if not urllib_parse.urlparse(path).scheme and not os.path.isabs(path): - path = get_converted_relative_path(path) - uri = uri or fil or path - - # Decide that scheme to use. - # 'file' - A file:// URI (possibly with VCS prefix). - # 'path' - local filesystem path. - # 'uri' - Any other URI. - if fil or (path and path.startswith("file:/")): - uri_scheme = "file" - else: - uri_scheme = "path" - - if not uri: - uri = path_to_url(path) - link_info = None # type: Optional[LinkInfo] - if uri and isinstance(uri, str): - link_info = cls.get_link_from_line(uri) - else: - raise ValueError( - "Failed parsing requirement from pipfile: {0!r}".format(pipfile) - ) - link = None # type: Optional[Link] - if link_info: - link = link_info.link - if link.url_without_fragment: - uri = link.url_without_fragment - extras = () # type: Optional[Tuple[str, ...]] - if "extras" in pipfile: - extras = tuple(pipfile["extras"]) # type: ignore - editable = pipfile["editable"] if "editable" in pipfile else False - arg_dict = { - "name": name, - "path": path, - "uri": uri, - "editable": editable, - "link": link, - "uri_scheme": uri_scheme, - "extras": extras if extras else None, - } - - extras_string = "" if not extras else extras_to_string(extras) - if editable and uri_scheme == "path": - line = "{0}{1}".format(path, extras_string) - else: - if name: - line_name = "{0}{1}".format(name, extras_string) - line = "{0}#egg={1}".format(link.url_without_fragment, line_name) - else: - if link: - line = link.url - elif uri and isinstance(uri, str): - line = uri - else: - raise ValueError( - "Failed parsing requirement from pipfile: {0!r}".format(pipfile) - ) - line = "{0}{1}".format(line, extras_string) - if "subdirectory" in pipfile: - arg_dict["subdirectory"] = pipfile["subdirectory"] - line = "{0}&subdirectory={1}".format(line, pipfile["subdirectory"]) - if editable: - line = "-e {0}".format(line) - arg_dict["parsed_line"] = Line(line=line, extras=extras) - arg_dict["setup_info"] = arg_dict["parsed_line"].setup_info - return cls(**arg_dict) # type: ignore - - @cached_property - def line_part(self): - # type: () -> str - link_url = None # type: Optional[str] - seed = None # type: Optional[str] - if self.link is not None: - link_url = self.link.url_without_fragment - is_vcs = getattr(self.link, "is_vcs", False) - if self.uri_scheme and self.uri_scheme == "path": - # We may need any one of these for passing to pip - seed = self.path or link_url or self.uri - elif (self.uri_scheme and self.uri_scheme == "file") or ( - (self.link.is_wheel or not is_vcs) and self.link.url - ): - seed = link_url or self.uri - # add egg fragments to remote artifacts (valid urls only) - if not self._has_hashed_name and self.is_remote_artifact and seed is not None: - seed += "#egg={0}".format(self.name) - editable = "-e " if self.editable else "" - if seed is None: - raise ValueError("Could not calculate url for {0!r}".format(self)) - return "{0}{1}".format(editable, seed) - - @property - def pipfile_part(self): - # type: () -> Dict[AnyStr, Dict[AnyStr, Any]] - excludes = [ - "_base_line", - "_has_hashed_name", - "setup_path", - "pyproject_path", - "uri_scheme", - "pyproject_requires", - "pyproject_backend", - "setup_info", - "parsed_line", - ] - pipfile_dict = self.dict() - for k in list(pipfile_dict.keys()): - if k in excludes: - pipfile_dict.pop(k) - elif not pipfile_dict.get(k): - pipfile_dict.pop(k) - if pipfile_dict.get("editable") is False: - pipfile_dict.pop("editable") - if pipfile_dict.get("line_part"): - pipfile_dict.pop("line_part") - name = pipfile_dict.pop("name", None) - if name is None: - if self.name: - name = self.name - elif self.parsed_line and self.parsed_line.name: - name = self.name = self.parsed_line.name - elif self.setup_info and self.setup_info.name: - name = self.name = self.setup_info.name - if "uri_scheme" in pipfile_dict: - pipfile_dict.pop("uri_scheme") - # For local paths and remote installable artifacts (zipfiles, etc) - collision_keys = {"file", "uri", "path"} - collision_order = ["file", "uri", "path"] # type: List[str] - key_match = next(iter(k for k in collision_order if k in pipfile_dict.keys())) - is_vcs = None - if self.link is not None: - is_vcs = getattr(self.link, "is_vcs", False) - if self.uri_scheme: - dict_key = self.uri_scheme - target_key = dict_key if dict_key in pipfile_dict else key_match - if target_key is not None: - winning_value = pipfile_dict.pop(target_key) - collisions = [k for k in collision_keys if k in pipfile_dict] - for key in collisions: - pipfile_dict.pop(key) - pipfile_dict[dict_key] = winning_value - elif ( - self.is_remote_artifact - or (is_vcs is not None and not is_vcs) - and (self.uri_scheme and self.uri_scheme == "file") - ): - dict_key = "file" - # Look for uri first because file is a uri format and this is designed - # to make sure we add file keys to the pipfile as a replacement of uri - if key_match is not None: - winning_value = pipfile_dict.pop(key_match) - pipfile_dict[dict_key] = winning_value - key_to_remove = (k for k in collision_keys if k in pipfile_dict) - for key in key_to_remove: - pipfile_dict.pop(key) - else: - collisions = [key for key in collision_order if key in pipfile_dict.keys()] - if len(collisions) > 1: - for k in collisions[1:]: - pipfile_dict.pop(k) - return {name: pipfile_dict} - - -class VCSRequirement(FileRequirement): - editable: Optional[bool] = None - uri: Optional[str] = None - path: Optional[str] = Field(default=None, validator=validate_path) - vcs: Optional[str] = Field(default=None, validator=validate_vcs) - ref: Optional[str] = None - _repo: Optional[VCSRepository] = None - _base_line: Optional[str] = None - parsed_line: Optional[Line] = None - uri_scheme: Optional[str] = None - name: str - link: Optional[Link] - req: Optional[PackagingRequirement] - - def __init__(self, **data): - super().__init__(**data) - if not self.uri and self.path: - self.uri = path_to_url(self.path) - if self.uri is not None: - split = urllib_parse.urlsplit(self.uri) - scheme, rest = split[0], split[1:] - vcs_type = "" - if "+" in scheme: - vcs_type, scheme = scheme.split("+", 1) - vcs_type = "{0}+".format(vcs_type) - new_uri = urllib_parse.urlunsplit((scheme,) + rest[:-1] + ("",)) - new_uri = "{0}{1}".format(vcs_type, new_uri) - self.uri = new_uri - - @cached_property - def url(self) -> str: - if self.link and self.link.url: - return self.link.url - elif self.uri: - return self.uri - raise ValueError("No valid url found for requirement {0!r}".format(self)) - - def get_link(self) -> Link: - uri = self.uri if self.uri else path_to_url(self.path) - vcs_uri = build_vcs_uri( - self.vcs, - add_ssh_scheme_to_git_uri(uri), - name=self.name, - ref=self.ref, - subdirectory=self.subdirectory, - extras=self.extras, - ) - return self.get_link_from_line(vcs_uri).link - - def get_name(self) -> str: - if self.link and self.link.egg_fragment: - return self.link.egg_fragment - if self.req and self.req.name: - return self.req.name - return super(VCSRequirement, self).get_name() - - @cached_property - def vcs_uri(self): - # type: () -> Optional[str] - uri = self.uri - if uri and not any(uri.startswith("{0}+".format(vcs)) for vcs in VCS_LIST): - if self.vcs: - uri = "{0}+{1}".format(self.vcs, uri) - return uri - - def get_requirement(self): - # type: () -> PackagingRequirement - name = None # type: Optional[str] - if self.name: - name = self.name - elif self.link and self.link.egg_fragment: - name = self.link.egg_fragment - url = None - if self.uri: - url = self.uri - elif self.link is not None: - url = self.link.url_without_fragment - if not name: - raise ValueError( - "pipenv requires an #egg fragment for version controlled " - "dependencies. Please install remote dependency " - "in the form {0}#egg=.".format(url) - ) - req = init_requirement(canonicalize_name(self.name)) - req.editable = self.editable - if not getattr(req, "url", None): - if url is not None: - url = add_ssh_scheme_to_git_uri(url) - elif self.uri is not None: - link = self.get_link_from_line(self.uri).link - if link: - url = link.url_without_fragment - if ( - url - and url.startswith("git+file:/") - and not url.startswith("git+file:///") - ): - url = url.replace("git+file:/", "git+file:///") - if url: - req.url = url - line = url if url else self.vcs_uri - if self.editable: - line = "-e {0}".format(line) - req.line = line - if self.ref: - req.revision = self.ref - if self.extras: - req.extras = self.extras - req.vcs = self.vcs - if self.path and self.link and self.link.scheme.startswith("file"): - req.local_file = True - req.path = self.path - req.link = self.link - if ( - self.link - and self.link.url_without_fragment - and self.uri - and self.uri != unquote(self.link.url_without_fragment) - and "git+ssh://" in self.link.url - and "git+git@" in self.uri - ): - req.line = self.uri - url = self.link.url_without_fragment - if ( - url - and url.startswith("git+file:/") - and not url.startswith("git+file:///") - ): - url = url.replace("git+file:/", "git+file:///") - req.url = url - return req - - @cached_property - def repo(self) -> VCSRepository: - if self._repo is None: - if self.parsed_line and self.parsed_line.vcsrepo: - self._repo = self.parsed_line.vcsrepo - else: - self._repo = self.get_vcs_repo() - if self.parsed_line: - self.parsed_line.set_vcsrepo(self._repo) - return self._repo - - def get_checkout_dir(self, src_dir=None) -> str: - if self.is_local: - path = self.path - if not path: - path = url_to_path(self.uri) - if path and os.path.exists(path): - checkout_dir = os.path.abspath(path) - return checkout_dir - return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) - - def get_vcs_repo(self, src_dir=None, checkout_dir=None): - # type: (Optional[str], str) -> VCSRepository - from .vcs import VCSRepository - - if checkout_dir is None: - checkout_dir = self.get_checkout_dir(src_dir=src_dir) - vcsrepo = VCSRepository( - url=expand_env_variables(self.url), - name=self.name, - ref=self.ref if self.ref else None, - checkout_directory=checkout_dir, - vcs_type=self.vcs, - subdirectory=self.subdirectory, - ) - if not self.is_local: - vcsrepo.obtain() - if self.subdirectory: - self.setup_path = os.path.join(checkout_dir, self.subdirectory, "setup.py") - self.pyproject_path = os.path.join( - checkout_dir, self.subdirectory, "pyproject.toml" - ) - result = get_pyproject(os.path.join(checkout_dir, self.subdirectory)) - else: - self.setup_path = os.path.join(checkout_dir, "setup.py") - self.pyproject_path = os.path.join(checkout_dir, "pyproject.toml") - result = get_pyproject(checkout_dir) - if result is not None: - self.pyproject_requires = tuple(result.get("build_requires", [])) - self.pyproject_backend = result.get("build_backend") - return vcsrepo - - @cached_property - def commit_hash(self) -> str: - return self.repo.commit_hash - - def update_repo(self, src_dir=None, ref=None) -> str: - if ref: - self.ref = ref - if not self.is_local and self.ref is not None: - self.repo.checkout_ref(self.ref) - repo_hash = self.commit_hash - if self.req: - self.req.revision = repo_hash - return repo_hash - - @contextmanager - def locked_vcs_repo( - self, src_dir: Optional[str] = None - ) -> Generator[VCSRepository, None, None]: - if not src_dir: - src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") - vcsrepo = self.get_vcs_repo(src_dir=src_dir) - if not self.req: - if self.parsed_line is not None: - self.req = self.parsed_line.requirement - else: - self.req = self.get_requirement() - revision = self.req.revision = vcsrepo.commit_hash - - # Remove potential ref in the end of uri after ref is parsed - if self.link and "@" in self.link.show_url and self.uri and "@" in self.uri: - uri, ref = split_ref_from_uri(self.uri) - checkout = revision - if checkout and ref and ref in checkout: - self.uri = uri - orig_repo = self._repo - self._repo = vcsrepo - if self.parsed_line: - self.parsed_line.set_vcsrepo(vcsrepo) - if self.parsed_line and self.parsed_line: - self.parsed_line.set_vcsrepo(vcsrepo) - if self.req and not self.editable: - if self.setup_info is None and self.parsed_line.setup_info: - self.setup_info = self.parsed_line.setup_info - if self.setup_info and not self.setup_info.version: - self.setup_info.build() - self.req.specifier = SpecifierSet("=={0}".format(self.setup_info.version)) - try: - yield self._repo - except Exception: - self._repo = orig_repo - raise - - @classmethod - def from_pipfile(cls, name, pipfile) -> Union["FileRequirement", "VCSRequirement"]: - creation_args = {} - pipfile_keys = [ - k - for k in ( - "ref", - "vcs", - "subdirectory", - "path", - "editable", - "file", - "uri", - "extras", - ) - + VCS_LIST - if k in pipfile - ] - for key in pipfile_keys: - if key == "extras" and key in pipfile: - extras = pipfile[key] - if isinstance(extras, (list, tuple)): - pipfile[key] = tuple(sorted({extra.lower() for extra in extras})) - else: - pipfile[key] = extras - if key in VCS_LIST and key in pipfile_keys: - creation_args["vcs"] = key - target = pipfile[key] - if isinstance(target, str): - drive, path = os.path.splitdrive(target) - if ( - not drive - and not os.path.exists(target) - and ( - is_valid_url(target) - or is_file_url(target) - or target.startswith("git@") - ) - ): - creation_args["uri"] = target - else: - creation_args["path"] = target - if os.path.isabs(target): - creation_args["uri"] = path_to_url(target) - elif key in pipfile_keys: - creation_args[key] = pipfile[key] - creation_args["name"] = name - cls_inst = cls(**creation_args) # type: ignore - return cls_inst - - @classmethod - def from_line( - cls, line, editable=None, extras=None, parsed_line=None - ) -> Union["FileRequirement", "VCSRequirement"]: - parsed_line = Line(line=line) - return vcs_req_from_parsed_line(parsed_line) - - @property - def line_part(self) -> str: - """requirements.txt compatible line part sans-extras.""" - if self.is_local: - base_link = self.link - if not self.link: - base_link = self.get_link() - if base_link and base_link.egg_fragment: - final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) - else: - final_format = "{0}" - base = final_format.format(self.vcs_uri) - elif self.parsed_line is not None and ( - self.parsed_line.is_direct_url and self.parsed_line.line_with_prefix - ): - return self.parsed_line.line_with_prefix - elif getattr(self, "_base_line", None) and (isinstance(self._base_line, str)): - base = self._base_line - else: - base = getattr(self, "link", self.get_link()).url - if base and self.extras and extras_to_string(self.extras) not in base: - if self.subdirectory: - base = "{0}".format(self.get_link().url) - else: - base = "{0}{1}".format(base, extras_to_string(sorted(self.extras))) - if "git+file:/" in base and "git+file:///" not in base: - base = base.replace("git+file:/", "git+file:///") - if self.editable and not base.startswith("-e "): - base = "-e {0}".format(base) - return base - - @staticmethod - def _choose_vcs_source(pipfile): - # type: (Dict[S, Union[S, Any]]) -> Dict[S, Union[S, Any]] - src_keys = [] - for key in ["path", "uri", "file"]: - if pipfile.get(key): - src_keys.append(key) - vcs_type = "" # type: Optional[str] - alt_type = "" # type: Optional[str] - vcs_value = "" # type: str - if src_keys: - chosen_key = next(iter(src_keys)) - vcs_type = pipfile.pop("vcs") - if chosen_key in pipfile: - vcs_value = pipfile[chosen_key] - alt_type, pipfile_url = split_vcs_method_from_uri(vcs_value) - if vcs_type is None: - vcs_type = alt_type - if vcs_type and pipfile_url: - pipfile[vcs_type] = pipfile_url - return pipfile - - @property - def pipfile_part(self): - # type: () -> Dict[S, Dict[S, Union[List[S], S, bool, PackagingRequirement, Link]]] - excludes = [ - "_repo", - "_base_line", - "setup_path", - "_has_hashed_name", - "pyproject_path", - "pyproject_requires", - "pyproject_backend", - "_setup_info", - "parsed_line", - "uri_scheme", - ] - filter_func = lambda k, v: bool(v) is True and k.name not in excludes # noqa - pipfile_dict = self.dict() - for k in list(pipfile_dict.keys()): - if k in excludes: - pipfile_dict.pop(k) - elif not pipfile_dict.get(k): - pipfile_dict.pop(k) - if pipfile_dict.get("editable") is False: - pipfile_dict.pop("editable") - if pipfile_dict.get("line_part"): - pipfile_dict.pop("line_part") - name = pipfile_dict.get("name") - if name is None: - if self.name: - name = self.name - elif self.parsed_line and self.parsed_line.name: - name = self.name = self.parsed_line.name - elif self.setup_info and self.setup_info.name: - name = self.name = self.setup_info.name - if "vcs" in pipfile_dict: - pipfile_dict = self._choose_vcs_source(pipfile_dict) - name, _ = _strip_extras(name) - return {name: pipfile_dict} # type: ignore - - -class Requirement(ReqLibBaseModel): - vcs: Optional[str] = Field(None, eq=True, order=True) - req: Optional[Any] = Field(None, eq=True, order=True) - markers: Optional[str] = Field("", eq=True, order=True) - index: Optional[str] = Field(None, eq=True, order=True) - editable: Optional[bool] = Field(None, eq=True, order=True) - hashes: Set[str] = set() - extras: Tuple[str, ...] = Field(tuple(), eq=True, order=True) - _line_instance: Optional[Line] = None - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - @validator('req') - def check_req_type(cls, v): - allowed_types = (VCSRequirement, NamedRequirement, FileRequirement) - if v is not None and not isinstance(v, allowed_types): - raise ValueError('req must be an instance of VCSRequirement, NamedRequirement, or FileRequirement') - return v - - def __hash__(self): - return hash(self.as_line()) - - @cached_property - def name(self) -> Optional[str]: - name = None - if self.req and self.req.name: - name = self.req.name - elif self.req and self.is_file_or_url and self.req.setup_info: - name = self.req.setup_info.name - return name - - @cached_property - def requirement(self) -> Optional[PackagingRequirement]: - if self.req: - return self.req.req - return None - - def add_hashes(self, hashes: set) -> "Requirement": - new_hashes = set() - if self.hashes is not None: - new_hashes |= set(self.hashes) - if isinstance(hashes, str): - new_hashes.add(hashes) - else: - new_hashes |= set(hashes) - self.hashes = new_hashes - - def get_hashes_as_pip(self, as_list: bool = False) -> Union[str, List[str]]: - if as_list: - return [HASH_STRING.format(h) for h in self.hashes] if self.hashes else [] - else: - return ( - "".join([HASH_STRING.format(h) for h in self.hashes]) - if self.hashes - else "" - ) - - @cached_property - def hashes_as_pip(self) -> str: - return self.get_hashes_as_pip() - - @cached_property - def extras_as_pip(self) -> str: - return ( - f"[{','.join(sorted([extra.lower() for extra in self.extras]))}]" - if self.extras - else "" - ) - - @cached_property - def commit_hash(self) -> Optional[str]: - if self.req is None or not isinstance(self.req, VCSRequirement): - return None - commit_hash = None - if self.req is not None: - with self.req.locked_vcs_repo() as repo: - commit_hash = repo.commit_hash - return commit_hash - - @cached_property - def get_specifiers(self) -> str: - if self.req and self.req.req and self.req.req.specifier: - return specs_to_string(self.req.req.specifier) - return "" - - def update_name_from_path(self, path: str) -> None: - metadata = get_metadata(path) - name = self.name - if metadata is not None: - metadata_name = metadata.get("name") - if metadata_name and metadata_name != "wheel": - name = metadata_name - if name is not None: - if self.req.name is None: - self.req.name = name - if self.req.req and self.req.req.name is None: - self.req.req.name = name - if self.req.parsed_line._name is None: - self.req.parsed_line.name = name - if self.req.setup_info and self.req.setup_info.name is None: - self.req.setup_info.name = name - - def get_line_instance(self) -> Line: - if self._line_instance is None: - line_parts = [] - local_editable = False - if self.req: - if self.req.line_part.startswith("-e "): - local_editable = True - line_parts.extend(self.req.line_part.split(" ", 1)) - else: - line_parts.append(self.req.line_part) - - version, fetched = self.get_version_from_setup_info() - if version is not None and not (self.is_file_or_url or self.is_vcs): - line_parts.append(f"=={version}") - if ( - self.is_file_or_url - and not local_editable - and not self.req.get_uri().startswith("file://") - # fix for file uri with egg names and extras - and not len(self.req.line_part.split("#")) > 1 - ): - line_parts.append(f"#egg={self.name}{self.extras_as_pip}") - elif not any(part for part in line_parts if self.extras_as_pip in part): - line_parts.append(self.extras_as_pip) - if self.specifiers and not (self.is_file_or_url or self.is_vcs): - line_parts.append(self.specifiers) - if self.markers and not self.is_file_or_url: - line_parts.append(" ; {0}".format(self.markers.replace('"', "'"))) - - if self.hashes_as_pip and not (self.editable or self.vcs or self.is_vcs): - line_parts.append(self.hashes_as_pip) - - if self.editable: - if line_parts[0] == "-e": - line = "".join(line_parts[1:]) - else: - line = "".join(line_parts) - if self.markers: - line = '"{0}"'.format(line) - line = "-e {0}".format(line) - else: - line = "".join(line_parts) - self._line_instance = Line(line=line) - return self._line_instance - - @property - def line_instance(self): - return self.get_line_instance() - - @cached_property - def specifiers(self) -> Optional[str]: - specs = self.get_specifiers - if specs: - return specs - elif self.req and isinstance(self.req, NamedRequirement) and self.req.version: - return "=={0}".format(self.req.version) - elif ( - self.req - and hasattr(self.req, "setup_info") - and self.req.setup_info - and self.req.setup_info.version - ): - return "=={0}".format(self.req.setup_info.version) - # TODO This creates circular call chain that eventuallys gets cached by cached_property - elif self.is_file_or_url or self.is_vcs: - version, fetched = self.get_version_from_setup_info() - if version is not None: - return "=={0}".format(version) - elif not fetched: - try: - setupinfo_dict = self.run_requires() - except Exception: - setupinfo_dict = None - if setupinfo_dict is not None: - return "=={0}".format(setupinfo_dict.get("version")) - elif self.req and hasattr(self.req, "setup_info") and self.req.setup_info.version: - return "=={0}".format(self.req.setup_info.version) - - @cached_property - def is_vcs(self) -> bool: - return isinstance(self.req, VCSRequirement) - - @cached_property - def is_file_or_url(self) -> bool: - return isinstance(self.req, FileRequirement) - - @cached_property - def is_named(self) -> bool: - return isinstance(self.req, NamedRequirement) - - @cached_property - def is_wheel(self) -> bool: - if ( - self.req - and not isinstance(self.req, NamedRequirement) - and (self.req.link is not None and self.req.link.is_wheel) - ): - return True - return False - - @cached_property - def normalized_name(self) -> str: - return canonicalize_name(self.name) - - def _copy(self): - return self.__class__(**self.__dict__) - - @classmethod - def from_line(cls, line, parse_setup_info=True) -> "Requirement": - if isinstance(line, Requirement): - return line - if isinstance(line, InstallRequirement): - line = format_requirement(line) - parsed_line = Line(line=line) - if parse_setup_info and not parsed_line.is_named and not parsed_line.is_wheel: - parsed_line.set_setup_info(parsed_line.get_setup_info()) - if ( - (parsed_line.is_file and parsed_line.is_installable) - or parsed_line.is_remote_url - ) and not parsed_line.is_vcs: - r = file_req_from_parsed_line(parsed_line) - elif parsed_line.is_vcs: - r = vcs_req_from_parsed_line(parsed_line) - elif line == "." and not is_installable_file(line): - raise RequirementError( - "Error parsing requirement %s -- are you sure it is installable?" % line - ) - else: - r = named_req_from_parsed_line(parsed_line) - req_markers = None - if parsed_line.markers: - req_markers = PackagingRequirement( - "fakepkg ; {0}".format(parsed_line.markers) - ) - if r is not None and r.req is not None: - r.req.marker = getattr(req_markers, "marker", None) if req_markers else None - args = { - "name": r.name, - "vcs": parsed_line.vcs, - "req": r, - "markers": parsed_line.markers, - "editable": parsed_line.editable, - } - if parsed_line.extras: - extras = tuple(sorted(dedup([extra.lower() for extra in parsed_line.extras]))) - args["extras"] = extras - if r is not None: - r.extras = extras - elif r is not None and r.extras is not None: - args["extras"] = tuple( - sorted(dedup([extra.lower() for extra in r.extras])) - ) # type: ignore - if r.req is not None: - r.req.extras = args["extras"] - if parsed_line.hashes: - args["hashes"] = tuple(parsed_line.hashes) # type: ignore - cls_inst = cls(**args) # type: ignore - return cls_inst - - @classmethod - def from_ireq(cls, ireq): - return cls.from_line(format_requirement(ireq)) - - @classmethod - def from_metadata(cls, name, version, extras, markers): - return cls.from_ireq( - make_install_requirement(name, version, extras=extras, markers=markers) - ) - - @classmethod - def from_pipfile(cls, name, pipfile): - from .markers import PipenvMarkers - - _pipfile = {} - if hasattr(pipfile, "keys"): - _pipfile = dict(pipfile).copy() - _pipfile["version"] = get_version(pipfile) - - # We ensure version contains an operator. Default to equals (==) - if _pipfile["version"] and COMPARE_OP.match(_pipfile["version"]) is None: - _pipfile["version"] = "=={}".format(_pipfile["version"]) - vcs = next(iter([vcs for vcs in VCS_LIST if vcs in _pipfile]), None) - if vcs: - _pipfile["vcs"] = vcs - r = VCSRequirement.from_pipfile(name, pipfile) - elif any(key in _pipfile for key in ["path", "file", "uri"]): - r = FileRequirement.from_pipfile(name, pipfile) - else: - r = NamedRequirement.from_pipfile(name, pipfile) - markers = PipenvMarkers.from_pipfile(name, _pipfile) - req_markers = None - if markers: - markers = str(markers) - req_markers = PackagingRequirement("fakepkg ; {0}".format(markers)) - if r.req is not None: - r.req.marker = req_markers.marker - extras = _pipfile.get("extras") - if r.req: - r.req.specifier = SpecifierSet(_pipfile["version"]) - r.req.extras = ( - tuple(sorted(dedup([extra.lower() for extra in extras]))) - if extras - else () - ) - args = { - "name": r.name, - "vcs": vcs, - "req": r, - "markers": markers, - "extras": tuple(_pipfile.get("extras", ())), - "editable": _pipfile.get("editable", False), - "index": _pipfile.get("index"), - } - if any(key in _pipfile for key in ["hash", "hashes"]): - args["hashes"] = _pipfile.get("hashes", [pipfile.get("hash")]) - cls_inst = Requirement(**args) - return cls_inst - - def as_line( - self, - sources=None, - include_hashes=True, - include_extras=True, - include_markers=True, - as_list=False, - ): - """Format this requirement as a line in requirements.txt. - - If ``sources`` provided, it should be an sequence of mappings, containing - all possible sources to be used for this requirement. - - If ``sources`` is omitted or falsy, no index information will be included - in the requirement line. - """ - - assert self.line_instance is not None - parts = self.line_instance.get_line( - with_prefix=True, - with_hashes=include_hashes, - with_markers=include_markers, - as_list=as_list, - ) - if sources and self.requirement and not (self.line_instance.is_local or self.vcs): - from ..utils import prepare_pip_source_args - - if self.index: - sources = [s for s in sources if s.get("name") == self.index] - source_list = prepare_pip_source_args(sources) - if as_list: - parts.extend(sources) - else: - index_string = " ".join(source_list) - parts = "{0} {1}".format(parts, index_string) - return parts - - @cached_property - def get_markers(self): - markers = self.markers - if markers: - fake_pkg = PackagingRequirement("fakepkg ; {0}".format(markers)) - markers = fake_pkg.marker - return markers - - @cached_property - def get_specifier(self) -> Union[Specifier, LegacySpecifier]: - try: - return Specifier(self.specifiers) - except InvalidSpecifier: - return LegacySpecifier(self.specifiers) - - @cached_property - def get_version(self): - return parse(self.get_specifier.version) - - def get_requirement(self) -> PackagingRequirement: - req_line = self.req.req.line - if req_line.startswith("-e "): - _, req_line = req_line.split(" ", 1) - req = init_requirement(self.name) - req.line = req_line - req.specifier = SpecifierSet(self.specifiers if self.specifiers else "") - if self.is_vcs or self.is_file_or_url: - req.url = getattr(self.req.req, "url", self.req.link.url_without_fragment) - req.marker = self.get_markers - req.extras = set(self.extras) if self.extras else set() - return req - - @cached_property - def constraint_line(self) -> str: - return self.as_line() - - @cached_property - def is_direct_url(self) -> bool: - return ( - self.is_file_or_url - and self.req.is_direct_url - or (self.line_instance.is_direct_url or self.req.parsed_line.is_direct_url) - ) - - def as_pipfile(self) -> Dict[str, Any]: - - good_keys = ( - "hashes", - "extras", - "markers", - "editable", - "version", - "index", - ) + VCS_LIST - req_dict = {} - for k, v in self.dict().items(): - if k in good_keys and v: - req_dict[k] = v - if req_dict.get("editable") is False: - req_dict.pop("editable") - - name = self.name - if "markers" in req_dict and req_dict["markers"]: - req_dict["markers"] = req_dict["markers"].replace('"', "'") - if not self.req.name: - name_carriers = (self.req, self, self.line_instance, self.req.parsed_line) - name_options = [ - getattr(carrier, "name", None) - for carrier in name_carriers - if carrier is not None - ] - req_name = next(iter(n for n in name_options if n is not None), None) - self.req.name = req_name - req_name = next(reversed(list(self.req.pipfile_part.keys()))) - dict_from_subreq = self.req.pipfile_part[req_name] - base_dict = { - k: v - for k, v in dict_from_subreq.items() - if k not in ["req", "link", "_setup_info"] - } - base_dict.update(req_dict) - file_conflicting_keys = ("path", "uri", "name") - if "file" in base_dict and any(k in base_dict for k in file_conflicting_keys): - conflicts = [k for k in file_conflicting_keys if k in base_dict] - for k in conflicts: - base_dict.pop(k) - vcs_conflicting_keys = ("path", "uri", "name") - if any(k in base_dict for k in VCS_LIST): - conflicts = [k for k in vcs_conflicting_keys if k in base_dict] - for k in conflicts: - base_dict.pop(k) - if "hashes" in base_dict: - _hashes = base_dict.pop("hashes") - hashes = [] - for _hash in _hashes: - try: - hashes.append(_hash.as_line()) - except AttributeError: - hashes.append(_hash) - base_dict["hashes"] = sorted(hashes) - if "extras" in base_dict: - base_dict["extras"] = list(base_dict["extras"]) - if "setup_info" in base_dict: - del base_dict["setup_info"] - if len(base_dict.keys()) == 1 and "version" in base_dict: - base_dict = base_dict.get("version") - - return {name: base_dict} - - def as_ireq(self) -> InstallRequirement: - if self.line_instance and self.line_instance.ireq: - return self.line_instance.ireq - elif getattr(self.req, "parsed_line", None) and self.req.parsed_line.ireq: - return self.req.parsed_line.ireq - kwargs = {"include_hashes": False} - if (self.is_file_or_url and self.req.is_local) or self.is_vcs: - kwargs["include_markers"] = False - ireq_line = self.as_line(**kwargs) - ireq = Line(line=ireq_line).ireq - if not getattr(ireq, "req", None): - ireq.req = self.req.req - if (self.is_file_or_url and self.req.is_local) or self.is_vcs: - if getattr(ireq, "req", None) and getattr(ireq.req, "marker", None): - ireq.req.marker = None - else: - ireq.req.extras = self.req.req.extras - if not ((self.is_file_or_url and self.req.is_local) or self.is_vcs): - ireq.req.marker = self.req.req.marker - return ireq - - @cached_property - def pipfile_entry(self) -> Tuple[str, Any]: - pipfile = self.as_pipfile() - last_key = next(reversed(list(pipfile.keys()))) - last_value = pipfile[last_key] - return last_key, last_value - - @cached_property - def ireq(self) -> InstallRequirement: - return self.as_ireq() - - def get_version_from_setup_info(self) -> Tuple[Optional[str], bool]: - if self.req and isinstance(self.req, FileRequirement): - self.req.parse_setup_info() - if self.req and hasattr(self.req, "setup_info") and self.req.setup_info: - return self.req.setup_info.version, True - elif self._line_instance and self._line_instance.setup_info: - return self._line_instance.setup_info.version, True - - return None, False - - def run_requires( - self, sources: Optional[List[str]] = None, finder: Optional[PackageFinder] = None - ) -> Dict[str, Any]: - if self.line_instance and self._line_instance.setup_info: - info_dict = self.line_instance.setup_info.as_dict() - else: - if not finder: - from .dependencies import get_finder - - finder = get_finder(sources=sources) - setup_info = SetupInfo.from_requirement(self, finder=finder) - if setup_info is None: - return {} - info_dict = setup_info.as_dict() - if self.req: - self.req.setup_info = setup_info - if self.req._has_hashed_name and info_dict.get("name"): - self.req.name = self.name = info_dict["name"] - if self.req.req.name != info_dict["name"]: - self.req.req.name = info_dict["name"] - return info_dict - - def merge_markers(self, markers: Union[str, Marker]) -> "Requirement": - if not markers: - return self - if not isinstance(markers, Marker): - markers = Marker(markers) - _markers = [] # type: List[Marker] - ireq = self.ireq - if ireq and ireq.markers: - ireq_marker = ireq.markers - _markers.append(str(ireq_marker)) - _markers.append(str(markers)) - marker_str = " and ".join([normalize_marker_str(m) for m in _markers if m]) - self.markers = marker_str - new_marker = Marker(marker_str) - new_ireq = getattr(self, "ireq", None) - if new_ireq and new_ireq.req: - new_ireq.req.marker = new_marker - if self.req.req: - self.req.req = new_marker - - -def file_req_from_parsed_line(parsed_line) -> FileRequirement: - path = parsed_line.relpath if parsed_line.relpath else parsed_line.path - pyproject_requires = None # type: Optional[Tuple[str, ...]] - if parsed_line.pyproject_requires is not None: - pyproject_requires = tuple(parsed_line.pyproject_requires) - pyproject_path = ( - Path(parsed_line.pyproject_toml) if parsed_line.pyproject_toml else None - ) - req_dict = { - "setup_path": parsed_line.setup_py, - "path": path, - "editable": parsed_line.editable, - "extras": parsed_line.extras, - "uri_scheme": parsed_line.preferred_scheme, - "link": parsed_line.link, - "uri": parsed_line.uri, - "pyproject_requires": pyproject_requires, - "pyproject_backend": parsed_line.pyproject_backend, - "pyproject_path": pyproject_path, - "parsed_line": parsed_line, - "req": parsed_line.requirement, - "_setup_info": parsed_line.setup_info, - } - if parsed_line.name is not None: - req_dict["name"] = parsed_line.name - return FileRequirement(**req_dict) # type: ignore - - -def vcs_req_from_parsed_line(parsed_line) -> VCSRequirement: - line = "{0}".format(parsed_line.line) - if parsed_line.editable: - line = "-e {0}".format(line) - if parsed_line.url is not None: - link = create_link( - build_vcs_uri( - vcs=parsed_line.vcs, - uri=parsed_line.url, - name=parsed_line.name, - ref=parsed_line.ref, - subdirectory=parsed_line.subdirectory, - extras=list(parsed_line.extras), - ) - ) - else: - link = parsed_line.link - pyproject_requires = () - if parsed_line.pyproject_requires is not None: - pyproject_requires = tuple(parsed_line.pyproject_requires) - vcs_dict = { - "setup_path": parsed_line.setup_py, - "path": parsed_line.path, - "editable": parsed_line.editable, - "vcs": parsed_line.vcs, - "ref": parsed_line.ref, - "subdirectory": parsed_line.subdirectory, - "extras": parsed_line.extras, - "uri_scheme": parsed_line.preferred_scheme, - "link": link, - "uri": parsed_line.uri, - "pyproject_requires": pyproject_requires, - "pyproject_backend": parsed_line.pyproject_backend, - "pyproject_path": Path(parsed_line.pyproject_toml) - if parsed_line.pyproject_toml - else None, - "parsed_line": parsed_line, - "req": parsed_line.requirement, - "base_line": line, - } - if parsed_line.name: - vcs_dict["name"] = parsed_line.name - return VCSRequirement(**vcs_dict) # type: ignore - - -def named_req_from_parsed_line(parsed_line): - # type: (Line) -> NamedRequirement - if parsed_line.name is not None: - return NamedRequirement( - name=parsed_line.name, - version=parsed_line.specifier, - req=parsed_line.requirement, - extras=parsed_line.extras, - editable=parsed_line.editable, - parsed_line=parsed_line, - ) - return NamedRequirement.from_line(parsed_line.line) diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py deleted file mode 100644 index ef77db38..00000000 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ /dev/null @@ -1,1849 +0,0 @@ -import ast -import atexit -import configparser -import contextlib -import errno -import locale -import os -import shutil -import stat -import subprocess as sp -import sys -import time -import uuid -import warnings -from functools import lru_cache -from itertools import count -from os import scandir -from pathlib import Path -from typing import Any, AnyStr, Callable, Dict, Generator, List, Optional, Tuple, Union, Iterable, Mapping -from urllib.parse import parse_qs, urlparse, urlunparse - -from pipenv.patched.pip._vendor.distlib.wheel import Wheel -from pipenv.vendor.pep517 import envbuild, wrappers - -from pipenv.patched.pip._internal.utils.temp_dir import TempDirectory, tempdir_kinds -from pipenv.patched.pip._internal.operations.prepare import File, _check_download_dir, unpack_vcs_link, get_file_url -from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.pip._internal.models.link import Link -from pipenv.patched.pip._internal.models.wheel import Wheel -from pipenv.patched.pip._internal.network.download import Downloader -from pipenv.patched.pip._internal.operations.prepare import unpack_url -from pipenv.patched.pip._internal.req.req_install import InstallRequirement -from pipenv.patched.pip._internal.utils.hashes import Hashes -from pipenv.patched.pip._internal.utils.unpacking import unpack_file -from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement -from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet -from pipenv.patched.pip._vendor.packaging.version import parse -from pipenv.patched.pip._vendor.pkg_resources import ( - DistInfoDistribution, - EggInfoDistribution, - PathMetadata, - Requirement, - distributions_from_metadata, - find_distributions, -) -from pipenv.patched.pip._vendor.platformdirs import user_cache_dir -from pipenv.patched.pip._vendor.pyparsing.core import cached_property -from pipenv.vendor.pydantic import Field - -from ..fileutils import cd, create_tracked_tempdir, temp_path, url_to_path -from ..utils import get_pip_command -from .common import ReqLibBaseModel -from .old_pip_utils import _copy_source_tree -from .utils import ( - HashableRequirement, - convert_to_hashable_requirement, - get_default_pyproject_backend, - get_name_variants, - get_pyproject, - init_requirement, - split_vcs_method_from_uri, - strip_extras_markers_from_requirement, -) - -CACHE_DIR = os.environ.get("PIPENV_CACHE_DIR", user_cache_dir("pipenv")) - - -def pep517_subprocess_runner(cmd, cwd=None, extra_environ=None) -> None: - """The default method of calling the wrapper subprocess.""" - env = os.environ.copy() - if extra_environ: - env.update(extra_environ) - - sp.run(cmd, cwd=cwd, env=env, stdout=sp.PIPE, stderr=sp.STDOUT) - - -class BuildEnv(envbuild.BuildEnvironment): - def pip_install(self, reqs): - python = os.environ.get("PIP_PYTHON_PATH", sys.executable) - cmd = [ - python, - "-m", - "pip", - "install", - "--ignore-installed", - "--prefix", - self.path, - ] + list(reqs) - - sp.run(cmd, stderr=sp.PIPE, stdout=sp.PIPE) - - -class HookCaller(wrappers.Pep517HookCaller): - def __init__(self, source_dir, build_backend, backend_path=None): - super().__init__(source_dir, build_backend, backend_path=backend_path) - self.source_dir = os.path.abspath(source_dir) - self.build_backend = build_backend - self._subprocess_runner = pep517_subprocess_runner - if backend_path: - backend_path = [ - wrappers.norm_and_check(self.source_dir, p) for p in backend_path - ] - self.backend_path = backend_path - - -def get_value_from_tuple(value, value_type): - try: - import winreg - except ImportError: - import _winreg as winreg - if value_type in (winreg.REG_SZ, winreg.REG_EXPAND_SZ): - if "\0" in value: - return value[: value.index("\0")] - return value - return None - - -def is_readonly_path(fn: os.PathLike) -> bool: - """check if a provided path exists and is readonly. - - permissions check is `bool(path.stat & stat.s_iread)` or `not - os.access(path, os.w_ok)` - """ - if os.path.exists(fn): - file_stat = os.stat(fn).st_mode - return not bool(file_stat & stat.s_iwrite) or not os.access(fn, os.w_ok) - return False - - -def query_registry_value(root, key_name, value): - try: - import winreg - except ImportError: - import _winreg as winreg - try: - with winreg.OpenKeyEx(root, key_name, 0, winreg.KEY_READ) as key: - return get_value_from_tuple(*winreg.QueryValueEx(key, value)) - except OSError: - return None - - -def _find_icacls_exe(): - if os.name == "nt": - paths = [ - os.path.expandvars(r"%windir%\{0}").format(subdir) - for subdir in ("system32", "SysWOW64") - ] - for path in paths: - icacls_path = next( - iter(fn for fn in os.listdir(path) if fn.lower() == "icacls.exe"), None - ) - if icacls_path is not None: - icacls_path = os.path.join(path, icacls_path) - return icacls_path - return None - - -def _walk_for_powershell(directory): - for _, dirs, files in os.walk(directory): - powershell = next( - iter(fn for fn in files if fn.lower() == "powershell.exe"), None - ) - if powershell is not None: - return os.path.join(directory, powershell) - for subdir in dirs: - powershell = _walk_for_powershell(os.path.join(directory, subdir)) - if powershell: - return powershell - return None - - -def _get_powershell_path(): - paths = [ - os.path.expandvars(r"%windir%\{0}\WindowsPowerShell").format(subdir) - for subdir in ("SysWOW64", "system32") - ] - powershell_path = next(iter(_walk_for_powershell(pth) for pth in paths), None) - if not powershell_path: - powershell_path = sp.run(["where", "powershell"]) - if powershell_path.stdout: - return powershell_path.stdout.strip() - - -def _get_sid_with_powershell(): - powershell_path = _get_powershell_path() - if not powershell_path: - return None - args = [ - powershell_path, - "-ExecutionPolicy", - "Bypass", - "-Command", - "Invoke-Expression '[System.Security.Principal.WindowsIdentity]::GetCurrent().user | Write-Host'", - ] - sid = sp.run(args, capture_output=True) - return sid.stdout.strip() - - -def _get_sid_from_registry(): - try: - import winreg - except ImportError: - import _winreg as winreg - var_names = ("%USERPROFILE%", "%HOME%") - current_user_home = next(iter(os.path.expandvars(v) for v in var_names if v), None) - root, subkey = ( - winreg.HKEY_LOCAL_MACHINE, - r"Software\Microsoft\Windows NT\CurrentVersion\ProfileList", - ) - subkey_names = [] - value = None - matching_key = None - try: - with winreg.OpenKeyEx(root, subkey, 0, winreg.KEY_READ) as key: - for i in count(): - key_name = winreg.EnumKey(key, i) - subkey_names.append(key_name) - value = query_registry_value( - root, r"{0}\{1}".format(subkey, key_name), "ProfileImagePath" - ) - if value and value.lower() == current_user_home.lower(): - matching_key = key_name - break - except OSError: - pass - if matching_key is not None: - return matching_key - - -def _get_current_user(): - fns = (_get_sid_from_registry, _get_sid_with_powershell) - for fn in fns: - result = fn() - if result: - return result - return None - - -def _wait_for_files(path): # pragma: no cover - """Retry with backoff up to 1 second to delete files from a directory. - - :param str path: The path to crawl to delete files from - :return: A list of remaining paths or None - :rtype: Optional[List[str]] - """ - timeout = 0.001 - remaining = [] - while timeout < 1.0: - remaining = [] - if os.path.isdir(path): - L = os.listdir(path) - for target in L: - _remaining = _wait_for_files(target) - if _remaining: - remaining.extend(_remaining) - continue - try: - os.unlink(path) - except FileNotFoundError as e: - if e.errno == errno.ENOENT: - return - except (OSError, IOError, PermissionError): # noqa:B014 - time.sleep(timeout) - timeout *= 2 - remaining.append(path) - else: - return - return remaining - - -def set_write_bit(fn: str) -> None: - """Set read-write permissions for the current user on the target path. Fail - silently if the path doesn't exist. - - :param str fn: The target filename or path - :return: None - """ - if not os.path.exists(fn): - return - file_stat = os.stat(fn).st_mode - os.chmod(fn, file_stat | stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) - if os.name == "nt": - user_sid = _get_current_user() - icacls_exe = _find_icacls_exe() or "icacls" - - if user_sid: - c = sp.run( - [ - icacls_exe, - "''{}''".format(fn), - "/grant", - "{}:WD".format(user_sid), - "/T", - "/C", - "/Q", - ], - capture_output=True, - # 2020-06-12 Yukihiko Shinoda - # There are 3 way to get system default encoding in Stack Overflow. - # see: https://stackoverflow.com/questions/37506535/how-to-get-the-system-default-encoding-in-python-2-x - # I investigated these way by using Shift-JIS Windows. - # >>> import locale - # >>> locale.getpreferredencoding() - # "cp932" (Shift-JIS) - # >>> import sys - # >>> sys.getdefaultencoding() - # "utf-8" - # >>> sys.stdout.encoding - # "UTF8" - encoding=locale.getpreferredencoding(), - ) - if not c.err and c.returncode == 0: - return - - if not os.path.isdir(fn): - for path in [fn, os.path.dirname(fn)]: - try: - os.chflags(path, 0) - except AttributeError: - pass - return None - for root, dirs, files in os.walk(fn, topdown=False): - for dir_ in [os.path.join(root, d) for d in dirs]: - set_write_bit(dir_) - for file_ in [os.path.join(root, f) for f in files]: - set_write_bit(file_) - - -def make_base_requirements(reqs) -> Tuple: - requirements = () - if not isinstance(reqs, (list, tuple, set)): - reqs = [reqs] - for req in reqs: - if isinstance(req, BaseRequirement): - requirements += (req,) - elif isinstance(req, Requirement): - requirements += (BaseRequirement.from_req(req),) - elif req and isinstance(req, str) and not req.startswith("#"): - requirements += (BaseRequirement.from_string(req),) - return requirements - - -def handle_remove_readonly(func, path, exc): - """Error handler for shutil.rmtree. - - Windows source repo folders are read-only by default, so this error handler - attempts to set them as writeable and then proceed with deletion. - - :param function func: The caller function - :param str path: The target path for removal - :param Exception exc: The raised exception - - This function will call check :func:`is_readonly_path` before attempting to call - :func:`set_write_bit` on the target path and try again. - """ - - PERM_ERRORS = (errno.EACCES, errno.EPERM, errno.ENOENT) - default_warning_message = "Unable to remove file due to permissions restriction: {!r}" - # split the initial exception out into its type, exception, and traceback - exc_type, exc_exception, exc_tb = exc - if is_readonly_path(path): - # Apply write permission and call original function - set_write_bit(path) - try: - func(path) - except ( # noqa:B014 - OSError, - IOError, - FileNotFoundError, - PermissionError, - ) as e: # pragma: no cover - if e.errno in PERM_ERRORS: - if e.errno == errno.ENOENT: - return - remaining = None - if os.path.isdir(path): - remaining = _wait_for_files(path) - if remaining: - warnings.warn( - default_warning_message.format(path), - ResourceWarning, - stacklevel=2, - ) - else: - func(path, ignore_errors=True) - return - - if exc_exception.errno in PERM_ERRORS: - set_write_bit(path) - remaining = _wait_for_files(path) - try: - func(path) - except (OSError, IOError, FileNotFoundError, PermissionError) as e: # noqa:B014 - if e.errno in PERM_ERRORS: - if e.errno != errno.ENOENT: # File still exists - warnings.warn( - default_warning_message.format(path), - ResourceWarning, - stacklevel=2, - ) - return - else: - raise exc_exception - - -def rmtree( - directory: str, ignore_errors: bool = False, onerror: Optional[Callable] = None -) -> None: - """Stand-in for :func:`~shutil.rmtree` with additional error-handling. - - This version of `rmtree` handles read-only paths, especially in the case of index - files written by certain source control systems. - - :param str directory: The target directory to remove - :param bool ignore_errors: Whether to ignore errors, defaults to False - :param func onerror: An error handling function, defaults to :func:`handle_remove_readonly` - - .. note:: - - Setting `ignore_errors=True` may cause this to silently fail to delete the path - """ - - if onerror is None: - onerror = handle_remove_readonly - try: - shutil.rmtree(directory, ignore_errors=ignore_errors, onerror=onerror) - except (IOError, OSError, FileNotFoundError, PermissionError) as exc: # noqa:B014 - pass - - -def suppress_unparsable(func, *args, **kwargs): - try: - return func(*args, **kwargs) - except Unparsable: - return None - - -class Unparsable(ValueError): - """Not able to parse from setup.py.""" - - -class SetupReader: - """Class that reads a setup.py file without executing it.""" - - @classmethod - def read_setup_py(cls, file: Path, raising: bool = True) -> "Dict[str, Any]": - - with file.open(encoding="utf-8-sig") as f: - content = f.read() - - body = ast.parse(content).body - - setup_call, body = cls._find_setup_call(body) - if not setup_call: - return {} - - if raising: - - def caller(func, *args, **kwargs): - return func(*args, **kwargs) - - else: - caller = suppress_unparsable - - return { - "name": caller(cls._find_single_string, setup_call, body, "name"), - "version": caller(cls._find_single_string, setup_call, body, "version"), - "install_requires": caller(cls._find_install_requires, setup_call, body), - "extras_require": caller(cls._find_extras_require, setup_call, body), - "python_requires": caller( - cls._find_single_string, setup_call, body, "python_requires" - ), - } - - @staticmethod - def read_setup_cfg(file: Path) -> "Dict[str, Any]": - parser = configparser.ConfigParser() - - parser.read(str(file)) - - name = None - version = None - if parser.has_option("metadata", "name"): - name = parser.get("metadata", "name") - - if parser.has_option("metadata", "version"): - version = parser.get("metadata", "version") - - install_requires = [] - extras_require: "Dict[str, List[str]]" = {} - python_requires = None - if parser.has_section("options"): - if parser.has_option("options", "install_requires"): - for dep in parser.get("options", "install_requires").split("\n"): - dep = dep.strip() - if not dep: - continue - - install_requires.append(dep) - - if parser.has_option("options", "python_requires"): - python_requires = parser.get("options", "python_requires") - - if parser.has_section("options.extras_require"): - for group in parser.options("options.extras_require"): - extras_require[group] = [] - deps = parser.get("options.extras_require", group) - for dep in deps.split("\n"): - dep = dep.strip() - if not dep: - continue - - extras_require[group].append(dep) - - return { - "name": name, - "version": version, - "install_requires": install_requires, - "extras_require": extras_require, - "python_requires": python_requires, - } - - @classmethod - def _find_setup_call( - cls, elements: "List[Any]" - ) -> "Tuple[Optional[ast.Call], Optional[List[Any]]]": - funcdefs = [] - for i, element in enumerate(elements): - if isinstance(element, ast.If) and i == len(elements) - 1: - # Checking if the last element is an if statement - # and if it is 'if __name__ == "__main__"' which - # could contain the call to setup() - test = element.test - if not isinstance(test, ast.Compare): - continue - - left = test.left - if not isinstance(left, ast.Name): - continue - - if left.id != "__name__": - continue - - setup_call, body = cls._find_sub_setup_call([element]) - if not setup_call: - continue - - return setup_call, body + elements - if not isinstance(element, ast.Expr): - if isinstance(element, ast.FunctionDef): - funcdefs.append(element) - - continue - - value = element.value - if not isinstance(value, ast.Call): - continue - - func = value.func - if not (isinstance(func, ast.Name) and func.id == "setup") and not ( - isinstance(func, ast.Attribute) - and isinstance(func.value, ast.Name) - and func.value.id == "setuptools" - and func.attr == "setup" - ): - continue - - return value, elements - - # Nothing, we inspect the function definitions - return cls._find_sub_setup_call(funcdefs) - - @classmethod - def _find_sub_setup_call( - cls, elements: "List[Any]" - ) -> "Tuple[Optional[ast.Call], Optional[List[Any]]]": - for element in elements: - if not isinstance(element, (ast.FunctionDef, ast.If)): - continue - - setup_call = cls._find_setup_call(element.body) - if setup_call != (None, None): - setup_call, body = setup_call - - body = elements + body - - return setup_call, body - - return None, None - - @classmethod - def _find_install_requires(cls, call: ast.Call, body: "Iterable[Any]") -> "List[str]": - value = cls._find_in_call(call, "install_requires") - if value is None: - # Trying to find in kwargs - kwargs = cls._find_call_kwargs(call) - - if kwargs is None: - return [] - - if not isinstance(kwargs, ast.Name): - raise Unparsable() - - variable = cls._find_variable_in_body(body, kwargs.id) - if not isinstance(variable, (ast.Dict, ast.Call)): - raise Unparsable() - - if isinstance(variable, ast.Call): - if not isinstance(variable.func, ast.Name): - raise Unparsable() - - if variable.func.id != "dict": - raise Unparsable() - - value = cls._find_in_call(variable, "install_requires") - else: - value = cls._find_in_dict(variable, "install_requires") - - if value is None: - return [] - - if isinstance(value, ast.List): - return [el.s for el in value.elts] - elif isinstance(value, ast.Name): - variable = cls._find_variable_in_body(body, value.id) - - if variable is not None and isinstance(variable, ast.List): - return [el.s for el in variable.elts] - - raise Unparsable() - - @classmethod - def _find_extras_require( - cls, call: ast.Call, body: "Iterable[Any]" - ) -> "Dict[str, List[str]]": - extras_require: "Dict[str, List[str]]" = {} - value = cls._find_in_call(call, "extras_require") - if value is None: - # Trying to find in kwargs - kwargs = cls._find_call_kwargs(call) - - if kwargs is None: - return extras_require - - if not isinstance(kwargs, ast.Name): - raise Unparsable() - - variable = cls._find_variable_in_body(body, kwargs.id) - if not isinstance(variable, (ast.Dict, ast.Call)): - raise Unparsable() - - if isinstance(variable, ast.Call): - if not isinstance(variable.func, ast.Name): - raise Unparsable() - - if variable.func.id != "dict": - raise Unparsable() - - value = cls._find_in_call(variable, "extras_require") - else: - value = cls._find_in_dict(variable, "extras_require") - - if value is None: - return extras_require - - if isinstance(value, ast.Dict): - for key, val in zip(value.keys, value.values): - if isinstance(val, ast.Name): - val = cls._find_variable_in_body(body, val.id) - - if isinstance(val, ast.List): - extras_require[key.s] = [e.s for e in val.elts] - else: - raise Unparsable() - elif isinstance(value, ast.Name): - variable = cls._find_variable_in_body(body, value.id) - - if variable is None or not isinstance(variable, ast.Dict): - raise Unparsable() - - for key, val in zip(variable.keys, variable.values): - if isinstance(val, ast.Name): - val = cls._find_variable_in_body(body, val.id) - - if isinstance(val, ast.List): - extras_require[key.s] = [e.s for e in val.elts] - else: - raise Unparsable() - else: - raise Unparsable() - - return extras_require - - @classmethod - def _find_single_string( - cls, call: ast.Call, body: "List[Any]", name: str - ) -> "Optional[str]": - value = cls._find_in_call(call, name) - if value is None: - # Trying to find in kwargs - kwargs = cls._find_call_kwargs(call) - if kwargs is None: - return None - - if not isinstance(kwargs, ast.Name): - raise Unparsable() - - variable = cls._find_variable_in_body(body, kwargs.id) - if not isinstance(variable, (ast.Dict, ast.Call)): - raise Unparsable() - - if isinstance(variable, ast.Call): - if not isinstance(variable.func, ast.Name): - raise Unparsable() - - if variable.func.id != "dict": - raise Unparsable() - - value = cls._find_in_call(variable, name) - else: - value = cls._find_in_dict(variable, name) - - if value is None: - return None - - if isinstance(value, ast.Str): - return value.s - elif isinstance(value, ast.Name): - variable = cls._find_variable_in_body(body, value.id) - - if variable is not None and isinstance(variable, ast.Str): - return variable.s - - raise Unparsable() - - @staticmethod - def _find_in_call(call: ast.Call, name: str) -> "Optional[Any]": - for keyword in call.keywords: - if keyword.arg == name: - return keyword.value - return None - - @staticmethod - def _find_call_kwargs(call: ast.Call) -> "Optional[Any]": - kwargs = None - for keyword in call.keywords: - if keyword.arg is None: - kwargs = keyword.value - - return kwargs - - @staticmethod - def _find_variable_in_body(body: "Iterable[Any]", name: str) -> "Optional[Any]": - for elem in body: - - if not isinstance(elem, (ast.Assign, ast.AnnAssign)): - continue - - if isinstance(elem, ast.AnnAssign): - if not isinstance(elem.target, ast.Name): - continue - if elem.value and elem.target.id == name: - return elem.value - else: - for target in elem.targets: - if not isinstance(target, ast.Name): - continue - if target.id == name: - return elem.value - return None - - @staticmethod - def _find_in_dict(dict_: ast.Dict, name: str) -> "Optional[Any]": - for key, val in zip(dict_.keys, dict_.values): - if isinstance(key, ast.Str) and key.s == name: - return val - return None - - -def setuptools_parse_setup_cfg(path): - try: - # v61.0.0 of setuptools deprecated setuptools.config.read_configuration - from setuptools.config.setupcfg import read_configuration - except ImportError: - from setuptools.config import read_configuration - - parsed = read_configuration(path) - results = parsed.get("metadata", {}) - results.update(parsed.get("options", {})) - if "install_requires" in results: - results["install_requires"] = make_base_requirements( - results.get("install_requires", []) - ) - if "extras_require" in results: - extras = {} - for extras_section, extras_reqs in results.get("extras_require", {}).items(): - new_reqs = tuple(make_base_requirements(extras_reqs)) - if new_reqs: - extras[extras_section] = new_reqs - results["extras_require"] = extras - if "setup_requires" in results: - results["setup_requires"] = make_base_requirements( - results.get("setup_requires", []) - ) - return results - - -def parse_setup_cfg(path: str) -> "Dict[str, Any]": - return SetupReader.read_setup_cfg(Path(path)) - - -def build_pep517(source_dir, build_dir, config_settings=None, dist_type="wheel"): - if config_settings is None: - config_settings = {} - result = get_pyproject(source_dir) or {} - hookcaller = HookCaller(source_dir, result.get("build_backend", get_default_pyproject_backend())) - if dist_type == "sdist": - get_requires_fn = hookcaller.get_requires_for_build_sdist - build_fn = hookcaller.build_sdist - else: - get_requires_fn = hookcaller.get_requires_for_build_wheel - build_fn = hookcaller.build_wheel - - with BuildEnv() as env: - env.pip_install(result.get("build_requires", [])) - reqs = get_requires_fn(config_settings) - env.pip_install(reqs) - return build_fn(build_dir, config_settings) - - -def _get_src_dir(root): - # type: (AnyStr) -> AnyStr - src = os.environ.get("PIP_SRC") - if src: - return src - virtual_env = os.environ.get("VIRTUAL_ENV") - if virtual_env is not None: - return os.path.join(virtual_env, "src") - if root is not None: - # Intentionally don't match pip's behavior here -- this is a temporary copy - src_dir = create_tracked_tempdir(prefix="requirementslib-", suffix="-src") - else: - src_dir = os.path.join(root, "src") - - os.makedirs(src_dir, mode=0o775) - return src_dir - - -@lru_cache() -def ensure_reqs(reqs): - # type: (List[Union[Requirement]]) -> List[Requirement] - - if not isinstance(reqs, Iterable): - raise TypeError("Expecting an Iterable, got %r" % reqs) - new_reqs = [] - for req in reqs: - if not req: - continue - if isinstance(req, str): - req = Requirement.parse("{0}".format(str(req))) - # req = strip_extras_markers_from_requirement(req) - new_reqs.append(req) - return new_reqs - - -def any_valid_values(data: "Dict[str, Any]", fields: "Iterable[str]") -> bool: - def is_valid(value: "Any") -> bool: - if isinstance(value, (list, tuple)): - return all(map(is_valid, value)) - elif isinstance(value, dict): - return all(map(is_valid, value.values())) - return isinstance(value, str) - - fields = [field for field in fields if field in data] - return fields and all(is_valid(data[field]) for field in fields) - - -def _prepare_wheel_building_kwargs( - ireq=None, # type: Optional[InstallRequirement] - src_root=None, # type: Optional[str] - src_dir=None, # type: Optional[str] - editable=False, # type: bool -): - # type: (...) -> Dict[str, str] - download_dir = os.path.join(CACHE_DIR, "pkgs") # type: str - os.makedirs(download_dir, exist_ok=True) - - wheel_download_dir = os.path.join(CACHE_DIR, "wheels") # type: str - os.makedirs(wheel_download_dir, exist_ok=True) - if src_dir is None: - if editable and src_root is not None: - src_dir = src_root - elif src_root is not None: - src_dir = _get_src_dir(root=src_root) # type: str - else: - src_dir = create_tracked_tempdir(prefix="reqlib-src") - - # Let's always resolve in isolation - if src_dir is None: - src_dir = create_tracked_tempdir(prefix="reqlib-src") - build_dir = create_tracked_tempdir(prefix="reqlib-build") - - return { - "build_dir": build_dir, - "src_dir": src_dir, - "download_dir": download_dir, - "wheel_download_dir": wheel_download_dir, - } - - -class ScandirCloser(object): - def __init__(self, path): - self.iterator = scandir(path) - - def __next__(self): - return next(iter(self.iterator)) - - def __iter__(self): - return self - - def next(self): - return self.__next__() - - def close(self): - if getattr(self.iterator, "close", None): - self.iterator.close() - else: - pass - - -def _is_venv_dir(path): - # type: (AnyStr) -> bool - if os.name == "nt": - return os.path.isfile(os.path.join(path, "Scripts/python.exe")) or os.path.isfile( - os.path.join(path, "Scripts/activate") - ) - else: - return os.path.isfile(os.path.join(path, "bin/python")) or os.path.isfile( - os.path.join(path, "bin/activate") - ) - - -def iter_metadata(path, pkg_name=None, metadata_type="egg-info") -> Generator: - if pkg_name is not None: - pkg_variants = get_name_variants(pkg_name) - dirs_to_search = [path] - while dirs_to_search: - p = dirs_to_search.pop(0) - # Skip when the directory is like a venv - if _is_venv_dir(p): - continue - with contextlib.closing(ScandirCloser(p)) as path_iterator: - for entry in path_iterator: - if entry.is_dir(): - entry_name, ext = os.path.splitext(entry.name) - if ext.endswith(metadata_type): - if pkg_name is None or entry_name.lower() in pkg_variants: - yield entry - elif not entry.name.endswith(metadata_type): - dirs_to_search.append(entry.path) - - -def find_egginfo(target, pkg_name=None): - # type: (AnyStr, Optional[AnyStr]) -> Generator - egg_dirs = ( - egg_dir - for egg_dir in iter_metadata(target, pkg_name=pkg_name) - if egg_dir is not None - ) - if pkg_name: - yield next(iter(eggdir for eggdir in egg_dirs if eggdir is not None), None) - else: - for egg_dir in egg_dirs: - yield egg_dir - - -def find_distinfo(target, pkg_name=None): - # type: (AnyStr, Optional[AnyStr]) -> Generator - dist_dirs = ( - dist_dir - for dist_dir in iter_metadata( - target, pkg_name=pkg_name, metadata_type="dist-info" - ) - if dist_dir is not None - ) - if pkg_name: - yield next(iter(dist for dist in dist_dirs if dist is not None), None) - else: - for dist_dir in dist_dirs: - yield dist_dir - - -def get_distinfo_dist(path, pkg_name=None) -> Optional[DistInfoDistribution]: - dist_dir = next(iter(find_distinfo(path, pkg_name=pkg_name)), None) - if dist_dir is not None: - metadata_dir = dist_dir.path - base_dir = os.path.dirname(metadata_dir) - dist = next(iter(find_distributions(base_dir)), None) - if dist is not None: - return dist - return None - - -def get_egginfo_dist(path, pkg_name=None) -> Optional[EggInfoDistribution]: - egg_dir = next(iter(find_egginfo(path, pkg_name=pkg_name)), None) - if egg_dir is not None: - metadata_dir = egg_dir.path - base_dir = os.path.dirname(metadata_dir) - path_metadata = PathMetadata(base_dir, metadata_dir) - dist_iter = distributions_from_metadata(path_metadata.egg_info) - dist = next(iter(dist_iter), None) - if dist is not None: - return dist - return None - - -def get_metadata(path, pkg_name=None, metadata_type=None): - wheel_allowed = metadata_type == "wheel" or metadata_type is None - egg_allowed = metadata_type == "egg" or metadata_type is None - dist = None # type: Optional[Union[DistInfoDistribution, EggInfoDistribution]] - dist = get_distinfo_dist(path, pkg_name=pkg_name) - if dist is None: - dist = get_egginfo_dist(path, pkg_name=pkg_name) - if dist is not None: - return get_metadata_from_dist(dist) - return {} - - -def get_extra_name_from_marker(marker): - if not marker: - raise ValueError("Invalid value for marker: {0!r}".format(marker)) - if not getattr(marker, "_markers", None): - raise TypeError("Expecting a marker instance, received {0!r}".format(marker)) - for elem in marker._markers: - if isinstance(elem, tuple) and elem[0].value == "extra": - return elem[2].value - return None - - -def get_metadata_from_wheel(wheel_path) -> Dict[Any, Any]: - if not isinstance(wheel_path, str): - raise TypeError("Expected string instance, received {0!r}".format(wheel_path)) - try: - dist = Wheel(wheel_path) - except Exception: - pass - metadata = dist.metadata - name = metadata.name - version = metadata.version - requires = [] - extras_keys = getattr(metadata, "extras", []) # type: List[str] - extras = {k: [] for k in extras_keys} # type: Dict[str, List[PackagingRequirement]] - for req in getattr(metadata, "run_requires", []): - parsed_req = init_requirement(req) - parsed_marker = parsed_req.marker - if parsed_marker: - extra = get_extra_name_from_marker(parsed_marker) - if extra is None: - requires.append(parsed_req) - continue - if extra not in extras: - extras[extra] = [] - parsed_req = strip_extras_markers_from_requirement(parsed_req) - extras[extra].append(parsed_req) - else: - requires.append(parsed_req) - return {"name": name, "version": version, "requires": requires, "extras": extras} - - -def get_metadata_from_dist(dist): - try: - requires = dist.requires() - except Exception: - requires = [] - try: - dep_map = dist._build_dep_map() - except Exception: - dep_map = {} - deps = [] # type: List[Requirement] - extras = {} - for k in dep_map.keys(): - if k is None: - deps.extend(dep_map.get(k)) - continue - else: - extra = None - _deps = dep_map.get(k) - if k.startswith(":python_version"): - marker = k.replace(":", "; ") - else: - if ":python_version" in k: - extra, _, marker = k.partition(":") - marker = "; {0}".format(marker) - else: - marker = "" - extra = "{0}".format(k) - _deps = ensure_reqs( - tuple(["{0}{1}".format(str(req), marker) for req in _deps]) - ) - if extra: - extras[extra] = _deps - else: - deps.extend(_deps) - requires.extend(deps) - return { - "name": dist.project_name, - "version": dist.version, - "requires": requires, - "extras": extras, - } - - -def ast_parse_setup_py(path: str, raising: bool = True) -> "Dict[str, Any]": - return SetupReader.read_setup_py(Path(path), raising) - - -def run_setup(script_path, egg_base=None): - """Run a `setup.py` script with a target **egg_base** if provided. - - :param script_path: The path to the `setup.py` script to run - :param Optional egg_base: The metadata directory to build in - :raises FileNotFoundError: If the provided `script_path` does not exist - :return: The metadata dictionary - :rtype: Dict[Any, Any] - """ - if not os.path.exists(script_path): - raise FileNotFoundError(script_path) - target_cwd = os.path.dirname(os.path.abspath(script_path)) - if egg_base is None: - egg_base = os.path.join(target_cwd, "reqlib-metadata") - with temp_path(), cd(target_cwd): - args = ["egg_info"] - if egg_base: - args += ["--egg-base", egg_base] - - python = os.environ.get("PIP_PYTHON_PATH", sys.executable) - sp.run( - [python, "setup.py"] + args, - capture_output=True, - ) - dist = get_metadata(egg_base, metadata_type="egg") - return dist - - -class BaseRequirement(ReqLibBaseModel): - name: str = "" - requirement: Optional[HashableRequirement] = None - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __setattr__(self, name, value): - if name == "requirement" and isinstance(value, Requirement): - value = convert_to_hashable_requirement(value) - super().__setattr__(name, value) - - def __str__(self) -> str: - return "{0}".format(str(self.requirement)) - - def __hash__(self): - return hash((self.name, str(self.requirement))) - - def as_dict(self) -> Dict[str, Optional[Requirement]]: - return {self.name: self.requirement} - - def as_tuple(self) -> Tuple[str, Optional[Requirement]]: - return (self.name, self.requirement) - - @classmethod - @lru_cache() - def from_string(cls, line: str) -> "HashableRequirement": - line = line.strip() - req = init_requirement(line) - return cls.from_req(req) - - @classmethod - @lru_cache() - def from_req(cls, req: Requirement) -> "HashableRequirement": - name = None - key = getattr(req, "key", None) - name = getattr(req, "name", None) - project_name = getattr(req, "project_name", None) - if key is not None: - name = key - if name is None: - name = project_name - hashable_req = convert_to_hashable_requirement(req) - return cls(name=name, requirement=hashable_req) - - -class SetupInfo(ReqLibBaseModel): - name: Optional[str] = None - base_dir: Optional[str] = None - _version: Optional[str] = None - _requirements: Optional[Tuple] = None - build_requires: Optional[Tuple] = None - build_backend: Optional[str] = None - setup_requires: Optional[Tuple] = None - python_requires: Optional[SpecifierSet] = None - _extras_requirements: Optional[Tuple] = None - setup_cfg: Optional[Path] = None - setup_py: Optional[Path] = None - pyproject: Optional[Path] = None - ireq: Optional[InstallRequirement] = None - extra_kwargs: Optional[Dict] = Field(default_factory=dict) - metadata: Optional[Tuple[str]] = None - _is_built: bool = False - _ran_setup: bool = False - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __init__(self, **data): - super().__init__(**data) - self._is_built = False - self._ran_setup = False - if not self.build_backend: - self.build_backend = "setuptools.build_meta:__legacy__" - if self._requirements is None: - self._requirements = () - self.get_initial_info() - self.get_info() - - def __hash__(self): - return hash( - ( - self.name, - self._version, - self._requirements, - self.build_requires, - self.build_backend, - self.setup_requires, - self.python_requires, - self._extras_requirements, - self.setup_cfg, - self.setup_py, - self.pyproject, - self.ireq, - ) - ) - - def __eq__(self, other): - if not isinstance(other, SetupInfo): - return NotImplemented - return ( - self.name == other.name - and self._version == other._version - and self._requirements == other._requirements - and self.build_requires == other.build_requires - ) - - @cached_property - def requires(self) -> Dict[str, HashableRequirement]: - if self._requirements is None: - self._requirements = () - return {req.name: req.requirement for req in self._requirements} - - @cached_property - def extras(self) -> Dict[str, Optional[Any]]: - if self._extras_requirements is None: - self._extras_requirements = () - extras_dict = {} - extras = set(self._extras_requirements) - for section, deps in extras: - if isinstance(deps, BaseRequirement): - extras_dict[section] = deps.requirement - elif isinstance(deps, (list, tuple)): - extras_dict[section] = [d.requirement for d in deps] - return extras_dict - - @property - def version(self) -> Optional[str]: - if not self._version: - info = self.as_dict() - self._version = info.get("version", None) - return self._version - - @property - def egg_base(self) -> str: - base = None # type: Optional[str] - if self.setup_py.exists(): - base = self.setup_py.parent - elif self.pyproject.exists(): - base = self.pyproject.parent - elif self.setup_cfg.exists(): - base = self.setup_cfg.parent - if base is None: - base = Path(self.base_dir) - if base is None: - base = Path(self.extra_kwargs["src_dir"]) - egg_base = base.joinpath("reqlib-metadata") - if not egg_base.exists(): - atexit.register(rmtree, egg_base.as_posix()) - egg_base.mkdir(parents=True, exist_ok=True) - return egg_base.as_posix() - - def update_from_dict(self, metadata: Dict[str, Any]) -> None: - name = metadata.get("name", self.name) - if isinstance(name, str): - self.name = self.name if self.name else name - version = metadata.get("version", None) - if version: - try: - parse(version) - except TypeError: - version = self.version if self.version else None - else: - version = version - if version: - self._version = version - build_requires = metadata.get("build_requires", []) - if self.build_requires is None: - self.build_requires = () - self.build_requires = tuple(set(self.build_requires) | set(build_requires)) - self._requirements = () if self._requirements is None else self._requirements - requirements = self._requirements - install_requires = make_base_requirements(metadata.get("install_requires", [])) - requirements += install_requires - setup_requires = make_base_requirements(metadata.get("setup_requires", [])) - if self.setup_requires is None: - self.setup_requires = () - self.setup_requires = tuple(self.setup_requires + setup_requires) - requirements += self.setup_requires - self.python_requires = metadata.get("python_requires", self.python_requires) - extras_require = metadata.get("extras_require", {}) - extras_tuples = [] - if self._extras_requirements is None: - self._extras_requirements = () - for section in set(extras_require) - {v[0] for v in self._extras_requirements}: - extras = extras_require[section] - extras_set = make_base_requirements(extras) - if self.ireq and self.ireq.extras and section in self.ireq.extras: - requirements += extras_set - extras_tuples.append((section, tuple(extras_set))) - self._extras_requirements += tuple(extras_tuples) - self.build_backend = metadata.get( - "build_backend", "setuptools.build_meta:__legacy__" - ) - self._requirements = requirements - - def get_extras_from_ireq(self) -> None: - if self.ireq and self.ireq.extras: - for extra in self.ireq.extras: - if extra in self.extras: - extras = make_base_requirements(self.extras[extra]) - self._requirements = self._requirements + extras - else: - extras = tuple(make_base_requirements(extra)) - self._extras_requirements += (extra, extras) - - def parse_setup_cfg(self) -> Dict[str, Any]: - if self.setup_cfg is not None and self.setup_cfg.exists(): - try: - parsed = setuptools_parse_setup_cfg(self.setup_cfg.as_posix()) - except Exception: - parsed = parse_setup_cfg(self.setup_cfg.as_posix()) - if not parsed: - return {} - return parsed - return {} - - def parse_setup_py(self) -> Dict[str, Any]: - if self.setup_py is not None and self.setup_py.exists(): - parsed = ast_parse_setup_py(self.setup_py.as_posix()) - if not parsed: - return {} - return parsed - return {} - - def run_setup(self) -> None: - if not self._ran_setup and self.setup_py is not None and self.setup_py.exists(): - dist = run_setup(self.setup_py.as_posix(), egg_base=self.egg_base) - target_cwd = self.setup_py.parent.as_posix() - with temp_path(), cd(target_cwd): - if not dist: - metadata = self.get_egg_metadata() - if metadata: - self.populate_metadata(metadata) - elif isinstance(dist, Mapping): - self.populate_metadata(dist) - self._ran_setup = True - - def build_wheel(self) -> str: - need_delete = False - if not self.pyproject.exists(): - if not self.build_requires: - build_requires = '"setuptools", "wheel"' - else: - build_requires = ", ".join( - ['"{0}"'.format(r) for r in self.build_requires] - ) - self.pyproject.write_text( - str( - """ -[build-system] -requires = [{0}] -build-backend = "{1}" - """.format( - build_requires, self.build_backend - ).strip() - ) - ) - need_delete = True - directory = self.base_dir - if self.ireq and self.ireq.link: - parsed = urlparse(str(self.ireq.link)) - subdir = parse_qs(parsed.fragment).get("subdirectory", []) - if subdir: - directory = f"{self.base_dir}/{subdir[0]}" - result = build_pep517( - directory, - self.extra_kwargs["build_dir"], - dist_type="wheel", - ) - if need_delete: - self.pyproject.unlink() - return result - - # noinspection PyPackageRequirements - def build_sdist(self) -> str: - need_delete = False - if not self.pyproject.exists(): - if not self.build_requires: - build_requires = '"setuptools", "wheel"' - else: - build_requires = ", ".join( - ['"{0}"'.format(r) for r in self.build_requires] - ) - self.pyproject.write_text( - str( - """ -[build-system] -requires = [{0}] -build-backend = "{1}" - """.format( - build_requires, self.build_backend - ).strip() - ) - ) - need_delete = True - result = build_pep517( - self.base_dir, - self.extra_kwargs["build_dir"], - dist_type="sdist", - ) - if need_delete: - self.pyproject.unlink() - return result - - def build(self) -> None: - if self._is_built: - return - metadata = None - try: - dist_path = self.build_wheel() - metadata = self.get_metadata_from_wheel( - os.path.join(self.extra_kwargs["build_dir"], dist_path) - ) - except Exception: - try: - dist_path = self.build_sdist() - metadata = self.get_egg_metadata(metadata_type="egg") - if metadata: - self.populate_metadata(metadata) - except Exception: - pass - if metadata: - self.populate_metadata(metadata) - if not self.metadata or not self.name: - metadata = self.get_egg_metadata() - if metadata: - self.populate_metadata(metadata) - if not self.metadata or not self.name: - self.run_setup() - self._is_built = True - - def get_metadata_from_wheel(self, wheel_path) -> Dict[Any, Any]: - """Given a path to a wheel, return the metadata from that wheel. - - :return: A dictionary of metadata from the provided wheel - :rtype: Dict[Any, Any] - """ - - metadata_dict = get_metadata_from_wheel(wheel_path) - return metadata_dict - - def get_egg_metadata(self, metadata_dir=None, metadata_type=None) -> Dict[Any, Any]: - """Given a metadata directory, return the corresponding metadata - dictionary. - - :param Optional[str] metadata_dir: Root metadata path, default: `os.getcwd()` - :param Optional[str] metadata_type: Type of metadata to search for, default None - :return: A metadata dictionary built from the metadata in the given location - """ - - package_indicators = [self.pyproject, self.setup_py, self.setup_cfg] - metadata_dirs = [] # type: List[str] - if any([fn is not None and fn.exists() for fn in package_indicators]): - metadata_dirs = [ - self.extra_kwargs["build_dir"], - self.egg_base, - self.extra_kwargs["src_dir"], - ] - if metadata_dir is not None: - metadata_dirs = [metadata_dir] + metadata_dirs - metadata = [ - get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) - for d in metadata_dirs - if os.path.exists(d) - ] - metadata = next(iter(d for d in metadata if d), None) - return metadata - - def populate_metadata(self, metadata) -> "SetupInfo": - """Populates the metadata dictionary from the supplied metadata.""" - - _metadata = () - for k, v in metadata.items(): - if k == "extras" and isinstance(v, dict): - extras = () - for extra, reqs in v.items(): - extras += ((extra, tuple(reqs)),) - _metadata += extras - elif isinstance(v, (list, tuple)): - _metadata += (k, tuple(v)) - else: - _metadata += (k, v) - self.metadata = _metadata - self.setup_requires = make_base_requirements(metadata.get("requires", ())) - self._requirements += tuple(self.setup_requires) - name = metadata.get("name") - if name: - self.name = name - version = metadata.get("version") - if version: - self._version = version - extras_require = metadata.get("extras", ()) - extras_tuples = [] - for section in set(extras_require): - extras = extras_require[section] - extras_set = make_base_requirements(extras) - if self.ireq and self.ireq.extras and section in self.ireq.extras: - self._requirements += extras_set - extras_tuples.append((section, tuple(extras_set))) - return self - - def run_pyproject(self) -> "SetupInfo": - """Populates the **pyproject.toml** metadata if available.""" - if self.pyproject and self.pyproject.exists(): - result = get_pyproject(self.pyproject.parent) - if result is not None: - if self.build_requires is None: - self.build_requires = () - if result.get("build_backend"): - self.build_backend = result.get("build_backend") - else: - self.build_backend = get_default_pyproject_backend() - if result.get("build_requires"): - self.build_requires = tuple(set(result.get("build_requires", [])) | set(self.build_requires)) - else: - self.build_requires = ("setuptools", "wheel") - if result.get("dependencies"): - self._requirements += make_base_requirements(tuple(set(result.get("dependencies", [])))) - return self - - def get_initial_info(self) -> Dict[str, Any]: - parse_setupcfg = False - parse_setuppy = False - self.run_pyproject() - self.run_setup() - if self.setup_cfg and self.setup_cfg.exists(): - parse_setupcfg = True - if self.setup_py and self.setup_py.exists(): - parse_setuppy = True - if ( - self.build_backend.startswith("setuptools") - and parse_setuppy - or parse_setupcfg - ): - parsed = {} - try: - with cd(self.base_dir): - if parse_setuppy: - parsed.update(self.parse_setup_py()) - if parse_setupcfg: - parsed.update(self.parse_setup_cfg()) - except Unparsable: - pass - else: - self.update_from_dict(parsed) - return self.as_dict() - - return self.as_dict() - - def get_info(self) -> None: - if self.metadata is None: - self.build() - - if self.setup_py and self.setup_py.exists(): - try: - self.run_setup() - except Exception: - metadata = self.get_egg_metadata() - if metadata: - self.populate_metadata(metadata) - if self.metadata is None or not self.name: - metadata = self.get_egg_metadata() - if metadata: - self.populate_metadata(metadata) - - def as_dict(self) -> Dict[str, Any]: - prop_dict = { - "name": self.name, - "version": self.version if self._version else None, - "base_dir": self.base_dir, - "ireq": self.ireq, - "build_backend": self.build_backend, - "build_requires": self.build_requires, - "requires": self.requires, - "setup_requires": self.setup_requires, - "python_requires": self.python_requires, - "extras": self.extras, - "extra_kwargs": self.extra_kwargs, - "setup_cfg": self.setup_cfg, - "setup_py": self.setup_py, - "pyproject": self.pyproject, - } - return {k: v for k, v in prop_dict.items() if v} - - @classmethod - def from_requirement(cls, requirement, finder=None) -> Optional["SetupInfo"]: - ireq = requirement.ireq - subdir = getattr(requirement.req, "subdirectory", None) - return cls.from_ireq(ireq, subdir=subdir, finder=finder) - - @classmethod - def from_ireq( - cls, ireq, subdir=None, finder=None, session=None - ) -> Optional["SetupInfo"]: - if not ireq: - return None - if not ireq.link: - return None - if ireq.link.is_wheel: - return None - if not session: - cmd = get_pip_command() - options, _ = cmd.parser.parse_args([]) - session = cmd._build_session(options) - vcs, uri = split_vcs_method_from_uri(ireq.link.url_without_fragment) - parsed = urlparse(uri) - if "file" in parsed.scheme: - url_path = parsed.path - if "@" in url_path: - url_path, _, _ = url_path.rpartition("@") - parsed = parsed._replace(path=url_path) - uri = urlunparse(parsed) - is_file = False - if ireq.link.scheme == "file" or uri.startswith("file://"): - is_file = True - kwargs = _prepare_wheel_building_kwargs(ireq) - is_artifact_or_vcs = getattr( - ireq.link, "is_vcs", getattr(ireq.link, "is_artifact", False) - ) - is_vcs = True if vcs else is_artifact_or_vcs - download_dir = None - if not (ireq.editable and is_file and is_vcs): - if ireq.is_wheel: - download_dir = kwargs["wheel_download_dir"] - else: - download_dir = kwargs["download_dir"] - if not ireq.source_dir: - if subdir: - directory = f"{kwargs['build_dir']}/{subdir}" - else: - directory = kwargs["build_dir"] - ensure_build_location(ireq, build_dir=directory, autodelete=False, parallel_builds=True) - if ireq.source_dir is None: - ireq.source_dir = ensure_build_location(ireq, build_dir=kwargs["src_dir"], autodelete=False, parallel_builds=True) - location = None - if ireq.source_dir: - location = ireq.source_dir - - if ireq.link.is_existing_dir(): - if os.path.isdir(location): - rmtree(location) - _copy_source_tree(ireq.link.file_path, location) - else: - unpack_url( - link=ireq.link, - location=location, - download=Downloader(session, "off"), - verbosity=1, - download_dir=download_dir, - hashes=ireq.hashes(True), - ) - created = cls.create( - ireq.source_dir, - subdirectory=subdir, - ireq=ireq, - kwargs=kwargs, - ) - return created - - @classmethod - def create( - cls, - base_dir: str, - subdirectory: Optional[str] = None, - ireq: Optional[InstallRequirement] = None, - kwargs: Optional[Dict[str, str]] = None, - ) -> Optional["SetupInfo"]: - if not base_dir or base_dir is None: - return None - - creation_kwargs = {"extra_kwargs": kwargs} - if not isinstance(base_dir, Path): - base_dir = Path(base_dir) - creation_kwargs["base_dir"] = base_dir.as_posix() - pyproject = base_dir.joinpath("pyproject.toml") - - if subdirectory is not None: - base_dir = base_dir.joinpath(subdirectory) - setup_py = base_dir.joinpath("setup.py") - setup_cfg = base_dir.joinpath("setup.cfg") - creation_kwargs["pyproject"] = pyproject - creation_kwargs["setup_py"] = setup_py - creation_kwargs["setup_cfg"] = setup_cfg - if ireq: - creation_kwargs["ireq"] = ireq - created = cls(**creation_kwargs) - created.get_initial_info() - return created - - -def ensure_build_location( - ireq, build_dir: str, autodelete: bool, parallel_builds: bool -) -> str: - assert build_dir is not None - if ireq._temp_build_dir is not None: - assert ireq._temp_build_dir.path - return ireq._temp_build_dir.path - if ireq.req is None: - # Some systems have /tmp as a symlink which confuses custom - # builds (such as numpy). Thus, we ensure that the real path - # is returned. - ireq._temp_build_dir = TempDirectory( - kind=tempdir_kinds.REQ_BUILD, globally_managed=False - ) - - return ireq._temp_build_dir.path - - # This is the only remaining place where we manually determine the path - # for the temporary directory. It is only needed for editables where - # it is the value of the --src option. - - # When parallel builds are enabled, add a UUID to the build directory - # name so multiple builds do not interfere with each other. - dir_name: str = canonicalize_name(ireq.name) - if parallel_builds: - dir_name = f"{dir_name}_{uuid.uuid4().hex}" - - if not os.path.exists(build_dir): - os.makedirs(build_dir) - actual_build_dir = os.path.join(build_dir, dir_name) - # `None` indicates that we respect the globally-configured deletion - # settings, which is what we actually want when auto-deleting. - delete_arg = None if autodelete else False - return TempDirectory( - path=actual_build_dir, - delete=delete_arg, - kind=tempdir_kinds.REQ_BUILD, - globally_managed=False, - ).path - - -def get_http_url( - link: Link, - download: Downloader, - download_dir: Optional[str] = None, - hashes: Optional[Hashes] = None, -) -> File: - temp_dir = TempDirectory(kind="unpack", globally_managed=False) - # If a download dir is specified, is the file already downloaded there? - already_downloaded_path = None - if download_dir: - already_downloaded_path = _check_download_dir(link, download_dir, hashes) - - if already_downloaded_path: - from_path = already_downloaded_path - content_type = None - else: - # let's download to a tmp dir - from_path, content_type = download(link, temp_dir.path) - if hashes: - hashes.check_against_path(from_path) - - return File(from_path, content_type) - - -def unpack_url( - link: Link, - location: str, - download: Downloader, - verbosity: int, - download_dir: Optional[str] = None, - hashes: Optional[Hashes] = None, -) -> Optional[File]: - """Unpack link into location, downloading if required. - - :param hashes: A Hashes object, one of whose embedded hashes must match, - or HashMismatch will be raised. If the Hashes is empty, no matches are - required, and unhashable types of requirements (like VCS ones, which - would ordinarily raise HashUnsupported) are allowed. - """ - # non-editable vcs urls - if link.is_vcs: - unpack_vcs_link(link, location, verbosity=verbosity) - return None - - assert not link.is_existing_dir() - - # file urls - if link.is_file: - file = get_file_url(link, download_dir, hashes=hashes) - - # http urls - else: - file = get_http_url( - link, - download, - download_dir, - hashes=hashes, - ) - - # unpack the archive to the build dir location. even when only downloading - # archives, they have to be unpacked to parse dependencies, except wheels - if not link.is_wheel: - unpack_file(file.path, location, file.content_type) - - return file diff --git a/pipenv/vendor/requirementslib/models/url.py b/pipenv/vendor/requirementslib/models/url.py deleted file mode 100644 index 14d7793c..00000000 --- a/pipenv/vendor/requirementslib/models/url.py +++ /dev/null @@ -1,484 +0,0 @@ -from typing import Dict, Optional, Text, Tuple, TypeVar, Union -from urllib.parse import quote -from urllib.parse import unquote as url_unquote -from urllib.parse import unquote_plus - -from pipenv.patched.pip._internal.models.link import Link -from pipenv.patched.pip._internal.req.constructors import _strip_extras -from pipenv.patched.pip._vendor.urllib3.util import parse_url as urllib3_parse -from pipenv.patched.pip._vendor.urllib3.util.url import Url -from pipenv.vendor.pydantic import Field - -from ..environment import MYPY_RUNNING -from ..utils import is_installable_file -from .common import ReqLibBaseModel -from .utils import DIRECT_URL_RE, extras_to_string, parse_extras_str, split_ref_from_uri - -if MYPY_RUNNING: - - _T = TypeVar("_T") - STRING_TYPE = Union[bytes, str, Text] - S = TypeVar("S", bytes, str, Text) - - -def _get_parsed_url(url) -> Url: - """This is a stand-in function for `urllib3.util.parse_url` - - The original function doesn't handle special characters very well, this simply splits - out the authentication section, creates the parsed url, then puts the authentication - section back in, bypassing validation. - - :return: The new, parsed URL object - :rtype: :class:`~urllib3.util.url.Url` - """ - - try: - parsed = urllib3_parse(url) - except ValueError: - scheme, _, url = url.partition("://") - auth, _, url = url.rpartition("@") - url = "{scheme}://{url}".format(scheme=scheme, url=url) - parsed = urllib3_parse(url)._replace(auth=auth) - if parsed.auth: - return parsed._replace(auth=url_unquote(parsed.auth)) - return parsed - - -class URI(ReqLibBaseModel): - host: Optional[str] = Field(...) - scheme: Optional[str] = Field( - "https", description="The URI Scheme, e.g. `salesforce`" - ) - port: Optional[int] = Field( - None, description="The numeric port of the url if specified" - ) - path: Optional[str] = Field("", description="The url path, e.g. `/path/to/endpoint`") - query: Optional[str] = Field( - "", description="Query parameters, e.g. `?variable=value...`" - ) - fragment: Optional[str] = Field( - "", description="URL Fragments, e.g. `#fragment=value`" - ) - subdirectory: Optional[str] = Field( - "", description="Subdirectory fragment, e.g. `&subdirectory=blah...`" - ) - ref: Optional[str] = Field("", description="VCS ref this URI points at, if available") - username: Optional[str] = Field( - "", description="The username if provided, parsed from `user:password@hostname`" - ) - password: Optional[str] = Field( - "", description="Password parsed from `user:password@hostname`", repr=False - ) - query_dict: Optional[Dict] = Field(default_factory=dict) - name: Optional[str] = Field( - "", - description="The name of the specified package in case it is a VCS URI with an egg fragment", - ) - extras: Optional[Tuple] = Field(default_factory=tuple) - is_direct_url: Optional[bool] = Field(False) - is_implicit_ssh: Optional[bool] = Field(False) - auth: Optional[str] = None - _fragment_dict: Optional[Dict] = Field(default_factory=dict) - _username_is_quoted: Optional[bool] = False - _password_is_quoted: Optional[bool] = False - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - # keep_untouched = (cached_property,) - - def __init__(self, **data): - super().__init__(**data) - self._parse_auth() - self._parse_query() - self._parse_fragment() - - def _parse_query(self) -> None: - query = self.query if self.query is not None else "" - query_dict = dict() - queries = query.split("&") - query_items = [] - subdirectory = self.subdirectory if self.subdirectory else None - for q in queries: - key, _, val = q.partition("=") - val = unquote_plus(val) - if key == "subdirectory" and not subdirectory: - subdirectory = val - else: - query_items.append((key, val)) - query_dict.update(query_items) - self.query_dict = query_dict - self.subdirectory = subdirectory - self.query = query - - def _parse_fragment(self) -> None: - subdirectory = self.subdirectory if self.subdirectory else "" - fragment = self.fragment if self.fragment else "" - if self.fragment is None: - return self - fragments = self.fragment.split("&") - fragment_items = {} - name = self.name if self.name else "" - extras = self.extras - for q in fragments: - key, _, val = q.partition("=") - val = unquote_plus(val) - fragment_items[key] = val - if key == "egg": - from .utils import parse_extras_str - - name, stripped_extras = _strip_extras(val) - if stripped_extras: - extras = tuple(parse_extras_str(stripped_extras)) - elif key == "subdirectory": - subdirectory = val - self.name = name - self.extras = extras - self.subdirectory = subdirectory - self.fragment = fragment - self._fragment_dict = fragment_items - - def _parse_auth(self) -> None: - if self.auth: - username, _, password = self.auth.partition(":") - username_is_quoted, password_is_quoted = False, False - quoted_username, quoted_password = "", "" - if password: - quoted_password = quote(password) - password_is_quoted = quoted_password != password - if username: - quoted_username = quote(username) - username_is_quoted = quoted_username != username - self.username = quoted_username - self.password = quoted_password - self._username_is_quoted = username_is_quoted - self._password_is_quoted = password_is_quoted - - def get_password(self, unquote=False, include_token=True) -> str: - password = self.password if self.password else "" - if password and unquote and self._password_is_quoted: - password = url_unquote(password) - return password - - def get_username(self, unquote=False) -> str: - username = self.username if self.username else "" - if username and unquote and self._username_is_quoted: - username = url_unquote(username) - return username - - @staticmethod - def parse_subdirectory(url_part): - # type: (str) -> Tuple[str, Optional[str]] - subdir = None - if "&subdirectory" in url_part: - url_part, _, subdir = url_part.rpartition("&") - if "#egg=" not in url_part: - subdir = "#{0}".format(subdir.strip()) - else: - subdir = "&{0}".format(subdir.strip()) - return url_part.strip(), subdir - - @classmethod - def get_parsed_url(cls, url): - # if there is a "#" in the auth section, this could break url parsing - maybe_auth = None - parsed_url = _get_parsed_url(url) - if "@" in url and "#" in url: - scheme = "{0}://".format(parsed_url.scheme) - if parsed_url.scheme == "file": - scheme = "{0}/".format(scheme) - url_without_scheme = url.replace(scheme, "") - maybe_auth, _, maybe_url = url_without_scheme.partition("@") - if "#" in maybe_auth and (not parsed_url.host or "." not in parsed_url.host): - new_parsed_url = _get_parsed_url("{0}{1}".format(scheme, maybe_url)) - new_parsed_url = new_parsed_url._replace(auth=maybe_auth) - return new_parsed_url - return parsed_url - - @classmethod - def parse(cls, url) -> "URI": - is_direct_url = False - name_with_extras = None - is_implicit_ssh = url.strip().startswith("git+git@") - if is_implicit_ssh: - from ..utils import add_ssh_scheme_to_git_uri - - url = add_ssh_scheme_to_git_uri(url) - direct_match = DIRECT_URL_RE.match(url) - if direct_match is not None: - is_direct_url = True - name_with_extras, _, url = url.partition("@") - name_with_extras = name_with_extras.strip() - url, ref = split_ref_from_uri(url.strip()) - if "file:/" in url and "file:///" not in url: - url = url.replace("file:/", "file:///") - parsed = cls.get_parsed_url(url) - # if there is a "#" in the auth section, this could break url parsing - if not (parsed.scheme and parsed.host): - # check if this is a file uri - if not ( - parsed.scheme - and parsed.path - and (parsed.scheme == "file" or parsed.scheme.endswith("+file")) - ): - raise ValueError("Failed parsing URL {0!r} - Not a valid url".format(url)) - parsed_dict = dict(parsed._asdict()).copy() - parsed_dict["is_direct_url"] = is_direct_url - parsed_dict["is_implicit_ssh"] = is_implicit_ssh - parsed_dict.update( - **update_url_name_and_fragment(name_with_extras, ref, parsed_dict) - ) # type: ignore - return cls(**parsed_dict) - - def to_string( - self, - escape_password=True, # type: bool - unquote=True, # type: bool - direct=None, # type: Optional[bool] - strip_ssh=False, # type: bool - strip_ref=False, # type: bool - strip_name=False, # type: bool - strip_subdir=False, # type: bool - ): - # type: (...) -> str - """Converts the current URI to a string, unquoting or escaping the - password as needed. - - :param escape_password: Whether to replace password with ``----``, default True - :param escape_password: bool, optional - :param unquote: Whether to unquote url-escapes in the password, default False - :param unquote: bool, optional - :param bool direct: Whether to format as a direct URL - :param bool strip_ssh: Whether to strip the SSH scheme from the url (git only) - :param bool strip_ref: Whether to drop the VCS ref (if present) - :param bool strip_name: Whether to drop the name and extras (if present) - :param bool strip_subdir: Whether to drop the subdirectory (if present) - :return: The reconstructed string representing the URI - :rtype: str - """ - - if direct is None: - direct = self.is_direct_url - if escape_password: - password = "----" if self.password else "" - if password: - username = self.get_username(unquote=unquote) - elif self.username: - username = "----" - else: - username = "" - else: - password = self.get_password(unquote=unquote) - username = self.get_username(unquote=unquote) - auth = "" - if username: - if password: - auth = "{username}:{password}@".format( - password=password, username=username - ) - else: - auth = "{username}@".format(username=username) - query = "" - if self.query: - query = "{query}?{self.query}".format(query=query, self=self) - subdir_prefix = "#" - if not direct: - if self.name and not strip_name: - fragment = "#egg={self.name_with_extras}".format(self=self) - subdir_prefix = "&" - elif not strip_name and ( - self.extras and self.scheme and self.scheme.startswith("file") - ): - from .utils import extras_to_string - - fragment = extras_to_string(self.extras) - else: - fragment = "" - query = "{query}{fragment}".format(query=query, fragment=fragment) - if self.subdirectory and not strip_subdir: - query = "{query}{subdir_prefix}subdirectory={self.subdirectory}".format( - query=query, subdir_prefix=subdir_prefix, self=self - ) - host_port_path = self.get_host_port_path(strip_ref=strip_ref) - url = "{self.scheme}://{auth}{host_port_path}{query}".format( - self=self, auth=auth, host_port_path=host_port_path, query=query - ) - if strip_ssh: - from ..utils import strip_ssh_from_git_uri - - url = strip_ssh_from_git_uri(url) - if self.name and direct and not strip_name: - return "{self.name_with_extras}@ {url}".format(self=self, url=url) - return url - - def get_host_port_path(self, strip_ref=False): - # type: (bool) -> str - host = self.host if self.host else "" - if self.port is not None: - host = "{host}:{self.port!s}".format(host=host, self=self) - path = "{self.path}".format(self=self) if self.path else "" - if self.ref and not strip_ref: - path = "{path}@{self.ref}".format(path=path, self=self) - return "{host}{path}".format(host=host, path=path) - - @property - def hidden_auth(self): - # type: () -> str - auth = "" - if self.username and self.password: - password = "****" - username = self.get_username(unquote=True) - auth = "{username}:{password}".format(username=username, password=password) - elif self.username and not self.password: - auth = "****" - return auth - - @property - def name_with_extras(self): - # type: () -> str - from .utils import extras_to_string - - if not self.name: - return "" - extras = extras_to_string(self.extras) - return "{self.name}{extras}".format(self=self, extras=extras) - - @property - def as_link(self): - # type: () -> Link - link = Link(self.to_string(escape_password=False, strip_ssh=False, direct=False)) - return link - - @property - def bare_url(self): - # type: () -> str - return self.to_string( - escape_password=False, - strip_ssh=self.is_implicit_ssh, - direct=False, - strip_name=True, - strip_ref=True, - strip_subdir=True, - ) - - @property - def url_without_fragment_or_ref(self): - # type: () -> str - return self.to_string( - escape_password=False, - strip_ssh=self.is_implicit_ssh, - direct=False, - strip_name=True, - strip_ref=True, - ) - - @property - def url_without_fragment(self): - # type: () -> str - return self.to_string( - escape_password=False, - strip_ssh=self.is_implicit_ssh, - direct=False, - strip_name=True, - ) - - @property - def url_without_ref(self): - # type: () -> str - return self.to_string( - escape_password=False, - strip_ssh=self.is_implicit_ssh, - direct=False, - strip_ref=True, - ) - - @property - def base_url(self): - # type: () -> str - return self.to_string( - escape_password=False, - strip_ssh=self.is_implicit_ssh, - direct=False, - unquote=False, - ) - - @property - def full_url(self): - # type: () -> str - return self.to_string(escape_password=False, strip_ssh=False, direct=False) - - @property - def secret(self): - # type: () -> str - return self.full_url - - @property - def safe_string(self): - # type: () -> str - return self.to_string(escape_password=True, unquote=True) - - @property - def unsafe_string(self): - # type: () -> str - return self.to_string(escape_password=False, unquote=True) - - @property - def uri_escape(self): - # type: () -> str - return self.to_string(escape_password=False, unquote=False) - - @property - def is_installable(self): - # type: () -> bool - return self.is_file_url and is_installable_file(self.bare_url) - - @property - def is_vcs(self): - # type: () -> bool - from ..utils import VCS_SCHEMES - - return self.scheme in VCS_SCHEMES - - @property - def is_file_url(self): - # type: () -> bool - return all([self.scheme, self.scheme == "file"]) - - def __str__(self): - # type: () -> str - return self.to_string(escape_password=True, unquote=True) - - -def update_url_name_and_fragment(name_with_extras, ref, parsed_dict): - # type: (Optional[str], Optional[str], Dict[str, Optional[str]]) -> Dict[str, Optional[str]] - if name_with_extras: - fragment = "" # type: Optional[str] - parsed_extras = () - name, extras = _strip_extras(name_with_extras) - if extras: - parsed_extras = parsed_extras + tuple(parse_extras_str(extras)) - if parsed_dict["fragment"] is not None: - fragment = "{0}".format(parsed_dict["fragment"]) - if fragment.startswith("egg="): - _, _, fragment_part = fragment.partition("=") - fragment_name, fragment_extras = _strip_extras(fragment_part) - name = name if name else fragment_name - if fragment_extras: - parsed_extras = parsed_extras + tuple( - parse_extras_str(fragment_extras) - ) - name_with_extras = "{0}{1}".format(name, extras_to_string(parsed_extras)) - elif ( - parsed_dict.get("path") is not None and "&subdirectory" in parsed_dict["path"] - ): - path, fragment = URI.parse_subdirectory(parsed_dict["path"]) # type: ignore - parsed_dict["path"] = path - elif ref is not None and "&subdirectory" in ref: - ref, fragment = URI.parse_subdirectory(ref) - parsed_dict["name"] = name - parsed_dict["extras"] = parsed_extras - if ref: - parsed_dict["ref"] = ref.strip() - return parsed_dict diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py deleted file mode 100644 index 319b27b2..00000000 --- a/pipenv/vendor/requirementslib/models/utils.py +++ /dev/null @@ -1,787 +0,0 @@ -import os -import re -import string -from functools import lru_cache -from pathlib import Path -from typing import ( - Any, - AnyStr, - Dict, - List, - Match, - Optional, - Set, - Text, - Tuple, - TypeVar, - Union, -) - -import pipenv.vendor.tomlkit as tomlkit -from pipenv.patched.pip._internal.models.link import Link -from pipenv.patched.pip._internal.req.constructors import install_req_from_line -from pipenv.patched.pip._internal.utils._jaraco_text import drop_comment, join_continuation, yield_lines -from pipenv.patched.pip._vendor.packaging.markers import InvalidMarker, Marker, Op, Value, Variable -from pipenv.patched.pip._vendor.packaging.requirements import Requirement as PackagingRequirement -from pipenv.patched.pip._vendor.packaging.specifiers import InvalidSpecifier, Specifier, SpecifierSet -from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name -from pipenv.patched.pip._vendor.packaging.version import parse as parse_version -from pipenv.patched.pip._vendor.pkg_resources import Requirement, get_distribution, safe_name -from pipenv.patched.pip._vendor.urllib3 import util as urllib3_util -from pipenv.patched.pip._vendor.urllib3.util import parse_url as urllib3_parse -from pipenv.vendor.plette.models import Package, PackageCollection -from pipenv.vendor.tomlkit.container import Container -from pipenv.vendor.tomlkit.items import AoT, Array, Bool, InlineTable, Item, String, Table - -from ..environment import MYPY_RUNNING -from ..fileutils import is_valid_url -from ..utils import VCS_LIST, is_star - -if MYPY_RUNNING: - from pipenv.patched.pip._vendor.packaging.markers import Marker as PkgResourcesMarker - from pipenv.patched.pip._vendor.packaging.markers import Op as PkgResourcesOp - from pipenv.patched.pip._vendor.packaging.markers import Value as PkgResourcesValue - from pipenv.patched.pip._vendor.packaging.markers import Variable as PkgResourcesVariable - from pipenv.patched.pip._vendor.urllib3.util.url import Url - - _T = TypeVar("_T") - TMarker = Union[Marker, PkgResourcesMarker] - TVariable = TypeVar("TVariable", PkgResourcesVariable, Variable) - TValue = TypeVar("TValue", PkgResourcesValue, Value) - TOp = TypeVar("TOp", PkgResourcesOp, Op) - MarkerTuple = Tuple[TVariable, TOp, TValue] - TRequirement = Union[PackagingRequirement, Requirement] - STRING_TYPE = Union[bytes, str, Text] - TOML_DICT_TYPES = Union[Container, Package, PackageCollection, Table, InlineTable] - S = TypeVar("S", bytes, str, Text) - - -TOML_DICT_OBJECTS = (Container, Package, Table, InlineTable, PackageCollection) -TOML_DICT_NAMES = [o.__class__.__name__ for o in TOML_DICT_OBJECTS] - -HASH_STRING = " --hash={0}" - -ALPHA_NUMERIC = r"[{0}{1}]".format(string.ascii_letters, string.digits) -PUNCTUATION = r"[\-_\.]" -ALPHANUM_PUNCTUATION = r"[{0}{1}\-_\.]".format(string.ascii_letters, string.digits) -NAME = r"{0}+{1}*{2}".format(ALPHANUM_PUNCTUATION, PUNCTUATION, ALPHA_NUMERIC) -REF = r"[{0}{1}\-\_\./]".format(string.ascii_letters, string.digits) -EXTRAS = r"(?P\[{0}(?:,{0})*\])".format(NAME) -NAME_WITH_EXTRAS = r"(?P{0}){1}?".format(NAME, EXTRAS) -NAME_RE = re.compile(NAME_WITH_EXTRAS) -SUBDIR_RE = r"(?:[&#]subdirectory=(?P.*))" -URL_NAME = r"(?:#egg={0})".format(NAME_WITH_EXTRAS) -REF_RE = r"(?:@(?P{0}+)?)".format(REF) -PATH_RE = r"(?P[:/])(?P[^ @]+){0}?".format(REF_RE) -PASS_RE = r"(?:(?<=:)(?P[^ ]+))" -AUTH_RE = r"(?:(?P[^ ]+)[:@]{0}?@)".format(PASS_RE) -HOST_RE = r"(?:{0}?(?P[^ ]+?\.?{1}+(?P:\d+)?))?".format( - AUTH_RE, ALPHA_NUMERIC -) -URL = r"(?P[^ ]+://){0}{1}".format(HOST_RE, PATH_RE) -URL_RE = re.compile(r"{0}(?:{1}?{2}?)?".format(URL, URL_NAME, SUBDIR_RE)) -DIRECT_URL_RE = re.compile(r"{0}\s?@\s?{1}".format(NAME_WITH_EXTRAS, URL)) - - -def filter_none(k, v) -> bool: - if v: - return True - return False - - -def filter_dict(dict_) -> Dict[AnyStr, Any]: - return {k: v for k, v in dict_.items() if filter_none(k, v)} - - -def create_link(link): - # type: (AnyStr) -> Link - - if not isinstance(link, str): - raise TypeError("must provide a string to instantiate a new link") - - return Link(link) - - -def tomlkit_value_to_python(toml_value): - # type: (Union[Array, AoT, TOML_DICT_TYPES, Item]) -> Union[List, Dict] - value_type = type(toml_value).__name__ - if ( - isinstance(toml_value, TOML_DICT_OBJECTS + (dict,)) - or value_type in TOML_DICT_NAMES - ): - return tomlkit_dict_to_python(toml_value) - elif isinstance(toml_value, AoT) or value_type == "AoT": - return [tomlkit_value_to_python(val) for val in toml_value._body] - elif isinstance(toml_value, Array) or value_type == "Array": - return [tomlkit_value_to_python(val) for val in list(toml_value)] - elif isinstance(toml_value, String) or value_type == "String": - return "{0!s}".format(toml_value) - elif isinstance(toml_value, Bool) or value_type == "Bool": - return toml_value.value - elif isinstance(toml_value, Item): - return toml_value.value - return toml_value - - -def tomlkit_dict_to_python(toml_dict): - # type: (TOML_DICT_TYPES) -> Dict - value_type = type(toml_dict).__name__ - if toml_dict is None: - raise TypeError("Invalid type NoneType when converting toml dict to python") - converted = None # type: Optional[Dict] - if isinstance(toml_dict, (InlineTable, Table)) or value_type in ( - "InlineTable", - "Table", - ): - converted = toml_dict.value - elif isinstance(toml_dict, (Package, PackageCollection)) or value_type in ( - "Package, PackageCollection" - ): - converted = toml_dict._data - if isinstance(converted, Container) or type(converted).__name__ == "Container": - converted = converted.value - elif isinstance(toml_dict, Container) or value_type == "Container": - converted = toml_dict.value - elif isinstance(toml_dict, dict): - converted = toml_dict.copy() - else: - raise TypeError( - "Invalid type for conversion: expected Container, Dict, or Table, " - "got {0!r}".format(toml_dict) - ) - if isinstance(converted, dict): - return {k: tomlkit_value_to_python(v) for k, v in converted.items()} - elif isinstance(converted, (TOML_DICT_OBJECTS)) or value_type in TOML_DICT_NAMES: - return tomlkit_dict_to_python(converted) - return converted - - -def get_url_name(url): - # type: (AnyStr) -> AnyStr - """Given a url, derive an appropriate name to use in a pipfile. - - :param str url: A url to derive a string from - :returns: The name of the corresponding pipfile entry - :rtype: Text - """ - if not isinstance(url, str): - raise TypeError("Expected a string, got {0!r}".format(url)) - return urllib3_util.parse_url(url).host - - -class HashableRequirement(Requirement): - def __hash__(self): - specifier_hash = hash(tuple((str(s),) for s in self.specifier)) - return hash( - ( - self.url, - specifier_hash, - frozenset(self.extras), - str(self.marker) if self.marker else None, - ) - ) - - @staticmethod - def parse(s): - (req,) = map( - HashableRequirement, join_continuation(map(drop_comment, yield_lines(s))) - ) - return req - - -def init_requirement(name): - if not isinstance(name, str): - raise TypeError("must supply a name to generate a requirement") - - req = HashableRequirement.parse(name) - req.vcs = None - req.local_file = None - req.revision = None - req.path = None - return req - - -def convert_to_hashable_requirement(req: Requirement) -> Optional[HashableRequirement]: - if req is None: - return None - - hashable_req = HashableRequirement(str(req)) - hashable_req.extras = req.extras - hashable_req.marker = req.marker - hashable_req.url = req.url - return hashable_req - - -def extras_to_string(extras) -> str: - """Turn a list of extras into a string.""" - if isinstance(extras, str): - if extras.startswith("["): - return extras - else: - extras = [extras] - if not extras: - return "" - return "[{0}]".format(",".join(sorted(set(extras)))) - - -def parse_extras_str(extras_str): - """Turn a string of extras into a parsed extras list. - - :param str extras_str: An extras string - :return: A sorted list of extras - :rtype: List[str] - """ - extras = Requirement.parse("fakepkg{0}".format(extras_to_string(extras_str))).extras - return sorted(dict.fromkeys([extra.lower() for extra in extras])) - - -def parse_extras_from_line(installable_line): - extras = [] - - # Extract the part within square brackets, if any - start = installable_line.find("[") - end = installable_line.find("]") - - if start != -1 and end != -1 and start < end: - extras_str = installable_line[start + 1 : end] - extras = extras_str.split(",") - - return extras - - -def specs_to_string(specs): - # type: (List[Union[STRING_TYPE, Specifier]]) -> AnyStr - """Turn a list of specifier tuples into a string. - - :param List[Union[Specifier, str]] specs: a list of specifiers to format - :return: A string of specifiers - :rtype: str - """ - - if specs: - if isinstance(specs, str): - return specs - try: - extras = ",".join(["".join(spec) for spec in specs]) - except TypeError: - extras = ",".join(["".join(spec._spec) for spec in specs]) # type: ignore - return extras - return "" - - -def build_vcs_uri( - vcs, - uri, - name=None, - ref=None, - subdirectory=None, - extras=None, -): - # type: (...) -> STRING_TYPE - if extras is None: - extras = [] - vcs_start = "" - if vcs is not None: - vcs_start = "{0}+".format(vcs) - if not uri.startswith(vcs_start): - uri = "{0}{1}".format(vcs_start, uri) - if ref: - uri = "{0}@{1}".format(uri, ref) - if name: - uri = "{0}#egg={1}".format(uri, name) - if extras: - extras_string = extras_to_string(extras) - uri = "{0}{1}".format(uri, extras_string) - if subdirectory: - uri = "{0}&subdirectory={1}".format(uri, subdirectory) - return uri - - -def _get_parsed_url(url): - # type: (S) -> Url - """This is a stand-in function for `urllib3.util.parse_url` - - The original function doesn't handle special characters very well, this simply splits - out the authentication section, creates the parsed url, then puts the authentication - section back in, bypassing validation. - - :return: The new, parsed URL object - :rtype: :class:`~urllib3.util.url.Url` - """ - - try: - parsed = urllib3_parse(url) - except ValueError: - scheme, _, url = url.partition("://") - auth, _, url = url.rpartition("@") - url = "{scheme}://{url}".format(scheme=scheme, url=url) - parsed = urllib3_parse(url)._replace(auth=auth) - return parsed - - -def convert_direct_url_to_url(direct_url): - # type: (AnyStr) -> AnyStr - """Converts direct URLs to standard, link-style URLs. - - Given a direct url as defined by *PEP 508*, convert to a :class:`Link` - compatible URL by moving the name and extras into an **egg_fragment**. - - :param str direct_url: A pep-508 compliant direct url. - :return: A reformatted URL for use with Link objects and :class:`InstallRequirement` objects. - :rtype: AnyStr - """ - direct_match = DIRECT_URL_RE.match(direct_url) # type: Optional[Match] - if direct_match is None: - url_match = URL_RE.match(direct_url) - if url_match or is_valid_url(direct_url): - return direct_url - match_dict = ( - {} - ) # type: Dict[STRING_TYPE, Union[Tuple[STRING_TYPE, ...], STRING_TYPE]] - if direct_match is not None: - match_dict = direct_match.groupdict() # type: ignore - if not match_dict: - raise ValueError( - "Failed converting value to normal URL, is it a direct URL? {0!r}".format( - direct_url - ) - ) - url_segments = [match_dict.get(s) for s in ("scheme", "host", "path", "pathsep")] - url = "" # type: STRING_TYPE - url = "".join([s for s in url_segments if s is not None]) # type: ignore - new_url = build_vcs_uri( - None, - url, - ref=match_dict.get("ref"), - name=match_dict.get("name"), - extras=match_dict.get("extras"), - subdirectory=match_dict.get("subdirectory"), - ) - return new_url - - -def get_version(pipfile_entry): - if str(pipfile_entry) == "{}" or is_star(pipfile_entry): - return "" - - if hasattr(pipfile_entry, "keys") and "version" in pipfile_entry: - if is_star(pipfile_entry.get("version")): - return "" - version = pipfile_entry.get("version") - if version is None: - version = "" - return version.strip().lstrip("(").rstrip(")") - - if isinstance(pipfile_entry, str): - return pipfile_entry.strip().lstrip("(").rstrip(")") - return "" - - -def strip_extras_markers_from_requirement(req): - # type: (TRequirement) -> TRequirement - """Strips extras markers from requirement instances. - - Given a :class:`~packaging.requirements.Requirement` instance with markers defining - *extra == 'name'*, strip out the extras from the markers and return the cleaned - requirement - - :param PackagingRequirement req: A packaging requirement to clean - :return: A cleaned requirement - :rtype: PackagingRequirement - """ - if req is None: - raise TypeError("Must pass in a valid requirement, received {0!r}".format(req)) - if getattr(req, "marker", None) is not None: - marker = req.marker # type: TMarker - marker._markers = _strip_extras_markers(marker._markers) - if not marker._markers: - req.marker = None - else: - req.marker = marker - return req - - -def _strip_extras_markers(marker): - # type: (Union[MarkerTuple, List[Union[MarkerTuple, str]]]) -> List[Union[MarkerTuple, str]] - if marker is None or not isinstance(marker, (list, tuple)): - raise TypeError("Expecting a marker type, received {0!r}".format(marker)) - markers_to_remove = [] - # iterate forwards and generate a list of indexes to remove first, then reverse the - # list so we can remove the text that normally occurs after (but we will already - # be past it in the loop) - for i, marker_list in enumerate(marker): - if isinstance(marker_list, list): - cleaned = _strip_extras_markers(marker_list) - if not cleaned: - markers_to_remove.append(i) - elif isinstance(marker_list, tuple) and marker_list[0].value == "extra": - markers_to_remove.append(i) - for i in reversed(markers_to_remove): - del marker[i] - if i > 0 and marker[i - 1] == "and": - del marker[i - 1] - return marker - - -@lru_cache() -def get_setuptools_version(): - # type: () -> Optional[STRING_TYPE] - - setuptools_dist = get_distribution(Requirement("setuptools")) - return getattr(setuptools_dist, "version", None) - - -def get_default_pyproject_backend(): - # type: () -> STRING_TYPE - st_version = get_setuptools_version() - if st_version is not None: - parsed_st_version = parse_version(st_version) - if parsed_st_version >= parse_version("40.8.0"): - return "setuptools.build_meta:__legacy__" - return "setuptools.build_meta" - - -def get_pyproject(path: Union[AnyStr, Path]) -> Optional[Dict[str, Union[List[AnyStr], AnyStr]]]: - """ - Given a base path, look for the corresponding ``pyproject.toml`` file - and return its build_requires and build_backend. - - :param path: The root path of the project, should be a directory (will be truncated) - :return: A dictionary with build requirements, build backend, and dependencies - """ - if not path: - return - - if not isinstance(path, Path): - path = Path(path) - - if not path.is_dir(): - path = path.parent - - pp_toml = path / "pyproject.toml" - - # Default values - requires = ["setuptools>=40.8", "wheel"] - backend = get_default_pyproject_backend() - dependencies = [] - - if pp_toml.exists(): - with open(pp_toml, encoding="utf-8") as fh: - pyproject_data = tomlkit.loads(fh.read()) - - # Extracting build system information - build_system = pyproject_data.get("build-system", None) - if build_system is not None: - requires = build_system.get("requires", requires) - backend = build_system.get("build-backend", backend) - - # Extracting project dependencies - project_data = pyproject_data.get("project", None) - if project_data is not None: - dependencies = project_data.get("dependencies", []) - - return {"build_requires": requires, "build_backend": backend, "dependencies": dependencies} - - -def split_markers_from_line(line): - # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] - """Split markers from a dependency.""" - quote_chars = ["'", '"'] - line_quote = next( - iter(quote for quote in quote_chars if line.startswith(quote)), None - ) - if line_quote and line.endswith(line_quote): - line = line.strip(line_quote) - marker_sep = " ; " - markers = None - if marker_sep in line: - line, markers = line.split(marker_sep, 1) - markers = markers.strip() if markers else None - return line, markers - - -def split_vcs_method_from_uri(uri): - # type: (AnyStr) -> Tuple[Optional[STRING_TYPE], STRING_TYPE] - """Split a vcs+uri formatted uri into (vcs, uri)""" - vcs_start = "{0}+" - vcs = next( - iter([vcs for vcs in VCS_LIST if uri.startswith(vcs_start.format(vcs))]), None - ) - if vcs: - vcs, uri = uri.split("+", 1) - return vcs, uri - - -def split_ref_from_uri(uri): - # type: (AnyStr) -> Tuple[AnyStr, Optional[AnyStr]] - """Given a path or URI, check for a ref and split it from the path if it is - present, returning a tuple of the original input and the ref or None. - - :param AnyStr uri: The path or URI to split - :returns: A 2-tuple of the path or URI and the ref - :rtype: Tuple[AnyStr, Optional[AnyStr]] - """ - if not isinstance(uri, str): - raise TypeError("Expected a string, received {0!r}".format(uri)) - parsed = _get_parsed_url(uri) - path = parsed.path if parsed.path else "" - scheme = parsed.scheme if parsed.scheme else "" - ref = None - schema_is_filelike = scheme in ("", "file") - if (not schema_is_filelike and "@" in path) or ( - schema_is_filelike and (re.match("^.*@[^/@]*$", path) or path.count("@") >= 2) - ): - path, _, ref = path.rpartition("@") - parsed = parsed._replace(path=path) - return (parsed.url, ref) - - -def validate_vcs(instance, attr_, value): - if value not in VCS_LIST: - raise ValueError("Invalid vcs {0!r}".format(value)) - - -def validate_path(instance, attr_, value): - if not os.path.exists(value): - raise ValueError("Invalid path {0!r}".format(value)) - - -def validate_specifiers(instance, attr_, value): - if value == "": - return True - try: - SpecifierSet(value) - except (InvalidMarker, InvalidSpecifier): - raise ValueError("Invalid Specifiers {0}".format(value)) - - -def key_from_req(req): - """Get an all-lowercase version of the requirement's name.""" - if hasattr(req, "key"): - # from pkg_resources, such as installed dists for pip-sync - key = req.key - else: - # from packaging, such as install requirements from requirements.txt - key = req.name - - key = key.replace("_", "-").lower() - return key - - -def _requirement_to_str_lowercase_name(requirement): - """Formats a packaging.requirements.Requirement with a lowercase name. - - This is simply a copy of - https://github.com/pypa/packaging/blob/16.8/packaging/requirements.py#L109-L124 - modified to lowercase the dependency name. - - Previously, we were invoking the original Requirement.__str__ method and - lower-casing the entire result, which would lowercase the name, *and* other, - important stuff that should not be lower-cased (such as the marker). See - this issue for more information: https://github.com/pypa/pipenv/issues/2113. - """ - parts = [requirement.name.lower()] - - if requirement.extras: - parts.append("[{0}]".format(",".join(sorted(requirement.extras)))) - - if requirement.specifier: - parts.append(str(requirement.specifier)) - - if requirement.url: - parts.append("@ {0}".format(requirement.url)) - - if requirement.marker: - parts.append(" ; {0}".format(requirement.marker)) - - return "".join(parts) - - -def format_requirement(ireq): - """Formats an `InstallRequirement` instance as a string. - - Generic formatter for pretty printing InstallRequirements to the terminal - in a less verbose way than using its `__str__` method. - - :param :class:`InstallRequirement` ireq: A pip **InstallRequirement** instance. - :return: A formatted string for prettyprinting - :rtype: str - """ - if ireq.editable: - line = "-e {}".format(ireq.link) - else: - line = _requirement_to_str_lowercase_name(ireq.req) - - if str(ireq.req.marker) != str(ireq.markers): - if not ireq.req.marker: - line = "{} ; {}".format(line, ireq.markers) - else: - name, markers = line.split(";", 1) - markers = markers.strip() - line = "{} ; ({}) and ({})".format(name, markers, ireq.markers) - - return line - - -def get_pinned_version(ireq): - """Get the pinned version of an InstallRequirement. - - An InstallRequirement is considered pinned if: - - - Is not editable - - It has exactly one specifier - - That specifier is "==" - - The version does not contain a wildcard - - Examples: - django==1.8 # pinned - django>1.8 # NOT pinned - django~=1.8 # NOT pinned - django==1.* # NOT pinned - - Raises `TypeError` if the input is not a valid InstallRequirement, or - `ValueError` if the InstallRequirement is not pinned. - """ - try: - specifier = ireq.specifier - except AttributeError: - raise TypeError("Expected InstallRequirement, not {}".format(type(ireq).__name__)) - - if getattr(ireq, "editable", False): - raise ValueError("InstallRequirement is editable") - if not specifier: - raise ValueError("InstallRequirement has no version specification") - if len(specifier._specs) != 1: - raise ValueError("InstallRequirement has multiple specifications") - - op, version = next(iter(specifier._specs))._spec - if op not in ("==", "===") or version.endswith(".*"): - raise ValueError("InstallRequirement not pinned (is {0!r})".format(op + version)) - - return version - - -def is_pinned_requirement(ireq): - """Returns whether an InstallRequirement is a "pinned" requirement. - - An InstallRequirement is considered pinned if: - - - Is not editable - - It has exactly one specifier - - That specifier is "==" - - The version does not contain a wildcard - - Examples: - django==1.8 # pinned - django>1.8 # NOT pinned - django~=1.8 # NOT pinned - django==1.* # NOT pinned - """ - - try: - get_pinned_version(ireq) - except (TypeError, ValueError): - return False - return True - - -def as_tuple(ireq): - """Pulls out the (name: str, version:str, extras:(str)) tuple from the - pinned InstallRequirement.""" - - if not is_pinned_requirement(ireq): - raise TypeError("Expected a pinned InstallRequirement, got {}".format(ireq)) - - name = key_from_req(ireq.req) - version = next(iter(ireq.specifier._specs))._spec[1] - extras = tuple(sorted(ireq.extras)) - return name, version, extras - - -def make_install_requirement( - name, version=None, extras=None, markers=None, constraint=False -): - """Generates an :class:`~pipenv.patched.pip._internal.req.req_install.InstallRequirement`. - - Create an InstallRequirement from the supplied metadata. - - :param name: The requirement's name. - :type name: str - :param version: The requirement version (must be pinned). - :type version: str. - :param extras: The desired extras. - :type extras: list[str] - :param markers: The desired markers, without a preceding semicolon. - :type markers: str - :param constraint: Whether to flag the requirement as a constraint, defaults to False. - :param constraint: bool, optional - :return: A generated InstallRequirement - :rtype: :class:`~pipenv.patched.pip._internal.req.req_install.InstallRequirement` - """ - requirement_string = "{0}".format(name) - if extras: - # Sort extras for stability - extras_string = "[{}]".format(",".join(sorted(extras))) - requirement_string = "{0}{1}".format(requirement_string, extras_string) - if version: - requirement_string = "{0}=={1}".format(requirement_string, str(version)) - if markers: - requirement_string = "{0} ; {1}".format(requirement_string, str(markers)) - return install_req_from_line(requirement_string, constraint=constraint) - - -def normalize_name(pkg) -> str: - """Given a package name, return its normalized, non-canonicalized form.""" - return pkg.replace("_", "-").lower() - - -def get_name_variants(pkg): - # type: (STRING_TYPE) -> Set[STRING_TYPE] - """Given a packager name, get the variants of its name for both the - canonicalized and "safe" forms. - - :param AnyStr pkg: The package to lookup - :returns: A list of names. - :rtype: Set - """ - - if not isinstance(pkg, str): - raise TypeError("must provide a string to derive package names") - - pkg = pkg.lower() - names = {safe_name(pkg), canonicalize_name(pkg), pkg.replace("-", "_")} - return names - - -def expand_env_variables(line): - # type: (AnyStr) -> AnyStr - """Expand the env vars in a line following pip's standard. - https://pip.pypa.io/en/stable/reference/pip_install/#id10. - - Matches environment variable-style values in '${MY_VARIABLE_1}' with - the variable name consisting of only uppercase letters, digits or - the '_' - """ - - def replace_with_env(match): - value = os.getenv(match.group(1)) - return value if value else match.group() - - return re.sub(r"\$\{([A-Z0-9_]+)\}", replace_with_env, line) - - -def tuple_to_dict(input_tuple): - result_dict = {} - i = 0 - while i < len(input_tuple): - key = input_tuple[i] - i += 1 - if i < len(input_tuple): - if isinstance(input_tuple[i], tuple): - value = tuple_to_dict(input_tuple[i]) - i += 1 - else: - value = input_tuple[i] - i += 1 - result_dict[key] = value - return result_dict diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py deleted file mode 100644 index b6d43b3b..00000000 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ /dev/null @@ -1,104 +0,0 @@ -import importlib -import os -import sys -from typing import Any, Optional, Tuple - -from pipenv.patched.pip._internal.vcs.versioncontrol import VcsSupport -from pipenv.patched.pip._vendor.pyparsing.core import cached_property -from pipenv.vendor.pydantic import Field - -from .common import ReqLibBaseModel -from .url import URI - - -class VCSRepository(ReqLibBaseModel): - url: str - name: str - checkout_directory: str - vcs_type: str - parsed_url: Optional[URI] = Field(default_factory=None) - subdirectory: Optional[str] = None - commit_sha: Optional[str] = None - ref: Optional[str] = None - repo_backend: Any = None - clone_log: Optional[str] = None - DEFAULT_RUN_ARGS: Optional[Any] = None - - class Config: - validate_assignment = True - arbitrary_types_allowed = True - allow_mutation = True - include_private_attributes = True - keep_untouched = (cached_property,) - - def __init__(self, **data): - super().__init__(**data) - self.parsed_url = self.get_parsed_url() - self.repo_backend = self.get_repo_backend() - - def get_parsed_url(self) -> URI: - return URI.parse(self.url) - - def get_repo_backend(self): - if self.DEFAULT_RUN_ARGS is None: - default_run_args = self.monkeypatch_pip() - else: - default_run_args = self.DEFAULT_RUN_ARGS - - VCS_SUPPORT = VcsSupport() - backend = VCS_SUPPORT.get_backend(self.vcs_type) - if backend.run_command.__func__.__defaults__ != default_run_args: - backend.run_command.__func__.__defaults__ = default_run_args - return backend - - @property - def is_local(self) -> bool: - url = self.url - if "+" in url: - url = url.split("+")[1] - return url.startswith("file") - - def obtain(self, verbosity=1) -> None: - if os.path.exists( - self.checkout_directory - ) and not self.repo_backend.is_repository_directory(self.checkout_directory): - self.repo_backend.unpack(self.checkout_directory) - elif not os.path.exists(self.checkout_directory): - self.repo_backend.obtain(self.checkout_directory, self.parsed_url, verbosity) - else: - if self.ref: - self.checkout_ref(self.ref) - if not self.commit_sha: - self.commit_sha = self.commit_hash - - def checkout_ref(self, ref: str) -> None: - rev_opts = self.repo_backend.make_rev_options(ref) - if not any( - [ - self.repo_backend.is_commit_id_equal(self.checkout_directory, ref), - self.repo_backend.is_commit_id_equal(self.checkout_directory, rev_opts), - self.is_local, - ] - ): - self.update(ref) - - def update(self, ref: str) -> None: - target_ref = self.repo_backend.make_rev_options(ref) - self.repo_backend.update(self.checkout_directory, self.url, target_ref) - self.commit_sha = self.commit_hash - - @cached_property - def commit_hash(self) -> str: - return self.repo_backend.get_revision(self.checkout_directory) - - @classmethod - def monkeypatch_pip(cls) -> Tuple[Any, ...]: - target_module = VcsSupport.__module__ - pip_vcs = importlib.import_module(target_module) - run_command_defaults = pip_vcs.VersionControl.run_command.__func__.__defaults__ - new_defaults = [False] + list(run_command_defaults)[1:] - new_defaults = tuple(new_defaults) - pip_vcs.VersionControl.run_command.__func__.__defaults__ = new_defaults - sys.modules[target_module] = pip_vcs - cls.DEFAULT_RUN_ARGS = new_defaults - return new_defaults diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index c9a14ae0..e4d86094 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -11,7 +11,6 @@ ptyprocess==0.7.0 pydantic==1.10.10 python-dotenv==1.0.0 pythonfinder==2.0.5 -requirementslib==3.0.0 ruamel.yaml==0.17.21 shellingham==1.5.0.post1 tomli==2.0.1 diff --git a/setup.py b/setup.py index b16c9b84..2a9b40f1 100644 --- a/setup.py +++ b/setup.py @@ -22,8 +22,7 @@ if sys.argv[-1] == "publish": required = [ "certifi", "setuptools>=67.0.0", - "virtualenv-clone>=0.2.5", - "virtualenv>=20.17.1", + "virtualenv>=20.24.2", ] extras = { "dev": [ @@ -31,7 +30,7 @@ extras = { "beautifulsoup4", "sphinx", "flake8>=3.3.0,<4.0", - "black;python_version>='3.7'", + "black==23.3.0", "parver", "invoke", ], diff --git a/tasks/release.py b/tasks/release.py index af73c11f..f828849c 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -9,7 +9,7 @@ import invoke from parver import Version from pipenv.__version__ import __version__ -from pipenv.vendor.requirementslib.utils import temp_environ +from pipenv.utils.shell import temp_environ from .vendoring import _get_git_root, drop_dir diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 5282288d..21b09307 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -15,7 +15,7 @@ import invoke import requests from urllib3.util import parse_url as urllib3_parse -from pipenv.vendor.requirementslib.fileutils import open_file +from pipenv.utils.fileutils import open_file TASK_NAME = "update" @@ -36,7 +36,6 @@ HARDCODED_LICENSE_URLS = { "pytoml": "https://github.com/avakar/pytoml/raw/master/LICENSE", "webencodings": "https://github.com/SimonSapin/python-webencodings/raw/" "master/LICENSE", - "requirementslib": "https://github.com/techalchemy/requirementslib/raw/master/LICENSE", "distlib": "https://github.com/vsajip/distlib/raw/master/LICENSE.txt", "pythonfinder": "https://raw.githubusercontent.com/techalchemy/pythonfinder/master/LICENSE.txt", "pipdeptree": "https://raw.githubusercontent.com/tox-dev/pipdeptree/main/LICENSE", diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 76f9a9d5..56d1ec71 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -12,11 +12,12 @@ from tempfile import TemporaryDirectory import subprocess import pytest -import requests -from pipenv.utils.processes import subprocess_run +from pipenv.patched.pip._vendor import requests from pipenv.vendor import tomlkit -from pipenv.vendor.requirementslib.utils import temp_environ -from pipenv.vendor.requirementslib.models.setup_info import handle_remove_readonly + +from pipenv.utils.processes import subprocess_run +from pipenv.utils.funktools import handle_remove_readonly +from pipenv.utils.shell import temp_environ log = logging.getLogger(__name__) warnings.simplefilter("default", category=ResourceWarning) @@ -173,7 +174,7 @@ class _Pipfile: @classmethod def get_fixture_path(cls, path, fixtures="test_artifacts"): - return Path(__file__).absolute().parent.parent / fixtures / path + return Path(__file__).resolve().parent.parent / fixtures / path class _PipenvInstance: @@ -284,17 +285,21 @@ class _PipenvInstance: return os.sep.join([self.path, 'Pipfile.lock']) -# Windows python3.8 fails without this patch. Additional details: https://bugs.python.org/issue42796 -def _rmtree_func(path, ignore_errors=True, onerror=None): - shutil_rmtree = _rmtree - if onerror is None: - onerror = handle_remove_readonly - try: - shutil_rmtree(path, ignore_errors=ignore_errors, onerror=onerror) - except (OSError, FileNotFoundError, PermissionError) as exc: - # Ignore removal failures where the file doesn't exist - if exc.errno != errno.ENOENT: - raise +if sys.version_info[:2] <= (3, 8): + # Windows python3.8 fails without this patch. Additional details: https://bugs.python.org/issue42796 + def _rmtree_func(path, ignore_errors=True, onerror=None): + shutil_rmtree = _rmtree + if onerror is None: + onerror = handle_remove_readonly + try: + shutil_rmtree(path, ignore_errors=ignore_errors, onerror=onerror) + except (OSError, FileNotFoundError, PermissionError) as exc: + # Ignore removal failures where the file doesn't exist + if exc.errno != errno.ENOENT: + raise +else: + _rmtree_func = _rmtree + @pytest.fixture() def pipenv_instance_pypi(capfdbinary, monkeypatch): diff --git a/tests/integration/test_cli.py b/tests/integration/test_cli.py index bf62f59f..c31fa40e 100644 --- a/tests/integration/test_cli.py +++ b/tests/integration/test_cli.py @@ -1,5 +1,6 @@ import os import re +import sys from pathlib import Path import pytest @@ -28,6 +29,7 @@ def test_pipenv_venv(pipenv_instance_pypi): @pytest.mark.cli +@pytest.mark.skipif(sys.version_info[:2] == (3, 8) and os.name == "nt", reason="Python 3.8 on Windows is not supported") def test_pipenv_py(pipenv_instance_pypi): with pipenv_instance_pypi() as p: c = p.pipenv('--python python') @@ -39,6 +41,7 @@ def test_pipenv_py(pipenv_instance_pypi): @pytest.mark.cli +@pytest.mark.skipif(os.name == 'nt' and sys.version_info[:2] == (3, 8), reason='Test issue with windows 3.8 CIs') def test_pipenv_site_packages(pipenv_instance_pypi): with pipenv_instance_pypi() as p: c = p.pipenv('--python python --site-packages') @@ -172,8 +175,8 @@ def test_pipenv_check_check_lockfile_categories(pipenv_instance_pypi, category): @pytest.mark.cli -def test_pipenv_clean(pipenv_instance_pypi): - with pipenv_instance_pypi() as p: +def test_pipenv_clean(pipenv_instance_private_pypi): + with pipenv_instance_private_pypi() as p: with open('setup.py', 'w') as f: f.write('from setuptools import setup; setup(name="empty")') c = p.pipenv('install -e .') diff --git a/tests/integration/test_import_requirements.py b/tests/integration/test_import_requirements.py index 39e7cbbe..07080fbb 100644 --- a/tests/integration/test_import_requirements.py +++ b/tests/integration/test_import_requirements.py @@ -1,9 +1,11 @@ import os -from pathlib import Path import tempfile +from unittest import mock import pytest +from pipenv.patched.pip._internal.operations.prepare import File + from pipenv.utils.requirements import import_requirements from pipenv.project import Project @@ -11,57 +13,69 @@ from pipenv.project import Project @pytest.mark.cli @pytest.mark.deploy @pytest.mark.system -def test_auth_with_pw_redacted(pipenv_instance_pypi): +@mock.patch("pipenv.utils.dependencies.unpack_url", mock.MagicMock(return_value=File("/some/path/to/project", content_type=None))) +@mock.patch("pipenv.utils.dependencies.find_package_name_from_directory") +def test_auth_with_pw_redacted(mock_find_package_name_from_directory, pipenv_instance_pypi): + mock_find_package_name_from_directory.return_value = "myproject" with pipenv_instance_pypi() as p: p.pipenv("run shell") project = Project() requirements_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) - requirements_file.write("""git+https://${AUTH_USER}:mypw1@github.com/user/myproject.git#egg=myproject""") + requirements_file.write("""git+https://${AUTH_USER}:mypw1@github.com/user/myproject.git@main#egg=myproject""") requirements_file.close() import_requirements(project, r=requirements_file.name) os.unlink(requirements_file.name) - assert p.pipfile["packages"]["myproject"] == {'git': 'https://${AUTH_USER}:****@github.com/user/myproject.git'} + assert p.pipfile["packages"]["myproject"] == {'git': 'git+https://${AUTH_USER}:****@github.com/user/myproject.git', 'ref': 'main'} @pytest.mark.cli @pytest.mark.deploy @pytest.mark.system -def test_auth_with_username_redacted(pipenv_instance_pypi): +@mock.patch("pipenv.utils.dependencies.unpack_url", mock.MagicMock(return_value=File("/some/path/to/project", content_type=None))) +@mock.patch("pipenv.utils.dependencies.find_package_name_from_directory") +def test_auth_with_username_redacted(mock_find_package_name_from_directory, pipenv_instance_pypi): + mock_find_package_name_from_directory.return_value = "myproject" with pipenv_instance_pypi() as p: p.pipenv("run shell") project = Project() requirements_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) - requirements_file.write("""git+https://username@github.com/user/myproject.git#egg=myproject""") + requirements_file.write("""git+https://username@github.com/user/myproject.git@main#egg=myproject""") requirements_file.close() import_requirements(project, r=requirements_file.name) os.unlink(requirements_file.name) - assert p.pipfile["packages"]["myproject"] == {'git': 'https://****@github.com/user/myproject.git'} + assert p.pipfile["packages"]["myproject"] == {'git': 'git+https://****@github.com/user/myproject.git', 'ref': 'main'} @pytest.mark.cli @pytest.mark.deploy @pytest.mark.system -def test_auth_with_pw_are_variables_passed_to_pipfile(pipenv_instance_pypi): +@mock.patch("pipenv.utils.dependencies.unpack_url", mock.MagicMock(return_value=File("/some/path/to/project", content_type=None))) +@mock.patch("pipenv.utils.dependencies.find_package_name_from_directory") +def test_auth_with_pw_are_variables_passed_to_pipfile(mock_find_package_name_from_directory, pipenv_instance_pypi): + mock_find_package_name_from_directory.return_value = "myproject" with pipenv_instance_pypi() as p: p.pipenv("run shell") project = Project() requirements_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) - requirements_file.write("""git+https://${AUTH_USER}:${AUTH_PW}@github.com/user/myproject.git#egg=myproject""") + requirements_file.write("""git+https://${AUTH_USER}:${AUTH_PW}@github.com/user/myproject.git@main#egg=myproject""") requirements_file.close() import_requirements(project, r=requirements_file.name) os.unlink(requirements_file.name) - assert p.pipfile["packages"]["myproject"] == {'git': 'https://${AUTH_USER}:${AUTH_PW}@github.com/user/myproject.git'} + assert p.pipfile["packages"]["myproject"] == {'git': 'git+https://${AUTH_USER}:${AUTH_PW}@github.com/user/myproject.git', 'ref': 'main'} @pytest.mark.cli @pytest.mark.deploy @pytest.mark.system -def test_auth_with_only_username_variable_passed_to_pipfile(pipenv_instance_pypi): +@mock.patch("pipenv.utils.dependencies.unpack_url", mock.MagicMock(return_value=File("/some/path/to/project", content_type=None))) +@mock.patch("pipenv.utils.dependencies.find_package_name_from_directory") +def test_auth_with_only_username_variable_passed_to_pipfile(mock_find_package_name_from_directory, pipenv_instance_pypi): + mock_find_package_name_from_directory.return_value = "myproject" with pipenv_instance_pypi() as p: p.pipenv("run shell") project = Project() requirements_file = tempfile.NamedTemporaryFile(mode="w+", delete=False) - requirements_file.write("""git+https://${AUTH_USER}@github.com/user/myproject.git#egg=myproject""") + requirements_file.write("""git+https://${AUTH_USER}@github.com/user/myproject.git@main#egg=myproject""") requirements_file.close() import_requirements(project, r=requirements_file.name) os.unlink(requirements_file.name) - assert p.pipfile["packages"]["myproject"] == {'git': 'https://${AUTH_USER}@github.com/user/myproject.git'} + assert p.pipfile["packages"]["myproject"] == {'git': 'git+https://${AUTH_USER}@github.com/user/myproject.git', 'ref': 'main'} diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index 1eef8712..3f3f1d3d 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -162,6 +162,7 @@ dataclasses-json = "==0.5.7" @pytest.mark.install @pytest.mark.resolver @pytest.mark.backup_resolver +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") def test_backup_resolver(pipenv_instance_private_pypi): with pipenv_instance_private_pypi() as p: with open(p.pipfile_path, "w") as f: @@ -298,10 +299,10 @@ def test_clean_on_empty_venv(pipenv_instance_pypi): @pytest.mark.basic @pytest.mark.install -def test_install_does_not_extrapolate_environ(pipenv_instance_pypi): +def test_install_does_not_extrapolate_environ(pipenv_instance_private_pypi): """Ensure environment variables are not expanded in lock file. """ - with temp_environ(), pipenv_instance_pypi() as p: + with temp_environ(), pipenv_instance_private_pypi() as p: os.environ["PYPI_URL"] = p.pypi with open(p.pipfile_path, "w") as f: @@ -321,7 +322,7 @@ name = 'mockpi' assert p.lockfile["_meta"]["sources"][0]["url"] == "${PYPI_URL}/simple" # Ensure package install does not extrapolate. - c = p.pipenv("install six") + c = p.pipenv("install six -v") assert c.returncode == 0 assert p.pipfile["source"][0]["url"] == "${PYPI_URL}/simple" assert p.lockfile["_meta"]["sources"][0]["url"] == "${PYPI_URL}/simple" @@ -528,9 +529,9 @@ def test_install_does_not_exclude_packaging(pipenv_instance_pypi): @pytest.mark.needs_internet def test_install_will_supply_extra_pip_args(pipenv_instance_pypi): with pipenv_instance_pypi() as p: - c = p.pipenv("""install dataclasses-json --extra-pip-args="--use-feature=truststore --proxy=test" """) + c = p.pipenv("""install -v dataclasses-json --extra-pip-args="--use-feature=truststore --proxy=test" """) assert c.returncode == 1 - assert "truststore feature" in c.stderr + assert "truststore feature" in c.stdout @pytest.mark.basic diff --git a/tests/integration/test_install_categories.py b/tests/integration/test_install_categories.py index 5d33fdf4..4fa39d77 100644 --- a/tests/integration/test_install_categories.py +++ b/tests/integration/test_install_categories.py @@ -84,7 +84,6 @@ def test_multiple_category_install_proceeds_in_order_specified(pipenv_instance_p """Ensure -e .[extras] installs. """ with pipenv_instance_private_pypi() as p: - #os.mkdir(os.path.join(p.path, "testpipenv")) setup_py = os.path.join(p.path, "setup.py") with open(setup_py, "w") as fh: contents = """ @@ -106,12 +105,12 @@ setup( with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] -testpipenv = {path = ".", editable = true} +testpipenv = {path = ".", editable = true, skip_resolver = true} [prereq] six = "*" """.strip()) - c = p.pipenv("lock") + c = p.pipenv("lock -v") assert c.returncode == 0 assert "testpipenv" in p.lockfile["default"] assert "testpipenv" not in p.lockfile["prereq"] diff --git a/tests/integration/test_install_markers.py b/tests/integration/test_install_markers.py index 3f0d2ee3..f72a98d3 100644 --- a/tests/integration/test_install_markers.py +++ b/tests/integration/test_install_markers.py @@ -28,7 +28,11 @@ fake_package = {} c = p.pipenv('install -v') assert c.returncode == 0 - assert 'markers' in p.lockfile['default']['fake-package'], p.lockfile["default"] + assert 'markers' in p.lockfile['default']['fake_package'], p.lockfile["default"] + assert p.lockfile['default']['fake_package']['markers'] == "os_name == 'splashwear'" + assert p.lockfile['default']['fake_package']['hashes'] == [ + 'sha256:1531e01a7f306f496721f425c8404f3cfd8d4933ee6daf4668fcc70059b133f3', + 'sha256:cf83dc3f6c34050d3360fbdf655b2652c56532e3028b1c95202611ba1ebdd624'] c = p.pipenv('run python -c "import fake_package;"') assert c.returncode == 1 diff --git a/tests/integration/test_install_misc.py b/tests/integration/test_install_misc.py index 503d409f..1fb0ff50 100644 --- a/tests/integration/test_install_misc.py +++ b/tests/integration/test_install_misc.py @@ -1,11 +1,15 @@ import pytest +from .conftest import DEFAULT_PRIVATE_PYPI_SERVER + + @pytest.mark.urls @pytest.mark.extras @pytest.mark.install -def test_install_uri_with_extras(pipenv_instance_private_pypi): - file_uri = "http://localhost:8080/packages/plette/plette-0.2.2-py2.py3-none-any.whl" - with pipenv_instance_private_pypi() as p: +def test_install_uri_with_extras(pipenv_instance_pypi): + server = DEFAULT_PRIVATE_PYPI_SERVER.replace("/simple", "") + file_uri = f"{server}/packages/plette/plette-0.2.2-py2.py3-none-any.whl" + with pipenv_instance_pypi() as p: with open(p.pipfile_path, 'w') as f: contents = f""" [[source]] @@ -20,3 +24,4 @@ plette = {{file = "{file_uri}", extras = ["validation"]}} c = p.pipenv("install") assert c.returncode == 0 assert "plette" in p.lockfile["default"] + assert "cerberus" in p.lockfile["default"] diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 583369d3..95d6cf7f 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -52,7 +52,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} c = p.pipenv(f"install {line}") assert c.returncode == 0 assert "testpipenv" in p.pipfile["packages"] - assert p.pipfile["packages"]["testpipenv"]["path"] == "." + assert p.pipfile["packages"]["testpipenv"]["file"] == "." assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] @@ -178,7 +178,7 @@ def test_local_package(pipenv_instance_private_pypi, testsroot): @pytest.mark.files @pytest.mark.local -def test_local_zip_file(pipenv_instance_private_pypi, testsroot): +def test_local_tar_gz_file(pipenv_instance_private_pypi, testsroot): file_name = "requests-2.19.1.tar.gz" with pipenv_instance_private_pypi() as p: @@ -275,3 +275,14 @@ def test_outdated_should_compare_postreleases_without_failing(pipenv_instance_pr c = p.pipenv("update --outdated") assert c.returncode != 0 assert "out-of-date" in c.stdout + + +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") +def test_install_remote_wheel_file_with_extras(pipenv_instance_pypi): + with pipenv_instance_pypi() as p: + c = p.pipenv("install fastapi[dev]@https://files.pythonhosted.org/packages/4e/1a/04887c641b67e6649bde845b9a631f73a7abfbe3afda83957e09b95d88eb/fastapi-0.95.2-py3-none-any.whl") + assert c.returncode == 0 + assert "ruff" in p.lockfile["default"] + assert "pre-commit" in p.lockfile["default"] + assert "uvicorn" in p.lockfile["default"] + diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 1ef124b0..e11ac38d 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -1,4 +1,5 @@ import os +import sys from pathlib import Path import pytest @@ -17,12 +18,13 @@ def test_basic_vcs_install_with_env_var(pipenv_instance_pypi): # edge case where normal package starts with VCS name shouldn't be flagged as vcs os.environ["GIT_HOST"] = "github.com" cli_runner = CliRunner(mix_stderr=False) - c = cli_runner.invoke(cli, "install git+https://${GIT_HOST}/benjaminp/six.git@1.11.0#egg=six gitdb2") + c = cli_runner.invoke(cli, "install -v git+https://${GIT_HOST}/benjaminp/six.git@1.11.0 gitdb2") assert c.exit_code == 0 assert all(package in p.pipfile["packages"] for package in ["six", "gitdb2"]) assert "git" in p.pipfile["packages"]["six"] assert p.lockfile["default"]["six"] == { - "git": "https://${GIT_HOST}/benjaminp/six.git", + "git": "git+https://${GIT_HOST}/benjaminp/six.git", + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", } assert "gitdb2" in p.lockfile["default"] @@ -43,7 +45,6 @@ def test_urls_work(pipenv_instance_pypi): dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - # now that we handle resolution with requirementslib, this will resolve to a name dep = p.lockfile["default"]["dataclasses-json"] assert "file" in dep, p.lockfile @@ -75,10 +76,10 @@ def test_file_urls_work(pipenv_instance_pypi): @pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_editable_vcs_install(pipenv_instance_pypi): +def test_vcs_install(pipenv_instance_pypi): with pipenv_instance_pypi() as p: c = p.pipenv( - "install -e git+https://github.com/lidatong/dataclasses-json.git#egg=dataclasses-json" + "install git+https://github.com/lidatong/dataclasses-json.git@v0.5.7" ) assert c.returncode == 0 assert "dataclasses-json" in p.pipfile["packages"] @@ -88,10 +89,10 @@ def test_editable_vcs_install(pipenv_instance_pypi): @pytest.mark.urls @pytest.mark.install @pytest.mark.needs_internet -def test_install_editable_git_tag(pipenv_instance_private_pypi): +def test_install_git_tag(pipenv_instance_private_pypi): with pipenv_instance_private_pypi() as p: c = p.pipenv( - "install -e git+https://github.com/benjaminp/six.git@1.11.0#egg=six" + "install git+https://github.com/benjaminp/six.git@1.11.0" ) assert c.returncode == 0 assert "six" in p.pipfile["packages"] @@ -99,7 +100,7 @@ def test_install_editable_git_tag(pipenv_instance_private_pypi): assert "git" in p.lockfile["default"]["six"] assert ( p.lockfile["default"]["six"]["git"] - == "https://github.com/benjaminp/six.git" + == "git+https://github.com/benjaminp/six.git" ) assert "ref" in p.lockfile["default"]["six"] @@ -108,6 +109,7 @@ def test_install_editable_git_tag(pipenv_instance_private_pypi): @pytest.mark.index @pytest.mark.install @pytest.mark.needs_internet +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") def test_install_named_index_alias(pipenv_instance_private_pypi): with pipenv_instance_private_pypi() as p: with open(p.pipfile_path, "w") as f: @@ -136,6 +138,7 @@ six = "*" @pytest.mark.index @pytest.mark.install @pytest.mark.needs_internet +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") def test_install_specifying_index_url(pipenv_instance_pypi): with pipenv_instance_pypi() as p: with open(p.pipfile_path, "w") as f: @@ -184,7 +187,7 @@ def test_install_local_vcs_not_in_lockfile(pipenv_instance_pypi): def test_get_vcs_refs(pipenv_instance_private_pypi): with pipenv_instance_private_pypi() as p: c = p.pipenv( - "install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six" + "install -e git+https://github.com/benjaminp/six.git@1.9.0" ) assert c.returncode == 0 assert "six" in p.pipfile["packages"] @@ -236,11 +239,8 @@ Jinja2 = {{ref = "2.11.0", git = "{jinja2_uri}"}} assert all(k in p.pipfile["packages"] for k in installed_packages) assert all(k.lower() in p.lockfile["default"] for k in installed_packages) assert all(k in p.lockfile["default"]["jinja2"] for k in ["ref", "git"]), str(p.lockfile["default"]) - assert p.lockfile["default"]["jinja2"].get("ref") is not None - assert ( - p.lockfile["default"]["jinja2"]["git"] - == jinja2_uri - ) + assert p.lockfile["default"]["jinja2"].get("ref") == "bbdafe33ce9f47e3cbfb9415619e354349f11243" + assert p.lockfile["default"]["jinja2"]["git"] == f"{jinja2_uri}" @pytest.mark.vcs @@ -249,8 +249,8 @@ Jinja2 = {{ref = "2.11.0", git = "{jinja2_uri}"}} @pytest.mark.needs_internet def test_vcs_can_use_markers(pipenv_instance_pypi): with pipenv_instance_pypi() as p: - path = p._pipfile.get_fixture_path("git/six/.git") - p._pipfile.install("six", {"git": f"{path.as_uri()}", "markers": "sys_platform == 'linux'"}) + path = p._pipfile.get_fixture_path("git/six/") + p._pipfile.install("six", {"git": f"{path.as_uri()}", "ref": "1.11.0", "markers": "sys_platform == 'linux'"}) assert "six" in p.pipfile["packages"] c = p.pipenv("install") assert c.returncode == 0 diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 14bca61c..29d57262 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -157,10 +157,7 @@ def test_resolve_skip_unmatched_requirements(pipenv_instance_pypi): p._pipfile.add("missing-package", {"markers": "os_name=='FakeOS'"}) c = p.pipenv("lock") assert c.returncode == 0 - assert ( - "Could not find a version of missing-package ; " - "os_name == 'FakeOS' that matches your environment" - ) in c.stderr + assert 'Could not find a matching version of missing-package; os_name == "FakeOS"' in c.stderr @pytest.mark.lock @@ -358,7 +355,7 @@ requests = {git = "%s@883caaf", editable = true} """.strip() % requests_uri) c = p.pipenv('lock') assert c.returncode == 0 - assert p.lockfile['default']['requests']['git'] == requests_uri + assert requests_uri in p.lockfile['default']['requests']['git'] assert p.lockfile['default']['requests']['ref'] == '883caaf145fbe93bd0d208a6b864de9146087312' @@ -509,21 +506,21 @@ def test_lock_nested_vcs_direct_url(pipenv_instance_pypi): assert "git" in p.lockfile["default"]["pep508-package"] assert "sibling-package" in p.lockfile["default"] assert "git" in p.lockfile["default"]["sibling-package"] - assert "subdirectory" in p.lockfile["default"]["sibling-package"] + assert "subdirectory" in p.lockfile["default"]["sibling-package"]["git"] assert "version" not in p.lockfile["default"]["sibling-package"] @pytest.mark.lock @pytest.mark.install -def test_lock_package_with_wildcard_version(pipenv_instance_pypi): - with pipenv_instance_pypi() as p: - c = p.pipenv("install 'six==1.11.*'") +def test_lock_package_with_compatible_release_specifier(pipenv_instance_private_pypi): + with pipenv_instance_private_pypi() as p: + c = p.pipenv("install six~=1.11") assert c.returncode == 0 assert "six" in p.pipfile["packages"] - assert p.pipfile["packages"]["six"] == "==1.11.*" + assert p.pipfile["packages"]["six"] == "~=1.11" assert "six" in p.lockfile["default"] assert "version" in p.lockfile["default"]["six"] - assert p.lockfile["default"]["six"]["version"] == "==1.11.0" + assert p.lockfile["default"]["six"]["version"] == "==1.12.0" @pytest.mark.lock @@ -649,4 +646,4 @@ dataclasses-json = {extras = ["dev"], version = "==0.5.7"} assert c.returncode == 0 assert "dataclasses-json" in p.pipfile["packages"] assert "dataclasses-json" in p.lockfile["default"] - assert "markers" not in p.lockfile["default"]["dataclasses-json"] + assert p.lockfile["default"]["dataclasses-json"].get("markers", "") is not None diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 9c40ca90..4279a3b0 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -6,7 +6,7 @@ import pytest from pipenv.project import Project from pipenv.utils.shell import temp_environ from pipenv.vendor.plette import Pipfile -from pipenv.vendor.requirementslib.fileutils import normalize_path +from pipenv.utils.fileutils import normalize_path @pytest.mark.project diff --git a/tests/integration/test_requirements.py b/tests/integration/test_requirements.py index 271374c5..ed0cdc5f 100644 --- a/tests/integration/test_requirements.py +++ b/tests/integration/test_requirements.py @@ -3,7 +3,8 @@ import os import pytest from pipenv.utils.shell import temp_environ -from pipenv.routines.requirements import requirements_from_deps +from pipenv.utils.requirements import requirements_from_lockfile + @pytest.mark.requirements def test_requirements_generates_requirements_from_lockfile(pipenv_instance_pypi): @@ -289,11 +290,11 @@ def test_requirements_generates_requirements_from_lockfile_without_env_var_expan }, True, True, - ["pyjwt[crypto] @ git+https://github.com/jpadilla/pyjwt.git@7665aa625506a11bae50b56d3e04413a3dc6fdf8"] + ["pyjwt[crypto]@ git+https://github.com/jpadilla/pyjwt.git@7665aa625506a11bae50b56d3e04413a3dc6fdf8"] ) ] ) def test_requirements_from_deps(deps, include_hashes, include_markers, expected): - result = requirements_from_deps(deps, include_hashes, include_markers) + result = requirements_from_lockfile(deps, include_hashes, include_markers) assert result == expected diff --git a/tests/integration/test_run.py b/tests/integration/test_run.py index 9337080d..0cb99d81 100644 --- a/tests/integration/test_run.py +++ b/tests/integration/test_run.py @@ -37,7 +37,6 @@ multicommand = "bash -c \"cd docs && make html\"" c = p.pipenv('run printfoo') assert c.returncode == 0 assert c.stdout.strip() == 'foo' - assert not c.stderr.strip() c = p.pipenv('run notfoundscript') assert c.returncode != 0 diff --git a/tests/integration/test_uninstall.py b/tests/integration/test_uninstall.py index 3956901a..06892bdc 100644 --- a/tests/integration/test_uninstall.py +++ b/tests/integration/test_uninstall.py @@ -1,4 +1,5 @@ import pytest +import sys from .conftest import DEFAULT_PRIVATE_PYPI_SERVER @@ -7,28 +8,20 @@ from pipenv.utils.shell import temp_environ @pytest.mark.uninstall @pytest.mark.install -def test_uninstall_requests(pipenv_instance_private_pypi): - # Uninstalling requests can fail even when uninstall Django below - # succeeds, if requests was de-vendored. - # See https://github.com/pypa/pipenv/issues/3644 for problems - # caused by devendoring - with pipenv_instance_private_pypi() as p: +def test_uninstall_requests(pipenv_instance_pypi): + with pipenv_instance_pypi() as p: c = p.pipenv("install requests") assert c.returncode == 0 assert "requests" in p.pipfile["packages"] - c = p.pipenv("run python -m requests.help") - assert c.returncode == 0 - c = p.pipenv("uninstall requests") assert c.returncode == 0 - assert "requests" not in p.pipfile["dev-packages"] - - c = p.pipenv("run python -m requests.help") - assert c.returncode > 0 + assert "requests" not in p.pipfile["packages"] + assert "requests" not in p.lockfile["default"] @pytest.mark.uninstall +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") def test_uninstall_django(pipenv_instance_private_pypi): with pipenv_instance_private_pypi() as p: c = p.pipenv("install Django") @@ -52,6 +45,7 @@ def test_uninstall_django(pipenv_instance_private_pypi): @pytest.mark.install @pytest.mark.uninstall +@pytest.mark.skipif(sys.version_info >= (3, 12), reason="Package does not work with Python 3.12") def test_mirror_uninstall(pipenv_instance_pypi): with temp_environ(), pipenv_instance_pypi() as p: @@ -132,8 +126,6 @@ six = "==1.12.0" assert "tablib" in p.lockfile["default"] assert "jinja2" in p.lockfile["develop"] assert "six" in p.lockfile["develop"] - - c = p.pipenv('run python -c "import jinja2"') assert c.returncode == 0 c = p.pipenv("uninstall --all-dev") diff --git a/tests/integration/test_update.py b/tests/integration/test_update.py index 7384453c..6cfd4bae 100644 --- a/tests/integration/test_update.py +++ b/tests/integration/test_update.py @@ -1,5 +1,6 @@ import pytest + @pytest.mark.parametrize("cmd_option", ["", "--dev"]) @pytest.mark.basic @pytest.mark.update @@ -7,5 +8,5 @@ def test_update_outdated_with_outdated_package(pipenv_instance_private_pypi, cmd with pipenv_instance_private_pypi() as p: package_name = "six" p.pipenv(f"install {cmd_option} {package_name}==1.11") - c = p.pipenv("update --outdated") + c = p.pipenv(f"update {package_name} {cmd_option} --outdated") assert f"Package '{package_name}' out-of-date:" in c.stdout diff --git a/tests/integration/test_windows.py b/tests/integration/test_windows.py index 1851c834..5fce3931 100644 --- a/tests/integration/test_windows.py +++ b/tests/integration/test_windows.py @@ -48,7 +48,7 @@ def test_local_path_windows(pipenv_instance_pypi): except OSError: whl = whl.absolute() with pipenv_instance_pypi() as p: - c = p.pipenv(f'install "{whl}"') + c = p.pipenv(f'install "{whl}" -v') assert c.returncode == 0 @@ -64,7 +64,7 @@ def test_local_path_windows_forward_slash(pipenv_instance_pypi): except OSError: whl = whl.absolute() with pipenv_instance_pypi() as p: - c = p.pipenv(f'install "{whl.as_posix()}"') + c = p.pipenv(f'install "{whl.as_posix()}" -v') assert c.returncode == 0 diff --git a/tests/pypi b/tests/pypi index f5530013..2a840e04 160000 --- a/tests/pypi +++ b/tests/pypi @@ -1 +1 @@ -Subproject commit f5530013426d6392d67cd1703f379d20a768c1cf +Subproject commit 2a840e0430ac944c6b404db3b12f6f77ace93b25 diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index c8a177e9..f5add9a3 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -25,11 +25,11 @@ DEP_PIP_PAIRS = [ "editable": True, } }, - "-e git+https://github.com/lidatong/dataclasses-json.git@v0.5.7#egg=dataclasses-json", + "dataclasses-json@ git+https://github.com/lidatong/dataclasses-json.git@v0.5.7", ), ( {"dataclasses-json": {"git": "https://github.com/lidatong/dataclasses-json.git", "ref": "v0.5.7"}}, - "git+https://github.com/lidatong/dataclasses-json.git@v0.5.7#egg=dataclasses-json", + "dataclasses-json@ git+https://github.com/lidatong/dataclasses-json.git@v0.5.7", ), ( # Extras in url @@ -39,7 +39,7 @@ DEP_PIP_PAIRS = [ "extras": ["pipenv"], } }, - "https://github.com/oz123/dparse/archive/refs/heads/master.zip#egg=dparse[pipenv]" + "dparse[pipenv] @ https://github.com/oz123/dparse/archive/refs/heads/master.zip", ), ( { @@ -50,7 +50,7 @@ DEP_PIP_PAIRS = [ "editable": False, } }, - "git+https://github.com/requests/requests.git@main#egg=requests[security]", + "requests[security]@ git+https://github.com/requests/requests.git@main", ), ] @@ -64,8 +64,6 @@ def mock_unpack(link, source_dir, download_dir, only_download=False, session=Non @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) @pytest.mark.needs_internet def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() assert dependencies.convert_deps_to_pip(deps) == [expected] @@ -136,13 +134,12 @@ def test_convert_deps_to_pip_one_way(): @pytest.mark.parametrize( "deps, expected", [ - ({"uvicorn": {}}, ["uvicorn"]), - ({"FooProject": {"path": ".", "editable": "true"}}, []), - ({"FooProject": {"version": "==1.2"}}, ["fooproject==1.2"]), - ({"uvicorn": {"extras": ["standard"]}}, []), - ({"uvicorn": {"extras": []}}, ["uvicorn"]), - ({"extras": {}}, ["extras"]), - ({"uvicorn[standard]": {}}, []) + ({"uvicorn": {}}, {"uvicorn"}), + ({"FooProject": {"path": ".", "editable": "true"}}, set()), + ({"FooProject": {"version": "==1.2"}}, {"fooproject==1.2"}), + ({"uvicorn": {"extras": ["standard"]}}, {"uvicorn"}), + ({"uvicorn": {"extras": []}}, {"uvicorn"}), + ({"extras": {}}, {"extras"}), ], ) def test_get_constraints_from_deps(deps, expected): @@ -209,7 +206,7 @@ class TestUtils: ) @pytest.mark.vcs def test_is_vcs(self, entry, expected): - from pipenv.vendor.requirementslib.utils import is_vcs + from pipenv.utils.requirementslib import is_vcs assert is_vcs(entry) is expected @pytest.mark.utils diff --git a/tests/unit/test_vendor.py b/tests/unit/test_vendor.py index 32e594b3..d6dc09e5 100644 --- a/tests/unit/test_vendor.py +++ b/tests/unit/test_vendor.py @@ -3,7 +3,7 @@ import pipenv # noqa import datetime -import os + import pytest import pytz @@ -43,9 +43,3 @@ from pipenv.vendor import tomlkit def test_token_date(dt, content): item = tomlkit.item(dt) assert item.as_string() == content - - -def test_dump_nonascii_string(): - content = 'name = "Stažené"\n' - toml_content = tomlkit.dumps(tomlkit.loads(content)) - assert toml_content == content