diff --git a/Pipfile b/Pipfile index db952be7..6b4924be 100644 --- a/Pipfile +++ b/Pipfile @@ -8,7 +8,7 @@ twine = "*" sphinx-click = "*" pytest-xdist = "*" click = "*" -pytest-pypy = {path = "./tests/pytest-pypi", editable = true} +pytest-pypi = {path = "./tests/pytest-pypi", editable = true} pytest-tap = "*" flaky = "*" stdeb = {version="*", markers="sys_platform == 'linux'"} diff --git a/Pipfile.lock b/Pipfile.lock index 46115fdc..54f07fe3 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -53,10 +53,10 @@ }, "atomicwrites": { "hashes": [ - "sha256:0312ad34fcad8fac3704d441f7b317e50af620823353ec657a53e981f92920c0", - "sha256:ec9ae8adaae229e4f8446952d204a3e4b5fdd2d099f9be3aaf556120135fb3ee" + "sha256:03472c30eb2c5d1ba9227e4c2ca66ab8287fbfbbda3888aa93dc2e28fc6811b4", + "sha256:75a9445bac02d8d058d5e1fe689654ba5a6556a1dfd8ce6ec55a0ed79866cfa6" ], - "version": "==1.2.1" + "version": "==1.3.0" }, "attrs": { "hashes": [ @@ -74,11 +74,11 @@ }, "beautifulsoup4": { "hashes": [ - "sha256:194ec62a25438adcb3fdb06378b26559eda1ea8a747367d34c33cef9c7f48d57", - "sha256:90f8e61121d6ae58362ce3bed8cd997efb00c914eae0ff3d363c32f9a9822d10", - "sha256:f0abd31228055d698bb392a826528ea08ebb9959e6bea17c606fd9c9009db938" + "sha256:034740f6cb549b4e932ae1ab975581e6103ac8f942200a0e9759065984391858", + "sha256:945065979fb8529dd2f37dbb58f00b661bdbcbebf954f93b32fdf5263ef35348", + "sha256:ba6d5c59906a85ac23dadfe5c88deaf3e179ef565f4898671253e50a78680718" ], - "version": "==4.6.3" + "version": "==4.7.1" }, "black": { "hashes": [ @@ -91,10 +91,10 @@ }, "bleach": { "hashes": [ - "sha256:48d39675b80a75f6d1c3bdbffec791cf0bbbab665cf01e20da701c77de278718", - "sha256:73d26f018af5d5adcdabf5c1c974add4361a9c76af215fe32fdec8a6fc5fb9b9" + "sha256:213336e49e102af26d9cde77dd2d0397afabc5a6bf2fed985dc35b5d1e285a16", + "sha256:3fdf7f77adcf649c9911387df51254b813185e32b2c6619f690b593a617e19fa" ], - "version": "==3.0.2" + "version": "==3.1.0" }, "bs4": { "hashes": [ @@ -111,10 +111,10 @@ }, "certifi": { "hashes": [ - "sha256:339dc09518b07e2fa7eda5450740925974815557727d6bd35d319c1524a04a4c", - "sha256:6d58c986d22b038c8c0df30d639f23a3e6d172a05c3583e766f4c0b785c0986a" + "sha256:47f9c83ef4c0c621eaef743f133f09fa8a74a9b75f037e8624f83bd1b6626cb7", + "sha256:993f830721089fef441cdfeb4b2c8c9df86f0c63239f06bd025a76a7daddb033" ], - "version": "==2018.10.15" + "version": "==2018.11.29" }, "cffi": { "hashes": [ @@ -217,9 +217,9 @@ }, "cursor": { "hashes": [ - "sha256:8ee9fe5b925e1001f6ae6c017e93682583d2b4d1ef7130a26cfcdf1651c0032c" + "sha256:7e728934f555a84a1c8b0850b66efcb580d092acc927b7d15dd43eb27dd4c4c5" ], - "version": "==1.2.0" + "version": "==1.3.1" }, "distlib": { "hashes": [ @@ -235,6 +235,13 @@ ], "version": "==0.14" }, + "entrypoints": { + "hashes": [ + "sha256:589f874b313739ad35be6e0cd7efde2a4e9b6fea91edcc34e58ecbb8dbe56d19", + "sha256:c70dd71abe5a8c85e55e12c19bd91ccfeec11a6e99044204511f9ed547d48451" + ], + "version": "==0.3" + }, "enum34": { "hashes": [ "sha256:2d81cbbe0e73112bdfe6ef8576f2238f2ba27dd0d55752a776c41d38b7da2850", @@ -242,7 +249,7 @@ "sha256:6bd0f6ad48ec2aa117d3d141940d484deccda84d4fcd884f5c3d93c23ecd8c79", "sha256:8ad8c4783bf61ded74527bffb48ed9b54166685e4230386a9ed9b1279e2df5b1" ], - "markers": "python_version < '3.4'", + "markers": "python_version < '3'", "version": "==1.1.6" }, "execnet": { @@ -306,10 +313,10 @@ }, "idna": { "hashes": [ - "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", - "sha256:684a38a6f903c1d71d6d5fac066b58d7768af4de2b832e426ec79c30daa94a16" + "sha256:c357b3f628cf53ae2c4c05627ecc484553142ca23264e593d327bcde5e9c3407", + "sha256:ea8b7f6188e6fa117537c3df7da9fc686d485087abf6ac197f9c46432f7e4a3c" ], - "version": "==2.7" + "version": "==2.8" }, "imagesize": { "hashes": [ @@ -415,25 +422,25 @@ }, "more-itertools": { "hashes": [ - "sha256:c187a73da93e7a8acc0001572aebc7e3c69daf7bf6881a2cea10650bd4420092", - "sha256:c476b5d3a34e12d40130bc2f935028b5f636df8f372dc2c1c01dc19681b2039e", - "sha256:fcbfeaea0be121980e15bc97b3817b5202ca73d0eae185b4550cbfce2a3ebb3d" + "sha256:0125e8f60e9e031347105eb1682cef932f5e97d7b9a1a28d9bf00c22a5daef40", + "sha256:590044e3942351a1bdb1de960b739ff4ce277960f2425ad4509446dbace8d9d1" ], - "version": "==4.3.0" + "markers": "python_version > '2.7'", + "version": "==6.0.0" }, "packaging": { "hashes": [ - "sha256:0886227f54515e592aaa2e5a553332c73962917f2831f1b0f9b9f4380a4b9807", - "sha256:f95a1e147590f204328170981833854229bb2912ac3d5f89e2a8ccd2834800c9" + "sha256:0c98a5d0be38ed775798ece1b9727178c4469d9c3b4ada66e8e6b7849f8732af", + "sha256:9e1cbf8c12b1f1ce0bb5344b8d7ecf66a6f8a6e91bcb0c84593ed6d3ab5c4ab3" ], - "version": "==18.0" + "version": "==19.0" }, "parso": { "hashes": [ - "sha256:35704a43a3c113cce4de228ddb39aab374b8004f4f2407d070b6a2ca784ce8a2", - "sha256:895c63e93b94ac1e1690f5fdd40b65f07c8171e3e53cbd7793b5b96c0e0a7f24" + "sha256:4580328ae3f548b358f4901e38c0578229186835f0fa0846e47369796dd5bcc9", + "sha256:68406ebd7eafe17f8e40e15a84b56848eccbf27d7c1feb89e93d8fca395706db" ], - "version": "==0.3.1" + "version": "==0.3.4" }, "parver": { "hashes": [ @@ -450,17 +457,17 @@ }, "pbr": { "hashes": [ - "sha256:f59d71442f9ece3dffc17bc36575768e1ee9967756e6b6535f0ee1f0054c3d68", - "sha256:f6d5b23f226a2ba58e14e49aa3b1bfaf814d0199144b95d78458212444de1387" + "sha256:a7953f66e1f82e4b061f43096a4bcc058f7d3d41de9b94ac871770e8bdd831a2", + "sha256:d717573351cfe09f49df61906cd272abaa759b3e91744396b804965ff7bff38b" ], - "version": "==5.1.1" + "version": "==5.1.2" }, "pep517": { "hashes": [ - "sha256:cc663a438fdfe2e88d8d3c5ef2203ac858de34e31b6609b1fc505d611490a926", - "sha256:f79bb08fb064dfc5b141204bfeb56a4141a6d504677fab4723036a464fc25cc1" + "sha256:43a7aa3902efd305a605c1028e4045968cd012831233ecab633a31d3ba4860a5", + "sha256:cb5ca55450b64e80744cd5c32f7d5b8928004042dfea50fdc3f96ad7f27cba96" ], - "version": "==0.3" + "version": "==0.5.0" }, "pip-shims": { "hashes": [ @@ -475,10 +482,10 @@ }, "pkginfo": { "hashes": [ - "sha256:5878d542a4b3f237e359926384f1dde4e099c9f5525d236b1840cf704fa8d474", - "sha256:a39076cb3eb34c333a0dd390b568e9e1e881c7bf2cc0aee12120636816f55aee" + "sha256:7424f2c8511c186cd5424bbf31045b77435b37a8d604990b79d4e70d741148bb", + "sha256:a6d9e40ca61ad3ebd0b72fbadd4fba16e4c0e4df0428c041e01e06eb6ee71f32" ], - "version": "==1.4.2" + "version": "==1.5.0.1" }, "plette": { "extras": [ @@ -492,10 +499,10 @@ }, "pluggy": { "hashes": [ - "sha256:447ba94990e8014ee25ec853339faf7b0fc8050cdc3289d4d71f7f410fb90095", - "sha256:bde19360a8ec4dfd8a20dcb811780a30998101f078fc7ded6162f0076f50508f" + "sha256:8ddc32f03971bfdf900a81961a48ccf2fb677cf7715108f85295c67405798616", + "sha256:980710797ff6a041e9a73a5787804f848996ecaa6f8a1b1e08224a5894f2074a" ], - "version": "==0.8.0" + "version": "==0.8.1" }, "py": { "hashes": [ @@ -506,10 +513,10 @@ }, "pycodestyle": { "hashes": [ - "sha256:cbc619d09254895b0d12c2c691e237b2e91e9b2ecf5e84c26b35400f93dcfb83", - "sha256:cbfca99bd594a10f674d0cd97a3d802a1fdef635d4361e1a2658de47ed261e3a" + "sha256:95a2219d12372f05704562a14ec30bc76b05a5b297b21a5dfe3f6fac3491ae56", + "sha256:e40a936c9a450ad81df37f549d676d127b1b66000a6c500caa2b085bc0ca976c" ], - "version": "==2.4.0" + "version": "==2.5.0" }, "pycparser": { "hashes": [ @@ -519,24 +526,24 @@ }, "pyflakes": { "hashes": [ - "sha256:9a7662ec724d0120012f6e29d6248ae3727d821bba522a0e6b356eff19126a49", - "sha256:f661252913bc1dbe7fcfcbf0af0db3f42ab65aabd1a6ca68fe5d466bace94dae" + "sha256:5e8c00e30c464c99e0b501dc160b13a14af7f27d4dffb529c556e30a159e231d", + "sha256:f277f9ca3e55de669fba45b7393a1449009cff5a37d1af10ebb76c52765269cd" ], - "version": "==2.0.0" + "version": "==2.1.0" }, "pygments": { "hashes": [ - "sha256:6301ecb0997a52d2d31385e62d0a4a4cf18d2f2da7054a5ddad5c366cd39cee7", - "sha256:82666aac15622bd7bb685a4ee7f6625dd716da3ef7473620c192c0168aae64fc" + "sha256:5ffada19f6203563680669ee7f53b64dabbeb100eb51b61996085e99c03b284a", + "sha256:e8218dd399a61674745138520d0d4cf2621d7e032439341bc3f647bff125818d" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pyparsing": { "hashes": [ - "sha256:40856e74d4987de5d01761a22d1621ae1c7f8774585acae358aa5c5936c6c90b", - "sha256:f353aab21fd474459d97b709e527b5571314ee5f067441dc9f88e33eecd96592" + "sha256:66c9268862641abcac4a96ba74506e594c884e3f57690a696d21ad8210ed667a", + "sha256:f6c5ef0d7480ad048c054c37632c67fca55299990fff127850181659eea33fc3" ], - "version": "==2.3.0" + "version": "==2.3.1" }, "pytest": { "hashes": [ @@ -548,14 +555,15 @@ }, "pytest-forked": { "hashes": [ - "sha256:e4500cd0509ec4a26535f7d4112a8cc0f17d3a41c29ffd4eab479d2a55b30805", - "sha256:f275cb48a73fc61a6710726348e1da6d68a978f0ec0c54ece5a5fae5977e5a08" + "sha256:5fe33fbd07d7b1302c95310803a5e5726a4ff7f19d5a542b7ce57c76fed8135f", + "sha256:d352aaced2ebd54d42a65825722cb433004b4446ab5d2044851d9cc7a00c9e38" ], - "version": "==0.2" + "version": "==1.0.2" }, - "pytest-pypy": { + "pytest-pypi": { "editable": true, - "path": "./tests/pytest-pypi" + "path": "./tests/pytest-pypi", + "version": "==0.1.1" }, "pytest-tap": { "hashes": [ @@ -596,24 +604,24 @@ }, "requests": { "hashes": [ - "sha256:65b3a120e4329e33c9889db89c80976c5272f56ea92d3e74da8a463992e3ff54", - "sha256:ea881206e59f41dbd0bd445437d792e43906703fff75ca8ff43ccdb11f33f263" + "sha256:502a824f31acdacb3a35b6690b5fbf0bc41d63a24a45c4004352b0242707598e", + "sha256:7bf2a778576d825600030a110f3c0e3e8edc51dfaafe1c146e39a2027784957b" ], - "version": "==2.20.1" + "version": "==2.21.0" }, "requests-toolbelt": { "hashes": [ - "sha256:42c9c170abc2cacb78b8ab23ac957945c7716249206f90874651971a4acff237", - "sha256:f6a531936c6fa4c6cfce1b9c10d5c4f498d16528d2a54a22ca00011205a187b5" + "sha256:380606e1d10dc85c3bd47bf5a6095f815ec007be7a8b69c878507068df059e6f", + "sha256:968089d4584ad4ad7c171454f0a5c6dac23971e9472521ea3b6d49d610aa6fc0" ], - "version": "==0.8.0" + "version": "==0.9.1" }, "requirementslib": { "hashes": [ - "sha256:c2c00c7bd3bd4984c97d10cd4d143efbe33b5ed9e55961bea30ca7a9a4927289", - "sha256:dc6b692e8dee03d6e90c29db1e337b0bf8152cce84a57f0fb4765e596afde4e0" + "sha256:c26feee79853dedddab550cf79fb2fa83b4bc1a16eab58f2c870e8314caa6cc5", + "sha256:d302b780afbd1d60f49d368b535929d8ff4b6d972797f3777c9560d48abdded7" ], - "version": "==1.3.3" + "version": "==1.4.0" }, "resolvelib": { "hashes": [ @@ -631,10 +639,10 @@ }, "six": { "hashes": [ - "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", - "sha256:832dc0e10feb1aa2c68dcc57dbb658f1c7e65b9b61af69048abc87a2db00a0eb" + "sha256:3350809f0555b11f552448330d0b52d5f24c91a322ea4a15ef22629740f3761c", + "sha256:d16a0141ec1a18405cd4ce8b4613101da75da0e9a7aec5bdd4fa804d0e0eba73" ], - "version": "==1.11.0" + "version": "==1.12.0" }, "snowballstemmer": { "hashes": [ @@ -643,6 +651,13 @@ ], "version": "==1.2.1" }, + "soupsieve": { + "hashes": [ + "sha256:afa56bf14907bb09403e5d15fbed6275caa4174d36b975226e3b67a3bb6e2c4b", + "sha256:eaed742b48b1f3e2d45ba6f79401b2ed5dc33b2123dfe216adb90d4bfa0ade26" + ], + "version": "==1.8" + }, "sphinx": { "hashes": [ "sha256:11f271e7a9398385ed730e90f0bb41dc3815294bdcd395b46ed2d033bc2e7d87", @@ -684,7 +699,8 @@ "toml": { "hashes": [ "sha256:229f81c57791a41d65e399fc06bf0848bab550a9dfd5ed66df18ce5f05e73d5c", - "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e" + "sha256:235682dd292d5899d361a811df37e04a8828a5b1da3115886b73cf81ebc9100e", + "sha256:f1db651f9657708513243e61e6cc67d101a39bad662eaa9b5546f789338e07a3" ], "version": "==0.10.0" }, @@ -702,10 +718,10 @@ }, "tqdm": { "hashes": [ - "sha256:3c4d4a5a41ef162dd61f1edb86b0e1c7859054ab656b2e7c7b77e7fbf6d9f392", - "sha256:5b4d5549984503050883bc126280b386f5f4ca87e6c023c5d015655ad75bdebb" + "sha256:d385c95361699e5cf7622485d9b9eae2d4864b21cd5a2374a9c381ffed701021", + "sha256:e22977e3ebe961f72362f6ddfb9197cc531c9737aaf5f607ef09740c849ecd05" ], - "version": "==4.28.1" + "version": "==4.31.1" }, "twine": { "hashes": [ @@ -733,26 +749,28 @@ }, "virtualenv": { "hashes": [ - "sha256:686176c23a538ecc56d27ed9d5217abd34644823d6391cbeb232f42bf722baad", - "sha256:f899fafcd92e1150f40c8215328be38ff24b519cd95357fa6e78e006c7638208" + "sha256:8b9abfc51c38b70f61634bf265e5beacf6fae11fc25d355d1871f49b8e45f0db", + "sha256:cceab52aa7d4df1e1871a70236eb2b89fcfe29b6b43510d9738689787c513261" ], - "version": "==16.1.0" + "version": "==16.4.0" }, "virtualenv-clone": { "hashes": [ - "sha256:afce268508aa5596c90dda234abe345deebc401a57d287bcbd76baa140a1aa58" + "sha256:217bd3f0880c9f85672c0bcc9ad9e0354ab7dfa89c2f117e63aa878b4279f5bf", + "sha256:316c8a05432a7adb5e461709759aca18c51433ffc2c33e2e80c9e51c452d339f", + "sha256:f2a07ed255f3abaceef8c8442512d8cdb2ba9f867e212d8a51680c7790a85033" ], - "version": "==0.4.0" + "version": "==0.5.1" }, "vistir": { "extras": [ "spinner" ], "hashes": [ - "sha256:3a1020fb7be000b268af96641ced9ead844b1f75840c41e20e473647688fc630", - "sha256:6d2005ad670f77bd9c9b5415c4e2a4a20dce5b0cf0e0d11598eb463b2e0ebe44" + "sha256:510408ec63a4b423967fd630bf0885c8d6a1d5d126f8bb1be6aba86a0da5e815", + "sha256:fc5cca7a14e92feaa6f85dd91da74d834904280a96a21190aecb4cd1d1048e0e" ], - "version": "==0.2.5" + "version": "==0.3.0" }, "webencodings": { "hashes": [ @@ -770,17 +788,17 @@ }, "wheel": { "hashes": [ - "sha256:029703bf514e16c8271c3821806a1c171220cc5bdd325cbf4e7da1e056a01db6", - "sha256:1e53cdb3f808d5ccd0df57f964263752aa74ea7359526d3da6c02114ec1e1d44" + "sha256:12363e6df5678ecf9daf8429f06f97e7106e701405898f24318ce7f0b79c611a", + "sha256:b79ffea026bc0dbd940868347ae9eee36789b6496b6623bd2dec7c7c540a8f99" ], - "version": "==0.32.3" + "version": "==0.33.0" }, "yaspin": { "hashes": [ - "sha256:36fdccc5e0637b5baa8892fe2c3d927782df7d504e9020f40eb2c1502518aa5a", - "sha256:8e52bf8079a48e2a53f3dfeec9e04addb900c101d1591c85df69cf677d3237e7" + "sha256:441f8a6761e347652d04614899fd0a9cfda7439e2d5682e664bd31230c656176", + "sha256:d3ebcf8162e0ef8bb5484b8751d5b6d2fbf0720112c81f64614c308576a03b1d" ], - "version": "==0.14.0" + "version": "==0.14.1" } } } diff --git a/news/3148.feature.rst b/news/3148.feature.rst new file mode 100644 index 00000000..e33434db --- /dev/null +++ b/news/3148.feature.rst @@ -0,0 +1 @@ +Added support for resolution of direct-url dependencies in ``setup.py`` files to respect ``PEP-508`` style URL dependencies. diff --git a/pipenv/core.py b/pipenv/core.py index 8fae27c8..08179fd9 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1,5 +1,5 @@ # -*- coding=utf-8 -*- - +from __future__ import absolute_import, print_function import json as simplejson import logging import os @@ -20,7 +20,7 @@ import dotenv import pipfile from . import environments, exceptions, pep508checker, progress -from ._compat import fix_utf8 +from ._compat import fix_utf8, decode_for_output from .cmdparse import Script from .environments import ( PIPENV_CACHE_DIR, PIPENV_COLORBLIND, PIPENV_DEFAULT_PYTHON_VERSION, @@ -675,7 +675,7 @@ def batch_install(deps_list, procs, failed_deps_queue, requirements_dir, no_deps=False, ignore_hashes=False, allow_global=False, blocking=False, pypi_mirror=None, nprocs=PIPENV_MAX_SUBPROCESS, retry=True): - + from .vendor.requirementslib.models.utils import strip_extras_markers_from_requirement failed = (not retry) if not failed: label = INSTALL_LABEL if os.name != "nt" else "" @@ -690,6 +690,10 @@ def batch_install(deps_list, procs, failed_deps_queue, trusted_hosts = [] # Install these because for dep in deps_list_bar: + if dep.req.req: + dep.req.req = strip_extras_markers_from_requirement(dep.req.req) + if dep.markers: + dep.markers = str(strip_extras_markers_from_requirement(dep.get_markers())) index = None if dep.index: index = project.find_source(dep.index) @@ -698,10 +702,18 @@ def batch_install(deps_list, procs, failed_deps_queue, trusted_hosts.append(urllib3_util.parse_url(index.get("url")).host) # Install the module. is_artifact = False + if no_deps: + link = getattr(dep.req, "link", None) + is_wheel = False + if link: + is_wheel = link.is_wheel if dep.is_file_or_url and (dep.is_direct_url or any( dep.req.uri.endswith(ext) for ext in ["zip", "tar.gz"] )): is_artifact = True + elif dep.is_vcs: + is_artifact = True + needs_deps = not no_deps if no_deps is True else is_artifact extra_indexes = [] if not index and indexes: @@ -714,24 +726,24 @@ def batch_install(deps_list, procs, failed_deps_queue, os.environ["PIP_USER"] = vistir.compat.fs_str("0") if "PYTHONHOME" in os.environ: del os.environ["PYTHONHOME"] - if no_deps: + if not needs_deps: link = getattr(dep.req, "link", None) is_wheel = False if link: is_wheel = link.is_wheel - is_non_editable_vcs = (dep.is_vcs and not dep.editable) - no_deps = not (dep.is_file_or_url and not (is_wheel or dep.editable)) + needs_deps = dep.is_file_or_url and not (is_wheel or dep.editable) c = pip_install( dep, ignore_hashes=any([ignore_hashes, dep.editable, dep.is_vcs]), allow_global=allow_global, - no_deps=no_deps, + no_deps=not needs_deps, block=any([dep.editable, dep.is_vcs, blocking]), index=index, requirements_dir=requirements_dir, pypi_mirror=pypi_mirror, trusted_hosts=trusted_hosts, - extra_indexes=extra_indexes + extra_indexes=extra_indexes, + use_pep517=not retry, ) if dep.is_vcs or dep.editable: c.block() @@ -814,6 +826,7 @@ def do_install_dependencies( else: install_kwargs["nprocs"] = 1 + # with project.environment.activated(): batch_install( deps_list, procs, failed_deps_queue, requirements_dir, **install_kwargs ) @@ -1016,9 +1029,12 @@ def do_lock( dev_packages = overwrite_dev(project.packages, dev_packages) # Resolve dev-package dependencies, with pip-tools. for is_dev in [True, False]: - pipfile_section = "dev_packages" if is_dev else "packages" + pipfile_section = "dev-packages" if is_dev else "packages" lockfile_section = "develop" if is_dev else "default" - packages = getattr(project, pipfile_section) + if project.pipfile_exists: + packages = project.parsed_pipfile.get(pipfile_section, {}) + else: + packages = getattr(project, pipfile_section.replace("-", "_")) if write: # Alert the user of progress. @@ -1264,12 +1280,12 @@ def pip_install( requirements_dir=None, extra_indexes=None, pypi_mirror=None, - trusted_hosts=None + trusted_hosts=None, + use_pep517=True ): from pipenv.patched.notpip._internal import logger as piplogger from .utils import Mapping from .vendor.urllib3.util import parse_url - src = [] write_to_tmpfile = False if requirement: @@ -1299,9 +1315,11 @@ def pip_install( f.write(vistir.misc.to_bytes(requirement.as_line())) r = f.name f.close() - # Install dependencies when a package is a VCS dependency. + if requirement and requirement.vcs: - no_deps = False + # Install dependencies when a package is a non-editable VCS dependency. + if not requirement.editable: + no_deps = False # Don't specify a source directory when using --system. if not allow_global and ("PIP_SRC" not in os.environ): src.extend(["--src", "{0}".format(project.virtualenv_src_location)]) @@ -1347,16 +1365,72 @@ def pip_install( create_mirror_source(pypi_mirror) if is_pypi_url(source["url"]) else source for source in sources ] + + line_kwargs = {"as_list": True, "include_hashes": not ignore_hashes} + + # Install dependencies when a package is a VCS dependency. + if requirement and requirement.vcs: + # Don't specify a source directory when using --system. + src_dir = None + if "PIP_SRC" in os.environ: + src_dir = os.environ["PIP_SRC"] + src = ["--src", os.environ["PIP_SRC"]] + if not requirement.editable: + no_deps = False + + if src_dir is not None: + repo = requirement.req.get_vcs_repo(src_dir=src_dir) + else: + repo = requirement.req.get_vcs_repo() + write_to_tmpfile = True + line_kwargs["include_markers"] = False + line_kwargs["include_hashes"] = False + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir(prefix="pipenv", + suffix="requirements") + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + line = "-e" if requirement.editable else "" + if requirement.editable or requirement.name is not None: + name = requirement.name + if requirement.extras: + name = "{0}{1}".format(name, requirement.extras_as_pip) + line = "-e {0}#egg={1}".format(vistir.path.path_to_url(repo.checkout_directory), requirement.name) + if repo.subdirectory: + line = "{0}&subdirectory={1}".format(line, repo.subdirectory) + else: + line = requirement.as_line(**line_kwargs) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + + # Create files for hash mode. + if write_to_tmpfile and not r: + if not requirements_dir: + requirements_dir = vistir.path.create_tracked_tempdir( + prefix="pipenv", suffix="requirements") + f = vistir.compat.NamedTemporaryFile( + prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir, + delete=False + ) + ignore_hashes = True if not requirement.hashes else ignore_hashes + line = requirement.as_line(include_hashes=not ignore_hashes) + line = "{0} {1}".format(line, " ".join(src)) + f.write(vistir.misc.to_bytes(line)) + r = f.name + f.close() + if (requirement and requirement.editable) and not r: - line_kwargs = {"as_list": True} - if requirement.markers: - line_kwargs["include_markers"] = False + line_kwargs["include_markers"] = False + line_kwargs["include_hashes"] = False install_reqs = requirement.as_line(**line_kwargs) if requirement.editable and install_reqs[0].startswith("-e "): req, install_reqs = install_reqs[0], install_reqs[1:] editable_opt, req = req.split(" ", 1) install_reqs = [editable_opt, req] + install_reqs - if not all(item.startswith("--hash") for item in install_reqs): + if not any(item.startswith("--hash") for item in install_reqs): ignore_hashes = True elif r: install_reqs = ["-r", r] @@ -1365,7 +1439,7 @@ def pip_install( ignore_hashes = True else: ignore_hashes = True if not requirement.hashes else False - install_reqs = requirement.as_line(as_list=True) + install_reqs = requirement.as_line(as_list=True, include_hashes=not ignore_hashes) if not requirement.markers: install_reqs = [escape_cmd(r) for r in install_reqs] elif len(install_reqs) > 1: @@ -1386,7 +1460,9 @@ def pip_install( pip_command.extend(prepare_pip_source_args(sources)) if not ignore_hashes: pip_command.append("--require-hashes") - + if not use_pep517: + pip_command.append("--no-build-isolation") + pip_command.append("--no-use-pep517") if environments.is_verbose(): click.echo("$ {0}".format(pip_command), err=True) cache_dir = vistir.compat.Path(PIPENV_CACHE_DIR) @@ -1405,6 +1481,8 @@ def pip_install( ) cmd = Script.parse(pip_command) pip_command = cmd.cmdify() + c = None + # with project.environment.activated(): c = delegator.run(pip_command, block=block, env=pip_config) return c @@ -1863,7 +1941,7 @@ def do_install( # This is for if the user passed in dependencies, then we want to make sure we else: - from .vendor.requirementslib import Requirement + from .vendor.requirementslib.models.requirements import Requirement # make a tuple of (display_name, entry) pkg_list = packages + ["-e {0}".format(pkg) for pkg in editable_packages] @@ -1914,6 +1992,15 @@ def do_install( pypi_mirror=pypi_mirror, ) if not c.ok: + sp.write_err(vistir.compat.fs_str( + "{0}: {1}".format( + crayons.red("WARNING"), + "Failed installing package {0}".format(pkg_line) + ), + )) + sp.write_err(vistir.compat.fs_str( + "Error text: {0}".format(c.out) + )) raise RuntimeError(c.err) except (ValueError, RuntimeError) as e: sp.write_err(vistir.compat.fs_str( @@ -1961,12 +2048,20 @@ def do_install( crayons.normal(fix_utf8("…"), bold=True), ) )) + # Add the package to the Pipfile. + try: + project.add_package_to_pipfile(pkg_requirement, dev) + except ValueError as e: + import traceback + sp.write_err( + "{0} {1}".format( + crayons.red("Error:", bold=True), traceback.format_exc() + ) + ) + sp.fail(environments.PIPENV_SPINNER_FAIL_TEXT.format( + "Failed adding package to Pipfile" + )) sp.ok(environments.PIPENV_SPINNER_OK_TEXT.format("Installation Succeeded")) - # Add the package to the Pipfile. - try: - project.add_package_to_pipfile(pkg_requirement, dev) - except ValueError as e: - raise exceptions.PipfileException(e) # Update project settings with pre preference. if pre: project.update_settings({"allow_prereleases": pre}) @@ -2165,7 +2260,7 @@ def do_shell(three=None, python=False, fancy=False, shell_args=None, pypi_mirror # Only set PIPENV_ACTIVE after finishing reading virtualenv_location # otherwise its value will be changed os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - + os.environ.pop("PIP_SHIMS_BASE_MODULE", None) if fancy: @@ -2304,6 +2399,8 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): load_dot_env() + previous_pip_shims_module = os.environ.pop("PIP_SHIMS_BASE_MODULE", None) + # Activate virtualenv under the current interpreter's environment inline_activate_virtual_environment() @@ -2311,10 +2408,9 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): # Only set PIPENV_ACTIVE after finishing reading virtualenv_location # such as in inline_activate_virtual_environment # otherwise its value will be changed + previous_pipenv_active_value = os.environ.get("PIPENV_ACTIVE") os.environ["PIPENV_ACTIVE"] = vistir.misc.fs_str("1") - os.environ.pop("PIP_SHIMS_BASE_MODULE", None) - try: script = project.build_script(command, args) cmd_string = ' '.join([script.command] + script.args) @@ -2322,10 +2418,21 @@ def do_run(command, args, three=None, python=False, pypi_mirror=None): click.echo(crayons.normal("$ {0}".format(cmd_string)), err=True) except ScriptEmptyError: click.echo("Can't run script {0!r}-it's empty?", err=True) + run_args = [script] + run_kwargs = {} if os.name == "nt": - do_run_nt(script) + run_fn = do_run_nt else: - do_run_posix(script, command=command) + run_fn = do_run_posix + run_kwargs = {"command": command} + try: + run_fn(*run_args, **run_kwargs) + finally: + os.environ.pop("PIPENV_ACTIVE", None) + if previous_pipenv_active_value is not None: + os.environ["PIPENV_ACTIVE"] = previous_pipenv_active_value + if previous_pip_shims_module is not None: + os.environ["PIP_SHIMS_BASE_MODULE"] = previous_pip_shims_module def do_check( diff --git a/pipenv/exceptions.py b/pipenv/exceptions.py index 0bf53929..80e3735b 100644 --- a/pipenv/exceptions.py +++ b/pipenv/exceptions.py @@ -4,12 +4,12 @@ import itertools import sys from pprint import pformat -from traceback import format_exception +from traceback import format_exception, format_tb import six from . import environments -from ._compat import fix_utf8 +from ._compat import fix_utf8, decode_for_output from .patched import crayons from .vendor.click._compat import get_text_stderr from .vendor.click.exceptions import ( @@ -25,8 +25,8 @@ def handle_exception(exc_type, exception, traceback, hook=sys.excepthook): hook(exc_type, exception, traceback) else: exc = format_exception(exc_type, exception, traceback) - lines = itertools.chain.from_iterable([l.splitlines() for l in exc]) - lines = list(lines)[-11:-1] + tb = format_tb(traceback, limit=-6) + lines = itertools.chain.from_iterable([frame.splitlines() for frame in tb]) for line in lines: line = line.strip("'").strip('"').strip("\n").strip() if not line.startswith("File"): @@ -307,3 +307,42 @@ class ResolutionFailure(PipenvException): ) ) super(ResolutionFailure, self).__init__(fix_utf8(message), extra=extra) + + +class RequirementError(PipenvException): + + def __init__(self, req=None): + from .utils import VCS_LIST + keys = ("name", "path",) + VCS_LIST + ("line", "uri", "url", "relpath") + if req is not None: + possible_display_values = [getattr(req, value, None) for value in keys] + req_value = next(iter( + val for val in possible_display_values if val is not None + ), None) + if not req_value: + getstate_fn = getattr(req, "__getstate__", None) + slots = getattr(req, "__slots__", None) + keys_fn = getattr(req, "keys", None) + if getstate_fn: + req_value = getstate_fn() + elif slots: + slot_vals = [ + (k, getattr(req, k, None)) for k in slots + if getattr(req, k, None) + ] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in slot_vals + ]) + elif keys_fn: + values = [(k, req.get(k)) for k in keys_fn() if req.get(k)] + req_value = "\n".join([ + " {0}: {1}".format(k, v) for k, v in values + ]) + else: + req_value = getattr(req.line_instance, "line", None) + message = "{0} {1}".format( + crayons.normal(decode_for_output("Failed creating requirement instance")), + crayons.white(decode_for_output("{0!r}".format(req_value))) + ) + extra = [crayons.normal(decode_for_output(str(req)))] + super(RequirementError, self).__init__(message, extra=extra) diff --git a/pipenv/project.py b/pipenv/project.py index edbcff87..68ec4666 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -681,7 +681,7 @@ class Project(object): ) name = self.name if self.name is not None else "Pipfile" - config_parser = ConfigOptionParser(name=self.name) + config_parser = ConfigOptionParser(name=name) config_parser.add_option_group(make_option_group(index_group, config_parser)) install = config_parser.option_groups[0] indexes = ( @@ -856,7 +856,8 @@ class Project(object): return self.pipfile_sources def find_source(self, source): - """given a source, find it. + """ + Given a source, find it. source can be a url or an index name. """ @@ -869,23 +870,34 @@ class Project(object): source = self.get_source(url=source) return source - def get_source(self, name=None, url=None): + def get_source(self, name=None, url=None, refresh=False): + from .utils import is_url_equal + def find_source(sources, name=None, url=None): source = None if name: - source = [s for s in sources if s.get("name") == name] + source = next(iter( + s for s in sources if "name" in s and s["name"] == name + ), None) elif url: - source = [s for s in sources if url.startswith(s.get("url"))] - if source: - return first(source) + source = next(iter( + s for s in sources + if "url" in s and is_url_equal(url, s.get("url", "")) + ), None) + if source is not None: + return source - found_source = find_source(self.sources, name=name, url=url) - if found_source: - return found_source - found_source = find_source(self.pipfile_sources, name=name, url=url) - if found_source: - return found_source - raise SourceNotFound(name or url) + sources = (self.sources, self.pipfile_sources) + if refresh: + self.clear_pipfile_cache() + sources = reversed(sources) + found = next( + iter(find_source(source, name=name, url=url) for source in sources), None + ) + target = next(iter(t for t in (name, url) if t is not None)) + if found is None: + raise SourceNotFound(target) + return found def get_package_name_in_pipfile(self, package_name, dev=False): """Get the equivalent package name in pipfile""" @@ -930,17 +942,17 @@ class Project(object): # Don't re-capitalize file URLs or VCSs. if not isinstance(package, Requirement): package = Requirement.from_line(package.strip()) - _, converted = package.pipfile_entry + req_name, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. if key not in p: p[key] = {} - name = self.get_package_name_in_pipfile(package.name, dev) + name = self.get_package_name_in_pipfile(req_name, dev) if name and is_star(converted): # Skip for wildcard version return # Add the package to the group. - p[key][name or pep423_name(package.name)] = converted + p[key][name or pep423_name(req_name)] = converted # Write Pipfile. self.write_toml(p) diff --git a/pipenv/utils.py b/pipenv/utils.py index fc30f791..ba33da0f 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -17,7 +17,6 @@ import toml import tomlkit from click import echo as click_echo -from first import first six.add_move(six.MovedAttribute("Mapping", "collections", "collections.abc")) # noqa six.add_move(six.MovedAttribute("Sequence", "collections", "collections.abc")) # noqa six.add_move(six.MovedAttribute("Set", "collections", "collections.abc")) # noqa @@ -30,24 +29,24 @@ import crayons import parse from . import environments -from .exceptions import PipenvUsageError +from .exceptions import PipenvUsageError, ResolutionFailure, RequirementError from .pep508checker import lookup from .vendor.urllib3 import util as urllib3_util +if environments.MYPY_RUNNING: + from typing import Tuple, Dict, Any, List, Union, Optional + from .vendor.requirementslib.models.requirements import Requirement, Line + from .project import Project + + logging.basicConfig(level=logging.ERROR) specifiers = [k for k in lookup.keys()] # List of version control systems we support. VCS_LIST = ("git", "svn", "hg", "bzr") SCHEME_LIST = ("http://", "https://", "ftp://", "ftps://", "file://") -requests_session = None - - -if environments.MYPY_RUNNING: - from typing import Tuple, Dict, Any, List, Union, Optional - from .vendor.requirementslib.models.requirements import Requirement, Line - from .project import Project +requests_session = None # type: ignore def _get_requests_session(): @@ -259,7 +258,7 @@ class Resolver(object): self.resolved_tree = set() self.hashes = {} self.clear = clear - self.pre = bool(pre) + self.pre = pre self.results = None self._pip_args = None self._constraints = None @@ -344,8 +343,11 @@ class Resolver(object): line = " ".join(remainder) req = Requirement.from_line(line) if url: - index_lookup[req.normalized_name] = project.get_source( - url=url, refresh=True).get("name") + try: + index_lookup[req.normalized_name] = project.get_source( + url=url, refresh=True).get("name") + except TypeError: + raise RequirementError(req=req) # strip the marker and re-add it later after resolution # but we will need a fallback in case resolution fails # eg pypiwin32 @@ -375,7 +377,10 @@ class Resolver(object): _, entry = req.pipfile_entry parsed_line = req.req.parsed_line # type: Line setup_info = None # type: Any - name = req.normalized_name + try: + name = req.normalized_name + except TypeError: + raise RequirementError(req=req) setup_info = req.req.setup_info locked_deps[pep423_name(name)] = entry requirements = [v for v in getattr(setup_info, "requires", {}).values()] @@ -869,7 +874,8 @@ def venv_resolve_deps( pypi_mirror=None, dev=False, pipfile=None, - lockfile=None + lockfile=None, + keep_outdated=False ): """ Resolve dependencies for a pipenv project, acts as a portal to the target environment. @@ -890,6 +896,7 @@ def venv_resolve_deps( :param pipfile: A Pipfile section to operate on, defaults to None :type pipfile: Optional[Dict[str, Union[str, Dict[str, bool, List[str]]]]] :param Dict[str, Any] lockfile: A project lockfile to mutate, defaults to None + :param bool keep_outdated: Whether to retain outdated dependencies and resolve with them in mind, defaults to False :raises RuntimeError: Raised on resolution failure :return: Nothing :rtype: None @@ -932,6 +939,8 @@ def venv_resolve_deps( cmd.append("--clear") if allow_global: cmd.append("--system") + if dev: + cmd.append("--dev") with temp_environ(): os.environ.update({fs_str(k): fs_str(val) for k, val in os.environ.items()}) os.environ["PIPENV_PACKAGES"] = str("\n".join(constraints)) @@ -941,6 +950,8 @@ def venv_resolve_deps( os.environ["PIPENV_REQ_DIR"] = fs_str(req_dir) os.environ["PIP_NO_INPUT"] = fs_str("1") os.environ["PIPENV_SITE_DIR"] = get_pipenv_sitedir() + if keep_outdated: + os.environ["PIPENV_KEEP_OUTDATED"] = fs_str("1") with create_spinner(text=fs_str("Locking...")) as sp: c = resolve(cmd, sp) results = c.out.strip() @@ -973,7 +984,6 @@ def resolve_deps( """Given a list of dependencies, return a resolved list of dependencies, using pip-tools -- and their hashes, using the warehouse API / pip. """ - index_lookup = {} markers_lookup = {} python_path = which("python", allow_global=allow_global) diff --git a/pipenv/vendor/requirementslib/models/pipfile.py b/pipenv/vendor/requirementslib/models/pipfile.py index 021b5d53..ba7eeed5 100644 --- a/pipenv/vendor/requirementslib/models/pipfile.py +++ b/pipenv/vendor/requirementslib/models/pipfile.py @@ -340,8 +340,8 @@ class Pipfile(object): if not os.path.exists(self.path_to("setup.py")): if not build_system or not build_system.get("requires"): build_system = { - "requires": ["setuptools>=38.2.5", "wheel"], - "build-backend": "setuptools.build_meta", + "requires": ["setuptools>=40.8", "wheel"], + "build-backend": "setuptools.build_meta:__legacy__", } self._build_system = build_system diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index d6c13819..18b83195 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -936,6 +936,8 @@ class Line(object): self.relpath = relpath self.path = path self.uri = uri + if prefer in ("path", "relpath") or uri.startswith("file"): + self.is_local = True if link.egg_fragment: name, extras = pip_shims.shims._strip_extras(link.egg_fragment) self.extras = tuple(sorted(set(parse_extras(extras)))) @@ -2550,7 +2552,7 @@ class Requirement(object): return True return False - @cached_property + @property def normalized_name(self): # type: () -> Text return canonicalize_name(self.name) @@ -2694,7 +2696,9 @@ class Requirement(object): parts.extend(hashes) else: parts.append(hashes) - if sources and not (self.requirement.local_file or self.vcs): + + is_local = self.is_file_or_url and self.req and self.req.is_local + if sources and self.requirement and not (is_local or self.vcs): from ..utils import prepare_pip_source_args if self.index: diff --git a/pipenv/vendor/requirementslib/models/setup_info.py b/pipenv/vendor/requirementslib/models/setup_info.py index b0b55d47..5f2ac1db 100644 --- a/pipenv/vendor/requirementslib/models/setup_info.py +++ b/pipenv/vendor/requirementslib/models/setup_info.py @@ -718,7 +718,7 @@ build-backend = "{1}" get_metadata(d, pkg_name=self.name, metadata_type=metadata_type) for d in metadata_dirs if os.path.exists(d) ] - metadata = next(iter(d for d in metadata if d is not None), None) + metadata = next(iter(d for d in metadata if d), None) if metadata is not None: self.populate_metadata(metadata) diff --git a/pipenv/vendor/requirementslib/models/utils.py b/pipenv/vendor/requirementslib/models/utils.py index c9852a7b..8e4836c0 100644 --- a/pipenv/vendor/requirementslib/models/utils.py +++ b/pipenv/vendor/requirementslib/models/utils.py @@ -338,7 +338,7 @@ def get_default_pyproject_backend(): st_version = get_setuptools_version() if st_version is not None: parsed_st_version = parse_version(st_version) - if parsed_st_version >= parse_version("40.6.0"): + if parsed_st_version >= parse_version("40.8.0"): return "setuptools.build_meta:__legacy__" return "setuptools.build_meta" @@ -366,7 +366,7 @@ def get_pyproject(path): if not pp_toml.exists(): if not setup_py.exists(): return None - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: pyproject_data = {} @@ -375,10 +375,10 @@ def get_pyproject(path): build_system = pyproject_data.get("build-system", None) if build_system is None: if setup_py.exists(): - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() else: - requires = ["setuptools>=40.6", "wheel"] + requires = ["setuptools>=40.8", "wheel"] backend = get_default_pyproject_backend() build_system = { "requires": requires, diff --git a/tasks/release.py b/tasks/release.py index 7c2e7aba..375d7302 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -87,7 +87,7 @@ def release(ctx, dry_run=False): log(f'Would commit with message: "Release v{version}"') else: ctx.run('towncrier') - ctx.run("git add CHANGELOG.rst news/") + ctx.run("git add CHANGELOG.rst news/ {0}".format(get_version_file(ctx).as_posix())) ctx.run("git rm CHANGELOG.draft.rst") ctx.run(f'git commit -m "Release v{version}"') diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index c9757965..d6ea9614 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -346,6 +346,26 @@ def post_install_cleanup(ctx, vendor_dir): remove_all(vendor_dir.glob('toml.py')) +@invoke.task +def apply_patches(ctx, patched=False, pre=False): + if patched: + vendor_dir = _get_patched_dir(ctx) + else: + vendor_dir = _get_vendor_dir(ctx) + log("Applying pre-patches...") + patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name + if pre: + if not patched: + pass + for patch in patch_dir.glob('*.patch'): + if not patch.name.startswith('_post'): + apply_patch(ctx, patch) + else: + patches = patch_dir.glob('*.patch' if not patched else '_post*.patch') + for patch in patches: + apply_patch(ctx, patch) + + def vendor(ctx, vendor_dir, package=None, rewrite=True): log('Reinstalling vendored libraries') is_patched = vendor_dir.name == 'patched' @@ -359,12 +379,8 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): # Apply pre-patches log("Applying pre-patches...") - patch_dir = Path(__file__).parent / 'patches' / vendor_dir.name if is_patched: - for patch in patch_dir.glob('*.patch'): - if not patch.name.startswith('_post'): - apply_patch(ctx, patch) - + apply_patches(ctx, patched=is_patched, pre=True) log("Removing scandir library files...") remove_all(vendor_dir.glob('*.so')) drop_dir(vendor_dir / 'setuptools') @@ -385,10 +401,7 @@ def vendor(ctx, vendor_dir, package=None, rewrite=True): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) if not package: - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) + apply_patches(ctx, patched=is_patched, pre=False) if is_patched: piptools_vendor = vendor_dir / 'piptools' / '_vendored' if piptools_vendor.exists(): diff --git a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch b/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch deleted file mode 100644 index c090e885..00000000 --- a/tasks/vendoring/patches/vendor/dotenv-windows-unicode.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git a/pipenv/vendor/dotenv/main.py b/pipenv/vendor/dotenv/main.py -index 3d1bd72f..75f49c4a 100644 ---- a/pipenv/vendor/dotenv/main.py -+++ b/pipenv/vendor/dotenv/main.py -@@ -94,6 +94,13 @@ class DotEnv(): - for k, v in self.dict().items(): - if k in os.environ and not override: - continue -+ # With Python 2 on Windows, ensuree environment variables are -+ # system strings to avoid "TypeError: environment can only contain -+ # strings" in Python's subprocess module. -+ if sys.version_info.major < 3 and sys.platform == 'win32': -+ from pipenv.utils import fs_str -+ k = fs_str(k) -+ v = fs_str(v) - os.environ[k] = v - - return True diff --git a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch index d479ebfa..5d5a381f 100644 --- a/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch +++ b/tasks/vendoring/patches/vendor/pipdeptree-updated-pip18.patch @@ -7,7 +7,7 @@ index 7820aa5..2082fc8 100644 from ordereddict import OrderedDict -try: -- from pipenv.patched.notpip._internal import get_installed_distributions +- from pipenv.patched.notpip._internal.utils.misc import get_installed_distributions - from pipenv.patched.notpip._internal.operations.freeze import FrozenRequirement -except ImportError: - from pipenv.patched.notpip import get_installed_distributions, FrozenRequirement @@ -17,3 +17,4 @@ index 7820aa5..2082fc8 100644 import pkg_resources # inline: + diff --git a/tasks/vendoring/patches/vendor/tomlkit-fix.patch b/tasks/vendoring/patches/vendor/tomlkit-fix.patch index 36e2f808..4aa6c16f 100644 --- a/tasks/vendoring/patches/vendor/tomlkit-fix.patch +++ b/tasks/vendoring/patches/vendor/tomlkit-fix.patch @@ -14,8 +14,10 @@ diff --git a/pipenv/vendor/tomlkit/container.py b/pipenv/vendor/tomlkit/containe index cb8af1d5..9b5db5cb 100644 --- a/pipenv/vendor/tomlkit/container.py +++ b/pipenv/vendor/tomlkit/container.py -@@ -1,13 +1,5 @@ +@@ -1,15 +1,7 @@ from __future__ import unicode_literals + + import copy -from typing import Any -from typing import Dict diff --git a/tasks/vendoring/patches/vendor/vistir-imports.patch b/tasks/vendoring/patches/vendor/vistir-imports.patch index 673efad8..544b5564 100644 --- a/tasks/vendoring/patches/vendor/vistir-imports.patch +++ b/tasks/vendoring/patches/vendor/vistir-imports.patch @@ -15,7 +15,7 @@ diff --git a/pipenv/vendor/vistir/compat.py b/pipenv/vendor/vistir/compat.py index 9ae33fdc..ec3b65cb 100644 --- a/pipenv/vendor/vistir/compat.py +++ b/pipenv/vendor/vistir/compat.py -@@ -31,11 +31,11 @@ if sys.version_info >= (3, 5): +@@ -43,11 +43,11 @@ if sys.version_info >= (3, 5): from functools import lru_cache else: from pathlib2 import Path @@ -29,12 +29,12 @@ index 9ae33fdc..ec3b65cb 100644 NamedTemporaryFile = _NamedTemporaryFile else: from tempfile import NamedTemporaryFile -@@ -44,7 +44,7 @@ else: +@@ -56,7 +56,7 @@ else: try: from weakref import finalize except ImportError: -- from backports.weakref import finalize -+ from pipenv.vendor.backports.weakref import finalize +- from backports.weakref import finalize # type: ignore ++ from pipenv.vendor.backports.weakref import finalize # type: ignore try: from functools import partialmethod diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index b5b5bc85..f96e6ec6 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -6,11 +6,11 @@ import warnings import pytest from vistir.compat import ResourceWarning, fs_str +from vistir.contextmanagers import temp_environ from vistir.path import mkdir_p from pipenv._compat import Path, TemporaryDirectory from pipenv.exceptions import VirtualenvActivationException -from pipenv.utils import temp_environ from pipenv.vendor import delegator, requests, toml, tomlkit from pytest_pypi.app import prepare_fixtures from pytest_pypi.app import prepare_packages as prepare_pypi_packages @@ -187,14 +187,21 @@ class _Pipfile(object): class _PipenvInstance(object): """An instance of a Pipenv Project...""" - def __init__(self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None): + def __init__( + self, pypi=None, pipfile=True, chdir=False, path=None, home_dir=None, + venv_root=None, ignore_virtualenvs=True, venv_in_project=True + ): self.pypi = pypi - self.original_umask = os.umask(0o007) + if ignore_virtualenvs: + os.environ["PIPENV_IGNORE_VIRTUALENVS"] = fs_str("1") + if venv_root: + os.environ["VIRTUAL_ENV"] = venv_root + if venv_in_project: + os.environ["PIPENV_VENV_IN_PROJECT"] = fs_str("1") + else: + os.environ.pop("PIPENV_VENV_IN_PROJECT", None) + self.original_dir = os.path.abspath(os.curdir) - os.environ["PIPENV_NOSPIN"] = fs_str("1") - os.environ["CI"] = fs_str("1") - warnings.simplefilter("ignore", category=ResourceWarning) - warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") path = path if path else os.environ.get("PIPENV_PROJECT_DIR", None) if not path: self._path = TemporaryDirectory(suffix='-project', prefix='pipenv-') @@ -224,10 +231,6 @@ class _PipenvInstance(object): self._pipfile = _Pipfile(Path(p_path)) def __enter__(self): - os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = fs_str('1') - os.environ['PIPENV_VENV_IN_PROJECT'] = fs_str('1') - os.environ['PIPENV_NOSPIN'] = fs_str('1') if self.chdir: os.chdir(self.path) return self @@ -243,7 +246,6 @@ class _PipenvInstance(object): except OSError as e: _warn_msg = warn_msg.format(e) warnings.warn(_warn_msg, ResourceWarning) - os.umask(self.original_umask) def pipenv(self, cmd, block=True): if self.pipfile_path and os.path.isfile(self.pipfile_path): @@ -290,7 +292,17 @@ class _PipenvInstance(object): @pytest.fixture() def PipenvInstance(): - yield _PipenvInstance + with temp_environ(): + original_umask = os.umask(0o007) + os.environ["PIPENV_NOSPIN"] = fs_str("1") + os.environ["CI"] = fs_str("1") + os.environ['PIPENV_DONT_USE_PYENV'] = fs_str('1') + warnings.simplefilter("ignore", category=ResourceWarning) + warnings.filterwarnings("ignore", category=ResourceWarning, message="unclosed.*") + try: + yield _PipenvInstance + finally: + os.umask(original_umask) @pytest.fixture(autouse=True) diff --git a/tests/integration/test_install_basic.py b/tests/integration/test_install_basic.py index f858f8ac..2f91486e 100644 --- a/tests/integration/test_install_basic.py +++ b/tests/integration/test_install_basic.py @@ -454,17 +454,18 @@ def test_rewrite_outline_table(PipenvInstance, pypi): with open(p.pipfile_path, 'w') as f: contents = """ [packages] -six = {version = "*", editable = true} +six = {version = "*"} [packages.requests] version = "*" +extras = ["socks"] """.strip() f.write(contents) - c = p.pipenv("install -e click") + c = p.pipenv("install plette[validation]") assert c.return_code == 0 with open(p.pipfile_path) as f: contents = f.read() assert "[packages.requests]" not in contents - assert 'six = {version = "*", editable = true}' in contents - assert 'requests = {version = "*"}' in contents - assert 'click = {' in contents + assert 'six = {version = "*"}' in contents + assert 'requests = {version = "*"' in contents + assert 'plette = {' in contents diff --git a/tests/integration/test_install_twists.py b/tests/integration/test_install_twists.py index 0879a265..af6dbd45 100644 --- a/tests/integration/test_install_twists.py +++ b/tests/integration/test_install_twists.py @@ -23,22 +23,21 @@ def test_local_extras_install(PipenvInstance, pypi): contents = """ from setuptools import setup, find_packages setup( -name='testpipenv', -version='0.1', -description='Pipenv Test Package', -author='Pipenv Test', -author_email='test@pipenv.package', -license='MIT', -packages=find_packages(), -install_requires=[], -extras_require={'dev': ['six']}, -zip_safe=False + name='testpipenv', + version='0.1', + description='Pipenv Test Package', + author='Pipenv Test', + author_email='test@pipenv.package', + license='MIT', + packages=find_packages(), + install_requires=[], + extras_require={'dev': ['six']}, + zip_safe=False ) """.strip() fh.write(contents) line = "-e .[dev]" - # pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} - project = Project() + pipfile = {"testpipenv": {"path": ".", "editable": True, "extras": ["dev"]}} with open(os.path.join(p.path, 'Pipfile'), 'w') as fh: fh.write(""" [packages] @@ -54,10 +53,11 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} assert "six" in p.lockfile["default"] c = p.pipenv("--rm") assert c.return_code == 0 + project = Project() project.write_toml({"packages": {}, "dev-packages": {}}) c = p.pipenv("install {0}".format(line)) assert c.return_code == 0 - assert "testpipenv" in p.pipfile["packages"] + assert "testpipenv" in p.pipfile["packages"], "{0}\n{1}\n\n{2}\n\n{3}".format(p.pipfile, Path(p.pipfile_path).read_text(), Path(os.getcwd()).joinpath("setup.py").read_text(), Path(os.path.join(os.getcwd(), "testpipenv.egg-info/PKG-INFO")).read_text()) assert p.pipfile["packages"]["testpipenv"]["path"] == "." assert p.pipfile["packages"]["testpipenv"]["extras"] == ["dev"] assert "six" in p.lockfile["default"] @@ -67,7 +67,7 @@ testpipenv = {path = ".", editable = true, extras = ["dev"]} @pytest.mark.local @pytest.mark.needs_internet @flaky -class TestDependencyLinks(object): +class TestDirectDependencies(object): """Ensure dependency_links are parsed and installed. This is needed for private repo dependencies. @@ -85,18 +85,15 @@ setup( version='0.1', packages=[], install_requires=[ - 'test-private-dependency' - ], - dependency_links=[ '{0}' - ] + ], ) """.strip().format(deplink) fh.write(contents) @staticmethod def helper_dependency_links_install_test(pipenv_instance, deplink): - TestDependencyLinks.helper_dependency_links_install_make_setup(pipenv_instance, deplink) + TestDirectDependencies.helper_dependency_links_install_make_setup(pipenv_instance, deplink) c = pipenv_instance.pipenv("install -v -e .") assert c.return_code == 0 assert "test-private-dependency" in pipenv_instance.lockfile["default"] @@ -107,19 +104,20 @@ setup( """Ensure dependency_links are parsed and installed (needed for private repo dependencies). """ with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: - os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+https://github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+https://github.com/atzannes/test-private-dependency@v0.1' ) @pytest.mark.needs_github_ssh def test_ssh_dependency_links_install(self, PipenvInstance, pypi): with temp_environ(), PipenvInstance(pypi=pypi, chdir=True) as p: os.environ['PIP_PROCESS_DEPENDENCY_LINKS'] = '1' - TestDependencyLinks.helper_dependency_links_install_test( + os.environ["PIP_NO_BUILD_ISOLATION"] = '1' + TestDirectDependencies.helper_dependency_links_install_test( p, - 'git+ssh://git@github.com/atzannes/test-private-dependency@v0.1#egg=test-private-dependency-v0.1' + 'test-private-dependency@ git+ssh://git@github.com/atzannes/test-private-dependency@v0.1' ) @@ -268,8 +266,8 @@ def test_local_zipfiles(PipenvInstance, pypi, testsroot): assert "file" in dep or "path" in dep assert c.return_code == 0 - key = [k for k in p.lockfile["default"].keys()][0] - dep = p.lockfile["default"][key] + # This now gets resolved to its name correctly + dep = p.lockfile["default"]["requests"] assert "file" in dep or "path" in dep diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 4dd35882..664f30cc 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -25,6 +25,7 @@ def test_basic_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "https://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } assert "gitdb2" in p.lockfile["default"] @@ -42,6 +43,7 @@ def test_git_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "git://github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } @@ -59,6 +61,7 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): assert p.lockfile["default"]["six"] == { "git": "ssh://git@github.com/benjaminp/six.git", "ref": "15e31431af97e5e64b80af0a3f598d382bcdd49a", + "version": "==1.11.0" } @@ -66,8 +69,9 @@ def test_ssh_vcs_install(PipenvInstance, pip_src_dir, pypi): @pytest.mark.urls @pytest.mark.needs_internet @flaky -def test_urls_work(PipenvInstance, pypi, pip_src_dir): +def test_urls_work(PipenvInstance, pypi): with PipenvInstance(pypi=pypi, chdir=True) as p: + # the library this installs is "django-cms" path = p._pipfile.get_url("django", "3.4.x.zip") c = p.pipenv( "install {0}".format(path) @@ -77,7 +81,8 @@ def test_urls_work(PipenvInstance, pypi, pip_src_dir): dep = list(p.pipfile["packages"].values())[0] assert "file" in dep, p.pipfile - dep = list(p.lockfile["default"].values())[0] + # now that we handle resolution with requirementslib, this will resolve to a name + dep = p.lockfile["default"]["django-cms"] assert "file" in dep, p.lockfile @@ -219,8 +224,8 @@ def test_get_vcs_refs(PipenvInstance, pip_src_dir): == "5efb522b0647f7467248273ec1b893d06b984a59" ) pipfile = Path(p.pipfile_path) - new_content = pipfile.read_bytes().replace(b"1.9.0", b"1.11.0") - pipfile.write_bytes(new_content) + new_content = pipfile.read_text().replace(u"1.9.0", u"1.11.0") + pipfile.write_text(new_content) c = p.pipenv("lock") assert c.return_code == 0 assert ( diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index d993b1a2..93b9d4f8 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -4,7 +4,7 @@ import sys import pytest from flaky import flaky - +from vistir.compat import Path from pipenv.utils import temp_environ @@ -585,6 +585,7 @@ six = "*" assert p.lockfile["default"]["six"]["index"] == "test" with open(p.pipfile_path, 'w') as f: f.write(contents.replace('name = "test"', 'name = "custom"')) - c = p.pipenv("lock") + c = p.pipenv("lock --clear") assert c.return_code == 0 - assert p.lockfile["default"]["six"]["index"] == "custom" + assert "index" in p.lockfile["default"]["six"] + assert p.lockfile["default"]["six"]["index"] == "custom", Path(p.lockfile_path).read_text() # p.lockfile["default"]["six"] diff --git a/tests/integration/test_project.py b/tests/integration/test_project.py index 42754521..237fd483 100644 --- a/tests/integration/test_project.py +++ b/tests/integration/test_project.py @@ -8,6 +8,7 @@ import pytest from pipenv.patched import pipfile from pipenv.project import Project from pipenv.utils import temp_environ +import pipenv.environments @pytest.mark.project @@ -168,25 +169,41 @@ def test_include_editable_packages(PipenvInstance, pypi, testsroot, pathlib_tmpd @pytest.mark.project @pytest.mark.virtualenv -def test_run_in_virtualenv(PipenvInstance, pypi, virtualenv): - with PipenvInstance(chdir=True, pypi=pypi) as p: - os.environ['PIPENV_IGNORE_VIRTUALENVS'] = '1' +def test_run_in_virtualenv_with_global_context(PipenvInstance, pypi, virtualenv): + with PipenvInstance(chdir=True, pypi=pypi, venv_root=virtualenv.as_posix(), ignore_virtualenvs=False, venv_in_project=False) as p: c = p.pipenv('run which pip') assert c.return_code == 0 - assert 'virtualenv' not in c.out - - os.environ.pop("PIPENV_IGNORE_VIRTUALENVS", None) - c = p.pipenv('run which pip') - assert c.return_code == 0 - assert 'virtualenv' in c.out + assert 'Creating a virtualenv' not in c.err project = Project() assert project.virtualenv_location == str(virtualenv) c = p.pipenv("run pip install click") assert c.return_code == 0 assert "Courtesy Notice" in c.err + c = p.pipenv("install six") + assert c.return_code == 0 c = p.pipenv('run python -c "import click;print(click.__file__)"') assert c.return_code == 0 assert c.out.strip().startswith(str(virtualenv)) c = p.pipenv("clean --dry-run") assert c.return_code == 0 assert "click" in c.out + + +@pytest.mark.project +@pytest.mark.virtualenv +def test_run_in_virtualenv(PipenvInstance, pypi): + with PipenvInstance(chdir=True, pypi=pypi) as p: + c = p.pipenv('run which pip') + assert c.return_code == 0 + assert 'Creating a virtualenv' in c.err + project = Project() + c = p.pipenv("run pip install click") + assert c.return_code == 0 + c = p.pipenv("install six") + assert c.return_code == 0 + c = p.pipenv('run python -c "import click;print(click.__file__)"') + assert c.return_code == 0 + assert c.out.strip().startswith(str(project.virtualenv_location)) + c = p.pipenv("clean --dry-run") + assert c.return_code == 0 + assert "click" in c.out diff --git a/tests/integration/test_sync.py b/tests/integration/test_sync.py index 8b8745e7..c9e5c057 100644 --- a/tests/integration/test_sync.py +++ b/tests/integration/test_sync.py @@ -21,11 +21,16 @@ def test_sync_error_without_lockfile(PipenvInstance, pypi): @pytest.mark.sync @pytest.mark.lock def test_mirror_lock_sync(PipenvInstance, pypi): - with temp_environ(), PipenvInstance(chdir=True) as p: + with temp_environ(), PipenvInstance(chdir=True, pypi=pypi) as p: mirror_url = os.environ.pop('PIPENV_TEST_INDEX', "https://pypi.kennethreitz.org/simple") assert 'pypi.org' not in mirror_url with open(p.pipfile_path, 'w') as f: f.write(""" +[[source]] +name = "pypi" +url = "https://pypi.org/simple" +verify_ssl = true + [packages] six = "*" """.strip()) diff --git a/tests/pytest-pypi/pytest_pypi/app.py b/tests/pytest-pypi/pytest_pypi/app.py index 4e013ab5..fa494ea8 100644 --- a/tests/pytest-pypi/pytest_pypi/app.py +++ b/tests/pytest-pypi/pytest_pypi/app.py @@ -1,5 +1,6 @@ import os import json +import io import sys import requests @@ -29,7 +30,26 @@ class Package(object): with open(os.path.join(path, 'api.json')) as f: return json.load(f) except FileNotFoundError: - pass + r = session.get('https://pypi.org/pypi/{0}/json'.format(self.name)) + response = r.json() + releases = response["releases"] + files = { + pkg for pkg_dir in self._package_dirs + for pkg in os.listdir(pkg_dir) + } + for release in list(releases.keys()): + values = ( + r for r in releases[release] if r["filename"] in files + ) + values = list(values) + if values: + releases[release] = values + else: + del releases[release] + response["releases"] = releases + with io.open(os.path.join(path, "api.json"), "w") as fh: + json.dump(response, fh, indent=4) + return response def __repr__(self): return "/json') def json_for_package(package): - try: - return jsonify(packages[package].json) - except Exception: - pass - - r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) - return jsonify(r.json()) + return jsonify(packages[package].json) + # try: + # except Exception: + # r = session.get('https://pypi.org/pypi/{0}/json'.format(package)) + # return jsonify(r.json()) if __name__ == '__main__': diff --git a/tests/unit/test_utils.py b/tests/unit/test_utils.py index ac5eb29b..17e5bab1 100644 --- a/tests/unit/test_utils.py +++ b/tests/unit/test_utils.py @@ -75,12 +75,20 @@ DEP_PIP_PAIRS = [ ] +def mock_unpack(link, source_dir, download_dir, only_download=False, session=None, + hashes=None, progress_bar="off"): + return + + @pytest.mark.utils @pytest.mark.parametrize("deps, expected", DEP_PIP_PAIRS) -def test_convert_deps_to_pip(deps, expected): - if expected.startswith("Django"): - expected = expected.lower() - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] +def test_convert_deps_to_pip(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + if expected.startswith("Django"): + expected = expected.lower() + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected] @pytest.mark.utils @@ -121,8 +129,11 @@ def test_convert_deps_to_pip(deps, expected): ), ], ) -def test_convert_deps_to_pip_one_way(deps, expected): - assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] +def test_convert_deps_to_pip_one_way(monkeypatch, deps, expected): + with monkeypatch.context() as m: + import pip_shims + # m.setattr(pip_shims.shims, "unpack_url", mock_unpack) + assert pipenv.utils.convert_deps_to_pip(deps, r=False) == [expected.lower()] @pytest.mark.skipif(isinstance(u"", str), reason="don't need to test if unicode is str")