diff --git a/.github/workflows/pypi_upload.yml b/.github/workflows/pypi_upload.yml new file mode 100644 index 00000000..fdbb728d --- /dev/null +++ b/.github/workflows/pypi_upload.yml @@ -0,0 +1,63 @@ +name: Create Release & Upload To PyPI + +on: + push: + # Sequence of patterns matched against refs/tags + tags: + - v?[0-9]+.[0-9]+.[0-9]+ # add .* to allow dev releases + +jobs: + build: + name: pipenv PyPI Upload + runs-on: ubuntu-latest + env: + CI: "1" + + steps: + - name: Checkout code + uses: actions/checkout@v1 + + - uses: webfactory/ssh-agent@v0.1.1 + with: + ssh-private-key: ${{ secrets.SSH_PRIVATE_KEY }} + + - name: Create Release + id: create_release + uses: actions/create-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + tag_name: ${{ github.ref }} + release_name: Release ${{ github.ref }} + draft: false + prerelease: false + + - name: Set up Python 3.7 + uses: actions/setup-python@v1 + with: + python-version: 3.7 + + - name: Install latest tools for build + run: | + python -m pip install --upgrade --upgrade-strategy=eager pip setuptools wheel invoke + python -m pip install . + python -m pipenv install --dev + - name: Build wheels + run: | + python -m pipenv runpython setup.py sdist bdist_wheel + # to upload to test pypi, pass repository_url: https://test.pypi.org/legacy/ and use secrets.TEST_PYPI_TOKEN + - name: Publish a Python distribution to PyPI + uses: pypa/gh-action-pypi-publish@master + with: + user: __token__ + password: ${{ secrets.TEST_PYPI_TOKEN }} + repository_url: https://test.pypi.org/legacy/ + packages_dir: dist/ + # git push https://${{ github.actor }}:${{ secrets.GITHUB_TOKEN }}@github.com/${{ github.repository }}.git HEAD:master + # we need to use a deploy key for this to get around branch protection as the default token fails + - name: Pre-bump + run: | + git config --local user.name 'Github Action' + git config --local user.email action@github.com + python -m pipenv run inv release.bump-version --dev --commit + git push git@github.com:${{ github.repository }}.git HEAD:master diff --git a/Pipfile.lock b/Pipfile.lock index 5d58f7dd..c02913fc 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -7,9 +7,9 @@ "requires": {}, "sources": [ { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true } ] }, @@ -193,30 +193,28 @@ }, "cryptography": { "hashes": [ - "sha256:02079a6addc7b5140ba0825f542c0869ff4df9a69c360e339ecead5baefa843c", - "sha256:1df22371fbf2004c6f64e927668734070a8953362cd8370ddd336774d6743595", - "sha256:369d2346db5934345787451504853ad9d342d7f721ae82d098083e1f49a582ad", - "sha256:3cda1f0ed8747339bbdf71b9f38ca74c7b592f24f65cdb3ab3765e4b02871651", - "sha256:44ff04138935882fef7c686878e1c8fd80a723161ad6a98da31e14b7553170c2", - "sha256:4b1030728872c59687badcca1e225a9103440e467c17d6d1730ab3d2d64bfeff", - "sha256:58363dbd966afb4f89b3b11dfb8ff200058fbc3b947507675c19ceb46104b48d", - "sha256:6ec280fb24d27e3d97aa731e16207d58bd8ae94ef6eab97249a2afe4ba643d42", - "sha256:7270a6c29199adc1297776937a05b59720e8a782531f1f122f2eb8467f9aab4d", - "sha256:73fd30c57fa2d0a1d7a49c561c40c2f79c7d6c374cc7750e9ac7c99176f6428e", - "sha256:7f09806ed4fbea8f51585231ba742b58cbcfbfe823ea197d8c89a5e433c7e912", - "sha256:90df0cc93e1f8d2fba8365fb59a858f51a11a394d64dbf3ef844f783844cc793", - "sha256:971221ed40f058f5662a604bd1ae6e4521d84e6cad0b7b170564cc34169c8f13", - "sha256:a518c153a2b5ed6b8cc03f7ae79d5ffad7315ad4569b2d5333a13c38d64bd8d7", - "sha256:b0de590a8b0979649ebeef8bb9f54394d3a41f66c5584fff4220901739b6b2f0", - "sha256:b43f53f29816ba1db8525f006fa6f49292e9b029554b3eb56a189a70f2a40879", - "sha256:d31402aad60ed889c7e57934a03477b572a03af7794fa8fb1780f21ea8f6551f", - "sha256:de96157ec73458a7f14e3d26f17f8128c959084931e8997b9e655a39c8fde9f9", - "sha256:df6b4dca2e11865e6cfbfb708e800efb18370f5a46fd601d3755bc7f85b3a8a2", - "sha256:ecadccc7ba52193963c0475ac9f6fa28ac01e01349a2ca48509667ef41ffd2cf", - "sha256:fb81c17e0ebe3358486cd8cc3ad78adbae58af12fc2bf2bc0bb84e8090fa5ce8" + "sha256:091d31c42f444c6f519485ed528d8b451d1a0c7bf30e8ca583a0cac44b8a0df6", + "sha256:18452582a3c85b96014b45686af264563e3e5d99d226589f057ace56196ec78b", + "sha256:1dfa985f62b137909496e7fc182dac687206d8d089dd03eaeb28ae16eec8e7d5", + "sha256:1e4014639d3d73fbc5ceff206049c5a9a849cefd106a49fa7aaaa25cc0ce35cf", + "sha256:22e91636a51170df0ae4dcbd250d318fd28c9f491c4e50b625a49964b24fe46e", + "sha256:3b3eba865ea2754738616f87292b7f29448aec342a7c720956f8083d252bf28b", + "sha256:651448cd2e3a6bc2bb76c3663785133c40d5e1a8c1a9c5429e4354201c6024ae", + "sha256:726086c17f94747cedbee6efa77e99ae170caebeb1116353c6cf0ab67ea6829b", + "sha256:844a76bc04472e5135b909da6aed84360f522ff5dfa47f93e3dd2a0b84a89fa0", + "sha256:88c881dd5a147e08d1bdcf2315c04972381d026cdb803325c03fe2b4a8ed858b", + "sha256:96c080ae7118c10fcbe6229ab43eb8b090fccd31a09ef55f83f690d1ef619a1d", + "sha256:a0c30272fb4ddda5f5ffc1089d7405b7a71b0b0f51993cb4e5dbb4590b2fc229", + "sha256:bb1f0281887d89617b4c68e8db9a2c42b9efebf2702a3c5bf70599421a8623e3", + "sha256:c447cf087cf2dbddc1add6987bbe2f767ed5317adb2d08af940db517dd704365", + "sha256:c4fd17d92e9d55b84707f4fd09992081ba872d1a0c610c109c18e062e06a2e55", + "sha256:d0d5aeaedd29be304848f1c5059074a740fa9f6f26b84c5b63e8b29e73dfc270", + "sha256:daf54a4b07d67ad437ff239c8a4080cfd1cc7213df57d33c97de7b4738048d5e", + "sha256:e993468c859d084d5579e2ebee101de8f5a27ce8e2159959b6673b418fd8c785", + "sha256:f118a95c7480f5be0df8afeb9a11bd199aa20afab7a96bcf20409b411a3a85f0" ], "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", - "version": "==2.8" + "version": "==2.9.2" }, "decorator": { "hashes": [ @@ -513,10 +511,10 @@ }, "pathspec": { "hashes": [ - "sha256:163b0632d4e31cef212976cf57b43d9fd6b0bac6e67c26015d611a647d5e7424", - "sha256:562aa70af2e0d434367d9790ad37aed893de47f1693e4201fd1d3dca15d19b96" - ], - "version": "==0.7.0" + "sha256:7d91249d21749788d07a2d0f94147accd8f845507400749ea19c1ec9054a12b0", + "sha256:da45173eb3a6f2a5a487efba21f050af2b41948be6ab52b6a1e3ff22bb8b7061" + ], + "version": "==0.8.0" }, "pbr": { "hashes": [ @@ -640,10 +638,10 @@ }, "readme-renderer": { "hashes": [ - "sha256:1b6d8dd1673a0b293766b4106af766b6eff3654605f9c4f239e65de6076bc222", - "sha256:e67d64242f0174a63c3b727801a2fff4c1f38ebe5d71d95ff7ece081945a6cd4" + "sha256:cbe9db71defedd2428a1589cdc545f9bd98e59297449f69d721ef8f1cfced68d", + "sha256:cc4957a803106e820d05d14f71033092537a22daa4f406dfbdd61177e0936376" ], - "version": "==25.0" + "version": "==26.0" }, "regex": { "hashes": [ diff --git a/RELEASING.md b/RELEASING.md new file mode 100644 index 00000000..3ba62317 --- /dev/null +++ b/RELEASING.md @@ -0,0 +1,139 @@ +# Releasing Pipenv + +We recognize that the pipenv release process is currently poorly managed. That's why this document seeks to streamline the release process, identify current complexities, and help eliminate existing single points of failure. + +## Development Install + +Any new release will require you to have the latest version of master installed in development mode: + +```bash +$ git checkout master +$ git fetch origin +$ git pull +$ pipenv install --dev +``` + +## Make new releases of related libraries + +A lot of pipenv depends on ancillary libraries. You may need to make new releases of: + + * [requirementslib](https://pypi.org/project/requirementslib/) + * [pip-shims](https://pypi.org/project/pip-shims/) + * [vistir](https://pypi.org/project/vistir/) + * [pythonfinder](https://pypi.org/project/pythonfinder/) + +## Updating Vendored Dependencies + +This is the most complex element of releasing new versions of pipenv due to existing patches on current code. Currently the largest patchsets are maintained against [pip](https://github.com/pypa/pip) and [pip-tools](https://github.com/jazzband/pip-tools). + +You can begin by reviewing vendored dependencies which can be found in `pipenv/vendor/vendor.txt`, a file which is consumed by the automated vendoring process. These dependencies may have minor patches applied which can be found in `tasks/vendoring/patches/vendor`. Check PyPI for updates to the specified packages and increment the versions as needed, making sure to capture all dependencies in case any were added. It would be *very bad* to release without necessary dependencies, obviously. + +Next you can consult `pipenv/patched/patched.txt` which enumerates the patched dependencies. Follow the same process, but be aware that you will need to rewrite patches for each dependency once you update (most likely) as they do tend to change somewhat substantially. + + +### Updating patches + +For larger libraries you can keep local clones of them and simply generate full patch sets in which you replace the updated path in pipenv when you are done making changes. Here is an example of a script used from inside a local clone of `pip` to generate a patch and copy it to pipenv's local patches directory. + +```bash +#!/usr/bin/bash +sed -i -r 's/([a-b]\/)(?:src\/)?(pip)/\1pipenv\/patched\/\2/g' diff.patch +cp diff.patch ../pipenv/tasks/vendoring/patches/patched/pip19.patch +``` + +Assuming patches are kept up to date and you are simply working on a modification that is relatively minor, here is a script you can use to pull in a single modified file from `pipenv` into a local clone of `pip-tools` and regenerate a patch from the updated version of that file: + +```bash +#!/bin/bash +cp ../pipenv/pipenv/patched/piptools/$@ piptools/$@ +sed -i 's/pipenv.patched.notpip/pip/g' piptools/$@ +git diff -p piptools/$@ > diff.patch +sed -i -r 's:([a-b]/)(piptools):\1pipenv/patched/\2:g' diff.patch +``` + +The resulting patch is then combined into the patchset by hand (make sure you don't alter the whitespace in the patch!) + + +## Updating Vendored Dependencies (continued) + +Okay, now that's done, it's time to update vendored dependencies. You can install pipenv itself by moving to the source directory (`cd pipenv`) and running `pip install -e .`. Then you can run `pipenv install --dev` to install the development dependencies into a virtual environment. + +Update the vendored dependencies by copying the `pipenv/vendor/vendor.txt` file to a new directory (e.g. `/tmp/vendor`) and unpinning all of the dependencies. Note there is a helper script for this: + +```bash +$ pipenv run inv vendoring.unpin-and-update-vendored +``` + +This should unpin all vendored and patched dependencies and resolve them; ideally you would keep the file formatted so that we can see what depends on what, but this will tell you what can be updated & provide the latest versions. + +To re-vendor and patch the vendored libraries, run the command: + +```bash +$ pipenv run inv vendoring.update +``` + +This will automatically remove the `./pipenv/vendor/` and `./pipenv/patched/` directories and re-download and patch the specified dependencies. It will also attempt to download any relevant licenses. Once this is completed, run `git status` and inspect the output -- look through the `git diff` for anything that may cause breakages. If any licenses have been deleted, you will need to determine why they were not replaced by the license download tooling. + +Make sure to read through any modified license files for changes -- note that we cannot redistribute code that is licensed under a [copyleft](https://en.wikipedia.org/wiki/Copyleft) license, such as the [GPL](https://en.wikipedia.org/wiki/GPL). Similarly, all vendored code **must** be licensed or it cannot be redistributed. If vendored libraries have become unlicensed or are no longer usable, suitable replacements will have to be found and potentially patched into the vendored dependencies. This may be a good time to consider simply including the dependency as an install requirement. + + +## Update Pipfile.lock + +Now we will need to update the lockfile. You will need to run the following: + +```bash +# use the latest python here +$ export PIPENV_PYTHON=3.8 +$ pipenv lock --dev +# Inspect the changes in a diff viewer, for example we should keep the python 2 dependencies to use for running tests +# on completion, stage the relevant changes +$ export PIPENV_PYTHON=2.7 +$ pipenv lock --keep-outdated --dev +# this helps avoid overwriting the entire lockfile and should introduce only the changes required to run tests on python 2 +# inspect the resulting lockfile and commit the changes +$ git commit +``` + +## Test locally + +Test pipenv locally. If tests pass, you can go ahead and make a PR to merge whatever you want to release. + +```bash +$ export PIPENV_PYTHON=3.8 +$ pipenv install --dev && pytest -ra tests +$ export PIPENV_PYTHON=2.7 +$ pipenv install --dev && pytest -ra tests +``` + +## Releasing + +1. Set a version: `pipenv run inv release.bump-version --trunc-month --pre --tag=a` - this will truncate the current month, creating an alpha pre-release, e.g. `2020.4.1a1` + a. **Note**: You can pass `--tag=b` or `--tag=rc` here as well +2. `make check` - This has the side-effect of producing wheels +3. `make tests`- Runs tests locally +4. `make upload-test` - This uploads artifacts to test-pypi +5. Consume the version on test pypi, ensure that version is functioning. +6. Push the pre-release to github & wait for CI to pass +7. Create a new tag, e.g. `v2020.4.1a1` and push it to github -- this can be achieved via `pipenv run inv release.tag-version --push` +8. The github action will automatically build and push the prerelease to `PyPI` +9. Once a release is pushed, the action will update `master` with a new `dev` version +10. Review any pull requests and issues that should be resolved before releasing the final version +11. The process is identical for releasing a standard release, except the `release.bump-version` command is called without any arguments. + + +If in doubt, follow the basic instructions below. + +## Uploading the release + +1. Get set up on [Test PyPI](https://test.pypi.org/) +2. [Use Test PyPI](https://packaging.python.org/guides/using-testpypi/) to upload the package, make sure the `README` renders, test that it installs okay, and so on +3. Get credentials to co-maintain the pipenv project on PyPI.org -- **SPOF alert** +4. Set the version number to [a pre-release identifier](https://www.python.org/dev/peps/pep-0440/#pre-release-separators) +5. Package and upload pipenv [to PyPI](https://pypi.org/project/pipenv/#history) as a pre-release/alpha +6. Publicize on distutils-sig, pypa-dev, and the relevant GitHub issue(s) +7. Wait a week, then update version number to a canonical release and re-release on PyPI.org + + +## Looking ahead + +Most of the pipenv related ecosystem libraries are using [github actions](https://github.com/sarugaku/vistir/blob/master/.github/workflows/pypi_upload.yml) to automate releases when tags are pushed. Most likely we will look to move in this direction and simplify the process. diff --git a/pipenv/core.py b/pipenv/core.py index ff8cc502..3b549c35 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -677,6 +677,9 @@ def _cleanup_procs(procs, failed_deps_queue, retry=True): click.echo(crayons.blue(c.out.strip() or c.err.strip())) # The Installation failed… if failed: + # If there is a mismatch in installed locations or the install fails + # due to wrongful disabling of pep517, we should allow for + # additional passes at installation if "does not match installed location" in c.err: project.environment.expand_egg_links() click.echo("{0}".format( @@ -687,6 +690,10 @@ def _cleanup_procs(procs, failed_deps_queue, retry=True): ) )) dep = c.dep.copy() + dep.use_pep517 = True + elif "Disabling PEP 517 processing is invalid" in c.err: + dep = c.dep.copy() + dep.use_pep517 = True elif not retry: # The Installation failed… # We echo both c.out and c.err because pip returns error details on out. @@ -698,6 +705,7 @@ def _cleanup_procs(procs, failed_deps_queue, retry=True): else: # Alert the user. dep = c.dep.copy() + dep.use_pep517 = False click.echo( "{0} {1}! Will try again.".format( crayons.red("An error occurred while installing"), @@ -760,7 +768,7 @@ def batch_install(deps_list, procs, failed_deps_queue, del os.environ["GIT_CONFIG"] use_pep517 = True if failed and not dep.is_vcs: - use_pep517 = False + use_pep517 = getattr(dep, "use_pep517", False) c = pip_install( dep, diff --git a/tasks/release.py b/tasks/release.py index ed1575d4..13c796dd 100644 --- a/tasks/release.py +++ b/tasks/release.py @@ -126,28 +126,20 @@ def drop_dist_dirs(ctx): @invoke.task def build_dists(ctx): drop_dist_dirs(ctx) - for py_version in ["3.6", "3.7", "3.8", "2.7"]: - env = {"PIPENV_PYTHON": py_version} - with ctx.cd(ROOT.as_posix()), temp_environ(): - executable = ctx.run( - "python -c 'import sys; print(sys.executable)'", hide=True - ).stdout.strip() - log("Building sdist using %s ...." % executable) - os.environ["PIPENV_PYTHON"] = py_version - ctx.run("pipenv install --dev", env=env) - ctx.run( - "pipenv run pip install -e . --upgrade --upgrade-strategy=eager", env=env - ) - log("Building wheel using python %s ...." % py_version) - tag_arg = "--python-tag py{}".format(py_version.replace(".", "")) - if py_version == "3.8": - ctx.run(f"pipenv run python setup.py sdist bdist_wheel {tag_arg}", env=env) - else: - ctx.run(f"pipenv run python setup.py bdist_wheel {tag_arg}", env=env) - if py_version in ("3.6", "2.7"): - # generate py2 / py3 generic untagged wheels - ctx.run(f"pipenv run python setup.py sdist bdist_wheel", env=env) - + py_version = ".".join(str(v) for v in sys.version_info[:2]) + env = {"PIPENV_PYTHON": py_version} + with ctx.cd(ROOT.as_posix()), temp_environ(): + executable = ctx.run( + "python -c 'import sys; print(sys.executable)'", hide=True + ).stdout.strip() + log("Building sdist using %s ...." % executable) + os.environ["PIPENV_PYTHON"] = py_version + ctx.run("pipenv install --dev", env=env) + ctx.run( + "pipenv run pip install -e . --upgrade --upgrade-strategy=eager", env=env + ) + log("Building wheel using python %s ...." % py_version) + ctx.run(f"pipenv run python setup.py sdist bdist_wheel", env=env) @invoke.task(build_dists) @@ -239,28 +231,63 @@ def tag_version(ctx, push=False): ctx.run("git push --tags") +def add_one_day(dt): + return dt + datetime.timedelta(days=1) + + +def date_offset(dt, month_offset=0, day_offset=0, truncate=False): + new_month = (dt.month + month_offset) % 12 + year_offset = month_offset // 12 + replace_args = { + "month": dt.month + month_offset, + "year": dt.year + year_offset, + } + log("Getting updated date from date: {0} using month offset: {1} and year offset {2}".format( + dt, new_month, replace_args["year"] + )) + if day_offset: + dt = dt + datetime.timedelta(days=day_offset) + log("updated date using day offset: {0} => {1}".format(day_offset, dt)) + if truncate: + log("Truncating...") + replace_args["day"] = 1 + return dt.replace(**replace_args) + + @invoke.task -def bump_version(ctx, dry_run=False, dev=False, pre=False, tag=None, commit=False): +def bump_version(ctx, dry_run=False, dev=False, pre=False, tag=None, commit=False, month_offset="0", trunc_month=False): current_version = Version.parse(__version__) today = datetime.date.today() + day_offset = 0 tomorrow = today + datetime.timedelta(days=1) - if pre and not tag: - print('Using "pre" requires a corresponding tag.') - return - if not (dev or pre or tag): - new_version = current_version.replace(release=today.timetuple()[:3]).clear( - pre=True, dev=True - ) + month_offset = int(month_offset) + if month_offset: + # if we are offsetting by a month, grab the first day of the month + trunc_month = True + else: + target_day = today + if dev or pre: + target_day = date_offset(today, day_offset=1) + target_day = date_offset( + today, + month_offset=month_offset, + day_offset=day_offset, + truncate=trunc_month + ) + log("target_day: {0}".format(target_day)) + target_timetuple = target_day.timetuple()[:3] + new_version = current_version.replace(release=target_timetuple) if pre and dev: raise RuntimeError("Can't use 'pre' and 'dev' together!") - if dev or pre: - new_version = current_version.replace(release=tomorrow.timetuple()[:3]).clear( - pre=True, dev=True - ) - if dev: - new_version = new_version.bump_dev() - else: - new_version = new_version.bump_pre(tag=tag) + if dev: + new_version = new_version.replace(pre=None).bump_dev() + elif pre: + if not tag: + print('Using "pre" requires a corresponding tag.') + return + new_version = new_version.bump_pre(tag=tag) + else: + new_version = new_version.replace(pre=None, dev=None) log("Updating version to %s" % new_version.normalize()) version = find_version(ctx) log("Found current version: %s" % version)