Add support for Python 3.8.7

Closes @W-7791264@.

CHANGELOG.md

@@ -2,6 +2,16 @@
 
 ## Unreleased
 
+- Python 3.8.7 is now available (CPython) (#1122).
+
+## v186 (2020-11-18)
+
+- Update the `BUILD_WITH_GEO_LIBRARIES` error message (#1121).
+- Switch NLTK feature detection away from `sp-grep` (#1119).
+- Switch Django collectstatic feature detection away from `sp-grep` (#1119).
+- Remove vendored `sp-grep` script (#1119).
+- Remove vendored `pip-diff` script (#1118).
+- Remove vendored `pip-grep` script (#1116).
 
 ## v185 (2020-11-12)
 

Makefile

@@ -1,9 +1,10 @@
 # These targets are not files
-.PHONY: check test builder-image buildenv deploy-runtimes tools
+.PHONY: check test compile builder-image buildenv deploy-runtimes tools
 
 STACK ?= heroku-18
 STACKS ?= heroku-16 heroku-18 heroku-20
 TEST_CMD ?= test/run-versions && test/run-features && test/run-deps
+FIXTURE ?= test/fixtures/requirements-standard
 ENV_FILE ?= builds/dockerenv.default
 BUILDER_IMAGE_PREFIX := heroku-python-build
 
@@ -12,7 +13,7 @@ STACK_IMAGE_TAG := heroku/$(subst -,:,$(STACK))-build
 
 check:
 	@shellcheck -x bin/compile bin/detect bin/release bin/test-compile bin/utils bin/warnings bin/default_pythons
-	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pip-uninstall bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
+	@shellcheck -x bin/steps/collectstatic bin/steps/eggpath-fix  bin/steps/eggpath-fix2 bin/steps/nltk bin/steps/pip-install bin/steps/pipenv bin/steps/pipenv-python-version bin/steps/python
 	@shellcheck -x bin/steps/hooks/*
 
 test:
@@ -21,6 +22,13 @@ test:
 	@docker run --rm -it -v $(PWD):/buildpack:ro -e "STACK=$(STACK)" "$(STACK_IMAGE_TAG)" bash -c 'cp -r /buildpack /buildpack_test && cd /buildpack_test && $(TEST_CMD)'
 	@echo
 
+compile:
+	@echo "Running compile using: STACK=$(STACK) FIXTURE=$(FIXTURE)"
+	@echo
+	@docker run --rm -it -v $(PWD):/src:ro -e "STACK=$(STACK)" -w /buildpack "$(STACK_IMAGE_TAG)" \
+		bash -c 'cp -r /src/{bin,vendor} /buildpack && cp -r /src/$(FIXTURE) /build && mkdir /cache /env && bin/compile /build /cache /env'
+	@echo
+
 builder-image:
 	@echo "Generating binary builder image for $(STACK)..."
 	@echo
@@ -51,8 +59,3 @@ endif
 			echo; \
 		done; \
 	done
-
-tools:
-	git clone https://github.com/kennethreitz/pip-pop.git
-	mv pip-pop/bin/* vendor/pip-pop/
-	rm -rf pip-pop

NOTICE

@@ -63,28 +63,3 @@ The Free Software Foundation may publish revised and/or new versions of the GNU
 Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License “or any later version” applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation.
 
 If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library.
-
-pip-pop license
----------------
-
-The MIT License (MIT)
-
-Copyright (c) 2014 Kenneth Reitz.
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in
-all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
-THE SOFTWARE.

README.md

@@ -63,7 +63,7 @@ Specify a Python Runtime
 Supported runtime options include:
 
 - `python-3.9.0`
-- `python-3.8.6`
+- `python-3.8.7`
 - `python-3.7.9`
 - `python-3.6.12`
 - `python-2.7.18`

bin/compile

@@ -68,24 +68,18 @@ DEFAULT_PYTHON_STACK="heroku-18"
 WARNINGS_LOG=$(mktemp)
 RECOMMENDED_PYTHON_VERSION=$DEFAULT_PYTHON_VERSION
 
-# The buildpack ships with a few executable tools (e.g. pip-grep, etc).
+# The buildpack ships with a few executable tools.
 # This installs them into the path, so we can execute them directly.
-export PATH=$PATH:$ROOT_DIR/vendor/:$ROOT_DIR/vendor/pip-pop
+export PATH=$PATH:$ROOT_DIR/vendor/
 
 # Set environment variables if they weren't set by the platform.
-# Note: this is legacy, for a deprecated build system known as Anvil.
-# This can likely be removed, with caution.
-[ ! "$SLUG_ID" ] && SLUG_ID="defaultslug"
-[ ! "$REQUEST_ID" ] && REQUEST_ID=$SLUG_ID
 [ ! "$STACK" ] && STACK=$DEFAULT_PYTHON_STACK
 
 # Sanitize externally-provided environment variables:
 # The following environment variables are either problematic or simply unneccessary
 # for the buildpack to have knowledge of, so we unset them, to keep the environment
 # as clean and pristine as possible.
-unset GIT_DIR PYTHONHOME PYTHONPATH
-unset RECEIVE_DATA RUN_KEY BUILD_INFO DEPLOY LOG_TOKEN
-unset CYTOKINE_LOG_FILE GEM_PATH
+unset PYTHONHOME PYTHONPATH
 
 # Import the utils script, which contains helper functions used throughout the buildpack.
 # shellcheck source=bin/utils
@@ -110,13 +104,15 @@ fi
 
 if [[ -f "${ENV_DIR}/BUILD_WITH_GEO_LIBRARIES" ]]; then
   mcount "failure.unsupported.BUILD_WITH_GEO_LIBRARIES"
-  puts-warn "The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported:"
-  puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-now-deprecated"
+  puts-warn "The Python buildpack's legacy BUILD_WITH_GEO_LIBRARIES functonality is"
+  puts-warn "no longer supported:"
+  puts-warn "https://devcenter.heroku.com/changelog-items/1947"
   puts-warn
-  puts-warn "For GDAL, GEOS and PROJ support, use the Geo buildpack alongside the Python buildpack:"
-  puts-warn "https://github.com/heroku/heroku-geo-buildpack"
+  puts-warn "To continue to use GDAL, GEOS or PROJ support, see the migration guide:"
+  puts-warn "https://help.heroku.com/D5INLB1A/python-s-build_with_geo_libraries-legacy-feature-is-no-longer-supported"
   puts-warn
-  puts-warn "To remove this error message, unset the BUILD_WITH_GEO_LIBRARIES variable using:"
+  puts-warn "Or if you no longer need those libraries, this message can be hidden by"
+  puts-warn "unsetting the BUILD_WITH_GEO_LIBRARIES environment variable, using:"
   puts-warn "heroku config:unset BUILD_WITH_GEO_LIBRARIES"
   exit 1
 fi

bin/default_pythons

@@ -7,7 +7,7 @@
 
 DEFAULT_PYTHON_VERSION="python-3.6.12"
 LATEST_39="python-3.9.0"
-LATEST_38="python-3.8.6"
+LATEST_38="python-3.8.7"
 LATEST_37="python-3.7.9"
 LATEST_36="python-3.6.12"
 LATEST_35="python-3.5.10"

bin/steps/collectstatic

@@ -20,8 +20,8 @@ MANAGE_FILE=${MANAGE_FILE:-fakepath}
 # Legacy file-based support for $DISABLE_COLLECTSTATIC
 [ -f .heroku/collectstatic_disabled ] && DISABLE_COLLECTSTATIC=1
 
-# Ensure that Django is explicitly specified in requirements.txt
-sp-grep -s django && DJANGO_INSTALLED=1
+# Ensure that Django is actually installed.
+is_module_available 'django' && DJANGO_INSTALLED=1
 
 
 if [ ! "$DISABLE_COLLECTSTATIC" ] && [ -f "$MANAGE_FILE" ] && [ "$DJANGO_INSTALLED" ]; then

bin/steps/nltk

@@ -14,7 +14,7 @@
 source "$BIN_DIR/utils"
 
 # Check that nltk was installed by pip, otherwise obviously not needed
-if sp-grep -s nltk; then
+if is_module_available 'nltk'; then
     puts-step "Downloading NLTK corpora…"
 
     nltk_packages_definition="$BUILD_DIR/nltk.txt"

bin/steps/pip-uninstall

@@ -1,28 +0,0 @@
-#!/usr/bin/env bash
-
-set +e
-# Install dependencies with Pip.
-# shellcheck source=bin/utils
-source "$BIN_DIR/utils"
-
-if [ ! "$SKIP_PIP_INSTALL" ]; then
-
-  if [[ -f .heroku/python/requirements-declared.txt ]]; then
-
-    cp .heroku/python/requirements-declared.txt requirements-declared.txt
-
-
-    if ! pip-diff --stale requirements-declared.txt requirements.txt --exclude setuptools pip wheel > .heroku/python/requirements-stale.txt; then
-      mcount "failure.bad-requirements"
-    fi
-
-    rm -rf requirements-declared.txt
-
-    if [[ -s .heroku/python/requirements-stale.txt ]]; then
-      puts-step "Uninstalling stale dependencies"
-      /app/.heroku/python/bin/pip uninstall -r .heroku/python/requirements-stale.txt -y --exists-action=w --disable-pip-version-check | cleanup | indent
-    fi
-  fi
-
-fi
-set -e

bin/steps/pipenv

@@ -69,7 +69,6 @@ if [ ! "$SKIP_PIPENV_INSTALL" ]; then
 
         else
             pipenv-to-pip Pipfile.lock > requirements.txt
-            "$BIN_DIR/steps/pip-uninstall"
             cp requirements.txt .heroku/python/requirements-declared.txt
             openssl dgst -sha256 Pipfile.lock > .heroku/python/Pipfile.lock.sha256
 

bin/utils

@@ -96,3 +96,11 @@ python_sqlite3_check() {
   ( python2_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_2" ) \
     || ( python3_check "$VERSION" && version_gte "$VERSION" "$MIN_PYTHON_3" )
 }
+
+is_module_available() {
+  # Returns 0 is the specified module exists, otherwise returns 1.
+  # Uses pkgutil rather than pkg_resources or pip's CLI, since pkgutil exists
+  # in the stdlib, and doesn't depend on the choice of package manager.
+  local module_name="${1}"
+  python -c "import sys, pkgutil; sys.exit(0 if pkgutil.find_loader('${module_name}') else 1)"
+}

builds/runtimes/python-3.8.7

@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+# Build Path: /app/.heroku/python/
+
+source $(dirname $0)/python3

test/fixtures/python3_8/runtime.txt

@@ -1 +1 @@
-python-3.8.6
+python-3.8.7

test/run-deps

@@ -21,7 +21,7 @@ testBuildWithGeoLibrariesWarning() {
   local env_dir="$(mktmpdir)"
   echo '1' > "${env_dir}/BUILD_WITH_GEO_LIBRARIES"
   compile 'gdal' '' "${env_dir}"
-  assertCaptured " !     The Python buildpack's BUILD_WITH_GEO_LIBRARIES functonality is no longer supported"
+  assertCaptured " !     The Python buildpack's legacy BUILD_WITH_GEO_LIBRARIES functonality"
   assertCapturedError
 }
 

vendor/pip-pop/docopt.py

@@ -1,581 +0,0 @@
-"""Pythonic command-line interface parser that will make you smile.
-
- * http://docopt.org
- * Repository and issue-tracker: https://github.com/docopt/docopt
- * Licensed under terms of MIT license (see LICENSE-MIT)
- * Copyright (c) 2013 Vladimir Keleshev, vladimir@keleshev.com
-
-"""
-import sys
-import re
-
-
-__all__ = ['docopt']
-__version__ = '0.6.1'
-
-
-class DocoptLanguageError(Exception):
-
-    """Error in construction of usage-message by developer."""
-
-
-class DocoptExit(SystemExit):
-
-    """Exit in case user invoked program with incorrect arguments."""
-
-    usage = ''
-
-    def __init__(self, message=''):
-        SystemExit.__init__(self, (message + '\n' + self.usage).strip())
-
-
-class Pattern(object):
-
-    def __eq__(self, other):
-        return repr(self) == repr(other)
-
-    def __hash__(self):
-        return hash(repr(self))
-
-    def fix(self):
-        self.fix_identities()
-        self.fix_repeating_arguments()
-        return self
-
-    def fix_identities(self, uniq=None):
-        """Make pattern-tree tips point to same object if they are equal."""
-        if not hasattr(self, 'children'):
-            return self
-        uniq = list(set(self.flat())) if uniq is None else uniq
-        for i, child in enumerate(self.children):
-            if not hasattr(child, 'children'):
-                assert child in uniq
-                self.children[i] = uniq[uniq.index(child)]
-            else:
-                child.fix_identities(uniq)
-
-    def fix_repeating_arguments(self):
-        """Fix elements that should accumulate/increment values."""
-        either = [list(child.children) for child in transform(self).children]
-        for case in either:
-            for e in [child for child in case if case.count(child) > 1]:
-                if type(e) is Argument or type(e) is Option and e.argcount:
-                    if e.value is None:
-                        e.value = []
-                    elif type(e.value) is not list:
-                        e.value = e.value.split()
-                if type(e) is Command or type(e) is Option and e.argcount == 0:
-                    e.value = 0
-        return self
-
-
-def transform(pattern):
-    """Expand pattern into an (almost) equivalent one, but with single Either.
-
-    Example: ((-a | -b) (-c | -d)) => (-a -c | -a -d | -b -c | -b -d)
-    Quirks: [-a] => (-a), (-a...) => (-a -a)
-
-    """
-    result = []
-    groups = [[pattern]]
-    while groups:
-        children = groups.pop(0)
-        parents = [Required, Optional, OptionsShortcut, Either, OneOrMore]
-        if any(t in map(type, children) for t in parents):
-            child = [c for c in children if type(c) in parents][0]
-            children.remove(child)
-            if type(child) is Either:
-                for c in child.children:
-                    groups.append([c] + children)
-            elif type(child) is OneOrMore:
-                groups.append(child.children * 2 + children)
-            else:
-                groups.append(child.children + children)
-        else:
-            result.append(children)
-    return Either(*[Required(*e) for e in result])
-
-
-class LeafPattern(Pattern):
-
-    """Leaf/terminal node of a pattern tree."""
-
-    def __init__(self, name, value=None):
-        self.name, self.value = name, value
-
-    def __repr__(self):
-        return '%s(%r, %r)' % (self.__class__.__name__, self.name, self.value)
-
-    def flat(self, *types):
-        return [self] if not types or type(self) in types else []
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        pos, match = self.single_match(left)
-        if match is None:
-            return False, left, collected
-        left_ = left[:pos] + left[pos + 1:]
-        same_name = [a for a in collected if a.name == self.name]
-        if type(self.value) in (int, list):
-            if type(self.value) is int:
-                increment = 1
-            else:
-                increment = ([match.value] if type(match.value) is str
-                             else match.value)
-            if not same_name:
-                match.value = increment
-                return True, left_, collected + [match]
-            same_name[0].value += increment
-            return True, left_, collected
-        return True, left_, collected + [match]
-
-
-class BranchPattern(Pattern):
-
-    """Branch/inner node of a pattern tree."""
-
-    def __init__(self, *children):
-        self.children = list(children)
-
-    def __repr__(self):
-        return '%s(%s)' % (self.__class__.__name__,
-                           ', '.join(repr(a) for a in self.children))
-
-    def flat(self, *types):
-        if type(self) in types:
-            return [self]
-        return sum([child.flat(*types) for child in self.children], [])
-
-
-class Argument(LeafPattern):
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if type(pattern) is Argument:
-                return n, Argument(self.name, pattern.value)
-        return None, None
-
-    @classmethod
-    def parse(class_, source):
-        name = re.findall('(<\S*?>)', source)[0]
-        value = re.findall('\[default: (.*)\]', source, flags=re.I)
-        return class_(name, value[0] if value else None)
-
-
-class Command(Argument):
-
-    def __init__(self, name, value=False):
-        self.name, self.value = name, value
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if type(pattern) is Argument:
-                if pattern.value == self.name:
-                    return n, Command(self.name, True)
-                else:
-                    break
-        return None, None
-
-
-class Option(LeafPattern):
-
-    def __init__(self, short=None, long=None, argcount=0, value=False):
-        assert argcount in (0, 1)
-        self.short, self.long, self.argcount = short, long, argcount
-        self.value = None if value is False and argcount else value
-
-    @classmethod
-    def parse(class_, option_description):
-        short, long, argcount, value = None, None, 0, False
-        options, _, description = option_description.strip().partition('  ')
-        options = options.replace(',', ' ').replace('=', ' ')
-        for s in options.split():
-            if s.startswith('--'):
-                long = s
-            elif s.startswith('-'):
-                short = s
-            else:
-                argcount = 1
-        if argcount:
-            matched = re.findall('\[default: (.*)\]', description, flags=re.I)
-            value = matched[0] if matched else None
-        return class_(short, long, argcount, value)
-
-    def single_match(self, left):
-        for n, pattern in enumerate(left):
-            if self.name == pattern.name:
-                return n, pattern
-        return None, None
-
-    @property
-    def name(self):
-        return self.long or self.short
-
-    def __repr__(self):
-        return 'Option(%r, %r, %r, %r)' % (self.short, self.long,
-                                           self.argcount, self.value)
-
-
-class Required(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        l = left
-        c = collected
-        for pattern in self.children:
-            matched, l, c = pattern.match(l, c)
-            if not matched:
-                return False, left, collected
-        return True, l, c
-
-
-class Optional(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        for pattern in self.children:
-            m, left, collected = pattern.match(left, collected)
-        return True, left, collected
-
-
-class OptionsShortcut(Optional):
-
-    """Marker/placeholder for [options] shortcut."""
-
-
-class OneOrMore(BranchPattern):
-
-    def match(self, left, collected=None):
-        assert len(self.children) == 1
-        collected = [] if collected is None else collected
-        l = left
-        c = collected
-        l_ = None
-        matched = True
-        times = 0
-        while matched:
-            # could it be that something didn't match but changed l or c?
-            matched, l, c = self.children[0].match(l, c)
-            times += 1 if matched else 0
-            if l_ == l:
-                break
-            l_ = l
-        if times >= 1:
-            return True, l, c
-        return False, left, collected
-
-
-class Either(BranchPattern):
-
-    def match(self, left, collected=None):
-        collected = [] if collected is None else collected
-        outcomes = []
-        for pattern in self.children:
-            matched, _, _ = outcome = pattern.match(left, collected)
-            if matched:
-                outcomes.append(outcome)
-        if outcomes:
-            return min(outcomes, key=lambda outcome: len(outcome[1]))
-        return False, left, collected
-
-
-class Tokens(list):
-
-    def __init__(self, source, error=DocoptExit):
-        self += source.split() if hasattr(source, 'split') else source
-        self.error = error
-
-    @staticmethod
-    def from_pattern(source):
-        source = re.sub(r'([\[\]\(\)\|]|\.\.\.)', r' \1 ', source)
-        source = [s for s in re.split('\s+|(\S*<.*?>)', source) if s]
-        return Tokens(source, error=DocoptLanguageError)
-
-    def move(self):
-        return self.pop(0) if len(self) else None
-
-    def current(self):
-        return self[0] if len(self) else None
-
-
-def parse_long(tokens, options):
-    """long ::= '--' chars [ ( ' ' | '=' ) chars ] ;"""
-    long, eq, value = tokens.move().partition('=')
-    assert long.startswith('--')
-    value = None if eq == value == '' else value
-    similar = [o for o in options if o.long == long]
-    if tokens.error is DocoptExit and similar == []:  # if no exact match
-        similar = [o for o in options if o.long and o.long.startswith(long)]
-    if len(similar) > 1:  # might be simply specified ambiguously 2+ times?
-        raise tokens.error('%s is not a unique prefix: %s?' %
-                           (long, ', '.join(o.long for o in similar)))
-    elif len(similar) < 1:
-        argcount = 1 if eq == '=' else 0
-        o = Option(None, long, argcount)
-        options.append(o)
-        if tokens.error is DocoptExit:
-            o = Option(None, long, argcount, value if argcount else True)
-    else:
-        o = Option(similar[0].short, similar[0].long,
-                   similar[0].argcount, similar[0].value)
-        if o.argcount == 0:
-            if value is not None:
-                raise tokens.error('%s must not have an argument' % o.long)
-        else:
-            if value is None:
-                if tokens.current() in [None, '--']:
-                    raise tokens.error('%s requires argument' % o.long)
-                value = tokens.move()
-        if tokens.error is DocoptExit:
-            o.value = value if value is not None else True
-    return [o]
-
-
-def parse_shorts(tokens, options):
-    """shorts ::= '-' ( chars )* [ [ ' ' ] chars ] ;"""
-    token = tokens.move()
-    assert token.startswith('-') and not token.startswith('--')
-    left = token.lstrip('-')
-    parsed = []
-    while left != '':
-        short, left = '-' + left[0], left[1:]
-        similar = [o for o in options if o.short == short]
-        if len(similar) > 1:
-            raise tokens.error('%s is specified ambiguously %d times' %
-                               (short, len(similar)))
-        elif len(similar) < 1:
-            o = Option(short, None, 0)
-            options.append(o)
-            if tokens.error is DocoptExit:
-                o = Option(short, None, 0, True)
-        else:  # why copying is necessary here?
-            o = Option(short, similar[0].long,
-                       similar[0].argcount, similar[0].value)
-            value = None
-            if o.argcount != 0:
-                if left == '':
-                    if tokens.current() in [None, '--']:
-                        raise tokens.error('%s requires argument' % short)
-                    value = tokens.move()
-                else:
-                    value = left
-                    left = ''
-            if tokens.error is DocoptExit:
-                o.value = value if value is not None else True
-        parsed.append(o)
-    return parsed
-
-
-def parse_pattern(source, options):
-    tokens = Tokens.from_pattern(source)
-    result = parse_expr(tokens, options)
-    if tokens.current() is not None:
-        raise tokens.error('unexpected ending: %r' % ' '.join(tokens))
-    return Required(*result)
-
-
-def parse_expr(tokens, options):
-    """expr ::= seq ( '|' seq )* ;"""
-    seq = parse_seq(tokens, options)
-    if tokens.current() != '|':
-        return seq
-    result = [Required(*seq)] if len(seq) > 1 else seq
-    while tokens.current() == '|':
-        tokens.move()
-        seq = parse_seq(tokens, options)
-        result += [Required(*seq)] if len(seq) > 1 else seq
-    return [Either(*result)] if len(result) > 1 else result
-
-
-def parse_seq(tokens, options):
-    """seq ::= ( atom [ '...' ] )* ;"""
-    result = []
-    while tokens.current() not in [None, ']', ')', '|']:
-        atom = parse_atom(tokens, options)
-        if tokens.current() == '...':
-            atom = [OneOrMore(*atom)]
-            tokens.move()
-        result += atom
-    return result
-
-
-def parse_atom(tokens, options):
-    """atom ::= '(' expr ')' | '[' expr ']' | 'options'
-             | long | shorts | argument | command ;
-    """
-    token = tokens.current()
-    result = []
-    if token in '([':
-        tokens.move()
-        matching, pattern = {'(': [')', Required], '[': [']', Optional]}[token]
-        result = pattern(*parse_expr(tokens, options))
-        if tokens.move() != matching:
-            raise tokens.error("unmatched '%s'" % token)
-        return [result]
-    elif token == 'options':
-        tokens.move()
-        return [OptionsShortcut()]
-    elif token.startswith('--') and token != '--':
-        return parse_long(tokens, options)
-    elif token.startswith('-') and token not in ('-', '--'):
-        return parse_shorts(tokens, options)
-    elif token.startswith('<') and token.endswith('>') or token.isupper():
-        return [Argument(tokens.move())]
-    else:
-        return [Command(tokens.move())]
-
-
-def parse_argv(tokens, options, options_first=False):
-    """Parse command-line argument vector.
-
-    If options_first:
-        argv ::= [ long | shorts ]* [ argument ]* [ '--' [ argument ]* ] ;
-    else:
-        argv ::= [ long | shorts | argument ]* [ '--' [ argument ]* ] ;
-
-    """
-    parsed = []
-    while tokens.current() is not None:
-        if tokens.current() == '--':
-            return parsed + [Argument(None, v) for v in tokens]
-        elif tokens.current().startswith('--'):
-            parsed += parse_long(tokens, options)
-        elif tokens.current().startswith('-') and tokens.current() != '-':
-            parsed += parse_shorts(tokens, options)
-        elif options_first:
-            return parsed + [Argument(None, v) for v in tokens]
-        else:
-            parsed.append(Argument(None, tokens.move()))
-    return parsed
-
-
-def parse_defaults(doc):
-    defaults = []
-    for s in parse_section('options:', doc):
-        # FIXME corner case "bla: options: --foo"
-        _, _, s = s.partition(':')  # get rid of "options:"
-        split = re.split('\n[ \t]*(-\S+?)', '\n' + s)[1:]
-        split = [s1 + s2 for s1, s2 in zip(split[::2], split[1::2])]
-        options = [Option.parse(s) for s in split if s.startswith('-')]
-        defaults += options
-    return defaults
-
-
-def parse_section(name, source):
-    pattern = re.compile('^([^\n]*' + name + '[^\n]*\n?(?:[ \t].*?(?:\n|$))*)',
-                         re.IGNORECASE | re.MULTILINE)
-    return [s.strip() for s in pattern.findall(source)]
-
-
-def formal_usage(section):
-    _, _, section = section.partition(':')  # drop "usage:"
-    pu = section.split()
-    return '( ' + ' '.join(') | (' if s == pu[0] else s for s in pu[1:]) + ' )'
-
-
-def extras(help, version, options, doc):
-    if help and any((o.name in ('-h', '--help')) and o.value for o in options):
-        print(doc.strip("\n"))
-        sys.exit()
-    if version and any(o.name == '--version' and o.value for o in options):
-        print(version)
-        sys.exit()
-
-
-class Dict(dict):
-    def __repr__(self):
-        return '{%s}' % ',\n '.join('%r: %r' % i for i in sorted(self.items()))
-
-
-def docopt(doc, argv=None, help=True, version=None, options_first=False):
-    """Parse `argv` based on command-line interface described in `doc`.
-
-    `docopt` creates your command-line interface based on its
-    description that you pass as `doc`. Such description can contain
-    --options, <positional-argument>, commands, which could be
-    [optional], (required), (mutually | exclusive) or repeated...
-
-    Parameters
-    ----------
-    doc : str
-        Description of your command-line interface.
-    argv : list of str, optional
-        Argument vector to be parsed. sys.argv[1:] is used if not
-        provided.
-    help : bool (default: True)
-        Set to False to disable automatic help on -h or --help
-        options.
-    version : any object
-        If passed, the object will be printed if --version is in
-        `argv`.
-    options_first : bool (default: False)
-        Set to True to require options precede positional arguments,
-        i.e. to forbid options and positional arguments intermix.
-
-    Returns
-    -------
-    args : dict
-        A dictionary, where keys are names of command-line elements
-        such as e.g. "--verbose" and "<path>", and values are the
-        parsed values of those elements.
-
-    Example
-    -------
-    >>> from docopt import docopt
-    >>> doc = '''
-    ... Usage:
-    ...     my_program tcp <host> <port> [--timeout=<seconds>]
-    ...     my_program serial <port> [--baud=<n>] [--timeout=<seconds>]
-    ...     my_program (-h | --help | --version)
-    ...
-    ... Options:
-    ...     -h, --help  Show this screen and exit.
-    ...     --baud=<n>  Baudrate [default: 9600]
-    ... '''
-    >>> argv = ['tcp', '127.0.0.1', '80', '--timeout', '30']
-    >>> docopt(doc, argv)
-    {'--baud': '9600',
-     '--help': False,
-     '--timeout': '30',
-     '--version': False,
-     '<host>': '127.0.0.1',
-     '<port>': '80',
-     'serial': False,
-     'tcp': True}
-
-    See also
-    --------
-    * For video introduction see http://docopt.org
-    * Full documentation is available in README.rst as well as online
-      at https://github.com/docopt/docopt#readme
-
-    """
-    argv = sys.argv[1:] if argv is None else argv
-
-    usage_sections = parse_section('usage:', doc)
-    if len(usage_sections) == 0:
-        raise DocoptLanguageError('"usage:" (case-insensitive) not found.')
-    if len(usage_sections) > 1:
-        raise DocoptLanguageError('More than one "usage:" (case-insensitive).')
-    DocoptExit.usage = usage_sections[0]
-
-    options = parse_defaults(doc)
-    pattern = parse_pattern(formal_usage(DocoptExit.usage), options)
-    # [default] syntax for argument is disabled
-    #for a in pattern.flat(Argument):
-    #    same_name = [d for d in arguments if d.name == a.name]
-    #    if same_name:
-    #        a.value = same_name[0].value
-    argv = parse_argv(Tokens(argv), list(options), options_first)
-    pattern_options = set(pattern.flat(Option))
-    for options_shortcut in pattern.flat(OptionsShortcut):
-        doc_options = parse_defaults(doc)
-        options_shortcut.children = list(set(doc_options) - pattern_options)
-        #if any_options:
-        #    options_shortcut.children += [Option(o.short, o.long, o.argcount)
-        #                    for o in argv if type(o) is Option]
-    extras(help, version, argv, doc)
-    matched, left, collected = pattern.fix().match(argv)
-    if matched and left == []:  # better error message if left?
-        return Dict((a.name, a.value) for a in (pattern.flat() + collected))
-    raise DocoptExit()

vendor/pip-pop/pip-diff

@@ -1,132 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  pip-diff (--fresh | --stale) <reqfile1> <reqfile2> [--exclude <package>...]
-  pip-diff (-h | --help)
-
-Options:
-  -h --help     Show this screen.
-  --fresh       List newly added packages.
-  --stale       List removed packages.
-"""
-import os
-from docopt import docopt
-
-try:  # pip >= 10
-    from pip._internal.req import parse_requirements
-    from pip._internal.download import PipSession as session
-
-    def PackageFinder(find_links, index_urls, session=None):
-        from pip._internal.index import PackageFinder
-        from pip._internal.models.search_scope import SearchScope
-        from pip._internal.models.selection_prefs import SelectionPreferences
-
-        search_scope = SearchScope.create(find_links, index_urls)
-        selection_prefs = SelectionPreferences(allow_yanked=False)
-        return PackageFinder.create(search_scope, selection_prefs, session=session)
-
-except ImportError:  # pip <= 9.0.3
-    from pip.req import parse_requirements
-    from pip.index import PackageFinder
-    from pip._vendor.requests import session
-
-requests = session()
-
-
-class Requirements(object):
-    def __init__(self, reqfile=None):
-        super(Requirements, self).__init__()
-        self.path = reqfile
-        self.requirements = []
-
-        if reqfile:
-            self.load(reqfile)
-
-    def __repr__(self):
-        return '<Requirements \'{}\'>'.format(self.path)
-
-    def load(self, reqfile):
-        if not os.path.exists(reqfile):
-            raise ValueError('The given requirements file does not exist.')
-
-        finder = PackageFinder([], [], session=requests)
-        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
-            if requirement.req:
-                if not getattr(requirement.req, 'name', None):
-                    # Prior to pip 8.1.2 the attribute `name` did not exist.
-                    requirement.req.name = requirement.req.project_name
-                    requirement.req.name = requirement.req.name.lower()
-                self.requirements.append(requirement.req)
-
-
-    def diff(self, requirements, ignore_versions=False, excludes=None):
-        r1 = self
-        r2 = requirements
-        results = {'fresh': [], 'stale': []}
-
-        # Generate fresh packages.
-        other_reqs = (
-            [r.name for r in r1.requirements]
-            if ignore_versions else r1.requirements
-        )
-
-        for req in r2.requirements:
-            r = req.name if ignore_versions else req
-
-            if r not in other_reqs and r not in excludes:
-                results['fresh'].append(req)
-
-        # Generate stale packages.
-        other_reqs = (
-            [r.name for r in r2.requirements]
-            if ignore_versions else r2.requirements
-        )
-
-        for req in r1.requirements:
-            r = req.name if ignore_versions else req
-
-            if r not in other_reqs and r not in excludes:
-                results['stale'].append(req)
-
-        return results
-
-
-def diff(r1, r2, include_fresh=False, include_stale=False, excludes=None):
-    include_versions = True if include_stale else False
-    excludes = excludes if len(excludes) else []
-
-    try:
-        r1 = Requirements(r1)
-        r2 = Requirements(r2)
-    except ValueError:
-        print('There was a problem loading the given requirements files.')
-        exit(os.EX_NOINPUT)
-
-    results = r1.diff(r2, ignore_versions=True, excludes=excludes)
-
-    if include_fresh:
-        for line in results['fresh']:
-            print(line.name if include_versions else line)
-
-    if include_stale:
-        for line in results['stale']:
-            print(line.name if include_versions else line)
-
-
-def main():
-    args = docopt(__doc__, version='pip-diff')
-
-    kwargs = {
-        'r1': args['<reqfile1>'],
-        'r2': args['<reqfile2>'],
-        'include_fresh': args['--fresh'],
-        'include_stale': args['--stale'],
-        'excludes': args['<package>']
-    }
-
-    diff(**kwargs)
-
-
-if __name__ == '__main__':
-    main()

vendor/pip-pop/pip-grep

@@ -1,94 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  pip-grep [-s] <reqfile> <package>...
-
-Options:
-  -h --help     Show this screen.
-"""
-import os
-import sys
-from docopt import docopt
-
-try:  # pip >= 10
-    from pip._internal.req import parse_requirements
-    from pip._internal.download import PipSession as session
-
-    def PackageFinder(find_links, index_urls, session=None):
-        from pip._internal.index import PackageFinder
-        from pip._internal.models.search_scope import SearchScope
-        from pip._internal.models.selection_prefs import SelectionPreferences
-
-        search_scope = SearchScope.create(find_links, index_urls)
-        selection_prefs = SelectionPreferences(allow_yanked=False)
-        return PackageFinder.create(search_scope, selection_prefs, session=session)
-
-except ImportError:  # pip <= 9.0.3
-    from pip.req import parse_requirements
-    from pip.index import PackageFinder
-    from pip._vendor.requests import session
-
-
-requests = session()
-
-
-class Requirements(object):
-    def __init__(self, reqfile=None):
-        super(Requirements, self).__init__()
-        self.path = reqfile
-        self.requirements = []
-
-        if reqfile:
-            self.load(reqfile)
-
-    def __repr__(self):
-        return '<Requirements \'{}\'>'.format(self.path)
-
-    def load(self, reqfile):
-        if not os.path.exists(reqfile):
-            raise ValueError('The given requirements file does not exist.')
-
-        finder = PackageFinder([], [], session=requests)
-        for requirement in parse_requirements(reqfile, finder=finder, session=requests):
-            if requirement.req:
-                if not getattr(requirement.req, 'name', None):
-                    # Prior to pip 8.1.2 the attribute `name` did not exist.
-                    requirement.req.name = requirement.req.project_name
-                self.requirements.append(requirement.req)
-
-
-def grep(reqfile, packages, silent=False):
-    try:
-        r = Requirements(reqfile)
-    except ValueError:
-        if not silent:
-            print('There was a problem loading the given requirement file.')
-        exit(os.EX_NOINPUT)
-
-    for req in r.requirements:
-        if req.name in packages:
-            if not silent:
-                print('Package {} found!'.format(req.name))
-            exit(0)
-
-    if not silent:
-        print('Not found.')
-
-    exit(1)
-
-
-def main():
-    args = docopt(__doc__, version='pip-grep')
-
-    kwargs = {'reqfile': args['<reqfile>'], 'packages': args['<package>'], 'silent': args['-s']}
-
-    grep(**kwargs)
-
-
-if __name__ == '__main__':
-    try:
-        main()
-    except Exception:
-        sys.exit(1)
-

vendor/pip-pop/sp-grep

@@ -1,38 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-
-"""Usage:
-  sp-grep [-s] <package>...
-
-Options:
-  -h --help     Show this screen.
-"""
-from docopt import docopt
-from pkg_resources import DistributionNotFound, get_distribution
-
-
-def has_any_distribution(names, silent=False):
-    for name in names:
-        try:
-            get_distribution(name)
-        except DistributionNotFound:
-            continue
-
-        if not silent:
-            print('Package {name} found!'.format(name=name))
-
-        exit(0)
-
-    if not silent:
-        print('Not found.')
-
-    exit(1)
-
-
-def main():
-    args = docopt(__doc__, version='sp-grep')
-    has_any_distribution(names=args['<package>'], silent=args['-s'])
-
-
-if __name__ == '__main__':
-    main()