kennethreitz/pipenv
1
0
Fork 0
mirror of https://github.com/kennethreitz/pipenv.git synced 2026-06-05 22:50:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Vendor new safety (#5217) (#5218)

Browse Source 
* Vendor new safety (#5217)

* Closest to correct vendoring of ruamel.

* Fix lint

* New pipenv check with minimal output.

* Remove decode_for_output use in pipenv check functions.

* Use click.secho where is possible.

* Rerun vendoring to get the latest safety version.

* skip this test for now.

Co-authored-by: Yeison Vargas <yeisonvargasjf@gmail.com>
This commit is contained in:
Matt Davis
2022-11-22 21:14:24 -05:00
committed by GitHub
parent 5db5472fd1
commit c58c371f6b
68 changed files with 18806 additions and 1086 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Pipfile
+1
View File
@@ -19,6 +19,7 @@ pytest-cov = "==3.*"
typing-extensions = "==4.*"
waitress = {version = "*", markers="sys_platform == 'win32'"}
gunicorn = {version = "*", markers="sys_platform == 'linux'"}
parse = "*"
[packages]
Pipfile.lock
Generated
+38 -29
View File
@@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "732f0dc59c2417ea9c448ef70f1d642487fa85ca8a743caa583076d0a6ccf32d"
"sha256": "ded711ae5f1cf2ba1a91d1ae79cc70609bfc9cf9b46789d6521246a5f540c57b"
},
"pipfile-spec": 6,
"requires": {},
@@ -24,10 +24,10 @@
},
"arpeggio": {
"hashes": [
"sha256:bfe349f252f82f82d84cb886f1d5081d1a31451e6045275e9f90b65d0daa06f1",
"sha256:fed68a1cb7f529cbd4d725597cc811b7506885fcdef17d4cdcf564341a1e210b"
"sha256:448e332deb0e9ccd04046f1c6c14529d197f41bc2fdb3931e43fc209042fbdd3",
"sha256:d6b03839019bb8a68785f9292ee6a36b1954eb84b925b84a6b8a5e1e26d3ed3d"
],
"version": "==1.10.2"
"version": "==2.0.0"
},
"atomicwrites": {
"hashes": [
@@ -217,11 +217,11 @@
},
"exceptiongroup": {
"hashes": [
"sha256:4d6c0aa6dd825810941c792f53d7b8d71da26f5e5f84f20f9508e8f2d33b140a",
"sha256:73866f7f842ede6cb1daa42c4af078e2035e5f7607f0e2c762cc51bb31bbe7b2"
"sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828",
"sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec"
],
"markers": "python_version < '3.11'",
"version": "==1.0.1"
"version": "==1.0.4"
},
"execnet": {
"hashes": [
@@ -268,16 +268,17 @@
"sha256:9dcc4547dbb1cb284accfb15ab5667a0e5d1881cc443e0677b4882a4067a807e",
"sha256:e0a968b5ba15f8a328fdfd7ab1fcb5af4470c28aaf7e55df02a99bc13138e6e8"
],
"index": "pypi",
"markers": "sys_platform == 'linux'",
"version": "==20.1.0"
},
"identify": {
"hashes": [
"sha256:48b7925fe122720088aeb7a6c34f17b27e706b72c61070f27fe3789094233440",
"sha256:7a214a10313b9489a0d61467db2856ae8d0b8306fc923e03a9effa53d8aedc58"
"sha256:906036344ca769539610436e40a684e170c3648b552194980bb7b617a8daeb9f",
"sha256:a390fb696e164dbddb047a0db26e57972ae52fbd037ae68797e5ae2f4492485d"
],
"markers": "python_version >= '3.7'",
"version": "==2.5.8"
"version": "==2.5.9"
},
"idna": {
"hashes": [
@@ -424,21 +425,28 @@
"markers": "python_version >= '3.6'",
"version": "==21.3"
},
"parse": {
"hashes": [
"sha256:9ff82852bcb65d139813e2a5197627a94966245c897796760a3a2a8eb66f020b"
],
"index": "pypi",
"version": "==1.19.0"
},
"parver": {
"hashes": [
"sha256:41a548c51b006a2f2522b54293cbfd2514bffa10774ece8430c9964a20cbd8b4",
"sha256:c902e0653bcce927cc156a7fd9b3a51924cbce3bf3d0bfd49fc282bfd0c5dfd3"
"sha256:c66d3347a4858643875ef959d8ba7a269d5964bfb690b0dd998b8f39da930be2",
"sha256:d4a3dbb93c53373ee9a0ba055e4858c44169b204b912e49d003ead95db9a9bca"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==0.3.1"
"markers": "python_version >= '3.7'",
"version": "==0.4"
},
"pathspec": {
"hashes": [
"sha256:46846318467efc4556ccfd27816e004270a9eeeeb4d062ce5e6fc7a87c573f93",
"sha256:7ace6161b621d31e7902eb6b5ae148d12cfd23f4a249b9ffb6b9fee12084323d"
"sha256:88c2606f2c1e818b978540f73ecc908e13999c6c3a383daf3705652ae79807a5",
"sha256:8f6bf73e5758fd365ef5d58ce09ac7c27d2833a8d7da51712eac6e27e35141b0"
],
"markers": "python_version >= '3.7'",
"version": "==0.10.1"
"version": "==0.10.2"
},
"pipenv": {
"editable": true,
@@ -450,11 +458,11 @@
},
"platformdirs": {
"hashes": [
"sha256:027d8e83a2d7de06bbac4e5ef7e023c02b863d7ea5d079477e722bb41ab25788",
"sha256:58c8abb07dcb441e6ee4b11d8df0ac856038f944ab98b7be6b27b2a3c7feef19"
"sha256:1006647646d80f16130f052404c6b901e80ee4ed6bef6792e1f238a8969106f7",
"sha256:af0276409f9a02373d540bf8480021a048711d572745aef4b7842dad245eba10"
],
"markers": "python_version >= '3.7'",
"version": "==2.5.2"
"version": "==2.5.4"
},
"pluggy": {
"hashes": [
@@ -637,11 +645,11 @@
},
"setuptools": {
"hashes": [
"sha256:d0b9a8433464d5800cbe05094acf5c6d52a91bfac9b52bcfc4d41382be5d5d31",
"sha256:e197a19aa8ec9722928f2206f8de752def0e4c9fc6953527360d1c36d94ddb2f"
"sha256:6211d2f5eddad8757bd0484923ca7c0a6302ebc4ab32ea5e94357176e0ca0840",
"sha256:d1eebf881c6114e51df1664bc2c9133d022f78d12d5f4f665b9191f084e2862d"
],
"markers": "python_version >= '3.7'",
"version": "==65.5.1"
"version": "==65.6.0"
},
"six": {
"hashes": [
@@ -732,16 +740,17 @@
},
"sphinxcontrib-spelling": {
"hashes": [
"sha256:a129d5dd0c00c9d414bcdf599afddb55bae01f1d543636e7ebb09491ba2babd4",
"sha256:a7ca90eea630c825657e1344000a11d7d7f547a865e0eebbed08d112beb073d1"
"sha256:56561c3f6a155b0946914e4de988729859315729dc181b5e4dc8a68fe78de35a",
"sha256:95a0defef8ffec6526f9e83b20cc24b08c9179298729d87976891840e3aa3064"
],
"index": "pypi",
"version": "==7.6.2"
"version": "==7.7.0"
},
"stdeb": {
"hashes": [
"sha256:08c22c9c03b28a140fe3ec5064b53a5288279f22e596ca06b0be698d50c93cf2"
],
"index": "pypi",
"markers": "sys_platform == 'linux'",
"version": "==0.10.0"
},
@@ -787,11 +796,11 @@
},
"virtualenv": {
"hashes": [
"sha256:186ca84254abcbde98180fd17092f9628c5fe742273c02724972a1d8a2035108",
"sha256:530b850b523c6449406dfba859d6345e48ef19b8439606c5d74d7d3c9e14d76e"
"sha256:8691e3ff9387f743e00f6bb20f70121f5e4f596cae754531f2b3b3a1b1ac696e",
"sha256:efd66b00386fdb7dbe4822d172303f40cd05e50e01740b19ea42425cbe653e29"
],
"markers": "python_version >= '3.6'",
"version": "==20.16.6"
"version": "==20.16.7"
},
"virtualenv-clone": {
"hashes": [
pipenv/cli/command.py
+35 -3
View File
@@ -440,7 +440,7 @@ def run(state, command, args):
"--db",
nargs=1,
default=lambda: os.environ.get("PIPENV_SAFETY_DB"),
help="Path to a local PyUp Safety vulnerabilities database."
help="Path or URL to a PyUp Safety vulnerabilities database."
" Default: ENV PIPENV_SAFETY_DB or None.",
)
@option(
@@ -451,7 +451,7 @@ def run(state, command, args):
)
@option(
"--output",
type=Choice(["default", "json", "full-report", "bare"]),
type=Choice(["default", "json", "full-report", "bare", "screen", "text", "minimal"]),
default="default",
help="Translates to --json, --full-report or --bare from PyUp Safety check",
)
@@ -464,6 +464,28 @@ def run(state, command, args):
@option(
"--quiet", is_flag=True, help="Quiet standard output, except vulnerability report."
)
@option("--policy-file", default="", help="Define the policy file to be used")
@option(
"--exit-code/--continue-on-error",
default=True,
help="Output standard exit codes. Default: --exit-code",
)
@option(
"--audit-and-monitor/--disable-audit-and-monitor",
default=True,
help="Send results back to pyup.io for viewing on your dashboard. Requires an API key.",
)
@option(
"--project",
default=None,
help="Project to associate this scan with on pyup.io. Defaults to a canonicalized github style name if available, otherwise unknown",
)
@option(
"--save-json",
default="",
help="Path to where output file will be placed, if the path is a directory, "
"Safety will use safety-report.json as filename. Default: empty",
)
@common_options
@system_option
@pass_state
@@ -472,9 +494,14 @@ def check(
db=None,
style=False,
ignore=None,
output="default",
output="screen",
key=None,
quiet=False,
exit_code=True,
policy_file="",
save_json="",
audit_and_monitor=True,
project=None,
**kwargs,
):
"""Checks for PyUp Safety security vulnerabilities and against PEP 508 markers provided in Pipfile."""
@@ -490,6 +517,11 @@ def check(
output=output,
key=key,
quiet=quiet,
exit_code=exit_code,
policy_file=policy_file,
save_json=save_json,
audit_and_monitor=audit_and_monitor,
safety_project=project,
pypi_mirror=state.pypi_mirror,
)
pipenv/core.py
+105 -39
View File
@@ -1,3 +1,4 @@
import io
import json as simplejson
import logging
import os
@@ -15,7 +16,7 @@ from posixpath import expandvars
from typing import Dict, List, Optional, Union
from pipenv import environments, exceptions, pep508checker
from pipenv._compat import decode_for_output, fix_utf8
from pipenv._compat import fix_utf8
from pipenv.patched.pip._internal.build_env import get_runnable_pip
from pipenv.patched.pip._internal.exceptions import PipError
from pipenv.patched.pip._internal.network.session import PipSession
@@ -2842,9 +2843,14 @@ def do_check(
system=False,
db=None,
ignore=None,
output="default",
output="screen",
key=None,
quiet=False,
exit_code=True,
policy_file="",
save_json="",
audit_and_monitor=True,
safety_project=None,
pypi_mirror=None,
):
import json
@@ -2860,9 +2866,7 @@ def do_check(
pypi_mirror=pypi_mirror,
)
if not quiet and not project.s.is_quiet():
click.echo(
click.style(decode_for_output("Checking PEP 508 requirements..."), bold=True)
)
click.secho("Checking PEP 508 requirements...", bold=True)
pep508checker_path = pep508checker.__file__.rstrip("cdo")
safety_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), "patched", "safety"
@@ -2879,7 +2883,7 @@ def do_check(
click.echo(
"{}\n{}\n{}".format(
click.style(
decode_for_output("Failed parsing pep508 results: "),
"Failed parsing pep508 results: ",
fg="white",
bold=True,
),
@@ -2913,12 +2917,11 @@ def do_check(
sys.exit(1)
else:
if not quiet and not project.s.is_quiet():
click.echo(click.style("Passed!", fg="green"))
click.secho("Passed!", fg="green")
if not quiet and not project.s.is_quiet():
click.echo(
click.style(
decode_for_output("Checking installed package safety..."), bold=True
)
click.secho(
"Checking installed packages for vulnerabilities...",
bold=True,
)
if ignore:
if not isinstance(ignore, (tuple, list)):
@@ -2926,57 +2929,120 @@ def do_check(
ignored = [["--ignore", cve] for cve in ignore]
if not quiet and not project.s.is_quiet():
click.echo(
"Notice: Ignoring CVE(s) {}".format(
click.style(", ".join(ignore), fg="yellow")
"Notice: Ignoring Vulnerabilit{} {}".format(
"ies" if len(ignored) > 1 else "y",
click.style(", ".join(ignore), fg="yellow"),
),
err=True,
)
else:
ignored = []
switch = output
if output == "default":
switch = "json"
options = [
"--audit-and-monitor" if audit_and_monitor else "--disable-audit-and-monitor",
"--exit-code" if exit_code else "--continue-on-error",
]
if output == "full-report":
options.append("--full-report")
elif output == "minimal":
options.append("--json")
elif output not in ["screen", "default"]:
options.append(f"--output={output}")
if save_json:
options.append(f"--save-json={save_json}")
if policy_file:
options.append(f"--policy-file={policy_file}")
if safety_project:
options.append(f"--project={safety_project}")
cmd = _cmd + [safety_path, "check"] + options
cmd = _cmd + [safety_path, "check", f"--{switch}"]
if db:
if not quiet and not project.s.is_quiet():
click.echo(click.style(f"Using local database {db}"))
click.echo(f"Using {db} database")
cmd.append(f"--db={db}")
elif key or project.s.PIPENV_PYUP_API_KEY:
cmd = cmd + [f"--key={key or project.s.PIPENV_PYUP_API_KEY}"]
else:
PIPENV_SAFETY_DB = (
"https://d2qjmgddvqvu75.cloudfront.net/aws/safety/pipenv/1.0.0/"
)
os.environ["SAFETY_ANNOUNCEMENTS_URL"] = f"{PIPENV_SAFETY_DB}announcements.json"
cmd.append(f"--db={PIPENV_SAFETY_DB}")
if ignored:
for cve in ignored:
cmd += cve
c = run_command(cmd, catch_exceptions=False, is_verbose=project.s.is_verbose())
if output == "default":
os.environ["SAFETY_CUSTOM_INTEGRATION"] = "True"
os.environ["SAFETY_SOURCE"] = "pipenv"
os.environ["SAFETY_PURE_YAML"] = "True"
from pipenv.patched.safety.cli import cli
sys.argv = cmd[1:]
if output == "minimal":
from contextlib import redirect_stderr, redirect_stdout
code = 0
with redirect_stdout(io.StringIO()) as out, redirect_stderr(io.StringIO()) as err:
try:
cli(prog_name="pipenv")
except SystemExit as exit_signal:
code = exit_signal.code
report = out.getvalue()
error = err.getvalue()
try:
results = simplejson.loads(c.stdout)
except (ValueError, json.JSONDecodeError):
raise exceptions.JSONParseError(c.stdout, c.stderr)
json_report = simplejson.loads(report)
except Exception:
raise exceptions.PipenvCmdError(
cmd_list_to_shell(c.args), c.stdout, c.stderr, c.returncode
cmd_list_to_shell(cmd), report, error, exit_code=code
)
for (package, resolved, installed, description, vuln, *_) in results:
meta = json_report.get("report_meta")
vulnerabilities_found = meta.get("vulnerabilities_found")
fg = "green"
message = "All good!"
db_type = "commercial" if meta.get("api_key", False) else "free"
if vulnerabilities_found >= 0:
fg = "red"
message = (
f"Scan was complete using Safety’s {db_type} vulnerability database."
)
click.echo()
click.secho(f"{vulnerabilities_found} vulnerabilities found.", fg=fg)
click.echo()
vulnerabilities = json_report.get("vulnerabilities", [])
for vuln in vulnerabilities:
click.echo(
"{}: {} {} resolved ({} installed)!".format(
click.style(vuln, bold=True),
click.style(package, fg="green"),
click.style(resolved, fg="yellow", bold=False),
click.style(installed, fg="yellow", bold=True),
"{}: {} {} open to vulnerability {} ({}). More info: {}".format(
click.style(vuln["vulnerability_id"], bold=True, fg="red"),
click.style(vuln["package_name"], fg="green"),
click.style(vuln["analyzed_version"], fg="yellow", bold=True),
click.style(vuln["vulnerability_id"], bold=True),
click.style(vuln["vulnerable_spec"], fg="yellow", bold=False),
click.style(vuln["more_info_url"], bold=True),
)
)
click.echo(f"{description}")
click.echo(f"{vuln['advisory']}")
click.echo()
if c.returncode == 0:
click.echo(click.style("All good!", fg="green"))
sys.exit(0)
else:
sys.exit(1)
else:
click.echo(c.stdout)
sys.exit(c.returncode)
click.secho(message, fg="white", bold=True)
sys.exit(code)
cli(prog_name="pipenv")
def do_graph(project, bare=False, json=False, json_tree=False, reverse=False):
pipenv/patched/patched.txt
+1 -1
View File
@@ -1,2 +1,2 @@
pip==22.3
safety==1.10.3
safety==2.3.2
pipenv/patched/pip/_internal/utils/misc.py
+1 -1
View File
@@ -603,7 +603,7 @@ def is_wheel_installed() -> bool:
Return whether the wheel package is installed.
"""
try:
import pipenv.vendor.wheel as wheel # noqa: F401
import wheel # noqa: F401
except ImportError:
return False
pipenv/patched/safety/VERSION
+1
View File
@@ -0,0 +1 @@
2.3.2
pipenv/patched/safety/__init__.py
+7 -1
View File
@@ -2,4 +2,10 @@
__author__ = """pyup.io"""
__email__ = 'support@pyup.io'
__version__ = '1.10.3'
import os
ROOT = os.path.dirname(os.path.abspath(__file__))
with open(os.path.join(ROOT, 'VERSION')) as version_file:
VERSION = version_file.read().strip()
pipenv/patched/safety/__main__.py
+1 -41
View File
@@ -1,48 +1,8 @@
"""Allow safety to be executable through `python -m safety`."""
from __future__ import absolute_import
import os
import sys
import sysconfig
PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
PIPENV_DIR = os.path.dirname(PATCHED_DIR)
VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor")
def get_site_packages():
prefixes = {sys.prefix, sysconfig.get_config_var('prefix')}
try:
prefixes.add(sys.real_prefix)
except AttributeError:
pass
form = sysconfig.get_path('purelib', expand=False)
py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info)
return {
form.format(base=prefix, py_version_short=py_version_short)
for prefix in prefixes
}
def insert_before_site_packages(*paths):
site_packages = get_site_packages()
index = None
for i, path in enumerate(sys.path):
if path in site_packages:
index = i
break
if index is None:
sys.path += list(paths)
else:
sys.path = sys.path[:index] + list(paths) + sys.path[index:]
def insert_pipenv_dirs():
insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR)
from pipenv.patched.safety.cli import cli
if __name__ == "__main__": # pragma: no cover
insert_pipenv_dirs()
from pipenv.patched.safety.cli import cli
cli(prog_name="safety")
pipenv/patched/safety/alerts/__init__.py
+42
View File
@@ -0,0 +1,42 @@
import sys
import json
from typing import Any
import pipenv.vendor.click as click
from dataclasses import dataclass
from . import github
from pipenv.patched.safety.util import SafetyPolicyFile
@dataclass
class Alert:
report: Any
key: str
policy: Any = None
requirements_files: Any = None
@click.group(help="Send alerts based on the results of a Safety scan.")
@click.option('--check-report', help='JSON output of Safety Check to work with.', type=click.File('r'), default=sys.stdin)
@click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml',
help="Define the policy file to be used")
@click.option("--key", envvar="SAFETY_API_KEY",
help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY "
"environment variable.", required=True)
@click.pass_context
def alert(ctx, check_report, policy_file, key):
with check_report:
# TODO: This breaks --help for subcommands
try:
safety_report = json.load(check_report)
except json.decoder.JSONDecodeError as e:
click.secho("Error decoding input JSON: {}".format(e.msg), fg='red')
sys.exit(1)
if not 'report_meta' in safety_report:
click.secho("You must pass in a valid Safety Check JSON report", fg='red')
sys.exit(1)
ctx.obj = Alert(report=safety_report, policy=policy_file if policy_file else {}, key=key)
alert.add_command(github.github_pr)
alert.add_command(github.github_issue)
pipenv/patched/safety/alerts/github.py
+298
View File
@@ -0,0 +1,298 @@
import re
import sys
import pipenv.vendor.click as click
try:
import github as pygithub
except ImportError:
pygithub = None
from . import utils, requirements
def create_branch(repo, base_branch, new_branch):
ref = repo.get_git_ref("heads/" + base_branch)
repo.create_git_ref(ref="refs/heads/" + new_branch, sha=ref.object.sha)
def delete_branch(repo, branch):
ref = repo.get_git_ref(f"heads/{branch}")
ref.delete()
@click.command()
@click.option('--repo', help='GitHub standard repo path (eg, my-org/my-project)')
@click.option('--token', help='GitHub Access Token')
@click.option('--base-url', help='Optional custom Base URL, if you\'re using GitHub enterprise', default=None)
@click.pass_obj
@utils.require_files_report
def github_pr(obj, repo, token, base_url):
"""
Create a GitHub PR to fix any vulnerabilities using PyUp's remediation data.
Normally, this is run by a GitHub action. If you're running this manually, ensure that your local repo is up to date and on HEAD - otherwise you'll see strange results.
"""
if pygithub is None:
click.secho("pygithub is not installed. Did you install Safety with GitHub support? Try pip install safety[github]", fg='red')
sys.exit(1)
# TODO: Improve access to our config in future.
branch_prefix = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('branch-prefix', 'pyup/')
pr_prefix = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('pr-prefix', '[PyUp] ')
assignees = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('assignees', [])
labels = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('labels', ['security'])
label_severity = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('label-severity', True)
ignore_cvss_severity_below = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('ignore-cvss-severity-below', 0)
ignore_cvss_unknown_severity = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('ignore-cvss-unknown-severity', False)
gh = pygithub.Github(token, **({"base_url": base_url} if base_url else {}))
repo = gh.get_repo(repo)
try:
self_user = gh.get_user().login
except pygithub.GithubException:
# If we're using a token from an action (or integration) we can't call `get_user()`. Fall back
# to assuming we're running under an action
self_user = "web-flow"
pulls = repo.get_pulls(state='open', sort='created', base=repo.default_branch)
pending_updates = set(obj.report['remediations'].keys())
# TODO: Refactor this loop into a fn to iterate over remediations nicely
for name, contents in obj.requirements_files.items():
raw_contents = contents
contents = contents.decode('utf-8') # TODO - encoding?
parsed_req_file = requirements.RequirementFile(name, contents)
for pkg, remediation in obj.report['remediations'].items():
if remediation['recommended_version'] is None:
print(f"The GitHub PR alerter only currently supports remediations that have a recommended_version: {pkg}")
continue
# We have a single remediation that can have multiple vulnerabilities
vulns = [x for x in obj.report['vulnerabilities'] if x['package_name'] == pkg and x['analyzed_version'] == remediation['current_version']]
if ignore_cvss_unknown_severity and all(x['severity'] is None for x in vulns):
print("All vulnerabilities have unknown severity, and ignore_cvss_unknown_severity is set.")
continue
highest_base_score = 0
for vuln in vulns:
if vuln['severity'] is not None:
highest_base_score = max(highest_base_score, (vuln['severity'].get('cvssv3', {}) or {}).get('base_score', 10))
if ignore_cvss_severity_below:
at_least_one_match = False
for vuln in vulns:
# Consider a None severity as a match, since it's controlled by a different flag
# If we can't find a base_score but we have severity data, assume it's critical for now.
if vuln['severity'] is None or (vuln['severity'].get('cvssv3', {}) or {}).get('base_score', 10) >= ignore_cvss_severity_below:
at_least_one_match = True
if not at_least_one_match:
print(f"None of the vulnerabilities found have a score greater than or equal to the ignore_cvss_severity_below of {ignore_cvss_severity_below}")
continue
for parsed_req in parsed_req_file.requirements:
if parsed_req.name == pkg:
updated_contents = parsed_req.update_version(contents, remediation['recommended_version'])
pending_updates.discard(pkg)
new_branch = branch_prefix + utils.generate_branch_name(pkg, remediation)
skip_create = False
# Few possible cases:
# 1. No existing PRs exist for this change (don't need to handle)
# 2. An existing PR exists, and it's out of date (eg, recommended 0.5.1 and we want 0.5.2)
# 3. An existing PR exists, and it's not mergable anymore (eg, needs a rebase)
# 4. An existing PR exists, and everything's up to date.
# 5. An existing PR exists, but it's not needed anymore (perhaps we've been updated to a later version)
# 6. No existing PRs exist, but a branch does exist (perhaps the PR was closed but a stale branch left behind)
# In any case, we only act if we've been the only committer to the branch.
for pr in pulls:
if not pr.head.ref.startswith(branch_prefix):
continue
authors = [commit.committer.login for commit in pr.get_commits()]
only_us = all([x == self_user for x in authors])
try:
_, pr_pkg, pr_ver = pr.head.ref.split('/')
except ValueError:
# It's possible that something weird has manually been done, so skip that
print('Found an invalid branch name on an open PR, that matches our prefix. Skipping.')
continue
if pr_pkg != pkg:
continue
# Case 4
if pr_pkg == pkg and pr_ver == remediation['recommended_version'] and pr.mergeable:
print(f"An up to date PR #{pr.number} for {pkg} was found, no action will be taken.")
skip_create = True
continue
if not only_us:
print(f"There are other committers on the PR #{pr.number} for {pkg}. No further action will be taken.")
continue
# Case 2
if pr_pkg == pkg and pr_ver != remediation['recommended_version']:
print(f"Closing stale PR #{pr.number} for {pkg} as a newer recommended version became")
pr.create_issue_comment("This PR has been replaced, since a newer recommended version became available.")
pr.edit(state='closed')
delete_branch(repo, pr.head.ref)
# Case 3
if not pr.mergeable:
print(f"Closing PR #{pr.number} for {pkg} as it has become unmergable and we were the only committer")
pr.create_issue_comment("This PR has been replaced since it became unmergable.")
pr.edit(state='closed')
delete_branch(repo, pr.head.ref)
if updated_contents == contents:
print(f"Couldn't update {pkg} to {remediation['recommended_version']}")
continue
if skip_create:
continue
try:
create_branch(repo, repo.default_branch, new_branch)
except pygithub.GithubException as e:
if e.data['message'] == "Reference already exists":
# There might be a stale branch. If the bot is the only committer, nuke it.
comparison = repo.compare(repo.default_branch, new_branch)
authors = [commit.committer.login for commit in comparison.commits]
only_us = all([x == self_user for x in authors])
if only_us:
delete_branch(repo, new_branch)
create_branch(repo, repo.default_branch, new_branch)
else:
print(f"The branch '{new_branch}' already exists - but there is no matching PR and this branch has committers other than us. This remediation will be skipped.")
continue
else:
raise e
try:
repo.update_file(
path=name,
message=utils.generate_commit_message(pkg, remediation),
content=updated_contents,
branch=new_branch,
sha=utils.git_sha1(raw_contents)
)
except pygithub.GithubException as e:
if "does not match" in e.data['message']:
click.secho(f"GitHub blocked a commit on our branch to the requirements file, {name}, as the local hash we computed didn't match the version on {repo.default_branch}. Make sure you're running safety against the latest code on your default branch.", fg='red')
continue
else:
raise e
pr = repo.create_pull(title=pr_prefix + utils.generate_title(pkg, remediation, vulns), body=utils.generate_body(pkg, remediation, vulns, api_key=obj.key), head=new_branch, base=repo.default_branch)
print(f"Created Pull Request to update {pkg}")
for assignee in assignees:
pr.add_to_assignees(assignee)
for label in labels:
pr.add_to_labels(label)
if label_severity:
score_as_label = utils.cvss3_score_to_label(highest_base_score)
if score_as_label:
pr.add_to_labels(score_as_label)
if len(pending_updates) > 0:
click.secho("The following remediations were not followed: {}".format(', '.join(pending_updates)), fg='red')
@click.command()
@click.option('--repo', help='GitHub standard repo path (eg, my-org/my-project)')
@click.option('--token', help='GitHub Access Token')
@click.option('--base-url', help='Optional custom Base URL, if you\'re using GitHub enterprise', default=None)
@click.pass_obj
@utils.require_files_report # TODO: For now, it can be removed in the future to support env scans.
def github_issue(obj, repo, token, base_url):
"""
Create a GitHub Issue for any vulnerabilities found using PyUp's remediation data.
Normally, this is run by a GitHub action. If you're running this manually, ensure that your local repo is up to date and on HEAD - otherwise you'll see strange results.
"""
if pygithub is None:
click.secho("pygithub is not installed. Did you install Safety with GitHub support? Try pip install safety[github]", fg='red')
sys.exit(1)
# TODO: Improve access to our config in future.
issue_prefix = obj.policy.get('alert', {}).get('security', {}).get('github-issue', {}).get('issue-prefix', '[PyUp] ')
assignees = obj.policy.get('alert', {}).get('security', {}).get('github-issue', {}).get('assignees', [])
labels = obj.policy.get('alert', {}).get('security', {}).get('github-issue', {}).get('labels', ['security'])
label_severity = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('label-severity', True)
ignore_cvss_severity_below = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('ignore-cvss-severity-below', 0)
ignore_cvss_unknown_severity = obj.policy.get('alert', {}).get('security', {}).get('github-pr', {}).get('ignore-cvss-unknown-severity', False)
gh = pygithub.Github(token, **({"base_url": base_url} if base_url else {}))
repo = gh.get_repo(repo)
issues = list(repo.get_issues(state='open', sort='created'))
ISSUE_TITLE_REGEX = re.escape(issue_prefix) + r"Security Vulnerability in (.+)"
for name, contents in obj.requirements_files.items():
raw_contents = contents
contents = contents.decode('utf-8') # TODO - encoding?
parsed_req_file = requirements.RequirementFile(name, contents)
for pkg, remediation in obj.report['remediations'].items():
if remediation['recommended_version'] is None:
print(f"The GitHub Issue alerter only currently supports remediations that have a recommended_version: {pkg}")
continue
# We have a single remediation that can have multiple vulnerabilities
vulns = [x for x in obj.report['vulnerabilities'] if x['package_name'] == pkg and x['analyzed_version'] == remediation['current_version']]
if ignore_cvss_unknown_severity and all(x['severity'] is None for x in vulns):
print("All vulnerabilities have unknown severity, and ignore_cvss_unknown_severity is set.")
continue
highest_base_score = 0
for vuln in vulns:
if vuln['severity'] is not None:
highest_base_score = max(highest_base_score, (vuln['severity'].get('cvssv3', {}) or {}).get('base_score', 10))
if ignore_cvss_severity_below:
at_least_one_match = False
for vuln in vulns:
# Consider a None severity as a match, since it's controlled by a different flag
# If we can't find a base_score but we have severity data, assume it's critical for now.
if vuln['severity'] is None or (vuln['severity'].get('cvssv3', {}) or {}).get('base_score', 10) >= ignore_cvss_severity_below:
at_least_one_match = True
if not at_least_one_match:
print(f"None of the vulnerabilities found have a score greater than or equal to the ignore_cvss_severity_below of {ignore_cvss_severity_below}")
continue
for parsed_req in parsed_req_file.requirements:
if parsed_req.name == pkg:
skip = False
for issue in issues:
match = re.match(ISSUE_TITLE_REGEX, issue.title)
if match:
if match.group(1) == pkg:
skip = True
# For now, we just skip issues if they already exist - we don't try and update them.
if skip:
print(f"An issue already exists for {pkg} - skipping")
continue
pr = repo.create_issue(title=issue_prefix + utils.generate_issue_title(pkg, remediation), body=utils.generate_issue_body(pkg, remediation, vulns, api_key=obj.key))
print(f"Created issue to update {pkg}")
for assignee in assignees:
pr.add_to_assignees(assignee)
for label in labels:
pr.add_to_labels(label)
if label_severity:
score_as_label = utils.cvss3_score_to_label(highest_base_score)
if score_as_label:
pr.add_to_labels(score_as_label)
pipenv/patched/safety/alerts/requirements.py
+339
View File
@@ -0,0 +1,339 @@
from __future__ import unicode_literals
from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
import pipenv.patched.pip._vendor.requests as requests
from datetime import datetime
from pipenv.vendor.dparse import parse, parser, updater, filetypes
from pipenv.vendor.dparse.dependencies import Dependency
from pipenv.vendor.dparse.parser import setuptools_parse_requirements_backport as parse_requirements
class RequirementFile(object):
def __init__(self, path, content, sha=None):
self.path = path
self.content = content
self.sha = sha
self._requirements = None
self._other_files = None
self._is_valid = None
self.is_pipfile = False
self.is_pipfile_lock = False
self.is_setup_cfg = False
def __str__(self):
return "RequirementFile(path='{path}', sha='{sha}', content='{content}')".format(
path=self.path,
content=self.content[:30] + "[truncated]" if len(self.content) > 30 else self.content,
sha=self.sha
)
@property
def is_valid(self):
if self._is_valid is None:
self._parse()
return self._is_valid
@property
def requirements(self):
if not self._requirements:
self._parse()
return self._requirements
@property
def other_files(self):
if not self._other_files:
self._parse()
return self._other_files
@staticmethod
def parse_index_server(line):
return parser.Parser.parse_index_server(line)
def _hash_parser(self, line):
return parser.Parser.parse_hashes(line)
def _parse_requirements_txt(self):
self.parse_dependencies(filetypes.requirements_txt)
def _parse_conda_yml(self):
self.parse_dependencies(filetypes.conda_yml)
def _parse_tox_ini(self):
self.parse_dependencies(filetypes.tox_ini)
def _parse_pipfile(self):
self.parse_dependencies(filetypes.pipfile)
self.is_pipfile = True
def _parse_pipfile_lock(self):
self.parse_dependencies(filetypes.pipfile_lock)
self.is_pipfile_lock = True
def _parse_setup_cfg(self):
self.parse_dependencies(filetypes.setup_cfg)
self.is_setup_cfg = True
def _parse(self):
self._requirements, self._other_files = [], []
if self.path.endswith('.yml') or self.path.endswith(".yaml"):
self._parse_conda_yml()
elif self.path.endswith('.ini'):
self._parse_tox_ini()
elif self.path.endswith("Pipfile"):
self._parse_pipfile()
elif self.path.endswith("Pipfile.lock"):
self._parse_pipfile_lock()
elif self.path.endswith('setup.cfg'):
self._parse_setup_cfg()
else:
self._parse_requirements_txt()
self._is_valid = len(self._requirements) > 0 or len(self._other_files) > 0
def parse_dependencies(self, file_type):
result = parse(
self.content,
path=self.path,
sha=self.sha,
file_type=file_type,
marker=(
("pyup: ignore file", "pyup:ignore file"), # file marker
("pyup: ignore", "pyup:ignore"), # line marker
)
)
for dep in result.dependencies:
req = Requirement(
name=dep.name,
specs=dep.specs,
line=dep.line,
lineno=dep.line_numbers[0] if dep.line_numbers else 0,
extras=dep.extras,
file_type=file_type,
)
req.index_server = dep.index_server
if self.is_pipfile:
req.pipfile = self.path
req.hashes = dep.hashes
self._requirements.append(req)
self._other_files = result.resolved_files
def iter_lines(self, lineno=0):
for line in self.content.splitlines()[lineno:]:
yield line
@classmethod
def resolve_file(cls, file_path, line):
return parser.Parser.resolve_file(file_path, line)
class Requirement(object):
def __init__(self, name, specs, line, lineno, extras, file_type):
self.name = name
self.key = name.lower()
self.specs = specs
self.line = line
self.lineno = lineno
self.index_server = None
self.extras = extras
self.hashes = []
self.file_type = file_type
self.pipfile = None
self.hashCmp = (
self.key,
self.specs,
frozenset(self.extras),
)
self._is_insecure = None
self._changelog = None
if len(self.specs._specs) == 1 and next(iter(self.specs._specs))._spec[0] == "~=":
# convert compatible releases to something more easily consumed,
# e.g. '~=1.2.3' is equivalent to '>=1.2.3,<1.3.0', while '~=1.2'
# is equivalent to '>=1.2,<2.0'
min_version = next(iter(self.specs._specs))._spec[1]
max_version = list(parse_version(min_version).release)
max_version[-1] = 0
max_version[-2] = max_version[-2] + 1
max_version = '.'.join(str(x) for x in max_version)
self.specs = SpecifierSet('>=%s,<%s' % (min_version, max_version))
def __eq__(self, other):
return (
isinstance(other, Requirement) and
self.hashCmp == other.hashCmp
)
def __ne__(self, other):
return not self == other
def __str__(self):
return "Requirement.parse({line}, {lineno})".format(line=self.line, lineno=self.lineno)
def __repr__(self):
return self.__str__()
@property
def is_pinned(self):
if len(self.specs._specs) == 1 and next(iter(self.specs._specs))._spec[0] == "==":
return True
return False
@property
def is_open_ranged(self):
if len(self.specs._specs) == 1 and next(iter(self.specs._specs))._spec[0] == ">=":
return True
return False
@property
def is_ranged(self):
return len(self.specs._specs) >= 1 and not self.is_pinned
@property
def is_loose(self):
return len(self.specs._specs) == 0
@staticmethod
def convert_semver(version):
semver = {'major': 0, "minor": 0, "patch": 0}
version = version.split(".")
# don't be overly clever here. repitition makes it more readable and works exactly how
# it is supposed to
try:
semver['major'] = int(version[0])
semver['minor'] = int(version[1])
semver['patch'] = int(version[2])
except (IndexError, ValueError):
pass
return semver
@property
def can_update_semver(self):
# return early if there's no update filter set
if "pyup: update" not in self.line:
return True
update = self.line.split("pyup: update")[1].strip().split("#")[0]
current_version = Requirement.convert_semver(next(iter(self.specs._specs))._spec[1])
next_version = Requirement.convert_semver(self.latest_version)
if update == "major":
if current_version['major'] < next_version['major']:
return True
elif update == 'minor':
if current_version['major'] < next_version['major'] \
or current_version['minor'] < next_version['minor']:
return True
return False
@property
def filter(self):
rqfilter = False
if "rq.filter:" in self.line:
rqfilter = self.line.split("rq.filter:")[1].strip().split("#")[0]
elif "pyup:" in self.line:
if "pyup: update" not in self.line:
rqfilter = self.line.split("pyup:")[1].strip().split("#")[0]
# unset the filter once the date set in 'until' is reached
if "until" in rqfilter:
rqfilter, until = [l.strip() for l in rqfilter.split("until")]
try:
until = datetime.strptime(until, "%Y-%m-%d")
if until < datetime.now():
rqfilter = False
except ValueError:
# wrong date formatting
pass
if rqfilter:
try:
rqfilter, = parse_requirements("filter " + rqfilter)
if len(rqfilter.specifier._specs) > 0:
return rqfilter.specifier
except ValueError:
pass
return False
@property
def version(self):
if self.is_pinned:
return next(iter(self.specs._specs))._spec[1]
specs = self.specs
if self.filter:
specs = SpecifierSet(
",".join(["".join(s._spec) for s in list(specs._specs) + list(self.filter._specs)])
)
return self.get_latest_version_within_specs(
specs,
versions=self.package.versions,
prereleases=self.prereleases
)
def get_hashes(self, version):
r = requests.get('https://pypi.org/pypi/{name}/{version}/json'.format(
name=self.key,
version=version
))
hashes = []
data = r.json()
for item in data.get("urls", {}):
sha256 = item.get("digests", {}).get("sha256", False)
if sha256:
hashes.append({"hash": sha256, "method": "sha256"})
return hashes
def update_version(self, content, version, update_hashes=True):
if self.file_type == filetypes.tox_ini:
updater_class = updater.ToxINIUpdater
elif self.file_type == filetypes.conda_yml:
updater_class = updater.CondaYMLUpdater
elif self.file_type == filetypes.requirements_txt:
updater_class = updater.RequirementsTXTUpdater
elif self.file_type == filetypes.pipfile:
updater_class = updater.PipfileUpdater
elif self.file_type == filetypes.pipfile_lock:
updater_class = updater.PipfileLockUpdater
elif self.file_type == filetypes.setup_cfg:
updater_class = updater.SetupCFGUpdater
else:
raise NotImplementedError
dep = Dependency(
name=self.name,
specs=self.specs,
line=self.line,
line_numbers=[self.lineno, ] if self.lineno != 0 else None,
dependency_type=self.file_type,
hashes=self.hashes,
extras=self.extras
)
hashes = []
if self.hashes and update_hashes:
hashes = self.get_hashes(version)
return updater_class.update(
content=content,
dependency=dep,
version=version,
hashes=hashes,
spec="=="
)
@classmethod
def parse(cls, s, lineno, file_type=filetypes.requirements_txt):
# setuptools requires a space before the comment. If this isn't the case, add it.
if "\t#" in s:
parsed, = parse_requirements(s.replace("\t#", "\t #"))
else:
parsed, = parse_requirements(s)
return cls(
name=parsed.name,
specs=parsed.specifier,
line=s,
lineno=lineno,
extras=parsed.extras,
file_type=file_type
)
pipenv/patched/safety/alerts/utils.py
+132
View File
@@ -0,0 +1,132 @@
import hashlib
import os
import sys
from functools import wraps
from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
from pathlib import Path
import pipenv.vendor.click as click
# Jinja2 will only be installed if the optional deps are installed.
# It's fine if our functions fail, but don't let this top level
# import error out.
try:
import jinja2
except ImportError:
jinja2 = None
import pipenv.patched.pip._vendor.requests as requests
def highest_base_score(vulns):
highest_base_score = 0
for vuln in vulns:
if vuln['severity'] is not None:
highest_base_score = max(highest_base_score, (vuln['severity'].get('cvssv3', {}) or {}).get('base_score', 10))
return highest_base_score
def generate_branch_name(pkg, remediation):
return pkg + "/" + remediation['recommended_version']
def generate_issue_title(pkg, remediation):
return f"Security Vulnerability in {pkg}"
def generate_title(pkg, remediation, vulns):
suffix = "y" if len(vulns) == 1 else "ies"
return f"Update {pkg} from {remediation['current_version']} to {remediation['recommended_version']} to fix {len(vulns)} vulnerabilit{suffix}"
def generate_body(pkg, remediation, vulns, *, api_key):
changelog = fetch_changelog(pkg, remediation['current_version'], remediation['recommended_version'], api_key=api_key)
p = Path(__file__).parent / 'templates'
env = jinja2.Environment(loader=jinja2.FileSystemLoader(Path(p)))
template = env.get_template('pr.jinja2')
overall_impact = cvss3_score_to_label(highest_base_score(vulns))
result = template.render({"pkg": pkg, "remediation": remediation, "vulns": vulns, "changelog": changelog, "overall_impact": overall_impact, "summary_changelog": False })
# GitHub has a PR body length limit of 65536. If we're going over that, skip the changelog and just use a link.
if len(result) > 65500:
return template.render({"pkg": pkg, "remediation": remediation, "vulns": vulns, "changelog": changelog, "overall_impact": overall_impact, "summary_changelog": True })
return result
def generate_issue_body(pkg, remediation, vulns, *, api_key):
changelog = fetch_changelog(pkg, remediation['current_version'], remediation['recommended_version'], api_key=api_key)
p = Path(__file__).parent / 'templates'
env = jinja2.Environment(loader=jinja2.FileSystemLoader(Path(p)))
template = env.get_template('issue.jinja2')
overall_impact = cvss3_score_to_label(highest_base_score(vulns))
result = template.render({"pkg": pkg, "remediation": remediation, "vulns": vulns, "changelog": changelog, "overall_impact": overall_impact, "summary_changelog": False })
# GitHub has a PR body length limit of 65536. If we're going over that, skip the changelog and just use a link.
if len(result) > 65500:
return template.render({"pkg": pkg, "remediation": remediation, "vulns": vulns, "changelog": changelog, "overall_impact": overall_impact, "summary_changelog": True })
def generate_commit_message(pkg, remediation):
return f"Update {pkg} from {remediation['current_version']} to {remediation['recommended_version']}"
def git_sha1(raw_contents):
return hashlib.sha1(b"blob " + str(len(raw_contents)).encode('ascii') + b"\0" + raw_contents).hexdigest()
def fetch_changelog(package, from_version, to_version, *, api_key):
from_version = parse_version(from_version)
to_version = parse_version(to_version)
changelog = {}
r = requests.get(
"https://pyup.io/api/v1/changelogs/{}/".format(package),
headers={"X-Api-Key": api_key}
)
if r.status_code == 200:
data = r.json()
if data:
# sort the changelog by release
sorted_log = sorted(data.items(), key=lambda v: parse_version(v[0]), reverse=True)
# go over each release and add it to the log if it's within the "upgrade
# range" e.g. update from 1.2 to 1.3 includes a changelog for 1.2.1 but
# not for 0.4.
for version, log in sorted_log:
parsed_version = parse_version(version)
if parsed_version > from_version and parsed_version <= to_version:
changelog[version] = log
return changelog
def cvss3_score_to_label(score):
if score >= 0.1 and score <= 3.9:
return 'low'
elif score >= 4.0 and score <= 6.9:
return 'medium'
elif score >= 7.0 and score <= 8.9:
return 'high'
elif score >= 9.0:
return 'critical'
return None
def require_files_report(func):
@wraps(func)
def inner(obj, *args, **kwargs):
if obj.report['report_meta']['scan_target'] != "files":
click.secho("This report was generated against an environment, but this alerter requires a file.", fg='red')
sys.exit(1)
files = obj.report['report_meta']['scanned']
obj.requirements_files = {}
for f in files:
if not os.path.exists(f):
cwd = os.getcwd()
click.secho("A requirements file scanned in the report, {}, does not exist (looking in {}).".format(f, cwd), fg='red')
sys.exit(1)
obj.requirements_files[f] = open(f, "rb").read()
return func(obj, *args, **kwargs)
return inner
pipenv/patched/safety/cli.py
+327 -140
View File
@@ -1,23 +1,56 @@
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import sys
import pipenv.vendor.click as click
from pipenv.patched.safety import __version__
from pipenv.patched.safety import safety
from pipenv.patched.safety.formatter import report, license_report
import itertools
from pipenv.patched.safety.util import read_requirements, read_vulnerabilities, get_proxy_dict, get_packages_licenses
from pipenv.patched.safety.errors import DatabaseFetchError, DatabaseFileNotFoundError, InvalidKeyError, TooManyRequestsError
try:
from json.decoder import JSONDecodeError
except ImportError:
JSONDecodeError = ValueError
import json
import logging
import os
import sys
import tempfile
import pipenv.vendor.click as click
from pipenv.patched.safety import safety
from pipenv.patched.safety.alerts import alert
from pipenv.patched.safety.constants import EXIT_CODE_VULNERABILITIES_FOUND, EXIT_CODE_OK, EXIT_CODE_FAILURE
from pipenv.patched.safety.errors import SafetyException, SafetyError
from pipenv.patched.safety.formatter import SafetyFormatter
from pipenv.patched.safety.output_utils import should_add_nl
from pipenv.patched.safety.safety import get_packages, read_vulnerabilities, fetch_policy, post_results
from pipenv.patched.safety.util import get_proxy_dict, get_packages_licenses, output_exception, \
MutuallyExclusiveOption, DependentOption, transform_ignore, SafetyPolicyFile, active_color_if_needed, \
get_processed_options, get_safety_version, json_alias, bare_alias, SafetyContext, is_a_remote_mirror, \
filter_announcements
LOG = logging.getLogger(__name__)
@click.group()
@click.version_option(version=__version__)
def cli():
pass
@click.option('--debug/--no-debug', default=False)
@click.option('--telemetry/--disable-telemetry', default=True, hidden=True)
@click.option('--disable-optional-telemetry-data', default=False, cls=MutuallyExclusiveOption,
mutually_exclusive=["telemetry", "disable-telemetry"], is_flag=True, show_default=True)
@click.version_option(version=get_safety_version())
@click.pass_context
def cli(ctx, debug, telemetry, disable_optional_telemetry_data):
"""
Safety checks Python dependencies for known security vulnerabilities and suggests the proper
remediations for vulnerabilities detected. Safety can be run on developer machines, in CI/CD pipelines and
on production systems.
"""
SafetyContext().safety_source = 'cli'
ctx.telemetry = telemetry and not disable_optional_telemetry_data
level = logging.CRITICAL
if debug:
level = logging.DEBUG
logging.basicConfig(format='%(asctime)s %(name)s => %(message)s', level=level)
LOG.info(f'Telemetry enabled: {ctx.telemetry}')
@ctx.call_on_close
def clean_up_on_close():
LOG.debug('Calling clean up on close function.')
safety.close_session()
@cli.command()
@@ -25,101 +58,197 @@ def cli():
help="API Key for pyup.io's vulnerability database. Can be set as SAFETY_API_KEY "
"environment variable. Default: empty")
@click.option("--db", default="",
help="Path to a local vulnerability database. Default: empty")
@click.option("--json/--no-json", default=False,
help="Output vulnerabilities in JSON format. Default: --no-json")
@click.option("--full-report/--short-report", default=False,
help='Full reports include a security advisory (if available). Default: '
'--short-report')
@click.option("--bare/--not-bare", default=False,
help='Output vulnerable packages only. '
'Useful in combination with other tools. '
'Default: --not-bare')
@click.option("--cache/--no-cache", default=False,
help="Cache requests to the vulnerability database locally. Default: --no-cache")
@click.option("--stdin/--no-stdin", default=False,
help="Read input from stdin. Default: --no-stdin")
@click.option("files", "--file", "-r", multiple=True, type=click.File(),
help="Path to a local or remote vulnerability database. Default: empty")
@click.option("--full-report/--short-report", default=False, cls=MutuallyExclusiveOption,
mutually_exclusive=["output", "json", "bare"],
with_values={"output": ['json', 'bare'], "json": [True, False], "bare": [True, False]},
help='Full reports include a security advisory (if available). Default: --short-report')
@click.option("--cache", is_flag=False, flag_value=60, default=0,
help="Cache requests to the vulnerability database locally. Default: 0 seconds",
hidden=True)
@click.option("--stdin", default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["files"],
help="Read input from stdin.", is_flag=True, show_default=True)
@click.option("files", "--file", "-r", multiple=True, type=click.File(), cls=MutuallyExclusiveOption,
mutually_exclusive=["stdin"],
help="Read input from one (or multiple) requirement files. Default: empty")
@click.option("ignore", "--ignore", "-i", multiple=True, type=str, default=[],
@click.option("--ignore", "-i", multiple=True, type=str, default=[], callback=transform_ignore,
help="Ignore one (or multiple) vulnerabilities by ID. Default: empty")
@click.option("--output", "-o", default="",
help="Path to where output file will be placed. Default: empty")
@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None,
help="Proxy host IP or DNS --proxy-host")
@click.option("proxyport", "--proxy-port", "-pp", multiple=False, type=int, default=80,
help="Proxy port number --proxy-port")
@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http',
@click.option('--json', default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["output", "bare"],
with_values={"output": ['screen', 'text', 'bare', 'json'], "bare": [True, False]}, callback=json_alias,
hidden=True, is_flag=True, show_default=True)
@click.option('--bare', default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["output", "json"],
with_values={"output": ['screen', 'text', 'bare', 'json'], "json": [True, False]}, callback=bare_alias,
hidden=True, is_flag=True, show_default=True)
@click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare'], case_sensitive=False),
default='screen', callback=active_color_if_needed, envvar='SAFETY_OUTPUT')
@click.option("--proxy-protocol", "-pr", type=click.Choice(['http', 'https']), default='https', cls=DependentOption, required_options=['proxy_host'],
help="Proxy protocol (https or http) --proxy-protocol")
def check(key, db, json, full_report, bare, stdin, files, cache, ignore, output, proxyprotocol, proxyhost, proxyport):
if files and stdin:
click.secho("Can't read from --stdin and --file at the same time, exiting", fg="red", file=sys.stderr)
sys.exit(-1)
@click.option("--proxy-host", "-ph", multiple=False, type=str, default=None,
help="Proxy host IP or DNS --proxy-host")
@click.option("--proxy-port", "-pp", multiple=False, type=int, default=80, cls=DependentOption, required_options=['proxy_host'],
help="Proxy port number --proxy-port")
@click.option("--exit-code/--continue-on-error", default=True,
help="Output standard exit codes. Default: --exit-code")
@click.option("--policy-file", type=SafetyPolicyFile(), default='.safety-policy.yml',
help="Define the policy file to be used")
@click.option("--audit-and-monitor/--disable-audit-and-monitor", default=True,
help="Send results back to pyup.io for viewing on your dashboard. Requires an API key.")
@click.option("--project", default=None,
help="Project to associate this scan with on pyup.io. Defaults to a canonicalized github style name if available, otherwise unknown")
@click.option("--save-json", default="", help="Path to where output file will be placed, if the path is a directory, "
"Safety will use safety-report.json as filename. Default: empty")
@click.pass_context
def check(ctx, key, db, full_report, stdin, files, cache, ignore, output, json, bare, proxy_protocol, proxy_host, proxy_port,
exit_code, policy_file, save_json, audit_and_monitor, project):
"""
Find vulnerabilities in Python dependencies at the target provided.
"""
LOG.info('Running check command')
if files:
packages = list(itertools.chain.from_iterable(read_requirements(f, resolve=True) for f in files))
elif stdin:
packages = list(read_requirements(sys.stdin))
else:
import pipenv.patched.pip._vendor.pkg_resources as pkg_resources
packages = [
d for d in pkg_resources.working_set
if d.key not in {"python", "wsgiref", "argparse"}
]
proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport)
try:
vulns = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_ids=ignore, proxy=proxy_dictionary)
output_report = report(vulns=vulns,
full=full_report,
json_report=json,
bare_report=bare,
checked_packages=len(packages),
db=db,
key=key)
packages = get_packages(files, stdin)
proxy_dictionary = get_proxy_dict(proxy_protocol, proxy_host, proxy_port)
if output:
with open(output, 'w+') as output_file:
output_file.write(output_report)
if key:
server_policies = fetch_policy(key=key, proxy=proxy_dictionary)
server_audit_and_monitor = server_policies["audit_and_monitor"]
server_safety_policy = server_policies["safety_policy"]
else:
click.secho(output_report, nl=False if bare and not vulns else True)
sys.exit(-1 if vulns else 0)
except InvalidKeyError:
click.secho("Your API Key '{key}' is invalid. See {link}".format(
key=key, link='https://goo.gl/O7Y1rS'),
fg="red",
file=sys.stderr)
sys.exit(-1)
except DatabaseFileNotFoundError:
click.secho("Unable to load vulnerability database from {db}".format(db=db), fg="red", file=sys.stderr)
sys.exit(-1)
except DatabaseFetchError:
click.secho("Unable to load vulnerability database", fg="red", file=sys.stderr)
sys.exit(-1)
server_audit_and_monitor = False
server_safety_policy = ""
if server_safety_policy and policy_file:
click.secho(
"Warning: both a local policy file '{policy_filename}' and a server sent policy are present. "
"Continuing with the local policy file.".format(policy_filename=policy_file['filename']),
fg="yellow",
file=sys.stderr
)
elif server_safety_policy:
with tempfile.NamedTemporaryFile(prefix='server-safety-policy-') as tmp:
tmp.write(server_safety_policy.encode('utf-8'))
tmp.seek(0)
policy_file = SafetyPolicyFile().convert(tmp.name, param=None, ctx=None)
LOG.info('Using server side policy file')
ignore_severity_rules = None
ignore, ignore_severity_rules, exit_code = get_processed_options(policy_file, ignore,
ignore_severity_rules, exit_code)
is_env_scan = not stdin and not files
params = {'stdin': stdin, 'files': files, 'policy_file': policy_file, 'continue_on_error': not exit_code,
'ignore_severity_rules': ignore_severity_rules, 'project': project, 'audit_and_monitor': server_audit_and_monitor and audit_and_monitor}
LOG.info('Calling the check function')
vulns, db_full = safety.check(packages=packages, key=key, db_mirror=db, cached=cache, ignore_vulns=ignore,
ignore_severity_rules=ignore_severity_rules, proxy=proxy_dictionary,
include_ignored=True, is_env_scan=is_env_scan, telemetry=ctx.parent.telemetry,
params=params)
LOG.debug('Vulnerabilities returned: %s', vulns)
LOG.debug('full database returned is None: %s', db_full is None)
LOG.info('Safety is going to calculate remediations')
remediations = safety.calculate_remediations(vulns, db_full)
announcements = []
if not db or is_a_remote_mirror(db):
LOG.info('Not local DB used, Getting announcements')
announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry)
json_report = None
if save_json or (server_audit_and_monitor and audit_and_monitor):
default_name = 'safety-report.json'
json_report = SafetyFormatter(output='json').render_vulnerabilities(announcements, vulns, remediations,
full_report, packages)
if server_audit_and_monitor and audit_and_monitor:
policy_contents = ''
if policy_file:
policy_contents = policy_file.get('raw', '')
r = post_results(key=key, proxy=proxy_dictionary, safety_json=json_report, policy_file=policy_contents)
SafetyContext().params['audit_and_monitor_url'] = r.get('url')
if save_json:
if os.path.isdir(save_json):
save_json = os.path.join(save_json, default_name)
with open(save_json, 'w+') as output_json_file:
output_json_file.write(json_report)
LOG.info('Safety is going to render the vulnerabilities report using %s output', output)
if json_report and output == 'json':
output_report = json_report
else:
output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
full_report, packages)
# Announcements are send to stderr if not terminal, it doesn't depend on "exit_code" value
stderr_announcements = filter_announcements(announcements=announcements, by_type='error')
if stderr_announcements and (not sys.stdout.isatty() and os.environ.get("SAFETY_OS_DESCRIPTION", None) != 'run'):
LOG.info('sys.stdout is not a tty, error announcements are going to be send to stderr')
click.secho(SafetyFormatter(output='text').render_announcements(stderr_announcements), fg="red",
file=sys.stderr)
found_vulns = list(filter(lambda v: not v.ignored, vulns))
LOG.info('Vulnerabilities found (Not ignored): %s', len(found_vulns))
LOG.info('All vulnerabilities found (ignored and Not ignored): %s', len(vulns))
click.secho(output_report, nl=should_add_nl(output, found_vulns), file=sys.stdout)
if exit_code and found_vulns:
LOG.info('Exiting with default code for vulnerabilities found')
sys.exit(EXIT_CODE_VULNERABILITIES_FOUND)
sys.exit(EXIT_CODE_OK)
except SafetyError as e:
LOG.exception('Expected SafetyError happened: %s', e)
output_exception(e, exit_code_output=exit_code)
except Exception as e:
LOG.exception('Unexpected Exception happened: %s', e)
exception = e if isinstance(e, SafetyException) else SafetyException(info=e)
output_exception(exception, exit_code_output=exit_code)
@cli.command()
@click.option("--full-report/--short-report", default=False,
@click.option("--full-report/--short-report", default=False, cls=MutuallyExclusiveOption, mutually_exclusive=["output"], with_values={"output": ['json', 'bare']},
help='Full reports include a security advisory (if available). Default: '
'--short-report')
@click.option("--bare/--not-bare", default=False,
help='Output vulnerable packages only. Useful in combination with other tools. '
'Default: --not-bare')
@click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare'], case_sensitive=False),
default='screen', callback=active_color_if_needed)
@click.option("file", "--file", "-f", type=click.File(), required=True,
help="Read input from an insecure report file. Default: empty")
def review(full_report, bare, file):
if full_report and bare:
click.secho("Can't choose both --bare and --full-report/--short-report", fg="red")
sys.exit(-1)
@click.pass_context
def review(ctx, full_report, output, file):
"""
Show an output from a previous exported JSON report.
"""
LOG.info('Running check command')
report = {}
try:
input_vulns = read_vulnerabilities(file)
except JSONDecodeError:
click.secho("Not a valid JSON file", fg="red")
sys.exit(-1)
report = read_vulnerabilities(file)
except SafetyError as e:
LOG.exception('Expected SafetyError happened: %s', e)
output_exception(e, exit_code_output=True)
except Exception as e:
LOG.exception('Unexpected Exception happened: %s', e)
exception = e if isinstance(e, SafetyException) else SafetyException(info=e)
output_exception(exception, exit_code_output=True)
vulns = safety.review(input_vulns)
output_report = report(vulns=vulns, full=full_report, bare_report=bare)
click.secho(output_report, nl=False if bare and not vulns else True)
params = {'file': file}
vulns, remediations, packages = safety.review(report, params=params)
announcements = safety.get_announcements(key=None, proxy=None, telemetry=ctx.parent.telemetry)
output_report = SafetyFormatter(output=output).render_vulnerabilities(announcements, vulns, remediations,
full_report, packages)
found_vulns = list(filter(lambda v: not v.ignored, vulns))
click.secho(output_report, nl=should_add_nl(output, found_vulns), file=sys.stdout)
sys.exit(EXIT_CODE_OK)
@cli.command()
@@ -128,15 +257,11 @@ def review(full_report, bare, file):
"environment variable. Default: empty")
@click.option("--db", default="",
help="Path to a local license database. Default: empty")
@click.option("--json/--no-json", default=False,
help="Output packages licenses in JSON format. Default: --no-json")
@click.option("--bare/--not-bare", default=False,
help='Output packages licenses names only. '
'Useful in combination with other tools. '
'Default: --not-bare')
@click.option("--cache/--no-cache", default=True,
@click.option('--output', "-o", type=click.Choice(['screen', 'text', 'json', 'bare'], case_sensitive=False),
default='screen')
@click.option("--cache", default=0,
help='Whether license database file should be cached.'
'Default: --cache')
'Default: 0 seconds')
@click.option("files", "--file", "-r", multiple=True, type=click.File(),
help="Read input from one (or multiple) requirement files. Default: empty")
@click.option("proxyhost", "--proxy-host", "-ph", multiple=False, type=str, default=None,
@@ -145,50 +270,112 @@ def review(full_report, bare, file):
help="Proxy port number --proxy-port")
@click.option("proxyprotocol", "--proxy-protocol", "-pr", multiple=False, type=str, default='http',
help="Proxy protocol (https or http) --proxy-protocol")
def license(key, db, json, bare, cache, files, proxyprotocol, proxyhost, proxyport):
@click.pass_context
def license(ctx, key, db, output, cache, files, proxyprotocol, proxyhost, proxyport):
"""
Find the open source licenses used by your Python dependencies.
"""
LOG.info('Running license command')
packages = get_packages(files, False)
if files:
packages = list(itertools.chain.from_iterable(read_requirements(f, resolve=True) for f in files))
else:
import pipenv.patched.pip._vendor.pkg_resources as pkg_resources
packages = [
d for d in pkg_resources.working_set
if d.key not in {"python", "wsgiref", "argparse"}
]
proxy_dictionary = get_proxy_dict(proxyprotocol, proxyhost, proxyport)
licenses_db = {}
try:
licenses_db = safety.get_licenses(key, db, cache, proxy_dictionary)
except InvalidKeyError as invalid_key_error:
if str(invalid_key_error):
message = str(invalid_key_error)
else:
message = "Your API Key '{key}' is invalid. See {link}".format(
key=key, link='https://goo.gl/O7Y1rS'
)
click.secho(message, fg="red", file=sys.stderr)
sys.exit(-1)
except DatabaseFileNotFoundError:
click.secho("Unable to load licenses database from {db}".format(db=db), fg="red", file=sys.stderr)
sys.exit(-1)
except TooManyRequestsError:
click.secho("Unable to load licenses database (Too many requests, please wait before another request)",
fg="red",
file=sys.stderr
)
sys.exit(-1)
except DatabaseFetchError:
click.secho("Unable to load licenses database", fg="red", file=sys.stderr)
sys.exit(-1)
filtered_packages_licenses = get_packages_licenses(packages, licenses_db)
output_report = license_report(
packages=packages,
licenses=filtered_packages_licenses,
json_report=json,
bare_report=bare
)
licenses_db = safety.get_licenses(key=key, db_mirror=db, cached=cache, proxy=proxy_dictionary,
telemetry=ctx.parent.telemetry)
except SafetyError as e:
LOG.exception('Expected SafetyError happened: %s', e)
output_exception(e, exit_code_output=False)
except Exception as e:
LOG.exception('Unexpected Exception happened: %s', e)
exception = e if isinstance(e, SafetyException) else SafetyException(info=e)
output_exception(exception, exit_code_output=False)
filtered_packages_licenses = get_packages_licenses(packages=packages, licenses_db=licenses_db)
announcements = []
if not db:
announcements = safety.get_announcements(key=key, proxy=proxy_dictionary, telemetry=ctx.parent.telemetry)
output_report = SafetyFormatter(output=output).render_licenses(announcements, filtered_packages_licenses)
click.secho(output_report, nl=True)
@cli.command()
@click.option("--path", default=".", help="Path where the generated file will be saved. Default: current directory")
@click.argument('name')
@click.pass_context
def generate(ctx, name, path):
"""Create a boilerplate supported file type.
NAME is the name of the file type to generate. Valid values are: policy_file
"""
if name != 'policy_file':
click.secho(f'This Safety version only supports "policy_file" generation. "{name}" is not supported.', fg='red',
file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
LOG.info('Running generate %s', name)
if not os.path.exists(path):
click.secho(f'The path "{path}" does not exist.', fg='red',
file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
policy = os.path.join(path, '.safety-policy.yml')
ROOT = os.path.dirname(os.path.abspath(__file__))
try:
with open(policy, "w") as f:
f.write(open(os.path.join(ROOT, 'safety-policy-template.yml')).read())
LOG.debug('Safety created the policy file.')
msg = f'A default Safety policy file has been generated! Review the file contents in the path {path} in the ' \
'file: .safety-policy.yml'
click.secho(msg, fg='green')
except Exception as exc:
if isinstance(exc, OSError):
LOG.debug('Unable to generate %s because: %s', name, exc.errno)
click.secho(f'Unable to generate {name}, because: {str(exc)} error.', fg='red',
file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
@cli.command()
@click.option("--path", default=".safety-policy.yml", help="Path where the generated file will be saved. Default: current directory")
@click.argument('name')
@click.pass_context
def validate(ctx, name, path):
"""Verify the validity of a supported file type.
NAME is the name of the file type to validate. Valid values are: policy_file
"""
if name != 'policy_file':
click.secho(f'This Safety version only supports "policy_file" validation. "{name}" is not supported.', fg='red',
file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
LOG.info('Running validate %s', name)
if not os.path.exists(path):
click.secho(f'The path "{path}" does not exist.', fg='red', file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
try:
values = SafetyPolicyFile().convert(path, None, None)
except Exception as e:
click.secho(str(e).lstrip(), fg='red', file=sys.stderr)
sys.exit(EXIT_CODE_FAILURE)
del values['raw']
click.secho(f'The Safety policy file was successfully parsed with the following values:', fg='green')
click.secho(json.dumps(values, indent=4, default=str))
cli.add_command(alert)
if __name__ == "__main__":
cli()
pipenv/patched/safety/constants.py
+22 -6
View File
@@ -2,21 +2,37 @@
import os
OPEN_MIRRORS = [
"https://raw.githubusercontent.com/pyupio/safety-db/master/data/",
"https://pyup.io/aws/safety/free/",
]
API_VERSION = 'v1/'
SAFETY_ENDPOINT = 'safety/'
API_BASE_URL = 'https://pyup.io/api/' + API_VERSION + SAFETY_ENDPOINT
API_MIRRORS = [
"https://pyup.io/api/v1/safety/"
API_BASE_URL
]
REQUEST_TIMEOUT = 5
CACHE_VALID_SECONDS = 60 * 60 * 2 # 2 hours
CACHE_LICENSES_VALID_SECONDS = 60 * 60 * 24 * 7 # one week
CACHE_FILE = os.path.join(
os.path.expanduser("~"),
".safety",
"cache.json"
)
# Colors
YELLOW = 'yellow'
RED = 'red'
GREEN = 'green'
# Exit codes
EXIT_CODE_OK = 0
EXIT_CODE_FAILURE = 1
EXIT_CODE_VULNERABILITIES_FOUND = 64
EXIT_CODE_INVALID_API_KEY = 65
EXIT_CODE_TOO_MANY_REQUESTS = 66
EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB = 67
EXIT_CODE_UNABLE_TO_FETCH_VULNERABILITY_DB = 68
EXIT_CODE_MALFORMED_DB = 69
pipenv/patched/safety/errors.py
+97 -5
View File
@@ -1,14 +1,106 @@
class DatabaseFetchError(Exception):
pass
from pipenv.patched.safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_INVALID_API_KEY, EXIT_CODE_TOO_MANY_REQUESTS, \
EXIT_CODE_UNABLE_TO_FETCH_VULNERABILITY_DB, EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB, EXIT_CODE_MALFORMED_DB
class SafetyException(Exception):
def __init__(self, message="Unhandled exception happened: {info}", info=""):
self.message = message.format(info=info)
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_FAILURE
class SafetyError(Exception):
def __init__(self, message="Unhandled Safety generic error"):
self.message = message
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_FAILURE
class MalformedDatabase(SafetyError):
def __init__(self, reason=None, fetched_from="server",
message="Sorry, something went wrong.\n" +
"Safety CLI can not read the data fetched from {fetched_from} because is malformed.\n"):
info = "Reason, {reason}".format(reason=reason)
self.message = message.format(fetched_from=fetched_from) + (info if reason else "")
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_MALFORMED_DB
class DatabaseFetchError(SafetyError):
def __init__(self, message="Unable to load vulnerability database"):
self.message = message
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_UNABLE_TO_FETCH_VULNERABILITY_DB
class DatabaseFileNotFoundError(DatabaseFetchError):
pass
def __init__(self, db=None, message="Unable to find vulnerability database in {db}"):
self.db = db
self.message = message.format(db=db)
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_UNABLE_TO_LOAD_LOCAL_VULNERABILITY_DB
class InvalidKeyError(DatabaseFetchError):
pass
def __init__(self, key=None, message="Your API Key '{key}' is invalid. See {link}.", reason=None):
self.key = key
self.link = 'https://bit.ly/3OY2wEI'
self.message = message.format(key=key, link=self.link) if key else message
info = f" Reason: {reason}"
self.message = self.message + (info if reason else "")
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_INVALID_API_KEY
class TooManyRequestsError(DatabaseFetchError):
pass
def __init__(self, reason=None,
message="Too many requests."):
info = f" Reason: {reason}"
self.message = message + (info if reason else "")
super().__init__(self.message)
def get_exit_code(self):
return EXIT_CODE_TOO_MANY_REQUESTS
class NetworkConnectionError(DatabaseFetchError):
def __init__(self, message="Check your network connection, unable to reach the server."):
self.message = message
super().__init__(self.message)
class RequestTimeoutError(DatabaseFetchError):
def __init__(self, message="Check your network connection, the request timed out."):
self.message = message
super().__init__(self.message)
class ServerError(DatabaseFetchError):
def __init__(self, reason=None,
message="Sorry, something went wrong.\n" + "Safety CLI can not connect to the server.\n" +
"Our engineers are working quickly to resolve the issue."):
info = f" Reason: {reason}"
self.message = message + (info if reason else "")
super().__init__(self.message)
pipenv/patched/safety/formatter.py
+44 -330
View File
@@ -1,342 +1,56 @@
# -*- coding: utf-8 -*-
import platform
import sys
import json
import os
import textwrap
import logging
from abc import ABCMeta, abstractmethod
from .util import get_packages_licenses
NOT_IMPLEMENTED = "You should implement this."
# python 2.7 compat
try:
FileNotFoundError
except NameError:
FileNotFoundError = IOError
try:
system = platform.system()
python_version = ".".join([str(i) for i in sys.version_info[0:2]])
# get_terminal_size exists on Python 3.4 but isn't working on windows
if system == "Windows" and python_version in ["3.4"]:
raise ImportError
from shutil import get_terminal_size
except ImportError:
# fallback for python < 3
import subprocess
from collections import namedtuple
def get_terminal_size():
size = namedtuple("_", ["rows", "columns"])
try:
rows, columns = subprocess.check_output(
['stty', 'size'],
stderr=subprocess.STDOUT
).split()
return size(rows=int(rows), columns=int(columns))
# this won't work
# - on windows (FileNotFoundError/OSError)
# - python 2.6 (AttributeError)
# - if the output is somehow mangled (ValueError)
except (ValueError, FileNotFoundError, OSError,
AttributeError, subprocess.CalledProcessError):
return size(rows=0, columns=0)
LOG = logging.getLogger(__name__)
def get_advisory(vuln):
return vuln.advisory if vuln.advisory else "No advisory found for this vulnerability."
class FormatterAPI:
"""
Strategy Abstract class, with all the render methods that the concrete implementations should support
"""
__metaclass__ = ABCMeta
@abstractmethod
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
raise NotImplementedError(NOT_IMPLEMENTED) # pragma: no cover
@abstractmethod
def render_licenses(self, announcements, licenses):
raise NotImplementedError(NOT_IMPLEMENTED) # pragma: no cover
@abstractmethod
def render_announcements(self, announcements):
raise NotImplementedError(NOT_IMPLEMENTED) # pragma: no cover
class SheetReport(object):
REPORT_BANNER = r"""
+==============================================================================+
| |
| /$$$$$$ /$$ |
| /$$__ $$ | $$ |
| /$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$ |
| /$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$ |
| | $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$ |
| \____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$ |
| /$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$ |
| |_______/ \_______/|__/ \_______/ \___/ \____ $$ |
| /$$ | $$ |
| | $$$$$$/ |
| by pyup.io \______/ |
| |
+==============================================================================+
""".strip()
class SafetyFormatter(FormatterAPI):
TABLE_HEADING = r"""
+============================+===========+==========================+==========+
| package | installed | affected | ID |
+============================+===========+==========================+==========+
""".strip()
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
LOG.info('Safety is going to render_vulnerabilities with format: %s', self.format)
return self.format.render_vulnerabilities(announcements, vulnerabilities, remediations, full, packages)
TABLE_HEADING_LICENSES = r"""
+=============================================+===========+====================+
| package | version | license |
+=============================================+===========+====================+
""".strip()
def render_licenses(self, announcements, licenses):
LOG.info('Safety is going to render_licenses with format: %s', self.format)
return self.format.render_licenses(announcements, licenses)
REPORT_HEADING = r"""
| REPORT |
""".strip()
def render_announcements(self, announcements):
LOG.info('Safety is going to render_announcements with format: %s', self.format)
return self.format.render_announcements(announcements)
REPORT_SECTION = r"""
+==============================================================================+
""".strip()
def __init__(self, output):
from pipenv.patched.safety.formatters.screen import ScreenReport
from pipenv.patched.safety.formatters.text import TextReport
from pipenv.patched.safety.formatters.json import JsonReport
from pipenv.patched.safety.formatters.bare import BareReport
REPORT_FOOTER = r"""
+==============================================================================+
""".strip()
self.format = ScreenReport()
@staticmethod
def render(vulns, full, checked_packages, used_db):
db_format_str = '{: <' + str(51 - len(str(checked_packages))) + '}'
status = "| checked {packages} packages, using {db} |".format(
packages=checked_packages,
db=db_format_str.format(used_db),
section=SheetReport.REPORT_SECTION
)
if vulns:
table = []
for n, vuln in enumerate(vulns):
table.append("| {:26} | {:9} | {:24} | {:8} |".format(
vuln.name[:26],
vuln.version[:9],
vuln.spec[:24],
vuln.vuln_id
))
if full:
table.append(SheetReport.REPORT_SECTION)
if vuln.cvssv2 is not None:
base_score = vuln.cvssv2.get("base_score", "None")
impact_score = vuln.cvssv2.get("impact_score", "None")
table.append("| {:76} |".format(
"CVSS v2 | BASE SCORE: {} | IMPACT SCORE: {}".format(
base_score,
impact_score,
)
))
table.append(SheetReport.REPORT_SECTION)
if vuln.cvssv3 is not None:
base_score = vuln.cvssv3.get("base_score", "None")
impact_score = vuln.cvssv3.get("impact_score", "None")
base_severity = vuln.cvssv3.get("base_severity", "None")
table.append("| {:76} |".format(
"CVSS v3 | BASE SCORE: {} | IMPACT SCORE: {} | BASE SEVERITY: {}".format(
base_score,
impact_score,
base_severity,
)
))
table.append(SheetReport.REPORT_SECTION)
advisory_lines = get_advisory(vuln).replace(
'\r', ''
).splitlines()
for line in advisory_lines:
if line == '':
table.append("| {:76} |".format(" "))
for wrapped_line in textwrap.wrap(line, width=76):
try:
table.append("| {:76} |".format(
wrapped_line.encode('utf-8')
))
except TypeError:
table.append("| {:76} |".format(
wrapped_line
))
# append the REPORT_SECTION only if this isn't the last entry
if n + 1 < len(vulns):
table.append(SheetReport.REPORT_SECTION)
return "\n".join(
[SheetReport.REPORT_BANNER, SheetReport.REPORT_HEADING, status, SheetReport.TABLE_HEADING,
"\n".join(table), SheetReport.REPORT_FOOTER]
)
else:
content = "| {:76} |".format("No known security vulnerabilities found.")
return "\n".join(
[SheetReport.REPORT_BANNER, SheetReport.REPORT_HEADING, status, SheetReport.REPORT_SECTION,
content, SheetReport.REPORT_FOOTER]
)
@staticmethod
def render_licenses(packages, packages_licenses):
heading = SheetReport.REPORT_HEADING.replace(" ", "", 12).replace(
"REPORT", " Packages licenses"
)
if not packages_licenses:
content = "| {:76} |".format("No packages licenses found.")
return "\n".join(
[SheetReport.REPORT_BANNER, heading, SheetReport.REPORT_SECTION,
content, SheetReport.REPORT_FOOTER]
)
table = []
iteration = 1
for pkg_license in packages_licenses:
max_char = last_char = 43 # defines a limit for package name.
current_line = 1
package = pkg_license['package']
license = pkg_license['license']
version = pkg_license['version']
license_line = int(int(len(package) / max_char) / 2) + 1 # Calc to get which line to add the license info.
table.append("| {:43} | {:9} | {:18} |".format(
package[:max_char],
version[:9] if current_line == license_line else "",
license[:18] if current_line == license_line else "",
))
long_name = True if len(package[max_char:]) > 0 else False
while long_name: # If the package has a long name, break it into multiple lines.
current_line += 1
table.append("| {:43} | {:9} | {:18} |".format(
package[last_char:last_char+max_char],
version[:9] if current_line == license_line else "",
license[:18] if current_line == license_line else "",
))
last_char = last_char+max_char
long_name = True if len(package[last_char:]) > 0 else False
if iteration != len(packages_licenses): # Do not add dashes "----" for last package.
table.append("|" + ("-" * 78) + "|")
iteration += 1
return "\n".join(
[SheetReport.REPORT_BANNER, heading, SheetReport.TABLE_HEADING_LICENSES,
"\n".join(table), SheetReport.REPORT_FOOTER]
)
class BasicReport(object):
"""Basic report, intented to be used for terminals with < 80 columns"""
@staticmethod
def render(vulns, full, checked_packages, used_db):
table = [
"safety report",
"checked {packages} packages, using {db}".format(
packages=checked_packages,
db=used_db
),
"---"
]
if vulns:
for vuln in vulns:
table.append("-> {}, installed {}, affected {}, id {}".format(
vuln.name,
vuln.version[:13],
vuln.spec[:27],
vuln.vuln_id
))
if full:
if vuln.cvssv2 is not None:
base_score = vuln.cvssv2.get("base_score", "None")
impact_score = vuln.cvssv2.get("impact_score", "None")
table.append("CVSS v2 -- BASE SCORE: {}, IMPACT SCORE: {}".format(
base_score,
impact_score,
))
if vuln.cvssv3 is not None:
base_score = vuln.cvssv3.get("base_score", "None")
impact_score = vuln.cvssv3.get("impact_score", "None")
base_severity = vuln.cvssv3.get("base_severity", "None")
table.append("CVSS v3 -- BASE SCORE: {}, IMPACT SCORE: {}, BASE SEVERITY: {}".format(
base_score,
impact_score,
base_severity,
))
table.append(get_advisory(vuln))
table.append("--")
else:
table.append("No known security vulnerabilities found.")
return "\n".join(
table
)
@staticmethod
def render_licenses(packages, packages_licenses):
table = [
"safety",
"packages licenses",
"---"
]
if not packages_licenses:
table.append("No packages licenses found.")
return "\n".join(table)
for pkg_license in packages_licenses:
text = pkg_license['package'] + \
", version " + pkg_license['version'] + \
", license " + pkg_license['license'] + "\n"
table.append(text)
return "\n".join(table)
class JsonReport(object):
"""Json report, for when the output is input for something else"""
@staticmethod
def render(vulns, full):
return json.dumps(vulns, indent=4, sort_keys=True)
@staticmethod
def render_licenses(packages_licenses):
return json.dumps(packages_licenses, indent=4, sort_keys=True)
class BareReport(object):
"""Bare report, for command line tools"""
@staticmethod
def render(vulns, full):
return " ".join(set([v.name for v in vulns]))
@staticmethod
def render_licenses(packages_licenses):
licenses = set([pkg_li.get('license') for pkg_li in packages_licenses])
if "N/A" in licenses:
licenses.remove("N/A")
sorted_licenses = sorted(licenses)
return " ".join(sorted_licenses)
def get_used_db(key, db):
key = key if key else os.environ.get("SAFETY_API_KEY", False)
if db:
return "local DB"
if key:
return "pyup.io's DB"
return "free DB (updated once a month)"
def report(vulns, full=False, json_report=False, bare_report=False, checked_packages=0, db=None, key=None):
if bare_report:
return BareReport.render(vulns, full=full)
if json_report:
return JsonReport.render(vulns, full=full)
size = get_terminal_size()
used_db = get_used_db(key=key, db=db)
if size.columns >= 80:
return SheetReport.render(vulns, full=full, checked_packages=checked_packages, used_db=used_db)
return BasicReport.render(vulns, full=full, checked_packages=checked_packages, used_db=used_db)
def license_report(packages, licenses, json_report=False, bare_report=False):
if json_report:
return JsonReport.render_licenses(packages_licenses=licenses)
elif bare_report:
return BareReport.render_licenses(packages_licenses=licenses)
size = get_terminal_size()
if size.columns >= 80:
return SheetReport.render_licenses(packages, licenses)
return BasicReport.render_licenses(packages, licenses)
if output == 'json':
self.format = JsonReport()
elif output == 'bare':
self.format = BareReport()
elif output == 'text':
self.format = TextReport()
pipenv/patched/safety/formatters/__init__.py
View File
pipenv/patched/safety/formatters/bare.py
+38
View File
@@ -0,0 +1,38 @@
from collections import namedtuple
from pipenv.patched.safety.formatter import FormatterAPI
from pipenv.patched.safety.util import get_basic_announcements
class BareReport(FormatterAPI):
"""Bare report, for command line tools"""
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
parsed_announcements = []
Announcement = namedtuple("Announcement", ["name"])
for announcement in get_basic_announcements(announcements):
normalized_message = "-".join(announcement.get('message', 'none').lower().split())
parsed_announcements.append(Announcement(name=normalized_message))
announcements_to_render = [announcement.name for announcement in parsed_announcements]
affected_packages = list(set([v.package_name for v in vulnerabilities if not v.ignored]))
return " ".join(announcements_to_render + affected_packages)
def render_licenses(self, announcements, packages_licenses):
parsed_announcements = []
for announcement in get_basic_announcements(announcements):
normalized_message = "-".join(announcement.get('message', 'none').lower().split())
parsed_announcements.append({'license': normalized_message})
announcements_to_render = [announcement.get('license') for announcement in parsed_announcements]
licenses = list(set([pkg_li.get('license') for pkg_li in packages_licenses]))
sorted_licenses = sorted(licenses)
return " ".join(announcements_to_render + sorted_licenses)
def render_announcements(self, announcements):
print('render_announcements bare')
pipenv/patched/safety/formatters/json.py
+84
View File
@@ -0,0 +1,84 @@
import logging
import json as json_parser
from pipenv.patched.pip._vendor.requests.models import PreparedRequest
from pipenv.patched.safety.formatter import FormatterAPI
from pipenv.patched.safety.output_utils import get_report_brief_info
from pipenv.patched.safety.util import get_basic_announcements
LOG = logging.getLogger(__name__)
class JsonReport(FormatterAPI):
"""Json report, for when the output is input for something else"""
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
remediations_recommended = len(remediations.keys())
LOG.debug('Rendering %s vulnerabilities, %s remediations with full_report: %s', len(vulnerabilities),
remediations_recommended, full)
vulns_ignored = [vuln.to_dict() for vuln in vulnerabilities if vuln.ignored]
vulns = [vuln.to_dict() for vuln in vulnerabilities if not vuln.ignored]
report = get_report_brief_info(as_dict=True, report_type=1, vulnerabilities_found=len(vulns),
vulnerabilities_ignored=len(vulns_ignored),
remediations_recommended=remediations_recommended)
remed = {}
for k, v in remediations.items():
if k not in remed:
remed[k] = {}
closest = v.get('closest_secure_version', {})
upgrade = closest.get('major', None)
downgrade = closest.get('minor', None)
recommended_version = None
if upgrade:
recommended_version = str(upgrade)
elif downgrade:
recommended_version = str(downgrade)
remed[k]['current_version'] = v.get('version', None)
remed[k]['vulnerabilities_found'] = v.get('vulns_found', 0)
remed[k]['recommended_version'] = recommended_version
remed[k]['other_recommended_versions'] = [other_v for other_v in v.get('secure_versions', []) if
other_v != recommended_version]
remed[k]['more_info_url'] = v.get('more_info_url', '')
# Use Request's PreparedRequest to handle parsing, joining etc the URL since we're adding query
# parameters and don't know what the server might send down.
if remed[k]['more_info_url']:
req = PreparedRequest()
req.prepare_url(remed[k]['more_info_url'], {'from': remed[k]['current_version'], 'to': recommended_version})
remed[k]['more_info_url'] = req.url
template = {
"report_meta": report,
"scanned_packages": {p.name: p.to_dict(short_version=True) for p in packages},
"affected_packages": {v.pkg.name: v.pkg.to_dict() for v in vulnerabilities},
"announcements": [{'type': item.get('type'), 'message': item.get('message')} for item in
get_basic_announcements(announcements)],
"vulnerabilities": vulns,
"ignored_vulnerabilities": vulns_ignored,
"remediations": remed
}
return json_parser.dumps(template, indent=4)
def render_licenses(self, announcements, licenses):
unique_license_types = set([lic['license'] for lic in licenses])
report = get_report_brief_info(as_dict=True, report_type=2, licenses_found=len(unique_license_types))
template = {
"report_meta": report,
"announcements": get_basic_announcements(announcements),
"licenses": licenses,
}
return json_parser.dumps(template, indent=4)
def render_announcements(self, announcements):
return json_parser.dumps({"announcements": get_basic_announcements(announcements)}, indent=4)
pipenv/patched/safety/formatters/screen.py
+143
View File
@@ -0,0 +1,143 @@
import pipenv.vendor.click as click
from pipenv.patched.safety.formatter import FormatterAPI
from pipenv.patched.safety.output_utils import build_announcements_section_content, format_long_text, \
add_empty_line, format_vulnerability, get_final_brief, \
build_report_brief_section, format_license, get_final_brief_license, build_remediation_section, \
build_primary_announcement
from pipenv.patched.safety.util import get_primary_announcement, get_basic_announcements, get_terminal_size
class ScreenReport(FormatterAPI):
DIVIDER_SECTIONS = '+' + '=' * (get_terminal_size().columns - 2) + '+'
REPORT_BANNER = DIVIDER_SECTIONS + '\n' + r"""
/$$$$$$ /$$
/$$__ $$ | $$
/$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$
/$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$
| $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$
\____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$
/$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$
|_______/ \_______/|__/ \_______/ \___/ \____ $$
/$$ | $$
| $$$$$$/
by pyup.io \______/
""" + DIVIDER_SECTIONS
ANNOUNCEMENTS_HEADING = format_long_text(click.style('ANNOUNCEMENTS', bold=True))
def __build_announcements_section(self, announcements):
announcements_section = []
basic_announcements = get_basic_announcements(announcements)
if basic_announcements:
announcements_content = build_announcements_section_content(basic_announcements)
announcements_section = [add_empty_line(), self.ANNOUNCEMENTS_HEADING, add_empty_line(),
announcements_content, add_empty_line(), self.DIVIDER_SECTIONS]
return announcements_section
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
announcements_section = self.__build_announcements_section(announcements)
primary_announcement = get_primary_announcement(announcements)
remediation_section = build_remediation_section(remediations)
end_content = []
if primary_announcement:
end_content = [add_empty_line(),
build_primary_announcement(primary_announcement, columns=get_terminal_size().columns),
self.DIVIDER_SECTIONS]
table = []
ignored = {}
total_ignored = 0
for n, vuln in enumerate(vulnerabilities):
if vuln.ignored:
total_ignored += 1
ignored[vuln.package_name] = ignored.get(vuln.package_name, 0) + 1
table.append(format_vulnerability(vuln, full))
report_brief_section = build_report_brief_section(primary_announcement=primary_announcement, report_type=1,
vulnerabilities_found=max(0, len(vulnerabilities)-total_ignored),
vulnerabilities_ignored=total_ignored,
remediations_recommended=len(remediations))
if vulnerabilities:
final_brief = get_final_brief(len(vulnerabilities), len(remediations), ignored, total_ignored)
return "\n".join(
[ScreenReport.REPORT_BANNER] + announcements_section + [report_brief_section,
add_empty_line(),
self.DIVIDER_SECTIONS,
format_long_text(
click.style('VULNERABILITIES FOUND',
bold=True, fg='red')),
self.DIVIDER_SECTIONS,
add_empty_line(),
"\n\n".join(table),
final_brief,
add_empty_line(),
self.DIVIDER_SECTIONS] +
remediation_section + end_content
)
else:
content = format_long_text(click.style("No known security vulnerabilities found.", bold=True, fg='green'))
return "\n".join(
[ScreenReport.REPORT_BANNER] + announcements_section + [report_brief_section,
self.DIVIDER_SECTIONS,
add_empty_line(),
content,
add_empty_line(),
self.DIVIDER_SECTIONS] +
end_content
)
def render_licenses(self, announcements, licenses):
unique_license_types = set([lic['license'] for lic in licenses])
report_brief_section = build_report_brief_section(primary_announcement=get_primary_announcement(announcements),
report_type=2, licenses_found=len(unique_license_types))
announcements_section = self.__build_announcements_section(announcements)
if not licenses:
content = format_long_text(click.style("No packages licenses found.", bold=True, fg='red'))
return "\n".join(
[ScreenReport.REPORT_BANNER] + announcements_section + [report_brief_section,
self.DIVIDER_SECTIONS,
add_empty_line(),
content,
add_empty_line(),
self.DIVIDER_SECTIONS]
)
table = []
for license in licenses:
table.append(format_license(license))
final_brief = get_final_brief_license(unique_license_types)
return "\n".join(
[ScreenReport.REPORT_BANNER] + announcements_section + [report_brief_section,
add_empty_line(),
self.DIVIDER_SECTIONS,
format_long_text(
click.style('LICENSES FOUND',
bold=True, fg='yellow')),
self.DIVIDER_SECTIONS,
add_empty_line(),
"\n".join(table),
final_brief,
add_empty_line(),
self.DIVIDER_SECTIONS]
)
def render_announcements(self, announcements):
return self.__build_announcements_section(announcements)
pipenv/patched/safety/formatters/text.py
+134
View File
@@ -0,0 +1,134 @@
import pipenv.vendor.click as click
from pipenv.patched.safety.formatter import FormatterAPI
from pipenv.patched.safety.output_utils import build_announcements_section_content, format_vulnerability, \
build_report_brief_section, get_final_brief_license, add_empty_line, get_final_brief, build_remediation_section, \
build_primary_announcement
from pipenv.patched.safety.util import get_primary_announcement, get_basic_announcements
class TextReport(FormatterAPI):
"""Basic report, intented to be used for terminals with < 80 columns"""
SMALL_DIVIDER_SECTIONS = '+' + '=' * 78 + '+'
TEXT_REPORT_BANNER = SMALL_DIVIDER_SECTIONS + '\n' + r"""
/$$$$$$ /$$
/$$__ $$ | $$
/$$$$$$$ /$$$$$$ | $$ \__//$$$$$$ /$$$$$$ /$$ /$$
/$$_____/ |____ $$| $$$$ /$$__ $$|_ $$_/ | $$ | $$
| $$$$$$ /$$$$$$$| $$_/ | $$$$$$$$ | $$ | $$ | $$
\____ $$ /$$__ $$| $$ | $$_____/ | $$ /$$| $$ | $$
/$$$$$$$/| $$$$$$$| $$ | $$$$$$$ | $$$$/| $$$$$$$
|_______/ \_______/|__/ \_______/ \___/ \____ $$
/$$ | $$
| $$$$$$/
by pyup.io \______/
""" + SMALL_DIVIDER_SECTIONS
def __build_announcements_section(self, announcements):
announcements_table = []
basic_announcements = get_basic_announcements(announcements)
if basic_announcements:
announcements_content = click.unstyle(build_announcements_section_content(basic_announcements,
columns=80,
start_line_decorator=' ' * 2,
end_line_decorator=''))
announcements_table = [add_empty_line(), 'ANNOUNCEMENTS', add_empty_line(),
announcements_content, add_empty_line(), self.SMALL_DIVIDER_SECTIONS]
return announcements_table
def render_vulnerabilities(self, announcements, vulnerabilities, remediations, full, packages):
primary_announcement = get_primary_announcement(announcements)
remediation_section = [click.unstyle(rem) for rem in build_remediation_section(remediations, columns=80)]
end_content = []
if primary_announcement:
end_content = [add_empty_line(),
build_primary_announcement(primary_announcement, columns=80, only_text=True),
self.SMALL_DIVIDER_SECTIONS]
announcement_section = self.__build_announcements_section(announcements)
ignored = {}
total_ignored = 0
for n, vuln in enumerate(vulnerabilities):
if vuln.ignored:
total_ignored += 1
ignored[vuln.package_name] = ignored.get(vuln.package_name, 0) + 1
report_brief_section = click.unstyle(
build_report_brief_section(columns=80, primary_announcement=primary_announcement,
vulnerabilities_found=max(0, len(vulnerabilities)-total_ignored),
vulnerabilities_ignored=total_ignored,
remediations_recommended=len(remediations)))
table = [self.TEXT_REPORT_BANNER] + announcement_section + [
report_brief_section,
'',
self.SMALL_DIVIDER_SECTIONS,
]
if vulnerabilities:
table += [" VULNERABILITIES FOUND", self.SMALL_DIVIDER_SECTIONS]
for vuln in vulnerabilities:
table.append('\n' + format_vulnerability(vuln, full, only_text=True, columns=80))
final_brief = click.unstyle(get_final_brief(len(vulnerabilities), len(remediations), ignored, total_ignored,
kwargs={'columns': 80}))
table += [final_brief, add_empty_line(), self.SMALL_DIVIDER_SECTIONS] + remediation_section + end_content
else:
table += [add_empty_line(), " No known security vulnerabilities found.", add_empty_line(),
self.SMALL_DIVIDER_SECTIONS] + end_content
return "\n".join(
table
)
def render_licenses(self, announcements, licenses):
unique_license_types = set([lic['license'] for lic in licenses])
report_brief_section = click.unstyle(
build_report_brief_section(columns=80, primary_announcement=get_primary_announcement(announcements),
licenses_found=len(unique_license_types)))
packages_licenses = licenses
announcements_table = self.__build_announcements_section(announcements)
final_brief = click.unstyle(
get_final_brief_license(unique_license_types, kwargs={'columns': 80}))
table = [self.TEXT_REPORT_BANNER] + announcements_table + [
report_brief_section,
self.SMALL_DIVIDER_SECTIONS,
" LICENSES",
self.SMALL_DIVIDER_SECTIONS,
add_empty_line(),
]
if not packages_licenses:
table.append(" No packages licenses found.")
table += [final_brief, add_empty_line(), self.SMALL_DIVIDER_SECTIONS]
return "\n".join(table)
for pkg_license in packages_licenses:
text = " {0}, version {1}, license {2}\n".format(pkg_license['package'], pkg_license['version'],
pkg_license['license'])
table.append(text)
table += [final_brief, add_empty_line(), self.SMALL_DIVIDER_SECTIONS]
return "\n".join(table)
def render_announcements(self, announcements):
rows = self.__build_announcements_section(announcements)
rows.insert(0, self.SMALL_DIVIDER_SECTIONS)
return '\n'.join(rows)
pipenv/patched/safety/models.py
+110
View File
@@ -0,0 +1,110 @@
from collections import namedtuple
from datetime import datetime
from typing import NamedTuple
class DictConverter(object):
def to_dict(self, **kwargs):
pass
announcement_nmt = namedtuple('Announcement', ['type', 'message'])
remediation_nmt = namedtuple('Remediation', ['Package', 'closest_secure_version', 'secure_versions',
'latest_package_version'])
cve_nmt = namedtuple('Cve', ['name', 'cvssv2', 'cvssv3'])
severity_nmt = namedtuple('Severity', ['source', 'cvssv2', 'cvssv3'])
vulnerability_nmt = namedtuple('Vulnerability',
['vulnerability_id', 'package_name', 'pkg', 'ignored', 'ignored_reason', 'ignored_expires',
'vulnerable_spec', 'all_vulnerable_specs', 'analyzed_version', 'advisory',
'is_transitive', 'published_date', 'fixed_versions',
'closest_versions_without_known_vulnerabilities', 'resources', 'CVE', 'severity',
'affected_versions', 'more_info_url'])
package_nmt = namedtuple('Package', ['name', 'version', 'found', 'insecure_versions', 'secure_versions',
'latest_version_without_known_vulnerabilities', 'latest_version', 'more_info_url'])
package_nmt.__new__.__defaults__ = (None,) * len(package_nmt._fields) # Ugly hack for now
RequirementFile = namedtuple('RequirementFile', ['path'])
class Package(package_nmt, DictConverter):
def to_dict(self, **kwargs):
if kwargs.get('short_version', False):
return {
'name': self.name,
'version': self.version,
}
return {'name': self.name,
'version': self.version,
'found': self.found,
'insecure_versions': self.insecure_versions,
'secure_versions': self.secure_versions,
'latest_version_without_known_vulnerabilities': self.latest_version_without_known_vulnerabilities,
'latest_version': self.latest_version,
'more_info_url': self.more_info_url
}
class Announcement(announcement_nmt):
pass
class Remediation(remediation_nmt, DictConverter):
def to_dict(self):
return {'package': self.Package.name,
'closest_secure_version': self.closest_secure_version,
'secure_versions': self.secure_versions,
'latest_package_version': self.latest_package_version
}
class CVE(cve_nmt, DictConverter):
def to_dict(self):
return {'name': self.name, 'cvssv2': self.cvssv2, 'cvssv3': self.cvssv3}
class Severity(severity_nmt, DictConverter):
def to_dict(self):
result = {'severity': {'source': self.source}}
result['severity']['cvssv2'] = self.cvssv2
result['severity']['cvssv3'] = self.cvssv3
return result
class Vulnerability(vulnerability_nmt):
def to_dict(self):
empty_list_if_none = ['fixed_versions', 'closest_versions_without_known_vulnerabilities', 'resources']
result = {
}
ignore = ['pkg']
for field, value in zip(self._fields, self):
if field in ignore:
continue
if value is None and field in empty_list_if_none:
value = []
if isinstance(value, CVE):
val = None
if value.name.startswith("CVE"):
val = value.name
result[field] = val
elif isinstance(value, DictConverter):
result.update(value.to_dict())
elif isinstance(value, datetime):
result[field] = str(value)
else:
result[field] = value
return result
def get_advisory(self):
return self.advisory.replace('\r', '') if self.advisory else "No advisory found for this vulnerability."
pipenv/patched/safety/output_utils.py
+693
View File
@@ -0,0 +1,693 @@
import json
import logging
import os
import textwrap
from datetime import datetime
import pipenv.vendor.click as click
from pipenv.patched.safety.constants import RED, YELLOW
from pipenv.patched.safety.util import get_safety_version, Package, get_terminal_size, \
SafetyContext, build_telemetry_data, build_git_data, is_a_remote_mirror
LOG = logging.getLogger(__name__)
def build_announcements_section_content(announcements, columns=get_terminal_size().columns,
start_line_decorator=' ', end_line_decorator=' '):
section = ''
for i, announcement in enumerate(announcements):
color = ''
if announcement.get('type') == 'error':
color = RED
elif announcement.get('type') == 'warning':
color = YELLOW
item = '{message}'.format(
message=format_long_text('* ' + announcement.get('message'), color, columns,
start_line_decorator, end_line_decorator))
section += '{item}'.format(item=item)
if i + 1 < len(announcements):
section += '\n'
return section
def add_empty_line():
return format_long_text('')
def style_lines(lines, columns, pre_processed_text='', start_line=' ' * 4, end_line=' ' * 4):
styled_text = pre_processed_text
for line in lines:
styled_line = ''
left_padding = ' ' * line.get('left_padding', 0)
for i, word in enumerate(line.get('words', [])):
if word.get('style', {}):
text = ''
if i == 0:
text = left_padding # Include the line padding in the word to avoid Github issues
left_padding = '' # Clean left padding to avoid be added two times
text += word.get('value', '')
styled_line += click.style(text=text, **word.get('style', {}))
else:
styled_line += word.get('value', '')
styled_text += format_long_text(styled_line, columns=columns, start_line_decorator=start_line,
end_line_decorator=end_line,
left_padding=left_padding, **line.get('format', {})) + '\n'
return styled_text
def format_vulnerability(vulnerability, full_mode, only_text=False, columns=get_terminal_size().columns):
common_format = {'left_padding': 3, 'format': {'sub_indent': ' ' * 3, 'max_lines': None}}
styled_vulnerability = [
{'words': [{'style': {'bold': True}, 'value': 'Vulnerability ID: '}, {'value': vulnerability.vulnerability_id}]},
]
vulnerability_spec = [
{'words': [{'style': {'bold': True}, 'value': 'Affected spec: '}, {'value': vulnerability.vulnerable_spec}]}]
cve = vulnerability.CVE
cvssv2_line = None
cve_lines = []
if cve:
if full_mode and cve.cvssv2:
b = cve.cvssv2.get("base_score", "-")
s = cve.cvssv2.get("impact_score", "-")
v = cve.cvssv2.get("vector_string", "-")
# Reset sub_indent as the left_margin is going to be applied in this case
cvssv2_line = {'format': {'sub_indent': ''}, 'words': [
{'value': f'CVSS v2, BASE SCORE {b}, IMPACT SCORE {s}, VECTOR STRING {v}'},
]}
if cve.cvssv3 and "base_severity" in cve.cvssv3.keys():
cvss_base_severity_style = {'bold': True}
base_severity = cve.cvssv3.get("base_severity", "-")
if base_severity.upper() in ['HIGH', 'CRITICAL']:
cvss_base_severity_style['fg'] = 'red'
b = cve.cvssv3.get("base_score", "-")
if full_mode:
s = cve.cvssv3.get("impact_score", "-")
v = cve.cvssv3.get("vector_string", "-")
cvssv3_text = f'CVSS v3, BASE SCORE {b}, IMPACT SCORE {s}, VECTOR STRING {v}'
else:
cvssv3_text = f'CVSS v3, BASE SCORE {b} '
cve_lines = [
{'words': [{'style': {'bold': True}, 'value': '{0} is '.format(cve.name)},
{'style': cvss_base_severity_style,
'value': f'{base_severity} SEVERITY => '},
{'value': cvssv3_text},
]},
]
if cvssv2_line:
cve_lines.append(cvssv2_line)
elif cve.name:
cve_lines = [
{'words': [{'style': {'bold': True}, 'value': cve.name}]}
]
advisory_format = {'sub_indent': ' ' * 3, 'max_lines': None} if full_mode else {'sub_indent': ' ' * 3,
'max_lines': 2}
basic_vuln_data_lines = [
{'format': advisory_format, 'words': [
{'style': {'bold': True}, 'value': 'ADVISORY: '},
{'value': vulnerability.advisory.replace('\n', '')}]}
]
if SafetyContext().key:
fixed_version_line = {'words': [
{'style': {'bold': True}, 'value': 'Fixed versions: '},
{'value': ', '.join(vulnerability.fixed_versions) if vulnerability.fixed_versions else 'No known fix'}
]}
basic_vuln_data_lines.append(fixed_version_line)
more_info_line = [{'words': [{'style': {'bold': True}, 'value': 'For more information, please visit '},
{'value': click.style(vulnerability.more_info_url)}]}]
vuln_title = f'-> Vulnerability found in {vulnerability.package_name} version {vulnerability.analyzed_version}\n'
styled_text = click.style(vuln_title, fg='red')
to_print = styled_vulnerability
if not vulnerability.ignored:
to_print += vulnerability_spec + basic_vuln_data_lines + cve_lines
else:
generic_reason = 'This vulnerability is being ignored'
if vulnerability.ignored_expires:
generic_reason += f" until {vulnerability.ignored_expires.strftime('%Y-%m-%d %H:%M:%S UTC')}. " \
f"See your configurations"
specific_reason = None
if vulnerability.ignored_reason:
specific_reason = [
{'words': [{'style': {'bold': True}, 'value': 'Reason: '}, {'value': vulnerability.ignored_reason}]}]
expire_section = [{'words': [
{'style': {'bold': True, 'fg': 'green'}, 'value': f'{generic_reason}.'}, ]}]
if specific_reason:
expire_section += specific_reason
to_print += expire_section
if cve:
to_print += more_info_line
to_print = [{**common_format, **line} for line in to_print]
content = style_lines(to_print, columns, styled_text, start_line='', end_line='', )
return click.unstyle(content) if only_text else content
def format_license(license, only_text=False, columns=get_terminal_size().columns):
to_print = [
{'words': [{'style': {'bold': True}, 'value': license['package']},
{'value': ' version {0} found using license '.format(license['version'])},
{'style': {'bold': True}, 'value': license['license']}
]
},
]
content = style_lines(to_print, columns, '-> ', start_line='', end_line='')
return click.unstyle(content) if only_text else content
def build_remediation_section(remediations, only_text=False, columns=get_terminal_size().columns, kwargs=None):
columns -= 2
left_padding = ' ' * 3
if not kwargs:
# Reset default params in the format_long_text func
kwargs = {'left_padding': '', 'columns': columns, 'start_line_decorator': '', 'end_line_decorator': '',
'sub_indent': left_padding}
END_SECTION = '+' + '=' * columns + '+'
if not remediations:
return []
content = ''
total_vulns = 0
total_packages = len(remediations.keys())
for pkg in remediations.keys():
total_vulns += remediations[pkg]['vulns_found']
upgrade_to = remediations[pkg]['closest_secure_version']['major']
downgrade_to = remediations[pkg]['closest_secure_version']['minor']
fix_version = None
if upgrade_to:
fix_version = str(upgrade_to)
elif downgrade_to:
fix_version = str(downgrade_to)
new_line = '\n'
other_options = [str(fix) for fix in remediations[pkg].get('secure_versions', []) if str(fix) != fix_version]
raw_recommendation = f"We recommend upgrading to version {upgrade_to} of {pkg}."
if other_options:
raw_other_options = ', '.join(other_options)
raw_pre_other_options = 'Other versions without known vulnerabilities are:'
if len(other_options) == 1:
raw_pre_other_options = 'Other version without known vulnerabilities is'
raw_recommendation = f"{raw_recommendation} {raw_pre_other_options} " \
f"{raw_other_options}"
remediation_content = [
f'{left_padding}The closest version with no known vulnerabilities is ' + click.style(upgrade_to, bold=True),
new_line,
click.style(f'{left_padding}{raw_recommendation}', bold=True, fg='green')
]
if not fix_version:
remediation_content = [new_line,
click.style(f'{left_padding}There is no known fix for this vulnerability.', bold=True, fg='yellow')]
text = 'vulnerabilities' if remediations[pkg]['vulns_found'] > 1 else 'vulnerability'
raw_rem_title = f"-> {pkg} version {remediations[pkg]['version']} was found, " \
f"which has {remediations[pkg]['vulns_found']} {text}"
remediation_title = click.style(raw_rem_title, fg=RED, bold=True)
content += new_line + format_long_text(remediation_title, **kwargs) + new_line
pre_content = remediation_content + [
f"{left_padding}For more information, please visit {remediations[pkg]['more_info_url']}",
f'{left_padding}Always check for breaking changes when upgrading packages.',
new_line]
for i, element in enumerate(pre_content):
content += format_long_text(element, **kwargs)
if i + 1 < len(pre_content):
content += '\n'
title = format_long_text(click.style(f'{left_padding}REMEDIATIONS', fg='green', bold=True), **kwargs)
body = [content]
if not is_using_api_key():
vuln_text = 'vulnerabilities were' if total_vulns != 1 else 'vulnerability was'
pkg_text = 'packages' if total_packages > 1 else 'package'
msg = "{0} {1} found in {2} {3}. " \
"For detailed remediation & fix recommendations, upgrade to a commercial license."\
.format(total_vulns, vuln_text, total_packages, pkg_text)
content = '\n' + format_long_text(msg, left_padding=' ', columns=columns) + '\n'
body = [content]
body.append(END_SECTION)
content = [title] + body
if only_text:
content = [click.unstyle(item) for item in content]
return content
def get_final_brief(total_vulns_found, total_remediations, ignored, total_ignored, kwargs=None):
if not kwargs:
kwargs = {}
total_vulns = max(0, total_vulns_found - total_ignored)
vuln_text = 'vulnerabilities' if total_ignored > 1 else 'vulnerability'
pkg_text = 'packages were' if len(ignored.keys()) > 1 else 'package was'
policy_file_text = ' using a safety policy file' if is_using_a_safety_policy_file() else ''
vuln_brief = f" {total_vulns} vulnerabilit{'y was' if total_vulns == 1 else 'ies were'} found."
ignored_text = f' {total_ignored} {vuln_text} from {len(ignored.keys())} {pkg_text} ignored.' if ignored else ''
remediation_text = f" {total_remediations} remediation{' was' if total_remediations == 1 else 's were'} " \
f"recommended." if is_using_api_key() else ''
raw_brief = f"Scan was completed{policy_file_text}.{vuln_brief}{ignored_text}{remediation_text}"
return format_long_text(raw_brief, start_line_decorator=' ', **kwargs)
def get_final_brief_license(licenses, kwargs=None):
if not kwargs:
kwargs = {}
licenses_text = ' Scan was completed.'
if licenses:
licenses_text = 'The following software licenses were present in your system: {0}'.format(', '.join(licenses))
return format_long_text("{0}".format(licenses_text), start_line_decorator=' ', **kwargs)
def format_long_text(text, color='', columns=get_terminal_size().columns, start_line_decorator=' ', end_line_decorator=' ', left_padding='', max_lines=None, styling=None, indent='', sub_indent=''):
if not styling:
styling = {}
if color:
styling.update({'fg': color})
columns -= len(start_line_decorator) + len(end_line_decorator)
formatted_lines = []
lines = text.replace('\r', '').splitlines()
for line in lines:
base_format = "{:" + str(columns) + "}"
if line == '':
empty_line = base_format.format(" ")
formatted_lines.append("{0}{1}{2}".format(start_line_decorator, empty_line, end_line_decorator))
wrapped_lines = textwrap.wrap(line, width=columns, max_lines=max_lines, initial_indent=indent, subsequent_indent=sub_indent, placeholder='...')
for wrapped_line in wrapped_lines:
try:
new_line = left_padding + wrapped_line.encode('utf-8')
except TypeError:
new_line = left_padding + wrapped_line
if styling:
new_line = click.style(new_line, **styling)
formatted_lines.append(f"{start_line_decorator}{new_line}{end_line_decorator}")
return "\n".join(formatted_lines)
def get_printable_list_of_scanned_items(scanning_target):
context = SafetyContext()
result = []
scanned_items_data = []
if scanning_target == 'environment':
locations = set([pkg.found for pkg in context.packages if isinstance(pkg, Package)])
for path in locations:
result.append([{'styled': False, 'value': '-> ' + path}])
scanned_items_data.append(path)
if len(locations) <= 0:
msg = 'No locations found in the environment'
result.append([{'styled': False, 'value': msg}])
scanned_items_data.append(msg)
elif scanning_target == 'stdin':
scanned_stdin = [pkg.name for pkg in context.packages if isinstance(pkg, Package)]
value = 'No found packages in stdin'
scanned_items_data = [value]
if len(scanned_stdin) > 0:
value = ', '.join(scanned_stdin)
scanned_items_data = scanned_stdin
result.append(
[{'styled': False, 'value': value}])
elif scanning_target == 'files':
for file in context.params.get('files', []):
result.append([{'styled': False, 'value': f'-> {file.name}'}])
scanned_items_data.append(file.name)
elif scanning_target == 'file':
file = context.params.get('file', None)
name = file.name if file else ''
result.append([{'styled': False, 'value': f'-> {name}'}])
scanned_items_data.append(name)
return result, scanned_items_data
REPORT_HEADING = format_long_text(click.style('REPORT', bold=True))
def build_report_brief_section(columns=None, primary_announcement=None, report_type=1, **kwargs):
if not columns:
columns = get_terminal_size().columns
styled_brief_lines = []
if primary_announcement:
styled_brief_lines.append(
build_primary_announcement(columns=columns, primary_announcement=primary_announcement))
for line in get_report_brief_info(report_type=report_type, **kwargs):
ln = ''
padding = ' ' * 2
for i, words in enumerate(line):
processed_words = words.get('value', '')
if words.get('style', False):
text = ''
if i == 0:
text = padding
padding = ''
text += processed_words
processed_words = click.style(text, bold=True)
ln += processed_words
styled_brief_lines.append(format_long_text(ln, color='', columns=columns, start_line_decorator='',
left_padding=padding, end_line_decorator='', sub_indent=' ' * 2))
return "\n".join([add_empty_line(), REPORT_HEADING, add_empty_line(), '\n'.join(styled_brief_lines)])
def build_report_for_review_vuln_report(as_dict=False):
ctx = SafetyContext()
report_from_file = ctx.review
packages = ctx.packages
if as_dict:
return report_from_file
policy_f_name = report_from_file.get('policy_file', None)
safety_policy_used = []
if policy_f_name:
safety_policy_used = [
{'style': False, 'value': '\nScanning using a security policy file'},
{'style': True, 'value': ' {0}'.format(policy_f_name)},
]
action_executed = [
{'style': True, 'value': 'Scanning dependencies'},
{'style': False, 'value': ' in your '},
{'style': True, 'value': report_from_file.get('scan_target', '-') + ':'},
]
scanned_items = []
for name in report_from_file.get('scanned', []):
scanned_items.append([{'styled': False, 'value': '-> ' + name}])
nl = [{'style': False, 'value': ''}]
using_sentence = build_using_sentence(report_from_file.get('api_key', None),
report_from_file.get('local_database_path_used', None))
scanned_count_sentence = build_scanned_count_sentence(packages)
old_timestamp = report_from_file.get('timestamp', None)
old_timestamp = [{'style': False, 'value': 'Report generated '}, {'style': True, 'value': old_timestamp}]
now = str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
current_timestamp = [{'style': False, 'value': 'Timestamp '}, {'style': True, 'value': now}]
brief_info = [[{'style': False, 'value': 'Safety '},
{'style': True, 'value': 'v' + report_from_file.get('safety_version', '-')},
{'style': False, 'value': ' is scanning for '},
{'style': True, 'value': 'Vulnerabilities'},
{'style': True, 'value': '...'}] + safety_policy_used, action_executed
] + [nl] + scanned_items + [nl] + [using_sentence] + [scanned_count_sentence] + [old_timestamp] + \
[current_timestamp]
return brief_info
def build_using_sentence(key, db):
key_sentence = []
custom_integration = os.environ.get('SAFETY_CUSTOM_INTEGRATION',
'false').lower() == 'true'
if key:
key_sentence = [{'style': True, 'value': 'an API KEY'},
{'style': False, 'value': ' and the '}]
db_name = 'PyUp Commercial'
elif db:
if is_a_remote_mirror(db):
if custom_integration:
return []
db_name = f"remote URL {db}"
else:
db_name = f"local file {db}"
else:
db_name = 'non-commercial'
database_sentence = [{'style': True, 'value': db_name + ' database'}]
return [{'style': False, 'value': 'Using '}] + key_sentence + database_sentence
def build_scanned_count_sentence(packages):
scanned_count = 'No packages found'
if len(packages) >= 1:
scanned_count = 'Found and scanned {0} {1}'.format(len(packages),
'packages' if len(packages) > 1 else 'package')
return [{'style': True, 'value': scanned_count}]
def add_warnings_if_needed(brief_info):
ctx = SafetyContext()
warnings = []
if ctx.packages:
if ctx.params.get('continue_on_error', False):
warnings += [[{'style': True,
'value': '* Continue-on-error is enabled, so returning successful (0) exit code in all cases.'}]]
if ctx.params.get('ignore_severity_rules', False) and not is_using_api_key():
warnings += [[{'style': True,
'value': '* Could not filter by severity, please upgrade your account to include severity data.'}]]
if warnings:
brief_info += [[{'style': False, 'value': ''}]] + warnings
def get_report_brief_info(as_dict=False, report_type=1, **kwargs):
LOG.info('get_report_brief_info: %s, %s, %s', as_dict, report_type, kwargs)
context = SafetyContext()
packages = [pkg for pkg in context.packages if isinstance(pkg, Package)]
brief_data = {}
command = context.command
if command == 'review':
review = build_report_for_review_vuln_report(as_dict)
return review
key = context.key
db = context.db_mirror
scanning_types = {'check': {'name': 'Vulnerabilities', 'action': 'Scanning dependencies', 'scanning_target': 'environment'}, # Files, Env or Stdin
'license': {'name': 'Licenses', 'action': 'Scanning licenses', 'scanning_target': 'environment'}, # Files or Env
'review': {'name': 'Report', 'action': 'Reading the report',
'scanning_target': 'file'}} # From file
targets = ['stdin', 'environment', 'files', 'file']
for target in targets:
if context.params.get(target, False):
scanning_types[command]['scanning_target'] = target
break
scanning_target = scanning_types.get(context.command, {}).get('scanning_target', '')
brief_data['scan_target'] = scanning_target
scanned_items, data = get_printable_list_of_scanned_items(scanning_target)
brief_data['scanned'] = data
nl = [{'style': False, 'value': ''}]
action_executed = [
{'style': True, 'value': scanning_types.get(context.command, {}).get('action', '')},
{'style': False, 'value': ' in your '},
{'style': True, 'value': scanning_target + ':'},
]
policy_file = context.params.get('policy_file', None)
safety_policy_used = []
brief_data['policy_file'] = policy_file.get('filename', '-') if policy_file else None
brief_data['policy_file_source'] = 'server' if brief_data['policy_file'] and 'server-safety-policy' in brief_data['policy_file'] else 'local'
if policy_file and policy_file.get('filename', False):
safety_policy_used = [
{'style': False, 'value': '\nScanning using a security policy file'},
{'style': True, 'value': ' {0}'.format(policy_file.get('filename', '-'))},
]
audit_and_monitor = []
if context.params.get('audit_and_monitor'):
logged_url = context.params.get('audit_and_monitor_url') if context.params.get('audit_and_monitor_url') else "https://pyup.io"
audit_and_monitor = [
{'style': False, 'value': '\nLogging scan results to'},
{'style': True, 'value': ' {0}'.format(logged_url)},
]
current_time = str(datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
brief_data['api_key'] = bool(key)
brief_data['local_database_path'] = db if db else None
brief_data['safety_version'] = get_safety_version()
brief_data['timestamp'] = current_time
brief_data['packages_found'] = len(packages)
# Vuln report
additional_data = []
if report_type == 1:
brief_data['vulnerabilities_found'] = kwargs.get('vulnerabilities_found', 0)
brief_data['vulnerabilities_ignored'] = kwargs.get('vulnerabilities_ignored', 0)
brief_data['remediations_recommended'] = 0
additional_data = [
[{'style': True, 'value': str(brief_data['vulnerabilities_found'])},
{'style': True, 'value': f' vulnerabilit{"y" if brief_data["vulnerabilities_found"] == 1 else "ies"} found'}],
[{'style': True, 'value': str(brief_data['vulnerabilities_ignored'])},
{'style': True, 'value': f' vulnerabilit{"y" if brief_data["vulnerabilities_ignored"] == 1 else "ies"} ignored'}],
]
if is_using_api_key():
brief_data['remediations_recommended'] = kwargs.get('remediations_recommended', 0)
additional_data.extend(
[[{'style': True, 'value': str(brief_data['remediations_recommended'])},
{'style': True, 'value':
f' remediation{"" if brief_data["remediations_recommended"] == 1 else "s"} recommended'}]])
elif report_type == 2:
brief_data['licenses_found'] = kwargs.get('licenses_found', 0)
additional_data = [
[{'style': True, 'value': str(brief_data['licenses_found'])},
{'style': True, 'value': f' license {"type" if brief_data["licenses_found"] == 1 else "types"} found'}],
]
brief_data['telemetry'] = build_telemetry_data()
brief_data['git'] = build_git_data()
brief_data['project'] = context.params.get('project', None)
brief_data['json_version'] = 1
using_sentence = build_using_sentence(key, db)
using_sentence_section = [nl] if not using_sentence else [nl] + [build_using_sentence(key, db)]
scanned_count_sentence = build_scanned_count_sentence(packages)
timestamp = [{'style': False, 'value': 'Timestamp '}, {'style': True, 'value': current_time}]
brief_info = [[{'style': False, 'value': 'Safety '},
{'style': True, 'value': 'v' + get_safety_version()},
{'style': False, 'value': ' is scanning for '},
{'style': True, 'value': scanning_types.get(context.command, {}).get('name', '')},
{'style': True, 'value': '...'}] + safety_policy_used + audit_and_monitor, action_executed
] + [nl] + scanned_items + using_sentence_section + [scanned_count_sentence] + [timestamp]
brief_info.extend(additional_data)
add_warnings_if_needed(brief_info)
LOG.info('Brief info data: %s', brief_data)
LOG.info('Brief info, styled output: %s', '\n\n LINE ---->\n ' + '\n\n LINE ---->\n '.join(map(str, brief_info)))
return brief_data if as_dict else brief_info
def build_primary_announcement(primary_announcement, columns=None, only_text=False):
lines = json.loads(primary_announcement.get('message'))
for line in lines:
if 'words' not in line:
raise ValueError('Missing words keyword')
if len(line['words']) <= 0:
raise ValueError('No words in this line')
for word in line['words']:
if 'value' not in word or not word['value']:
raise ValueError('Empty word or without value')
message = style_lines(lines, columns, start_line='', end_line='')
return click.unstyle(message) if only_text else message
def is_using_api_key():
return bool(SafetyContext().key)
def is_using_a_safety_policy_file():
return bool(SafetyContext().params.get('policy_file', None))
def should_add_nl(output, found_vulns):
if output == 'bare' and not found_vulns:
return False
return True
pipenv/patched/safety/safety-policy-template.yml
+81
View File
@@ -0,0 +1,81 @@
# Safety Security and License Configuration file
# We recommend checking this file into your source control in the root of your Python project
# If this file is named .safety-policy.yml and is in the same directory where you run `safety check` it will be used by default.
# Otherwise, you can use the flag `safety check --policy-file <path-to-this-file>` to specify a custom location and name for the file.
# To validate and review your policy file, run the validate command: `safety validate policy_file --path <path-to-this-file>`
security: # configuration for the `safety check` command
ignore-cvss-severity-below: 0 # A severity number between 0 and 10. Some helpful reference points: 9=ignore all vulnerabilities except CRITICAL severity. 7=ignore all vulnerabilities except CRITICAL & HIGH severity. 4=ignore all vulnerabilities except CRITICAL, HIGH & MEDIUM severity.
ignore-cvss-unknown-severity: False # True or False. We recommend you set this to False.
ignore-vulnerabilities: # Here you can list multiple specific vulnerabilities you want to ignore (optionally for a time period)
# We recommend making use of the optional `reason` and `expires` keys for each vulnerability that you ignore.
25853: # Example vulnerability ID
reason: we don't use the vulnerable function # optional, for internal note purposes to communicate with your team. This reason will be reported in the Safety reports
expires: '2022-10-21' # datetime string - date this ignore will expire, best practice to use this variable
continue-on-vulnerability-error: False # Suppress non-zero exit codes when vulnerabilities are found. Enable this in pipelines and CI/CD processes if you want to pass builds that have vulnerabilities. We recommend you set this to False.
alert: # configuration for the `safety alert` command
security:
# Configuration specific to Safety's GitHub Issue alerting
github-issue:
# Same as for security - these allow controlling if this alert will fire based
# on severity information.
# default: not set
# ignore-cvss-severity-below: 6
# ignore-cvss-unknown-severity: False
# Add a label to pull requests with the cvss severity, if available
# default: true
# label-severity: True
# Add a label to pull requests, default is 'security'
# requires private repo permissions, even on public repos
# default: security
# labels:
# - security
# Assign users to pull requests, default is not set
# requires private repo permissions, even on public repos
# default: empty
# assignees:
# - example-user
# Prefix to give issues when creating them. Note that changing
# this might cause duplicate issues to be created.
# default: "[PyUp] "
# issue-prefix: "[PyUp] "
# Configuration specific to Safety's GitHub PR alerting
github-pr:
# Same as for security - these allow controlling if this alert will fire based
# on severity information.
# default: not set
# ignore-cvss-severity-below: 6
# ignore-cvss-unknown-severity: False
# Set the default branch (ie, main, master)
# default: empty, the default branch on GitHub
branch: ''
# Add a label to pull requests with the cvss severity, if available
# default: true
# label-severity: True
# Add a label to pull requests, default is 'security'
# requires private repo permissions, even on public repos
# default: security
# labels:
# - security
# Assign users to pull requests, default is not set
# requires private repo permissions, even on public repos
# default: empty
# assignees:
# - example-user
# Configure the branch prefix for PRs created by this alert.
# NB: Changing this will likely cause duplicate PRs.
# default: pyup/
branch-prefix: pyup/
# Set a global prefix for PRs
# default: "[PyUp] "
pr-prefix: "[PyUp] "
pipenv/patched/safety/safety.py
+478 -78
View File
@@ -1,42 +1,57 @@
# -*- coding: utf-8 -*-
import errno
import itertools
import json
import logging
import os
import sys
import time
from collections import namedtuple
from datetime import datetime
import pipenv.patched.pip._vendor.requests as requests
from pipenv.patched.pip._vendor.packaging.specifiers import SpecifierSet
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
from pipenv.patched.pip._vendor.packaging.version import parse as parse_version, Version, LegacyVersion, parse
from .constants import (API_MIRRORS, CACHE_FILE, CACHE_LICENSES_VALID_SECONDS,
CACHE_VALID_SECONDS, OPEN_MIRRORS, REQUEST_TIMEOUT)
from .constants import (API_MIRRORS, CACHE_FILE, OPEN_MIRRORS, REQUEST_TIMEOUT, API_BASE_URL)
from .errors import (DatabaseFetchError, DatabaseFileNotFoundError,
InvalidKeyError, TooManyRequestsError)
from .util import RequirementFile
InvalidKeyError, TooManyRequestsError, NetworkConnectionError,
RequestTimeoutError, ServerError, MalformedDatabase)
from .models import Vulnerability, CVE, Severity
from .util import RequirementFile, read_requirements, Package, build_telemetry_data, sync_safety_context, SafetyContext, \
validate_expiration_date, is_a_remote_mirror
session = requests.session()
LOG = logging.getLogger(__name__)
class Vulnerability(namedtuple("Vulnerability",
["name", "spec", "version", "advisory", "vuln_id", "cvssv2", "cvssv3"])):
pass
def get_from_cache(db_name):
def get_from_cache(db_name, cache_valid_seconds=0):
LOG.debug('Trying to get from cache...')
if os.path.exists(CACHE_FILE):
LOG.info('Cache file path: %s', CACHE_FILE)
with open(CACHE_FILE) as f:
try:
data = json.loads(f.read())
LOG.debug('Trying to get the %s from the cache file', db_name)
LOG.debug('Databases in CACHE file: %s', ', '.join(data))
if db_name in data:
if "cached_at" in data[db_name]:
if 'licenses.json' in db_name:
# Getting the specific cache time for the licenses db.
cache_valid_seconds = CACHE_LICENSES_VALID_SECONDS
else:
cache_valid_seconds = CACHE_VALID_SECONDS
LOG.debug('db_name %s', db_name)
if "cached_at" in data[db_name]:
if data[db_name]["cached_at"] + cache_valid_seconds > time.time():
LOG.debug('Getting the database from cache at %s, cache setting: %s',
data[db_name]["cached_at"], cache_valid_seconds)
return data[db_name]["db"]
LOG.debug('Cached file is too old, it was cached at %s', data[db_name]["cached_at"])
else:
LOG.debug('There is not the cached_at key in %s database', data[db_name])
except json.JSONDecodeError:
pass
LOG.debug('JSONDecodeError trying to get the cached database.')
else:
LOG.debug("Cache file doesn't exist...")
return False
@@ -58,7 +73,9 @@ def write_to_cache(db_name, data):
os.makedirs(os.path.dirname(CACHE_FILE))
with open(CACHE_FILE, "w") as _:
_.write(json.dumps({}))
LOG.debug('Cache file created')
except OSError as exc: # Guard against race condition
LOG.debug('Unable to create the cache file because: %s', exc.errno)
if exc.errno != errno.EEXIST:
raise
@@ -66,6 +83,7 @@ def write_to_cache(db_name, data):
try:
cache = json.loads(f.read())
except json.JSONDecodeError:
LOG.debug('JSONDecodeError in the local cache, dumping the full cache file.')
cache = {}
with open(CACHE_FILE, "w") as f:
@@ -74,51 +92,134 @@ def write_to_cache(db_name, data):
"db": data
}
f.write(json.dumps(cache))
LOG.debug('Safety updated the cache file for %s database.', db_name)
def fetch_database_url(mirror, db_name, key, cached, proxy):
def fetch_database_url(mirror, db_name, key, cached, proxy, telemetry=True):
headers = {}
if key:
headers["X-Api-Key"] = key
if not proxy:
proxy = {}
if cached:
cached_data = get_from_cache(db_name=db_name)
cached_data = get_from_cache(db_name=db_name, cache_valid_seconds=cached)
if cached_data:
LOG.info('Database %s returned from cache.', db_name)
return cached_data
url = mirror + db_name
r = requests.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy)
if r.status_code == 200:
telemetry_data = {'telemetry': json.dumps(build_telemetry_data(telemetry=telemetry))}
try:
r = session.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy, params=telemetry_data)
except requests.exceptions.ConnectionError:
raise NetworkConnectionError()
except requests.exceptions.Timeout:
raise RequestTimeoutError()
except requests.exceptions.RequestException:
raise DatabaseFetchError()
if r.status_code == 403:
raise InvalidKeyError(key=key, reason=r.text)
if r.status_code == 429:
raise TooManyRequestsError(reason=r.text)
if r.status_code != 200:
raise ServerError(reason=r.reason)
try:
data = r.json()
if cached:
write_to_cache(db_name, data)
return data
elif r.status_code == 403:
raise InvalidKeyError()
elif r.status_code == 429:
raise TooManyRequestsError()
except json.JSONDecodeError as e:
raise MalformedDatabase(reason=e)
if cached:
LOG.info('Writing %s to cache because cached value was %s', db_name, cached)
write_to_cache(db_name, data)
return data
def fetch_policy(key, proxy):
url = f"{API_BASE_URL}policy/"
headers = {"X-Api-Key": key}
if not proxy:
proxy = {}
try:
LOG.debug(f'Getting policy')
r = session.get(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy)
LOG.debug(r.text)
return r.json()
except:
import pipenv.vendor.click as click
LOG.exception("Error fetching policy")
click.secho(
"Warning: couldn't fetch policy from pyup.io.",
fg="yellow",
file=sys.stderr
)
return {"safety_policy": "", "audit_and_monitor": False}
def post_results(key, proxy, safety_json, policy_file):
url = f"{API_BASE_URL}result/"
headers = {"X-Api-Key": key}
if not proxy:
proxy = {}
# safety_json is in text form already. policy_file is a text YAML
audit_report = {
"safety_json": json.loads(safety_json),
"policy_file": policy_file
}
try:
LOG.debug(f'Posting results: {audit_report}')
r = session.post(url=url, timeout=REQUEST_TIMEOUT, headers=headers, proxies=proxy, json=audit_report)
LOG.debug(r.text)
return r.json()
except:
import pipenv.vendor.click as click
LOG.exception("Error posting results")
click.secho(
"Warning: couldn't upload results to pyup.io.",
fg="yellow",
file=sys.stderr
)
return {}
def fetch_database_file(path, db_name):
full_path = os.path.join(path, db_name)
if not os.path.exists(full_path):
raise DatabaseFileNotFoundError()
raise DatabaseFileNotFoundError(db=path)
with open(full_path) as f:
return json.loads(f.read())
def fetch_database(full=False, key=False, db=False, cached=False, proxy={}):
if db:
def fetch_database(full=False, key=False, db=False, cached=0, proxy=None, telemetry=True):
if key:
mirrors = API_MIRRORS
elif db:
mirrors = [db]
else:
mirrors = API_MIRRORS if key else OPEN_MIRRORS
mirrors = OPEN_MIRRORS
db_name = "insecure_full.json" if full else "insecure.json"
for mirror in mirrors:
# mirror can either be a local path or a URL
if mirror.startswith("http://") or mirror.startswith("https://"):
data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy)
if is_a_remote_mirror(mirror):
data = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy, telemetry=telemetry)
else:
data = fetch_database_file(mirror, db_name=db_name)
if data:
@@ -133,12 +234,97 @@ def get_vulnerabilities(pkg, spec, db):
yield entry
def check(packages, key, db_mirror, cached, ignore_ids, proxy):
key = key if key else os.environ.get("SAFETY_API_KEY", False)
db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy)
def get_vulnerability_from(vuln_id, cve, data, specifier, db, name, pkg, ignore_vulns):
base_domain = db.get('$meta', {}).get('base_domain')
pkg_meta = db.get('$meta', {}).get('packages', {}).get(name, {})
insecure_versions = pkg_meta.get("insecure_versions", [])
secure_versions = pkg_meta.get("secure_versions", [])
latest_version_without_known_vulnerabilities = pkg_meta.get("latest_secure_version", None)
latest_version = pkg_meta.get("latest_version", None)
pkg_refreshed = pkg._replace(insecure_versions=insecure_versions, secure_versions=secure_versions,
latest_version_without_known_vulnerabilities=latest_version_without_known_vulnerabilities,
latest_version=latest_version,
more_info_url=f"{base_domain}{pkg_meta.get('more_info_path', '')}")
ignored = (ignore_vulns and vuln_id in ignore_vulns and (
not ignore_vulns[vuln_id]['expires'] or ignore_vulns[vuln_id]['expires'] > datetime.utcnow()))
more_info_url = f"{base_domain}{data.get('more_info_path', '')}"
severity = None
if cve and (cve.cvssv2 or cve.cvssv3):
severity = Severity(source=cve.name, cvssv2=cve.cvssv2, cvssv3=cve.cvssv3)
return Vulnerability(
vulnerability_id=vuln_id,
package_name=name,
pkg=pkg_refreshed,
ignored=ignored,
ignored_reason=ignore_vulns.get(vuln_id, {}).get('reason', None) if ignore_vulns else None,
ignored_expires=ignore_vulns.get(vuln_id, {}).get('expires', None) if ignore_vulns else None,
vulnerable_spec=specifier,
all_vulnerable_specs=data.get("specs", []),
analyzed_version=pkg_refreshed.version,
advisory=data.get("advisory"),
is_transitive=data.get("transitive", False),
published_date=data.get("published_date"),
fixed_versions=[ver for ver in data.get("fixed_versions", []) if ver],
closest_versions_without_known_vulnerabilities=data.get("closest_secure_versions", []),
resources=data.get("vulnerability_resources"),
CVE=cve,
severity=severity,
affected_versions=data.get("affected_versions", []),
more_info_url=more_info_url
)
def get_cve_from(data, db_full):
cve_data = data.get("cve", '')
if not cve_data:
return None
cve_id = cve_data.split(",")[0].strip()
cve_meta = db_full.get("$meta", {}).get("cve", {}).get(cve_id, {})
return CVE(name=cve_id, cvssv2=cve_meta.get("cvssv2", None),
cvssv3=cve_meta.get("cvssv3", None))
def ignore_vuln_if_needed(vuln_id, cve, ignore_vulns, ignore_severity_rules):
if not ignore_severity_rules or not isinstance(ignore_vulns, dict):
return
severity = None
if cve:
if cve.cvssv2 and cve.cvssv2.get("base_score", None):
severity = cve.cvssv2.get("base_score", None)
if cve.cvssv3 and cve.cvssv3.get("base_score", None):
severity = cve.cvssv3.get("base_score", None)
ignore_severity_below = float(ignore_severity_rules.get('ignore-cvss-severity-below', 0.0))
ignore_unknown_severity = bool(ignore_severity_rules.get('ignore-cvss-unknown-severity', False))
if severity:
if float(severity) < ignore_severity_below:
reason = 'Ignored by severity rule in policy file, {0} < {1}'.format(float(severity),
ignore_severity_below)
ignore_vulns[vuln_id] = {'reason': reason, 'expires': None}
elif ignore_unknown_severity:
reason = 'Unknown CVSS severity, ignored by severity rule in policy file.'
ignore_vulns[vuln_id] = {'reason': reason, 'expires': None}
@sync_safety_context
def check(packages, key=False, db_mirror=False, cached=0, ignore_vulns=None, ignore_severity_rules=None, proxy=None,
include_ignored=False, is_env_scan=True, telemetry=True, params=None, project=None):
SafetyContext().command = 'check'
db = fetch_database(key=key, db=db_mirror, cached=cached, proxy=proxy, telemetry=telemetry)
db_full = None
vulnerable_packages = frozenset(db.keys())
vulnerable = []
vulnerabilities = []
for pkg in packages:
# Ignore recursive files not resolved
if isinstance(pkg, RequirementFile):
@@ -146,7 +332,7 @@ def check(packages, key, db_mirror, cached, ignore_ids, proxy):
# normalize the package name, the safety-db is converting underscores to dashes and uses
# lowercase
name = pkg.key.replace("_", "-").lower()
name = canonicalize_name(pkg.name)
if name in vulnerable_packages:
# we have a candidate here, build the spec set
@@ -154,51 +340,175 @@ def check(packages, key, db_mirror, cached, ignore_ids, proxy):
spec_set = SpecifierSet(specifiers=specifier)
if spec_set.contains(pkg.version):
if not db_full:
db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy)
db_full = fetch_database(full=True, key=key, db=db_mirror, cached=cached, proxy=proxy,
telemetry=telemetry)
for data in get_vulnerabilities(pkg=name, spec=specifier, db=db_full):
vuln_id = data.get("id").replace("pyup.io-", "")
cve_id = data.get("cve")
if cve_id:
cve_id = cve_id.split(",")[0].strip()
if vuln_id and vuln_id not in ignore_ids:
cve_meta = db_full.get("$meta", {}).get("cve", {}).get(cve_id, {})
vulnerable.append(
Vulnerability(
name=name,
spec=specifier,
version=pkg.version,
advisory=data.get("advisory"),
vuln_id=vuln_id,
cvssv2=cve_meta.get("cvssv2", None),
cvssv3=cve_meta.get("cvssv3", None),
)
)
return vulnerable
cve = get_cve_from(data, db_full)
ignore_vuln_if_needed(vuln_id, cve, ignore_vulns, ignore_severity_rules)
vulnerability = get_vulnerability_from(vuln_id, cve, data, specifier, db_full, name, pkg,
ignore_vulns)
should_add_vuln = not (vulnerability.is_transitive and is_env_scan)
if (include_ignored or vulnerability.vulnerability_id not in ignore_vulns) and should_add_vuln:
vulnerabilities.append(vulnerability)
return vulnerabilities, db_full
def review(vulnerabilities):
def precompute_remediations(remediations, package_metadata, vulns,
ignored_vulns):
for vuln in vulns:
if vuln.ignored:
ignored_vulns.add(vuln.vulnerability_id)
continue
if vuln.package_name in remediations.keys():
remediations[vuln.package_name]['vulns_found'] = remediations[vuln.package_name].get('vulns_found', 0) + 1
else:
vulns_count = 1
package_metadata[vuln.package_name] = {'insecure_versions': vuln.pkg.insecure_versions,
'secure_versions': vuln.pkg.secure_versions, 'version': vuln.pkg.version}
remediations[vuln.package_name] = {'vulns_found': vulns_count, 'version': vuln.pkg.version,
'more_info_url': vuln.pkg.more_info_url}
def get_closest_ver(versions, version):
results = {'minor': None, 'major': None}
if not version or not versions:
return results
sorted_versions = sorted(versions, key=lambda ver: parse_version(ver), reverse=True)
for v in sorted_versions:
index = parse_version(v)
current_v = parse_version(version)
if index > current_v:
results['major'] = index
if index < current_v:
results['minor'] = index
break
return results
def compute_sec_ver_for_user(package, ignored_vulns, db_full):
pkg_meta = db_full.get('$meta', {}).get('packages', {}).get(package, {})
versions = set(pkg_meta.get("insecure_versions", []) + pkg_meta.get("secure_versions", []))
affected_versions = []
for vuln in db_full.get(package, []):
vuln_id = vuln.get('id', None)
if vuln_id and vuln_id not in ignored_vulns:
affected_versions += vuln.get('affected_versions', [])
affected_v = set(affected_versions)
sec_ver_for_user = list(versions.difference(affected_v))
return sorted(sec_ver_for_user, key=lambda ver: parse_version(ver), reverse=True)
def compute_sec_ver(remediations, package_metadata, ignored_vulns, db_full):
"""
Compute the secure_versions and the closest_secure_version for each remediation using the affected_versions
of each no ignored vulnerability of the same package, there is only a remediation for each package.
"""
for pkg_name in remediations.keys():
pkg = package_metadata.get(pkg_name, {})
if not ignored_vulns:
secure_v = pkg.get('secure_versions', [])
else:
secure_v = compute_sec_ver_for_user(package=pkg_name, ignored_vulns=ignored_vulns, db_full=db_full)
remediations[pkg_name]['secure_versions'] = secure_v
remediations[pkg_name]['closest_secure_version'] = get_closest_ver(secure_v,
pkg.get('version', None))
def calculate_remediations(vulns, db_full):
remediations = {}
package_metadata = {}
ignored_vulns = set()
if not db_full:
return remediations
precompute_remediations(remediations, package_metadata, vulns, ignored_vulns)
compute_sec_ver(remediations, package_metadata, ignored_vulns, db_full)
return remediations
@sync_safety_context
def review(report=None, params=None):
SafetyContext().command = 'review'
vulnerable = []
vulnerabilities = report.get('vulnerabilities', []) + report.get('ignored_vulnerabilities', [])
remediations = {}
for key, value in report.get('remediations', {}).items():
recommended = value.get('recommended_version', None)
secure_v = value.get('other_recommended_versions', [])
major = None
if recommended:
secure_v.append(recommended)
major = parse(recommended)
remediations[key] = {'vulns_found': value.get('vulnerabilities_found', 0),
'version': value.get('current_version'),
'secure_versions': secure_v,
'closest_secure_version': {'major': major, 'minor': None},
# minor isn't supported in review
'more_info_url': value.get('more_info_url')}
packages = report.get('scanned_packages', [])
pkgs = {pkg_name: Package(**pkg_values) for pkg_name, pkg_values in packages.items()}
ctx = SafetyContext()
found_packages = list(pkgs.values())
ctx.packages = found_packages
ctx.review = report.get('report_meta', [])
ctx.key = ctx.review.get('api_key', False)
cvssv2 = None
cvssv3 = None
for vuln in vulnerabilities:
current_vuln = {
"name": vuln[0],
"spec": vuln[1],
"version": vuln[2],
"advisory": vuln[3],
"vuln_id": vuln[4],
"cvssv2": None,
"cvssv3": None
}
vulnerable.append(
Vulnerability(**current_vuln)
)
return vulnerable
vuln['pkg'] = pkgs.get(vuln.get('package_name', None))
XVE_ID = vuln.get('CVE', None) # Trying to get first the CVE ID
severity = vuln.get('severity', None)
if severity and severity.get('source', False):
cvssv2 = severity.get('cvssv2', None)
cvssv3 = severity.get('cvssv3', None)
# Trying to get the PVE ID if it exists, otherwise it will be the same CVE ID of above
XVE_ID = severity.get('source', False)
vuln['severity'] = Severity(source=XVE_ID, cvssv2=cvssv2, cvssv3=cvssv3)
else:
vuln['severity'] = None
ignored_expires = vuln.get('ignored_expires', None)
if ignored_expires:
vuln['ignored_expires'] = validate_expiration_date(ignored_expires)
vuln['CVE'] = CVE(name=XVE_ID, cvssv2=cvssv2, cvssv3=cvssv3) if XVE_ID else None
vulnerable.append(Vulnerability(**vuln))
return vulnerable, remediations, found_packages
def get_licenses(key, db_mirror, cached, proxy):
@sync_safety_context
def get_licenses(key=False, db_mirror=False, cached=0, proxy=None, telemetry=True):
key = key if key else os.environ.get("SAFETY_API_KEY", False)
if not key and not db_mirror:
raise InvalidKeyError("The API-KEY was not provided.")
raise InvalidKeyError(message="The API-KEY was not provided.")
if db_mirror:
mirrors = [db_mirror]
else:
@@ -208,10 +518,100 @@ def get_licenses(key, db_mirror, cached, proxy):
for mirror in mirrors:
# mirror can either be a local path or a URL
if mirror.startswith("http://") or mirror.startswith("https://"):
licenses = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy)
if is_a_remote_mirror(mirror):
licenses = fetch_database_url(mirror, db_name=db_name, key=key, cached=cached, proxy=proxy,
telemetry=telemetry)
else:
licenses = fetch_database_file(mirror, db_name=db_name)
if licenses:
return licenses
raise DatabaseFetchError()
def get_announcements(key, proxy, telemetry=True):
LOG.info('Getting announcements')
announcements = []
headers = {}
if key:
headers["X-Api-Key"] = key
url = f"{API_BASE_URL}announcements/"
method = 'post'
data = build_telemetry_data(telemetry=telemetry)
request_kwargs = {'headers': headers, 'proxies': proxy, 'timeout': 3}
data_keyword = 'json'
source = os.environ.get('SAFETY_ANNOUNCEMENTS_URL', None)
if source:
LOG.debug(f'Getting the announcement from a different source: {source}')
url = source
method = 'get'
data = {
'telemetry': json.dumps(data)}
data_keyword = 'params'
request_kwargs[data_keyword] = data
request_kwargs['url'] = url
LOG.debug(f'Telemetry data sent: {data}')
try:
request_func = getattr(session, method)
r = request_func(**request_kwargs)
LOG.debug(r.text)
except Exception as e:
LOG.info('Unexpected but HANDLED Exception happened getting the announcements: %s', e)
return announcements
if r.status_code == 200:
try:
announcements = r.json()
if 'announcements' in announcements.keys():
announcements = announcements.get('announcements', [])
else:
LOG.info('There is not announcements key in the JSON response, is this a wrong structure?')
announcements = []
except json.JSONDecodeError as e:
LOG.info('Unexpected but HANDLED Exception happened decoding the announcement response: %s', e)
LOG.info('Announcements fetched')
return announcements
def get_packages(files=False, stdin=False):
if files:
return list(itertools.chain.from_iterable(read_requirements(f, resolve=True) for f in files))
if stdin:
return list(read_requirements(sys.stdin))
import pipenv.patched.pip._vendor.pkg_resources as pkg_resources
return [
Package(name=d.key, version=d.version, found=d.location, insecure_versions=[], secure_versions=[],
latest_version=None, latest_version_without_known_vulnerabilities=None, more_info_url=None) for d in
pkg_resources.working_set
if d.key not in {"python", "wsgiref", "argparse"}
]
def read_vulnerabilities(fh):
try:
data = json.load(fh)
except json.JSONDecodeError as e:
raise MalformedDatabase(reason=e, fetched_from=fh.name)
except TypeError as e:
raise MalformedDatabase(reason=e, fetched_from=fh.name)
return data
def close_session():
LOG.debug('Closing requests session.')
session.close()
pipenv/patched/safety/util.py
+601 -103
View File
@@ -1,33 +1,39 @@
from pipenv.vendor.dparse.parser import setuptools_parse_requirements_backport as _parse_requirements
from collections import namedtuple
from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
import pipenv.vendor.click as click
import sys
import json
import logging
import os
Package = namedtuple("Package", ["key", "version"])
RequirementFile = namedtuple("RequirementFile", ["path"])
import platform
import sys
from datetime import datetime
from difflib import SequenceMatcher
from threading import Lock
from typing import List
import pipenv.vendor.click as click
from pipenv.vendor.click import BadParameter
from pipenv.vendor.dparse import parse, filetypes
from pipenv.patched.pip._vendor.packaging.utils import canonicalize_name
from pipenv.patched.pip._vendor.packaging.version import parse as parse_version
from pipenv.vendor.ruamel.yaml import YAML
from pipenv.vendor.ruamel.yaml.error import MarkedYAMLError
from pipenv.patched.safety.constants import EXIT_CODE_FAILURE, EXIT_CODE_OK
from pipenv.patched.safety.models import Package, RequirementFile
LOG = logging.getLogger(__name__)
def read_vulnerabilities(fh):
return json.load(fh)
def is_a_remote_mirror(mirror):
return mirror.startswith("http://") or mirror.startswith("https://")
def iter_lines(fh, lineno=0):
for line in fh.readlines()[lineno:]:
yield line
def is_supported_by_parser(path):
supported_types = (".txt", ".in", ".yml", ".ini", "Pipfile",
"Pipfile.lock", "setup.cfg", "poetry.lock")
return path.endswith(supported_types)
def parse_line(line):
if line.startswith('-e') or line.startswith('http://') or line.startswith('https://'):
if "#egg=" in line:
line = line.split("#egg=")[-1]
if ' --hash' in line:
line = line.split(" --hash")[0]
return _parse_requirements(line)
def read_requirements(fh, resolve=False):
def read_requirements(fh, resolve=True):
"""
Reads requirements from a file like object and (optionally) from referenced files.
:param fh: file like object to read from
@@ -35,88 +41,52 @@ def read_requirements(fh, resolve=False):
:return: generator
"""
is_temp_file = not hasattr(fh, 'name')
for num, line in enumerate(iter_lines(fh)):
line = line.strip()
if not line:
# skip empty lines
continue
if line.startswith('#') or \
line.startswith('-i') or \
line.startswith('--index-url') or \
line.startswith('--extra-index-url') or \
line.startswith('-f') or line.startswith('--find-links') or \
line.startswith('--no-index') or line.startswith('--allow-external') or \
line.startswith('--allow-unverified') or line.startswith('-Z') or \
line.startswith('--always-unzip'):
# skip unsupported lines
continue
elif line.startswith('-r') or line.startswith('--requirement'):
# got a referenced file here, try to resolve the path
# if this is a tempfile, skip
if is_temp_file:
continue
path = None
found = 'temp_file'
file_type = filetypes.requirements_txt
# strip away the recursive flag
prefixes = ["-r", "--requirement"]
filename = line.strip()
for prefix in prefixes:
if filename.startswith(prefix):
filename = filename[len(prefix):].strip()
if not is_temp_file and is_supported_by_parser(fh.name):
LOG.debug('not temp and a compatible file')
path = fh.name
found = path
file_type = None
# if there is a comment, remove it
if " #" in filename:
filename = filename.split(" #")[0].strip()
req_file_path = os.path.join(os.path.dirname(fh.name), filename)
if resolve:
# recursively yield the resolved requirements
if os.path.exists(req_file_path):
with open(req_file_path) as _fh:
for req in read_requirements(_fh, resolve=True):
yield req
else:
yield RequirementFile(path=req_file_path)
else:
try:
parseable_line = line
# multiline requirements are not parseable
if "\\" in line:
parseable_line = line.replace("\\", "")
for next_line in iter_lines(fh, num + 1):
parseable_line += next_line.strip().replace("\\", "")
line += "\n" + next_line
if "\\" in next_line:
continue
break
req, = parse_line(parseable_line)
if len(req.specifier._specs) == 1 and \
next(iter(req.specifier._specs))._spec[0] == "==":
yield Package(key=req.name, version=next(iter(req.specifier._specs))._spec[1])
else:
try:
fname = fh.name
except AttributeError:
fname = line
LOG.debug(f'Path: {path}')
LOG.debug(f'File Type: {file_type}')
LOG.debug('Trying to parse file using dparse...')
content = fh.read()
LOG.debug(f'Content: {content}')
dependency_file = parse(content, path=path, resolve=resolve,
file_type=file_type)
LOG.debug(f'Dependency file: {dependency_file.serialize()}')
LOG.debug(f'Parsed, dependencies: {[dep.serialize() for dep in dependency_file.resolved_dependencies]}')
for dep in dependency_file.resolved_dependencies:
try:
spec = next(iter(dep.specs))._spec
except StopIteration:
click.secho(
f"Warning: unpinned requirement '{dep.name}' found in {path}, "
"unable to check.",
fg="yellow",
file=sys.stderr
)
return
click.secho(
"Warning: unpinned requirement '{req}' found in {fname}, "
"unable to check.".format(req=req.name,
fname=fname),
fg="yellow",
file=sys.stderr
)
except ValueError:
continue
version = spec[1]
if spec[0] == '==':
yield Package(name=dep.name, version=version,
found=found,
insecure_versions=[],
secure_versions=[], latest_version=None,
latest_version_without_known_vulnerabilities=None,
more_info_url=None)
def get_proxy_dict(proxyprotocol, proxyhost, proxyport):
proxy_dictionary = {}
if proxyhost is not None:
if proxyprotocol in ["http", "https"]:
proxy_dictionary = {proxyprotocol: "{0}://{1}:{2}".format(proxyprotocol, proxyhost, str(proxyport))}
else:
click.secho("Proxy Protocol should be http or https only.", fg="red")
sys.exit(-1)
return proxy_dictionary
def get_proxy_dict(proxy_protocol, proxy_host, proxy_port):
if proxy_protocol and proxy_host and proxy_port:
# Safety only uses https request, so only https dict will be passed to requests
return {'https': f"{proxy_protocol}://{proxy_host}:{str(proxy_port)}"}
return None
def get_license_name_by_id(license_id, db):
@@ -126,13 +96,541 @@ def get_license_name_by_id(license_id, db):
return name
return None
def get_packages_licenses(packages, licenses_db):
"""Get the licenses for the specified packages based on their version.
def get_flags_from_context():
flags = {}
context = click.get_current_context(silent=True)
if context:
for option in context.command.params:
flags_per_opt = option.opts + option.secondary_opts
for flag in flags_per_opt:
flags[flag] = option.name
return flags
def get_used_options():
flags = get_flags_from_context()
used_options = {}
for arg in sys.argv:
cleaned_arg = arg if '=' not in arg else arg.split('=')[0]
if cleaned_arg in flags:
option_used = flags.get(cleaned_arg)
if option_used in used_options:
used_options[option_used][cleaned_arg] = used_options[option_used].get(cleaned_arg, 0) + 1
else:
used_options[option_used] = {cleaned_arg: 1}
return used_options
def get_safety_version():
from pipenv.patched.safety import VERSION
return VERSION
def get_primary_announcement(announcements):
for announcement in announcements:
if announcement.get('type', '').lower() == 'primary_announcement':
try:
from pipenv.patched.safety.output_utils import build_primary_announcement
build_primary_announcement(announcement, columns=80)
except Exception as e:
LOG.debug(f'Failed to build primary announcement: {str(e)}')
return None
return announcement
return None
def get_basic_announcements(announcements):
return [announcement for announcement in announcements if
announcement.get('type', '').lower() != 'primary_announcement']
def filter_announcements(announcements, by_type='error'):
return [announcement for announcement in announcements if
announcement.get('type', '').lower() == by_type]
def build_telemetry_data(telemetry=True):
context = SafetyContext()
body = {
'os_type': os.environ.get("SAFETY_OS_TYPE", None) or platform.system(),
'os_release': os.environ.get("SAFETY_OS_RELEASE", None) or platform.release(),
'os_description': os.environ.get("SAFETY_OS_DESCRIPTION", None) or platform.platform(),
'python_version': platform.python_version(),
'safety_command': context.command,
'safety_options': get_used_options()
} if telemetry else {}
body['safety_version'] = get_safety_version()
body['safety_source'] = os.environ.get("SAFETY_SOURCE", None) or context.safety_source
LOG.debug(f'Telemetry body built: {body}')
return body
def build_git_data():
import subprocess
def git_command(commandline):
return subprocess.run(commandline, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL).stdout.decode('utf-8').strip()
try:
is_git = git_command(["git", "rev-parse", "--is-inside-work-tree"])
except Exception:
is_git = False
if is_git == "true":
result = {
"branch": "",
"tag": "",
"commit": "",
"dirty": "",
"origin": ""
}
try:
result['branch'] = git_command(["git", "symbolic-ref", "--short", "-q", "HEAD"])
result['tag'] = git_command(["git", "describe", "--tags", "--exact-match"])
commit = git_command(["git", "describe", '--match=""', '--always', '--abbrev=40', '--dirty'])
result['dirty'] = commit.endswith('-dirty')
result['commit'] = commit.split("-dirty")[0]
result['origin'] = git_command(["git", "remote", "get-url", "origin"])
except Exception:
pass
return result
else:
return {
"error": "not-git-repo"
}
def output_exception(exception, exit_code_output=True):
click.secho(str(exception), fg="red", file=sys.stderr)
if exit_code_output:
exit_code = EXIT_CODE_FAILURE
if hasattr(exception, 'get_exit_code'):
exit_code = exception.get_exit_code()
else:
exit_code = EXIT_CODE_OK
sys.exit(exit_code)
def get_processed_options(policy_file, ignore, ignore_severity_rules, exit_code):
if policy_file:
security = policy_file.get('security', {})
source = click.get_current_context().get_parameter_source("exit_code")
if not ignore:
ignore = security.get('ignore-vulnerabilities', {})
if source == click.core.ParameterSource.DEFAULT:
exit_code = not security.get('continue-on-vulnerability-error', False)
ignore_cvss_below = security.get('ignore-cvss-severity-below', 0.0)
ignore_cvss_unknown = security.get('ignore-cvss-unknown-severity', False)
ignore_severity_rules = {'ignore-cvss-severity-below': ignore_cvss_below,
'ignore-cvss-unknown-severity': ignore_cvss_unknown}
return ignore, ignore_severity_rules, exit_code
class MutuallyExclusiveOption(click.Option):
def __init__(self, *args, **kwargs):
self.mutually_exclusive = set(kwargs.pop('mutually_exclusive', []))
self.with_values = kwargs.pop('with_values', {})
help = kwargs.get('help', '')
if self.mutually_exclusive:
ex_str = ', '.join(["{0} with values {1}".format(item, self.with_values.get(item)) if item in self.with_values else item for item in self.mutually_exclusive])
kwargs['help'] = help + (
' NOTE: This argument is mutually exclusive with '
' arguments: [' + ex_str + '].'
)
super(MutuallyExclusiveOption, self).__init__(*args, **kwargs)
def handle_parse_result(self, ctx, opts, args):
m_exclusive_used = self.mutually_exclusive.intersection(opts)
option_used = m_exclusive_used and self.name in opts
exclusive_value_used = False
for used in m_exclusive_used:
value_used = opts.get(used, None)
if not isinstance(value_used, List):
value_used = [value_used]
if value_used and set(self.with_values.get(used, [])).intersection(value_used):
exclusive_value_used = True
if option_used and (not self.with_values or exclusive_value_used):
options = ', '.join(self.opts)
prohibited = ''.join(["\n * --{0} with {1}".format(item, self.with_values.get(
item)) if item in self.with_values else f"\n * {item}" for item in self.mutually_exclusive])
raise click.UsageError(
f"Illegal usage: `{options}` is mutually exclusive with: {prohibited}"
)
return super(MutuallyExclusiveOption, self).handle_parse_result(
ctx,
opts,
args
)
class DependentOption(click.Option):
def __init__(self, *args, **kwargs):
self.required_options = set(kwargs.pop('required_options', []))
help = kwargs.get('help', '')
if self.required_options:
ex_str = ', '.join(self.required_options)
kwargs['help'] = help + (
' NOTE: This argument requires the following flags '
' [' + ex_str + '].'
)
super(DependentOption, self).__init__(*args, **kwargs)
def handle_parse_result(self, ctx, opts, args):
missing_required_arguments = self.required_options.difference(opts) and self.name in opts
if missing_required_arguments:
raise click.UsageError(
"Illegal usage: `{}` needs the "
"arguments `{}`.".format(
self.name,
', '.join(missing_required_arguments)
)
)
return super(DependentOption, self).handle_parse_result(
ctx,
opts,
args
)
def transform_ignore(ctx, param, value):
if isinstance(value, tuple):
return dict(zip(value, [{'reason': '', 'expires': None} for _ in range(len(value))]))
return {}
def active_color_if_needed(ctx, param, value):
if value == 'screen':
ctx.color = True
color = os.environ.get("SAFETY_COLOR", None)
if color is not None:
color = color.lower()
if color == '1' or color == 'true':
ctx.color = True
elif color == '0' or color == 'false':
ctx.color = False
return value
def json_alias(ctx, param, value):
if value:
os.environ['SAFETY_OUTPUT'] = 'json'
return value
def bare_alias(ctx, param, value):
if value:
os.environ['SAFETY_OUTPUT'] = 'bare'
return value
def get_terminal_size():
from shutil import get_terminal_size as t_size
# get_terminal_size can report 0, 0 if run from pseudo-terminal prior Python 3.11 versions
columns = t_size().columns or 80
lines = t_size().lines or 24
return os.terminal_size((columns, lines))
def validate_expiration_date(expiration_date):
d = None
if expiration_date:
try:
d = datetime.strptime(expiration_date, '%Y-%m-%d')
except ValueError as e:
pass
try:
d = datetime.strptime(expiration_date, '%Y-%m-%d %H:%M:%S')
except ValueError as e:
pass
return d
class SafetyPolicyFile(click.ParamType):
"""
Custom Safety Policy file to hold validations
"""
name = "filename"
envvar_list_splitter = os.path.pathsep
def __init__(
self,
mode: str = "r",
encoding: str = None,
errors: str = "strict",
pure: bool = os.environ.get('SAFETY_PURE_YAML', 'false').lower() == 'true'
) -> None:
self.mode = mode
self.encoding = encoding
self.errors = errors
self.basic_msg = '\n' + click.style('Unable to load the Safety Policy file "{name}".', fg='red')
self.pure = pure
def to_info_dict(self):
info_dict = super().to_info_dict()
info_dict.update(mode=self.mode, encoding=self.encoding)
return info_dict
def fail_if_unrecognized_keys(self, used_keys, valid_keys, param=None, ctx=None, msg='{hint}', context_hint=''):
for keyword in used_keys:
if keyword not in valid_keys:
match = None
max_ratio = 0.0
if isinstance(keyword, str):
for option in valid_keys:
ratio = SequenceMatcher(None, keyword, option).ratio()
if ratio > max_ratio:
match = option
max_ratio = ratio
maybe_msg = f' Maybe you meant: {match}' if max_ratio > 0.7 else \
f' Valid keywords in this level are: {", ".join(valid_keys)}'
self.fail(msg.format(hint=f'{context_hint}"{keyword}" is not a valid keyword.{maybe_msg}'), param, ctx)
def fail_if_wrong_bool_value(self, keyword, value, msg='{hint}'):
if value is not None and not isinstance(value, bool):
self.fail(msg.format(hint=f"'{keyword}' value needs to be a boolean. "
"You can use True, False, TRUE, FALSE, true or false"))
def convert(self, value, param, ctx):
try:
if hasattr(value, "read") or hasattr(value, "write"):
return value
msg = self.basic_msg.format(name=value) + '\n' + click.style('HINT:', fg='yellow') + ' {hint}'
f, _ = click.types.open_stream(
value, self.mode, self.encoding, self.errors, atomic=False
)
filename = ''
try:
raw = f.read()
yaml = YAML(typ='safe', pure=self.pure)
safety_policy = yaml.load(raw)
filename = f.name
f.close()
except Exception as e:
show_parsed_hint = isinstance(e, MarkedYAMLError)
hint = str(e)
if show_parsed_hint:
hint = f'{str(e.problem).strip()} {str(e.context).strip()} {str(e.context_mark).strip()}'
self.fail(msg.format(name=value, hint=hint), param, ctx)
if not safety_policy or not isinstance(safety_policy, dict) or not safety_policy.get('security', None):
self.fail(
msg.format(hint='you are missing the security root tag'), param, ctx)
security_config = safety_policy.get('security', {})
security_keys = ['ignore-cvss-severity-below', 'ignore-cvss-unknown-severity', 'ignore-vulnerabilities',
'continue-on-vulnerability-error']
used_keys = security_config.keys()
self.fail_if_unrecognized_keys(used_keys, security_keys, param=param, ctx=ctx, msg=msg,
context_hint='"security" -> ')
ignore_cvss_security_below = security_config.get('ignore-cvss-severity-below', None)
if ignore_cvss_security_below:
limit = 0.0
try:
limit = float(ignore_cvss_security_below)
except ValueError as e:
self.fail(msg.format(hint="'ignore-cvss-severity-below' value needs to be an integer or float."))
if limit < 0 or limit > 10:
self.fail(msg.format(hint="'ignore-cvss-severity-below' needs to be a value between 0 and 10"))
continue_on_vulnerability_error = security_config.get('continue-on-vulnerability-error', None)
self.fail_if_wrong_bool_value('continue-on-vulnerability-error', continue_on_vulnerability_error, msg)
ignore_cvss_unknown_severity = security_config.get('ignore-cvss-unknown-severity', None)
self.fail_if_wrong_bool_value('ignore-cvss-unknown-severity', ignore_cvss_unknown_severity, msg)
ignore_vulns = safety_policy.get('security', {}).get('ignore-vulnerabilities', {})
if ignore_vulns:
if not isinstance(ignore_vulns, dict):
self.fail(msg.format(hint="Vulnerability IDs under the 'ignore-vulnerabilities' key, need to "
"follow the convention 'ID_NUMBER:', probably you are missing a colon."))
normalized = {}
for ignored_vuln_id, config in ignore_vulns.items():
ignored_vuln_config = config if config else {}
if not isinstance(ignored_vuln_config, dict):
self.fail(
msg.format(hint=f"Wrong configuration under the vulnerability with ID: {ignored_vuln_id}"))
context_msg = f'"security" -> "ignore-vulnerabilities" -> "{ignored_vuln_id}" -> '
self.fail_if_unrecognized_keys(ignored_vuln_config.keys(), ['reason', 'expires'], param=param,
ctx=ctx, msg=msg, context_hint=context_msg)
reason = ignored_vuln_config.get('reason', '')
reason = str(reason) if reason else None
expires = ignored_vuln_config.get('expires', '')
expires = str(expires) if expires else None
try:
if int(ignored_vuln_id) < 0:
raise ValueError('Negative Vulnerability ID')
except ValueError as e:
self.fail(msg.format(
hint=f"vulnerability id {ignored_vuln_id} under the 'ignore-vulnerabilities' root needs to "
f"be a positive integer")
)
# Validate expires
d = validate_expiration_date(expires)
if expires and not d:
self.fail(msg.format(hint=f"{context_msg}expires: \"{expires}\" isn't a valid format "
f"for the expires keyword, "
"valid options are: YYYY-MM-DD or "
"YYYY-MM-DD HH:MM:SS")
)
normalized[str(ignored_vuln_id)] = {'reason': reason, 'expires': d}
safety_policy['security']['ignore-vulnerabilities'] = normalized
safety_policy['filename'] = filename
safety_policy['raw'] = raw
else:
safety_policy['security']['ignore-vulnerabilities'] = {}
return safety_policy
except BadParameter as expected_e:
raise expected_e
except Exception as e:
# Don't fail in the default case
if ctx and isinstance(e, OSError):
source = ctx.get_parameter_source("policy_file")
if e.errno == 2 and source == click.core.ParameterSource.DEFAULT and value == '.safety-policy.yml':
return None
problem = click.style("Policy file YAML is not valid.")
hint = click.style("HINT: ", fg='yellow') + str(e)
self.fail(f"{problem}\n{hint}", param, ctx)
def shell_complete(
self, ctx: "Context", param: "Parameter", incomplete: str
):
"""Return a special completion marker that tells the completion
system to use the shell to provide file path completions.
:param ctx: Invocation context for this command.
:param param: The parameter that is requesting completion.
:param incomplete: Value being completed. May be empty.
.. versionadded:: 8.0
"""
from pipenv.vendor.click.shell_completion import CompletionItem
return [CompletionItem(incomplete, type="file")]
class SingletonMeta(type):
_instances = {}
_lock = Lock()
def __call__(cls, *args, **kwargs):
with cls._lock:
if cls not in cls._instances:
instance = super().__call__(*args, **kwargs)
cls._instances[cls] = instance
return cls._instances[cls]
class SafetyContext(metaclass=SingletonMeta):
packages = None
key = False
db_mirror = False
cached = None
ignore_vulns = None
ignore_severity_rules = None
proxy = None
include_ignored = False
telemetry = None
files = None
stdin = None
is_env_scan = None
command = None
review = None
params = {}
safety_source = 'code'
def sync_safety_context(f):
def new_func(*args, **kwargs):
ctx = SafetyContext()
for attr in dir(ctx):
if attr in kwargs:
setattr(ctx, attr, kwargs.get(attr))
return f(*args, **kwargs)
return new_func
@sync_safety_context
def get_packages_licenses(packages=None, licenses_db=None):
"""Get the licenses for the specified packages based on their version.
:param packages: packages list
:param licenses_db: the licenses db in the raw form.
:return: list of objects with the packages and their respectives licenses.
"""
SafetyContext().command = 'license'
if not packages:
packages = []
if not licenses_db:
licenses_db = {}
packages_licenses_db = licenses_db.get('packages', {})
filtered_packages_licenses = []
@@ -141,7 +639,7 @@ def get_packages_licenses(packages, licenses_db):
if isinstance(pkg, RequirementFile):
continue
# normalize the package name
pkg_name = pkg.key.replace("_", "-").lower()
pkg_name = canonicalize_name(pkg.name)
# packages may have different licenses depending their version.
pkg_licenses = packages_licenses_db.get(pkg_name, [])
version_requested = parse_version(pkg.version)
@@ -160,7 +658,7 @@ def get_packages_licenses(packages, licenses_db):
if license_id:
license_name = get_license_name_by_id(license_id, licenses_db)
if not license_id or not license_name:
license_name = "N/A"
license_name = "unknown"
filtered_packages_licenses.append({
"package": pkg_name,
pipenv/vendor/HISTORY.rst
Vendored
-237
View File
@@ -1,237 +0,0 @@
Release History
===============
2.0.0 / 2021-05-25
------------------
* Drop Python 2.7 and 3.5 support
* Make ``termcolor`` an external dependency
* Run CI tests under Ubuntu 20.04
* Update dependencies
1.5.0 / 2021-03-21
------------------
* Update cli-spinners to ``v2.6.0``
* Update dependencies
1.4.1 / 2021-02-28
------------------
* Fix timer round-up behavior (#118)
1.4.0 / 2021-02-21
------------------
* Add spinner timer (#99, #108)
* fix(#107): use ``poetry_core`` as build backend
* fix(#34): allow ``write()`` to print non-string objects
* Update dependencies
1.3.0 / 2021-01-17
------------------
* Optimization: wait of stop event instead of sleep
* Update dependencies
1.2.0 / 2020-10-19
------------------
* Update cli-spinners to ``v2.5.0``
* Add support for Python 3.9
1.1.0 / 2020-10-04
------------------
* Add ``hidden()`` context manager #68
* fix(#70): ``hidden()`` exceptions handling
* Replace coveralls.io with codecov.io
* Update dependencies
1.0.0 / 2020-08-02
------------------
* "Stabilize" yaspin; ``1.*`` branch will contain stable release with Python 2
support. Drop Python 2 and switch to Python 3 completely is planned for versions
``2.*``.
0.18.0 / 2020-07-21
-------------------
* Update cli-spinners to ``v2.4.0``
* Update dependencies
* fix(#59): remove ``tests/`` and ``examples/`` from wheels distribution
0.17.0 / 2020-05-08
-------------------
* Migrate to ``poetry`` for dependencies management, building and publishing project
* Add tests for Python 3.8
* Deprecate support for Python 3.4
* Run tests under Ubuntu 18.04
* Update dev dependencies to the most recent ones (compatible with Python 2.7)
* Remove Tox from the project (use CI for tests under different versions of Python)
0.16.0 / 2020-01-11
-------------------
* Allow use inside zip bundled package
* Code improvements
0.15.0 / 2019-08-09
-------------------
* Update cli-spinners to v2.2.0
0.14.3 / 2019-05-12
-------------------
* fix(#29): race condition between spinner thread and ``write()``
0.14.2 / 2019-04-27
-------------------
* fix: remove extra ``\b`` written to stdout. Fixes ``write()`` in rxvt terminal
0.14.1 / 2019-01-28
-------------------
* fix(#26): traceback on PYTHONOPTIMIZE=2
0.14.0 / 2018-09-05
-------------------
* Support for handling POSIX signals
* New function in public API: ``kbi_safe_yaspin``
0.13.0 / 2018-08-14
-------------------
* API improvements: ``spinner``, ``color``, ``on_color``, ``attrs`` and ``side`` argument values are handled via ``__getattr__``
* New ``yaspin`` arguments: ``on_color``, ``attrs``
* ``right=False`` argument replaced with ``side="left"``
* ``Yaspin.right`` replaced with ``Yaspin.side``
* ``reverse`` argument replaced with ``reversal``
* ``Yaspin.reverse`` replaced with ``Yaspin.reversal``
* Remove default text stripping in ``Yaspin._freeze``
0.12.0 / 2018-07-16
-------------------
* Add support for Python 3.7
* Drop support for Python 2.6 and 3.3
* dev: Migrate to Pipfile
* dev: Speedup local unittests with pytest-xdist
0.11.1 / 2018-07-10
-------------------
* fix(#16): remove default text stripping in ``Yaspin.write`` to allow printing of the hierarchical text
0.11.0 / 2018-06-23
-------------------
* Update cli-spinners to v1.3.1
0.10.0 / 2018-03-23
-------------------
* New ``hide`` and ``show`` methods to toggle the display of the spinner
0.9.0 / 2018-02-26
------------------
* New ``write`` method for writing text into terminal without breaking the spinner
0.8.0 / 2017-12-31
------------------
* Speedup reading spinners collection with simplejson
0.7.1 / 2017-12-02
------------------
* fix(#7): handling bytes sequences in ``Spinner.frames``
0.7.0 / 2017-11-28
------------------
* Reverse spinner support
0.6.0 / 2017-11-26
------------------
* Right spinner support
0.5.0 / 2017-11-24
------------------
* Colors support
0.4.2 / 2017-11-17
------------------
* RST vs PyPI episode 2
0.4.1 / 2017-11-17
------------------
* RST vs PyPI episode 1
0.4.0 / 2017-11-17
------------------
* Support for success and failure finalizers
0.3.0 / 2017-11-14
------------------
* Support for changing spinner properties on the fly
0.2.0 / 2017-11-10
------------------
* Support all spinners from `cli-spinners`_
* API changes:
- ``yaspin.spinner`` -> ``yaspin.yaspin``
0.1.0 / 2017-10-31
------------------
* First version
.. _cli-spinners: https://github.com/sindresorhus/cli-spinners
pipenv/vendor/LICENSE
Vendored
-21
View File
@@ -1,21 +0,0 @@
MIT License
Copyright (c) 2021 Pavlo Dmytrenko
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
pipenv/vendor/ruamel.yaml-0.17.21-py3.9-nspkg.pth
Vendored
+1
View File
@@ -0,0 +1 @@
import sys, types, os;has_mfs = sys.version_info > (3, 5);p = os.path.join(sys._getframe(1).f_locals['sitedir'], *('ruamel',));importlib = has_mfs and __import__('importlib.util');has_mfs and __import__('importlib.machinery');m = has_mfs and sys.modules.setdefault('ruamel', importlib.util.module_from_spec(importlib.machinery.PathFinder.find_spec('ruamel', [os.path.dirname(p)])));m = m or sys.modules.setdefault('ruamel', types.ModuleType('ruamel'));mp = (m or []) and m.__dict__.setdefault('__path__',[]);(p not in mp) and mp.append(p)
pipenv/vendor/ruamel.yaml-0.17.21-py3.9-nspkg.pth.LICENSE
Vendored
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014-2022 Anthon van der Neut, Ruamel bvba
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
pipenv/vendor/ruamel.yaml.LICENSE
Vendored
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2014-2022 Anthon van der Neut, Ruamel bvba
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
pipenv/vendor/ruamel/yaml/__init__.py
Vendored
+57
View File
@@ -0,0 +1,57 @@
# coding: utf-8
if False: # MYPY
from typing import Dict, Any # NOQA
_package_data = dict(
full_package_name='ruamel.yaml',
version_info=(0, 17, 21),
__version__='0.17.21',
version_timestamp='2022-02-12 09:49:22',
author='Anthon van der Neut',
author_email='a.van.der.neut@ruamel.eu',
description='ruamel.yaml is a YAML parser/emitter that supports roundtrip preservation of comments, seq/map flow style, and map key order', # NOQA
entry_points=None,
since=2014,
extras_require={
':platform_python_implementation=="CPython" and python_version<"3.11"': ['ruamel.yaml.clib>=0.2.6'], # NOQA
'jinja2': ['ruamel.yaml.jinja2>=0.2'],
'docs': ['ryd'],
},
classifiers=[
'Programming Language :: Python :: 3 :: Only',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: 3.8',
'Programming Language :: Python :: 3.9',
'Programming Language :: Python :: 3.10',
'Programming Language :: Python :: Implementation :: CPython',
'Topic :: Software Development :: Libraries :: Python Modules',
'Topic :: Text Processing :: Markup',
'Typing :: Typed',
],
keywords='yaml 1.2 parser round-trip preserve quotes order config',
read_the_docs='yaml',
supported=[(3, 5)], # minimum
tox=dict(
env='*f', # f for 3.5
fl8excl='_test/lib',
),
# universal=True,
python_requires='>=3',
rtfd='yaml',
) # type: Dict[Any, Any]
version_info = _package_data['version_info']
__version__ = _package_data['__version__']
try:
from .cyaml import * # NOQA
__with_libyaml__ = True
except (ImportError, ValueError): # for Jython
__with_libyaml__ = False
from pipenv.vendor.ruamel.yaml.main import * # NOQA
pipenv/vendor/ruamel/yaml/anchor.py
Vendored
+20
View File
@@ -0,0 +1,20 @@
# coding: utf-8
if False: # MYPY
from typing import Any, Dict, Optional, List, Union, Optional, Iterator # NOQA
anchor_attrib = '_yaml_anchor'
class Anchor:
__slots__ = 'value', 'always_dump'
attrib = anchor_attrib
def __init__(self):
# type: () -> None
self.value = None
self.always_dump = False
def __repr__(self):
# type: () -> Any
ad = ', (always dump)' if self.always_dump else ""
return 'Anchor({!r}{})'.format(self.value, ad)
pipenv/vendor/ruamel/yaml/comments.py
Vendored
+1267
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/compat.py
Vendored
+268
View File
@@ -0,0 +1,268 @@
# coding: utf-8
# partially from package six by Benjamin Peterson
import sys
import os
import io
import traceback
from abc import abstractmethod
import collections.abc
# fmt: off
if False: # MYPY
from typing import Any, Dict, Optional, List, Union, BinaryIO, IO, Text, Tuple # NOQA
from typing import Optional # NOQA
# fmt: on
_DEFAULT_YAML_VERSION = (1, 2)
try:
from collections import OrderedDict
except ImportError:
from ordereddict import OrderedDict # type: ignore
# to get the right name import ... as ordereddict doesn't do that
class ordereddict(OrderedDict): # type: ignore
if not hasattr(OrderedDict, 'insert'):
def insert(self, pos, key, value):
# type: (int, Any, Any) -> None
if pos >= len(self):
self[key] = value
return
od = ordereddict()
od.update(self)
for k in od:
del self[k]
for index, old_key in enumerate(od):
if pos == index:
self[key] = value
self[old_key] = od[old_key]
PY2 = sys.version_info[0] == 2
PY3 = sys.version_info[0] == 3
# replace with f-strings when 3.5 support is dropped
# ft = '42'
# assert _F('abc {ft!r}', ft=ft) == 'abc %r' % ft
# 'abc %r' % ft -> _F('abc {ft!r}' -> f'abc {ft!r}'
def _F(s, *superfluous, **kw):
# type: (Any, Any, Any) -> Any
if superfluous:
raise TypeError
return s.format(**kw)
StringIO = io.StringIO
BytesIO = io.BytesIO
if False: # MYPY
# StreamType = Union[BinaryIO, IO[str], IO[unicode], StringIO]
# StreamType = Union[BinaryIO, IO[str], StringIO] # type: ignore
StreamType = Any
StreamTextType = StreamType # Union[Text, StreamType]
VersionType = Union[List[int], str, Tuple[int, int]]
builtins_module = 'builtins'
def with_metaclass(meta, *bases):
# type: (Any, Any) -> Any
"""Create a base class with a metaclass."""
return meta('NewBase', bases, {})
DBG_TOKEN = 1
DBG_EVENT = 2
DBG_NODE = 4
_debug = None # type: Optional[int]
if 'RUAMELDEBUG' in os.environ:
_debugx = os.environ.get('RUAMELDEBUG')
if _debugx is None:
_debug = 0
else:
_debug = int(_debugx)
if bool(_debug):
class ObjectCounter:
def __init__(self):
# type: () -> None
self.map = {} # type: Dict[Any, Any]
def __call__(self, k):
# type: (Any) -> None
self.map[k] = self.map.get(k, 0) + 1
def dump(self):
# type: () -> None
for k in sorted(self.map):
sys.stdout.write('{} -> {}'.format(k, self.map[k]))
object_counter = ObjectCounter()
# used from yaml util when testing
def dbg(val=None):
# type: (Any) -> Any
global _debug
if _debug is None:
# set to true or false
_debugx = os.environ.get('YAMLDEBUG')
if _debugx is None:
_debug = 0
else:
_debug = int(_debugx)
if val is None:
return _debug
return _debug & val
class Nprint:
def __init__(self, file_name=None):
# type: (Any) -> None
self._max_print = None # type: Any
self._count = None # type: Any
self._file_name = file_name
def __call__(self, *args, **kw):
# type: (Any, Any) -> None
if not bool(_debug):
return
out = sys.stdout if self._file_name is None else open(self._file_name, 'a')
dbgprint = print # to fool checking for print statements by dv utility
kw1 = kw.copy()
kw1['file'] = out
dbgprint(*args, **kw1)
out.flush()
if self._max_print is not None:
if self._count is None:
self._count = self._max_print
self._count -= 1
if self._count == 0:
dbgprint('forced exit\n')
traceback.print_stack()
out.flush()
sys.exit(0)
if self._file_name:
out.close()
def set_max_print(self, i):
# type: (int) -> None
self._max_print = i
self._count = None
def fp(self, mode='a'):
# type: (str) -> Any
out = sys.stdout if self._file_name is None else open(self._file_name, mode)
return out
nprint = Nprint()
nprintf = Nprint('/var/tmp/ruamel.yaml.log')
# char checkers following production rules
def check_namespace_char(ch):
# type: (Any) -> bool
if '\x21' <= ch <= '\x7E': # ! to ~
return True
if '\xA0' <= ch <= '\uD7FF':
return True
if ('\uE000' <= ch <= '\uFFFD') and ch != '\uFEFF': # excl. byte order mark
return True
if '\U00010000' <= ch <= '\U0010FFFF':
return True
return False
def check_anchorname_char(ch):
# type: (Any) -> bool
if ch in ',[]{}':
return False
return check_namespace_char(ch)
def version_tnf(t1, t2=None):
# type: (Any, Any) -> Any
"""
return True if ruamel version_info < t1, None if t2 is specified and bigger else False
"""
from pipenv.vendor.ruamel.yaml import version_info # NOQA
if version_info < t1:
return True
if t2 is not None and version_info < t2:
return None
return False
class MutableSliceableSequence(collections.abc.MutableSequence): # type: ignore
__slots__ = ()
def __getitem__(self, index):
# type: (Any) -> Any
if not isinstance(index, slice):
return self.__getsingleitem__(index)
return type(self)([self[i] for i in range(*index.indices(len(self)))]) # type: ignore
def __setitem__(self, index, value):
# type: (Any, Any) -> None
if not isinstance(index, slice):
return self.__setsingleitem__(index, value)
assert iter(value)
# nprint(index.start, index.stop, index.step, index.indices(len(self)))
if index.step is None:
del self[index.start : index.stop]
for elem in reversed(value):
self.insert(0 if index.start is None else index.start, elem)
else:
range_parms = index.indices(len(self))
nr_assigned_items = (range_parms[1] - range_parms[0] - 1) // range_parms[2] + 1
# need to test before changing, in case TypeError is caught
if nr_assigned_items < len(value):
raise TypeError(
'too many elements in value {} < {}'.format(nr_assigned_items, len(value))
)
elif nr_assigned_items > len(value):
raise TypeError(
'not enough elements in value {} > {}'.format(
nr_assigned_items, len(value)
)
)
for idx, i in enumerate(range(*range_parms)):
self[i] = value[idx]
def __delitem__(self, index):
# type: (Any) -> None
if not isinstance(index, slice):
return self.__delsingleitem__(index)
# nprint(index.start, index.stop, index.step, index.indices(len(self)))
for i in reversed(range(*index.indices(len(self)))):
del self[i]
@abstractmethod
def __getsingleitem__(self, index):
# type: (Any) -> Any
raise IndexError
@abstractmethod
def __setsingleitem__(self, index, value):
# type: (Any, Any) -> None
raise IndexError
@abstractmethod
def __delsingleitem__(self, index):
# type: (Any) -> None
raise IndexError
pipenv/vendor/ruamel/yaml/composer.py
Vendored
+243
View File
@@ -0,0 +1,243 @@
# coding: utf-8
import warnings
from pipenv.vendor.ruamel.yaml.error import MarkedYAMLError, ReusedAnchorWarning
from pipenv.vendor.ruamel.yaml.compat import _F, nprint, nprintf # NOQA
from pipenv.vendor.ruamel.yaml.events import (
StreamStartEvent,
StreamEndEvent,
MappingStartEvent,
MappingEndEvent,
SequenceStartEvent,
SequenceEndEvent,
AliasEvent,
ScalarEvent,
)
from pipenv.vendor.ruamel.yaml.nodes import MappingNode, ScalarNode, SequenceNode
if False: # MYPY
from typing import Any, Dict, Optional, List # NOQA
__all__ = ['Composer', 'ComposerError']
class ComposerError(MarkedYAMLError):
pass
class Composer:
def __init__(self, loader=None):
# type: (Any) -> None
self.loader = loader
if self.loader is not None and getattr(self.loader, '_composer', None) is None:
self.loader._composer = self
self.anchors = {} # type: Dict[Any, Any]
@property
def parser(self):
# type: () -> Any
if hasattr(self.loader, 'typ'):
self.loader.parser
return self.loader._parser
@property
def resolver(self):
# type: () -> Any
# assert self.loader._resolver is not None
if hasattr(self.loader, 'typ'):
self.loader.resolver
return self.loader._resolver
def check_node(self):
# type: () -> Any
# Drop the STREAM-START event.
if self.parser.check_event(StreamStartEvent):
self.parser.get_event()
# If there are more documents available?
return not self.parser.check_event(StreamEndEvent)
def get_node(self):
# type: () -> Any
# Get the root node of the next document.
if not self.parser.check_event(StreamEndEvent):
return self.compose_document()
def get_single_node(self):
# type: () -> Any
# Drop the STREAM-START event.
self.parser.get_event()
# Compose a document if the stream is not empty.
document = None # type: Any
if not self.parser.check_event(StreamEndEvent):
document = self.compose_document()
# Ensure that the stream contains no more documents.
if not self.parser.check_event(StreamEndEvent):
event = self.parser.get_event()
raise ComposerError(
'expected a single document in the stream',
document.start_mark,
'but found another document',
event.start_mark,
)
# Drop the STREAM-END event.
self.parser.get_event()
return document
def compose_document(self):
# type: (Any) -> Any
# Drop the DOCUMENT-START event.
self.parser.get_event()
# Compose the root node.
node = self.compose_node(None, None)
# Drop the DOCUMENT-END event.
self.parser.get_event()
self.anchors = {}
return node
def return_alias(self, a):
# type: (Any) -> Any
return a
def compose_node(self, parent, index):
# type: (Any, Any) -> Any
if self.parser.check_event(AliasEvent):
event = self.parser.get_event()
alias = event.anchor
if alias not in self.anchors:
raise ComposerError(
None,
None,
_F('found undefined alias {alias!r}', alias=alias),
event.start_mark,
)
return self.return_alias(self.anchors[alias])
event = self.parser.peek_event()
anchor = event.anchor
if anchor is not None: # have an anchor
if anchor in self.anchors:
# raise ComposerError(
# "found duplicate anchor %r; first occurrence"
# % (anchor), self.anchors[anchor].start_mark,
# "second occurrence", event.start_mark)
ws = (
'\nfound duplicate anchor {!r}\nfirst occurrence {}\nsecond occurrence '
'{}'.format((anchor), self.anchors[anchor].start_mark, event.start_mark)
)
warnings.warn(ws, ReusedAnchorWarning)
self.resolver.descend_resolver(parent, index)
if self.parser.check_event(ScalarEvent):
node = self.compose_scalar_node(anchor)
elif self.parser.check_event(SequenceStartEvent):
node = self.compose_sequence_node(anchor)
elif self.parser.check_event(MappingStartEvent):
node = self.compose_mapping_node(anchor)
self.resolver.ascend_resolver()
return node
def compose_scalar_node(self, anchor):
# type: (Any) -> Any
event = self.parser.get_event()
tag = event.tag
if tag is None or tag == '!':
tag = self.resolver.resolve(ScalarNode, event.value, event.implicit)
node = ScalarNode(
tag,
event.value,
event.start_mark,
event.end_mark,
style=event.style,
comment=event.comment,
anchor=anchor,
)
if anchor is not None:
self.anchors[anchor] = node
return node
def compose_sequence_node(self, anchor):
# type: (Any) -> Any
start_event = self.parser.get_event()
tag = start_event.tag
if tag is None or tag == '!':
tag = self.resolver.resolve(SequenceNode, None, start_event.implicit)
node = SequenceNode(
tag,
[],
start_event.start_mark,
None,
flow_style=start_event.flow_style,
comment=start_event.comment,
anchor=anchor,
)
if anchor is not None:
self.anchors[anchor] = node
index = 0
while not self.parser.check_event(SequenceEndEvent):
node.value.append(self.compose_node(node, index))
index += 1
end_event = self.parser.get_event()
if node.flow_style is True and end_event.comment is not None:
if node.comment is not None:
nprint(
'Warning: unexpected end_event commment in sequence '
'node {}'.format(node.flow_style)
)
node.comment = end_event.comment
node.end_mark = end_event.end_mark
self.check_end_doc_comment(end_event, node)
return node
def compose_mapping_node(self, anchor):
# type: (Any) -> Any
start_event = self.parser.get_event()
tag = start_event.tag
if tag is None or tag == '!':
tag = self.resolver.resolve(MappingNode, None, start_event.implicit)
node = MappingNode(
tag,
[],
start_event.start_mark,
None,
flow_style=start_event.flow_style,
comment=start_event.comment,
anchor=anchor,
)
if anchor is not None:
self.anchors[anchor] = node
while not self.parser.check_event(MappingEndEvent):
# key_event = self.parser.peek_event()
item_key = self.compose_node(node, None)
# if item_key in node.value:
# raise ComposerError("while composing a mapping",
# start_event.start_mark,
# "found duplicate key", key_event.start_mark)
item_value = self.compose_node(node, item_key)
# node.value[item_key] = item_value
node.value.append((item_key, item_value))
end_event = self.parser.get_event()
if node.flow_style is True and end_event.comment is not None:
node.comment = end_event.comment
node.end_mark = end_event.end_mark
self.check_end_doc_comment(end_event, node)
return node
def check_end_doc_comment(self, end_event, node):
# type: (Any, Any) -> None
if end_event.comment and end_event.comment[1]:
# pre comments on an end_event, no following to move to
if node.comment is None:
node.comment = [None, None]
assert not isinstance(node, ScalarEvent)
# this is a post comment on a mapping node, add as third element
# in the list
node.comment.append(end_event.comment[1])
end_event.comment[1] = None
pipenv/vendor/ruamel/yaml/configobjwalker.py
Vendored
+14
View File
@@ -0,0 +1,14 @@
# coding: utf-8
import warnings
from pipenv.vendor.ruamel.yaml.util import configobj_walker as new_configobj_walker
if False: # MYPY
from typing import Any # NOQA
def configobj_walker(cfg):
# type: (Any) -> Any
warnings.warn('configobj_walker has moved to ruamel.util, please update your code')
return new_configobj_walker(cfg)
pipenv/vendor/ruamel/yaml/constructor.py
Vendored
+1845
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/cyaml.py
Vendored
+183
View File
@@ -0,0 +1,183 @@
# coding: utf-8
from _ruamel_yaml import CParser, CEmitter # type: ignore
from pipenv.vendor.ruamel.yaml.constructor import Constructor, BaseConstructor, SafeConstructor
from pipenv.vendor.ruamel.yaml.representer import Representer, SafeRepresenter, BaseRepresenter
from pipenv.vendor.ruamel.yaml.resolver import Resolver, BaseResolver
if False: # MYPY
from typing import Any, Union, Optional # NOQA
from pipenv.vendor.ruamel.yaml.compat import StreamTextType, StreamType, VersionType # NOQA
__all__ = ['CBaseLoader', 'CSafeLoader', 'CLoader', 'CBaseDumper', 'CSafeDumper', 'CDumper']
# this includes some hacks to solve the usage of resolver by lower level
# parts of the parser
class CBaseLoader(CParser, BaseConstructor, BaseResolver): # type: ignore
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
CParser.__init__(self, stream)
self._parser = self._composer = self
BaseConstructor.__init__(self, loader=self)
BaseResolver.__init__(self, loadumper=self)
# self.descend_resolver = self._resolver.descend_resolver
# self.ascend_resolver = self._resolver.ascend_resolver
# self.resolve = self._resolver.resolve
class CSafeLoader(CParser, SafeConstructor, Resolver): # type: ignore
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
CParser.__init__(self, stream)
self._parser = self._composer = self
SafeConstructor.__init__(self, loader=self)
Resolver.__init__(self, loadumper=self)
# self.descend_resolver = self._resolver.descend_resolver
# self.ascend_resolver = self._resolver.ascend_resolver
# self.resolve = self._resolver.resolve
class CLoader(CParser, Constructor, Resolver): # type: ignore
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
CParser.__init__(self, stream)
self._parser = self._composer = self
Constructor.__init__(self, loader=self)
Resolver.__init__(self, loadumper=self)
# self.descend_resolver = self._resolver.descend_resolver
# self.ascend_resolver = self._resolver.ascend_resolver
# self.resolve = self._resolver.resolve
class CBaseDumper(CEmitter, BaseRepresenter, BaseResolver): # type: ignore
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
CEmitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
encoding=encoding,
allow_unicode=allow_unicode,
line_break=line_break,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
)
self._emitter = self._serializer = self._representer = self
BaseRepresenter.__init__(
self,
default_style=default_style,
default_flow_style=default_flow_style,
dumper=self,
)
BaseResolver.__init__(self, loadumper=self)
class CSafeDumper(CEmitter, SafeRepresenter, Resolver): # type: ignore
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
self._emitter = self._serializer = self._representer = self
CEmitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
encoding=encoding,
allow_unicode=allow_unicode,
line_break=line_break,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
)
self._emitter = self._serializer = self._representer = self
SafeRepresenter.__init__(
self, default_style=default_style, default_flow_style=default_flow_style
)
Resolver.__init__(self)
class CDumper(CEmitter, Representer, Resolver): # type: ignore
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
CEmitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
encoding=encoding,
allow_unicode=allow_unicode,
line_break=line_break,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
)
self._emitter = self._serializer = self._representer = self
Representer.__init__(
self, default_style=default_style, default_flow_style=default_flow_style
)
Resolver.__init__(self)
pipenv/vendor/ruamel/yaml/dumper.py
Vendored
+219
View File
@@ -0,0 +1,219 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.emitter import Emitter
from pipenv.vendor.ruamel.yaml.serializer import Serializer
from pipenv.vendor.ruamel.yaml.representer import (
Representer,
SafeRepresenter,
BaseRepresenter,
RoundTripRepresenter,
)
from pipenv.vendor.ruamel.yaml.resolver import Resolver, BaseResolver, VersionedResolver
if False: # MYPY
from typing import Any, Dict, List, Union, Optional # NOQA
from pipenv.vendor.ruamel.yaml.compat import StreamType, VersionType # NOQA
__all__ = ['BaseDumper', 'SafeDumper', 'Dumper', 'RoundTripDumper']
class BaseDumper(Emitter, Serializer, BaseRepresenter, BaseResolver):
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (Any, StreamType, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
Emitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
block_seq_indent=block_seq_indent,
dumper=self,
)
Serializer.__init__(
self,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
dumper=self,
)
BaseRepresenter.__init__(
self,
default_style=default_style,
default_flow_style=default_flow_style,
dumper=self,
)
BaseResolver.__init__(self, loadumper=self)
class SafeDumper(Emitter, Serializer, SafeRepresenter, Resolver):
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
Emitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
block_seq_indent=block_seq_indent,
dumper=self,
)
Serializer.__init__(
self,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
dumper=self,
)
SafeRepresenter.__init__(
self,
default_style=default_style,
default_flow_style=default_flow_style,
dumper=self,
)
Resolver.__init__(self, loadumper=self)
class Dumper(Emitter, Serializer, Representer, Resolver):
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Any, Optional[bool], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
Emitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
block_seq_indent=block_seq_indent,
dumper=self,
)
Serializer.__init__(
self,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
dumper=self,
)
Representer.__init__(
self,
default_style=default_style,
default_flow_style=default_flow_style,
dumper=self,
)
Resolver.__init__(self, loadumper=self)
class RoundTripDumper(Emitter, Serializer, RoundTripRepresenter, VersionedResolver):
def __init__(
self,
stream,
default_style=None,
default_flow_style=None,
canonical=None,
indent=None,
width=None,
allow_unicode=None,
line_break=None,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
block_seq_indent=None,
top_level_colon_align=None,
prefix_colon=None,
):
# type: (StreamType, Any, Optional[bool], Optional[int], Optional[int], Optional[int], Optional[bool], Any, Any, Optional[bool], Optional[bool], Any, Any, Any, Any, Any) -> None # NOQA
Emitter.__init__(
self,
stream,
canonical=canonical,
indent=indent,
width=width,
allow_unicode=allow_unicode,
line_break=line_break,
block_seq_indent=block_seq_indent,
top_level_colon_align=top_level_colon_align,
prefix_colon=prefix_colon,
dumper=self,
)
Serializer.__init__(
self,
encoding=encoding,
explicit_start=explicit_start,
explicit_end=explicit_end,
version=version,
tags=tags,
dumper=self,
)
RoundTripRepresenter.__init__(
self,
default_style=default_style,
default_flow_style=default_flow_style,
dumper=self,
)
VersionedResolver.__init__(self, loader=self)
pipenv/vendor/ruamel/yaml/emitter.py
Vendored
+1772
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/error.py
Vendored
+332
View File
@@ -0,0 +1,332 @@
# coding: utf-8
import warnings
import textwrap
from pipenv.vendor.ruamel.yaml.compat import _F
if False: # MYPY
from typing import Any, Dict, Optional, List, Text # NOQA
__all__ = [
'FileMark',
'StringMark',
'CommentMark',
'YAMLError',
'MarkedYAMLError',
'ReusedAnchorWarning',
'UnsafeLoaderWarning',
'MarkedYAMLWarning',
'MarkedYAMLFutureWarning',
]
class StreamMark:
__slots__ = 'name', 'index', 'line', 'column'
def __init__(self, name, index, line, column):
# type: (Any, int, int, int) -> None
self.name = name
self.index = index
self.line = line
self.column = column
def __str__(self):
# type: () -> Any
where = _F(
' in "{sname!s}", line {sline1:d}, column {scolumn1:d}',
sname=self.name,
sline1=self.line + 1,
scolumn1=self.column + 1,
)
return where
def __eq__(self, other):
# type: (Any) -> bool
if self.line != other.line or self.column != other.column:
return False
if self.name != other.name or self.index != other.index:
return False
return True
def __ne__(self, other):
# type: (Any) -> bool
return not self.__eq__(other)
class FileMark(StreamMark):
__slots__ = ()
class StringMark(StreamMark):
__slots__ = 'name', 'index', 'line', 'column', 'buffer', 'pointer'
def __init__(self, name, index, line, column, buffer, pointer):
# type: (Any, int, int, int, Any, Any) -> None
StreamMark.__init__(self, name, index, line, column)
self.buffer = buffer
self.pointer = pointer
def get_snippet(self, indent=4, max_length=75):
# type: (int, int) -> Any
if self.buffer is None: # always False
return None
head = ""
start = self.pointer
while start > 0 and self.buffer[start - 1] not in '\0\r\n\x85\u2028\u2029':
start -= 1
if self.pointer - start > max_length / 2 - 1:
head = ' ... '
start += 5
break
tail = ""
end = self.pointer
while end < len(self.buffer) and self.buffer[end] not in '\0\r\n\x85\u2028\u2029':
end += 1
if end - self.pointer > max_length / 2 - 1:
tail = ' ... '
end -= 5
break
snippet = self.buffer[start:end]
caret = '^'
caret = '^ (line: {})'.format(self.line + 1)
return (
' ' * indent
+ head
+ snippet
+ tail
+ '\n'
+ ' ' * (indent + self.pointer - start + len(head))
+ caret
)
def __str__(self):
# type: () -> Any
snippet = self.get_snippet()
where = _F(
' in "{sname!s}", line {sline1:d}, column {scolumn1:d}',
sname=self.name,
sline1=self.line + 1,
scolumn1=self.column + 1,
)
if snippet is not None:
where += ':\n' + snippet
return where
def __repr__(self):
# type: () -> Any
snippet = self.get_snippet()
where = _F(
' in "{sname!s}", line {sline1:d}, column {scolumn1:d}',
sname=self.name,
sline1=self.line + 1,
scolumn1=self.column + 1,
)
if snippet is not None:
where += ':\n' + snippet
return where
class CommentMark:
__slots__ = ('column',)
def __init__(self, column):
# type: (Any) -> None
self.column = column
class YAMLError(Exception):
pass
class MarkedYAMLError(YAMLError):
def __init__(
self,
context=None,
context_mark=None,
problem=None,
problem_mark=None,
note=None,
warn=None,
):
# type: (Any, Any, Any, Any, Any, Any) -> None
self.context = context
self.context_mark = context_mark
self.problem = problem
self.problem_mark = problem_mark
self.note = note
# warn is ignored
def __str__(self):
# type: () -> Any
lines = [] # type: List[str]
if self.context is not None:
lines.append(self.context)
if self.context_mark is not None and (
self.problem is None
or self.problem_mark is None
or self.context_mark.name != self.problem_mark.name
or self.context_mark.line != self.problem_mark.line
or self.context_mark.column != self.problem_mark.column
):
lines.append(str(self.context_mark))
if self.problem is not None:
lines.append(self.problem)
if self.problem_mark is not None:
lines.append(str(self.problem_mark))
if self.note is not None and self.note:
note = textwrap.dedent(self.note)
lines.append(note)
return '\n'.join(lines)
class YAMLStreamError(Exception):
pass
class YAMLWarning(Warning):
pass
class MarkedYAMLWarning(YAMLWarning):
def __init__(
self,
context=None,
context_mark=None,
problem=None,
problem_mark=None,
note=None,
warn=None,
):
# type: (Any, Any, Any, Any, Any, Any) -> None
self.context = context
self.context_mark = context_mark
self.problem = problem
self.problem_mark = problem_mark
self.note = note
self.warn = warn
def __str__(self):
# type: () -> Any
lines = [] # type: List[str]
if self.context is not None:
lines.append(self.context)
if self.context_mark is not None and (
self.problem is None
or self.problem_mark is None
or self.context_mark.name != self.problem_mark.name
or self.context_mark.line != self.problem_mark.line
or self.context_mark.column != self.problem_mark.column
):
lines.append(str(self.context_mark))
if self.problem is not None:
lines.append(self.problem)
if self.problem_mark is not None:
lines.append(str(self.problem_mark))
if self.note is not None and self.note:
note = textwrap.dedent(self.note)
lines.append(note)
if self.warn is not None and self.warn:
warn = textwrap.dedent(self.warn)
lines.append(warn)
return '\n'.join(lines)
class ReusedAnchorWarning(YAMLWarning):
pass
class UnsafeLoaderWarning(YAMLWarning):
text = """
The default 'Loader' for 'load(stream)' without further arguments can be unsafe.
Use 'load(stream, Loader=ruamel.yaml.Loader)' explicitly if that is OK.
Alternatively include the following in your code:
import warnings
warnings.simplefilter('ignore', ruamel.error.UnsafeLoaderWarning)
In most other cases you should consider using 'safe_load(stream)'"""
pass
warnings.simplefilter('once', UnsafeLoaderWarning)
class MantissaNoDotYAML1_1Warning(YAMLWarning):
def __init__(self, node, flt_str):
# type: (Any, Any) -> None
self.node = node
self.flt = flt_str
def __str__(self):
# type: () -> Any
line = self.node.start_mark.line
col = self.node.start_mark.column
return """
In YAML 1.1 floating point values should have a dot ('.') in their mantissa.
See the Floating-Point Language-Independent Type for YAML™ Version 1.1 specification
( http://yaml.org/type/float.html ). This dot is not required for JSON nor for YAML 1.2
Correct your float: "{}" on line: {}, column: {}
or alternatively include the following in your code:
import warnings
warnings.simplefilter('ignore', ruamel.error.MantissaNoDotYAML1_1Warning)
""".format(
self.flt, line, col
)
warnings.simplefilter('once', MantissaNoDotYAML1_1Warning)
class YAMLFutureWarning(Warning):
pass
class MarkedYAMLFutureWarning(YAMLFutureWarning):
def __init__(
self,
context=None,
context_mark=None,
problem=None,
problem_mark=None,
note=None,
warn=None,
):
# type: (Any, Any, Any, Any, Any, Any) -> None
self.context = context
self.context_mark = context_mark
self.problem = problem
self.problem_mark = problem_mark
self.note = note
self.warn = warn
def __str__(self):
# type: () -> Any
lines = [] # type: List[str]
if self.context is not None:
lines.append(self.context)
if self.context_mark is not None and (
self.problem is None
or self.problem_mark is None
or self.context_mark.name != self.problem_mark.name
or self.context_mark.line != self.problem_mark.line
or self.context_mark.column != self.problem_mark.column
):
lines.append(str(self.context_mark))
if self.problem is not None:
lines.append(self.problem)
if self.problem_mark is not None:
lines.append(str(self.problem_mark))
if self.note is not None and self.note:
note = textwrap.dedent(self.note)
lines.append(note)
if self.warn is not None and self.warn:
warn = textwrap.dedent(self.warn)
lines.append(warn)
return '\n'.join(lines)
pipenv/vendor/ruamel/yaml/events.py
Vendored
+196
View File
@@ -0,0 +1,196 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.compat import _F
# Abstract classes.
if False: # MYPY
from typing import Any, Dict, Optional, List # NOQA
SHOW_LINES = False
def CommentCheck():
# type: () -> None
pass
class Event:
__slots__ = 'start_mark', 'end_mark', 'comment'
def __init__(self, start_mark=None, end_mark=None, comment=CommentCheck):
# type: (Any, Any, Any) -> None
self.start_mark = start_mark
self.end_mark = end_mark
# assert comment is not CommentCheck
if comment is CommentCheck:
comment = None
self.comment = comment
def __repr__(self):
# type: () -> Any
if True:
arguments = []
if hasattr(self, 'value'):
# if you use repr(getattr(self, 'value')) then flake8 complains about
# abuse of getattr with a constant. When you change to self.value
# then mypy throws an error
arguments.append(repr(self.value)) # type: ignore
for key in ['anchor', 'tag', 'implicit', 'flow_style', 'style']:
v = getattr(self, key, None)
if v is not None:
arguments.append(_F('{key!s}={v!r}', key=key, v=v))
if self.comment not in [None, CommentCheck]:
arguments.append('comment={!r}'.format(self.comment))
if SHOW_LINES:
arguments.append(
'({}:{}/{}:{})'.format(
self.start_mark.line,
self.start_mark.column,
self.end_mark.line,
self.end_mark.column,
)
)
arguments = ', '.join(arguments) # type: ignore
else:
attributes = [
key
for key in ['anchor', 'tag', 'implicit', 'value', 'flow_style', 'style']
if hasattr(self, key)
]
arguments = ', '.join(
[_F('{k!s}={attr!r}', k=key, attr=getattr(self, key)) for key in attributes]
)
if self.comment not in [None, CommentCheck]:
arguments += ', comment={!r}'.format(self.comment)
return _F(
'{self_class_name!s}({arguments!s})',
self_class_name=self.__class__.__name__,
arguments=arguments,
)
class NodeEvent(Event):
__slots__ = ('anchor',)
def __init__(self, anchor, start_mark=None, end_mark=None, comment=None):
# type: (Any, Any, Any, Any) -> None
Event.__init__(self, start_mark, end_mark, comment)
self.anchor = anchor
class CollectionStartEvent(NodeEvent):
__slots__ = 'tag', 'implicit', 'flow_style', 'nr_items'
def __init__(
self,
anchor,
tag,
implicit,
start_mark=None,
end_mark=None,
flow_style=None,
comment=None,
nr_items=None,
):
# type: (Any, Any, Any, Any, Any, Any, Any, Optional[int]) -> None
NodeEvent.__init__(self, anchor, start_mark, end_mark, comment)
self.tag = tag
self.implicit = implicit
self.flow_style = flow_style
self.nr_items = nr_items
class CollectionEndEvent(Event):
__slots__ = ()
# Implementations.
class StreamStartEvent(Event):
__slots__ = ('encoding',)
def __init__(self, start_mark=None, end_mark=None, encoding=None, comment=None):
# type: (Any, Any, Any, Any) -> None
Event.__init__(self, start_mark, end_mark, comment)
self.encoding = encoding
class StreamEndEvent(Event):
__slots__ = ()
class DocumentStartEvent(Event):
__slots__ = 'explicit', 'version', 'tags'
def __init__(
self,
start_mark=None,
end_mark=None,
explicit=None,
version=None,
tags=None,
comment=None,
):
# type: (Any, Any, Any, Any, Any, Any) -> None
Event.__init__(self, start_mark, end_mark, comment)
self.explicit = explicit
self.version = version
self.tags = tags
class DocumentEndEvent(Event):
__slots__ = ('explicit',)
def __init__(self, start_mark=None, end_mark=None, explicit=None, comment=None):
# type: (Any, Any, Any, Any) -> None
Event.__init__(self, start_mark, end_mark, comment)
self.explicit = explicit
class AliasEvent(NodeEvent):
__slots__ = 'style'
def __init__(self, anchor, start_mark=None, end_mark=None, style=None, comment=None):
# type: (Any, Any, Any, Any, Any) -> None
NodeEvent.__init__(self, anchor, start_mark, end_mark, comment)
self.style = style
class ScalarEvent(NodeEvent):
__slots__ = 'tag', 'implicit', 'value', 'style'
def __init__(
self,
anchor,
tag,
implicit,
value,
start_mark=None,
end_mark=None,
style=None,
comment=None,
):
# type: (Any, Any, Any, Any, Any, Any, Any, Any) -> None
NodeEvent.__init__(self, anchor, start_mark, end_mark, comment)
self.tag = tag
self.implicit = implicit
self.value = value
self.style = style
class SequenceStartEvent(CollectionStartEvent):
__slots__ = ()
class SequenceEndEvent(CollectionEndEvent):
__slots__ = ()
class MappingStartEvent(CollectionStartEvent):
__slots__ = ()
class MappingEndEvent(CollectionEndEvent):
__slots__ = ()
pipenv/vendor/ruamel/yaml/loader.py
Vendored
+75
View File
@@ -0,0 +1,75 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.reader import Reader
from pipenv.vendor.ruamel.yaml.scanner import Scanner, RoundTripScanner
from pipenv.vendor.ruamel.yaml.parser import Parser, RoundTripParser
from pipenv.vendor.ruamel.yaml.composer import Composer
from pipenv.vendor.ruamel.yaml.constructor import (
BaseConstructor,
SafeConstructor,
Constructor,
RoundTripConstructor,
)
from pipenv.vendor.ruamel.yaml.resolver import VersionedResolver
if False: # MYPY
from typing import Any, Dict, List, Union, Optional # NOQA
from pipenv.vendor.ruamel.yaml.compat import StreamTextType, VersionType # NOQA
__all__ = ['BaseLoader', 'SafeLoader', 'Loader', 'RoundTripLoader']
class BaseLoader(Reader, Scanner, Parser, Composer, BaseConstructor, VersionedResolver):
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
self.comment_handling = None
Reader.__init__(self, stream, loader=self)
Scanner.__init__(self, loader=self)
Parser.__init__(self, loader=self)
Composer.__init__(self, loader=self)
BaseConstructor.__init__(self, loader=self)
VersionedResolver.__init__(self, version, loader=self)
class SafeLoader(Reader, Scanner, Parser, Composer, SafeConstructor, VersionedResolver):
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
self.comment_handling = None
Reader.__init__(self, stream, loader=self)
Scanner.__init__(self, loader=self)
Parser.__init__(self, loader=self)
Composer.__init__(self, loader=self)
SafeConstructor.__init__(self, loader=self)
VersionedResolver.__init__(self, version, loader=self)
class Loader(Reader, Scanner, Parser, Composer, Constructor, VersionedResolver):
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
self.comment_handling = None
Reader.__init__(self, stream, loader=self)
Scanner.__init__(self, loader=self)
Parser.__init__(self, loader=self)
Composer.__init__(self, loader=self)
Constructor.__init__(self, loader=self)
VersionedResolver.__init__(self, version, loader=self)
class RoundTripLoader(
Reader,
RoundTripScanner,
RoundTripParser,
Composer,
RoundTripConstructor,
VersionedResolver,
):
def __init__(self, stream, version=None, preserve_quotes=None):
# type: (StreamTextType, Optional[VersionType], Optional[bool]) -> None
# self.reader = Reader.__init__(self, stream)
self.comment_handling = None # issue 385
Reader.__init__(self, stream, loader=self)
RoundTripScanner.__init__(self, loader=self)
RoundTripParser.__init__(self, loader=self)
Composer.__init__(self, loader=self)
RoundTripConstructor.__init__(self, preserve_quotes=preserve_quotes, loader=self)
VersionedResolver.__init__(self, version, loader=self)
pipenv/vendor/ruamel/yaml/main.py
Vendored
+1667
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/nodes.py
Vendored
+135
View File
@@ -0,0 +1,135 @@
# coding: utf-8
import sys
from pipenv.vendor.ruamel.yaml.compat import _F
if False: # MYPY
from typing import Dict, Any, Text # NOQA
class Node:
__slots__ = 'tag', 'value', 'start_mark', 'end_mark', 'comment', 'anchor'
def __init__(self, tag, value, start_mark, end_mark, comment=None, anchor=None):
# type: (Any, Any, Any, Any, Any, Any) -> None
self.tag = tag
self.value = value
self.start_mark = start_mark
self.end_mark = end_mark
self.comment = comment
self.anchor = anchor
def __repr__(self):
# type: () -> Any
value = self.value
# if isinstance(value, list):
# if len(value) == 0:
# value = '<empty>'
# elif len(value) == 1:
# value = '<1 item>'
# else:
# value = f'<{len(value)} items>'
# else:
# if len(value) > 75:
# value = repr(value[:70]+' ... ')
# else:
# value = repr(value)
value = repr(value)
return _F(
'{class_name!s}(tag={self_tag!r}, value={value!s})',
class_name=self.__class__.__name__,
self_tag=self.tag,
value=value,
)
def dump(self, indent=0):
# type: (int) -> None
if isinstance(self.value, str):
sys.stdout.write(
'{}{}(tag={!r}, value={!r})\n'.format(
' ' * indent, self.__class__.__name__, self.tag, self.value
)
)
if self.comment:
sys.stdout.write(' {}comment: {})\n'.format(' ' * indent, self.comment))
return
sys.stdout.write(
'{}{}(tag={!r})\n'.format(' ' * indent, self.__class__.__name__, self.tag)
)
if self.comment:
sys.stdout.write(' {}comment: {})\n'.format(' ' * indent, self.comment))
for v in self.value:
if isinstance(v, tuple):
for v1 in v:
v1.dump(indent + 1)
elif isinstance(v, Node):
v.dump(indent + 1)
else:
sys.stdout.write('Node value type? {}\n'.format(type(v)))
class ScalarNode(Node):
"""
styles:
? -> set() ? key, no value
" -> double quoted
' -> single quoted
| -> literal style
> -> folding style
"""
__slots__ = ('style',)
id = 'scalar'
def __init__(
self, tag, value, start_mark=None, end_mark=None, style=None, comment=None, anchor=None
):
# type: (Any, Any, Any, Any, Any, Any, Any) -> None
Node.__init__(self, tag, value, start_mark, end_mark, comment=comment, anchor=anchor)
self.style = style
class CollectionNode(Node):
__slots__ = ('flow_style',)
def __init__(
self,
tag,
value,
start_mark=None,
end_mark=None,
flow_style=None,
comment=None,
anchor=None,
):
# type: (Any, Any, Any, Any, Any, Any, Any) -> None
Node.__init__(self, tag, value, start_mark, end_mark, comment=comment)
self.flow_style = flow_style
self.anchor = anchor
class SequenceNode(CollectionNode):
__slots__ = ()
id = 'sequence'
class MappingNode(CollectionNode):
__slots__ = ('merge',)
id = 'mapping'
def __init__(
self,
tag,
value,
start_mark=None,
end_mark=None,
flow_style=None,
comment=None,
anchor=None,
):
# type: (Any, Any, Any, Any, Any, Any, Any) -> None
CollectionNode.__init__(
self, tag, value, start_mark, end_mark, flow_style, comment, anchor
)
self.merge = None
pipenv/vendor/ruamel/yaml/parser.py
Vendored
+884
View File
@@ -0,0 +1,884 @@
# coding: utf-8
# The following YAML grammar is LL(1) and is parsed by a recursive descent
# parser.
#
# stream ::= STREAM-START implicit_document? explicit_document*
# STREAM-END
# implicit_document ::= block_node DOCUMENT-END*
# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
# block_node_or_indentless_sequence ::=
# ALIAS
# | properties (block_content |
# indentless_block_sequence)?
# | block_content
# | indentless_block_sequence
# block_node ::= ALIAS
# | properties block_content?
# | block_content
# flow_node ::= ALIAS
# | properties flow_content?
# | flow_content
# properties ::= TAG ANCHOR? | ANCHOR TAG?
# block_content ::= block_collection | flow_collection | SCALAR
# flow_content ::= flow_collection | SCALAR
# block_collection ::= block_sequence | block_mapping
# flow_collection ::= flow_sequence | flow_mapping
# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)*
# BLOCK-END
# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
# block_mapping ::= BLOCK-MAPPING_START
# ((KEY block_node_or_indentless_sequence?)?
# (VALUE block_node_or_indentless_sequence?)?)*
# BLOCK-END
# flow_sequence ::= FLOW-SEQUENCE-START
# (flow_sequence_entry FLOW-ENTRY)*
# flow_sequence_entry?
# FLOW-SEQUENCE-END
# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
# flow_mapping ::= FLOW-MAPPING-START
# (flow_mapping_entry FLOW-ENTRY)*
# flow_mapping_entry?
# FLOW-MAPPING-END
# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
#
# FIRST sets:
#
# stream: { STREAM-START <}
# explicit_document: { DIRECTIVE DOCUMENT-START }
# implicit_document: FIRST(block_node)
# block_node: { ALIAS TAG ANCHOR SCALAR BLOCK-SEQUENCE-START
# BLOCK-MAPPING-START FLOW-SEQUENCE-START FLOW-MAPPING-START }
# flow_node: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START FLOW-MAPPING-START }
# block_content: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START
# FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
# flow_content: { FLOW-SEQUENCE-START FLOW-MAPPING-START SCALAR }
# block_collection: { BLOCK-SEQUENCE-START BLOCK-MAPPING-START }
# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
# block_sequence: { BLOCK-SEQUENCE-START }
# block_mapping: { BLOCK-MAPPING-START }
# block_node_or_indentless_sequence: { ALIAS ANCHOR TAG SCALAR
# BLOCK-SEQUENCE-START BLOCK-MAPPING-START FLOW-SEQUENCE-START
# FLOW-MAPPING-START BLOCK-ENTRY }
# indentless_sequence: { ENTRY }
# flow_collection: { FLOW-SEQUENCE-START FLOW-MAPPING-START }
# flow_sequence: { FLOW-SEQUENCE-START }
# flow_mapping: { FLOW-MAPPING-START }
# flow_sequence_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START
# FLOW-MAPPING-START KEY }
# flow_mapping_entry: { ALIAS ANCHOR TAG SCALAR FLOW-SEQUENCE-START
# FLOW-MAPPING-START KEY }
# need to have full path with import, as pkg_resources tries to load parser.py in __init__.py
# only to not do anything with the package afterwards
# and for Jython too
from pipenv.vendor.ruamel.yaml.error import MarkedYAMLError
from pipenv.vendor.ruamel.yaml.tokens import * # NOQA
from pipenv.vendor.ruamel.yaml.events import * # NOQA
from pipenv.vendor.ruamel.yaml.scanner import Scanner, RoundTripScanner, ScannerError # NOQA
from pipenv.vendor.ruamel.yaml.scanner import BlankLineComment
from pipenv.vendor.ruamel.yaml.comments import C_PRE, C_POST, C_SPLIT_ON_FIRST_BLANK
from pipenv.vendor.ruamel.yaml.compat import _F, nprint, nprintf # NOQA
if False: # MYPY
from typing import Any, Dict, Optional, List, Optional # NOQA
__all__ = ['Parser', 'RoundTripParser', 'ParserError']
def xprintf(*args, **kw):
# type: (Any, Any) -> Any
return nprintf(*args, **kw)
pass
class ParserError(MarkedYAMLError):
pass
class Parser:
# Since writing a recursive-descendant parser is a straightforward task, we
# do not give many comments here.
DEFAULT_TAGS = {'!': '!', '!!': 'tag:yaml.org,2002:'}
def __init__(self, loader):
# type: (Any) -> None
self.loader = loader
if self.loader is not None and getattr(self.loader, '_parser', None) is None:
self.loader._parser = self
self.reset_parser()
def reset_parser(self):
# type: () -> None
# Reset the state attributes (to clear self-references)
self.current_event = self.last_event = None
self.tag_handles = {} # type: Dict[Any, Any]
self.states = [] # type: List[Any]
self.marks = [] # type: List[Any]
self.state = self.parse_stream_start # type: Any
def dispose(self):
# type: () -> None
self.reset_parser()
@property
def scanner(self):
# type: () -> Any
if hasattr(self.loader, 'typ'):
return self.loader.scanner
return self.loader._scanner
@property
def resolver(self):
# type: () -> Any
if hasattr(self.loader, 'typ'):
return self.loader.resolver
return self.loader._resolver
def check_event(self, *choices):
# type: (Any) -> bool
# Check the type of the next event.
if self.current_event is None:
if self.state:
self.current_event = self.state()
if self.current_event is not None:
if not choices:
return True
for choice in choices:
if isinstance(self.current_event, choice):
return True
return False
def peek_event(self):
# type: () -> Any
# Get the next event.
if self.current_event is None:
if self.state:
self.current_event = self.state()
return self.current_event
def get_event(self):
# type: () -> Any
# Get the next event and proceed further.
if self.current_event is None:
if self.state:
self.current_event = self.state()
# assert self.current_event is not None
# if self.current_event.end_mark.line != self.peek_event().start_mark.line:
xprintf('get_event', repr(self.current_event), self.peek_event().start_mark.line)
self.last_event = value = self.current_event
self.current_event = None
return value
# stream ::= STREAM-START implicit_document? explicit_document*
# STREAM-END
# implicit_document ::= block_node DOCUMENT-END*
# explicit_document ::= DIRECTIVE* DOCUMENT-START block_node? DOCUMENT-END*
def parse_stream_start(self):
# type: () -> Any
# Parse the stream start.
token = self.scanner.get_token()
self.move_token_comment(token)
event = StreamStartEvent(token.start_mark, token.end_mark, encoding=token.encoding)
# Prepare the next state.
self.state = self.parse_implicit_document_start
return event
def parse_implicit_document_start(self):
# type: () -> Any
# Parse an implicit document.
if not self.scanner.check_token(DirectiveToken, DocumentStartToken, StreamEndToken):
self.tag_handles = self.DEFAULT_TAGS
token = self.scanner.peek_token()
start_mark = end_mark = token.start_mark
event = DocumentStartEvent(start_mark, end_mark, explicit=False)
# Prepare the next state.
self.states.append(self.parse_document_end)
self.state = self.parse_block_node
return event
else:
return self.parse_document_start()
def parse_document_start(self):
# type: () -> Any
# Parse any extra document end indicators.
while self.scanner.check_token(DocumentEndToken):
self.scanner.get_token()
# Parse an explicit document.
if not self.scanner.check_token(StreamEndToken):
version, tags = self.process_directives()
if not self.scanner.check_token(DocumentStartToken):
raise ParserError(
None,
None,
_F(
"expected '<document start>', but found {pt!r}",
pt=self.scanner.peek_token().id,
),
self.scanner.peek_token().start_mark,
)
token = self.scanner.get_token()
start_mark = token.start_mark
end_mark = token.end_mark
# if self.loader is not None and \
# end_mark.line != self.scanner.peek_token().start_mark.line:
# self.loader.scalar_after_indicator = False
event = DocumentStartEvent(
start_mark, end_mark, explicit=True, version=version, tags=tags,
comment=token.comment
) # type: Any
self.states.append(self.parse_document_end)
self.state = self.parse_document_content
else:
# Parse the end of the stream.
token = self.scanner.get_token()
event = StreamEndEvent(token.start_mark, token.end_mark, comment=token.comment)
assert not self.states
assert not self.marks
self.state = None
return event
def parse_document_end(self):
# type: () -> Any
# Parse the document end.
token = self.scanner.peek_token()
start_mark = end_mark = token.start_mark
explicit = False
if self.scanner.check_token(DocumentEndToken):
token = self.scanner.get_token()
end_mark = token.end_mark
explicit = True
event = DocumentEndEvent(start_mark, end_mark, explicit=explicit)
# Prepare the next state.
if self.resolver.processing_version == (1, 1):
self.state = self.parse_document_start
else:
self.state = self.parse_implicit_document_start
return event
def parse_document_content(self):
# type: () -> Any
if self.scanner.check_token(
DirectiveToken, DocumentStartToken, DocumentEndToken, StreamEndToken
):
event = self.process_empty_scalar(self.scanner.peek_token().start_mark)
self.state = self.states.pop()
return event
else:
return self.parse_block_node()
def process_directives(self):
# type: () -> Any
yaml_version = None
self.tag_handles = {}
while self.scanner.check_token(DirectiveToken):
token = self.scanner.get_token()
if token.name == 'YAML':
if yaml_version is not None:
raise ParserError(
None, None, 'found duplicate YAML directive', token.start_mark
)
major, minor = token.value
if major != 1:
raise ParserError(
None,
None,
'found incompatible YAML document (version 1.* is required)',
token.start_mark,
)
yaml_version = token.value
elif token.name == 'TAG':
handle, prefix = token.value
if handle in self.tag_handles:
raise ParserError(
None,
None,
_F('duplicate tag handle {handle!r}', handle=handle),
token.start_mark,
)
self.tag_handles[handle] = prefix
if bool(self.tag_handles):
value = yaml_version, self.tag_handles.copy() # type: Any
else:
value = yaml_version, None
if self.loader is not None and hasattr(self.loader, 'tags'):
self.loader.version = yaml_version
if self.loader.tags is None:
self.loader.tags = {}
for k in self.tag_handles:
self.loader.tags[k] = self.tag_handles[k]
for key in self.DEFAULT_TAGS:
if key not in self.tag_handles:
self.tag_handles[key] = self.DEFAULT_TAGS[key]
return value
# block_node_or_indentless_sequence ::= ALIAS
# | properties (block_content | indentless_block_sequence)?
# | block_content
# | indentless_block_sequence
# block_node ::= ALIAS
# | properties block_content?
# | block_content
# flow_node ::= ALIAS
# | properties flow_content?
# | flow_content
# properties ::= TAG ANCHOR? | ANCHOR TAG?
# block_content ::= block_collection | flow_collection | SCALAR
# flow_content ::= flow_collection | SCALAR
# block_collection ::= block_sequence | block_mapping
# flow_collection ::= flow_sequence | flow_mapping
def parse_block_node(self):
# type: () -> Any
return self.parse_node(block=True)
def parse_flow_node(self):
# type: () -> Any
return self.parse_node()
def parse_block_node_or_indentless_sequence(self):
# type: () -> Any
return self.parse_node(block=True, indentless_sequence=True)
def transform_tag(self, handle, suffix):
# type: (Any, Any) -> Any
return self.tag_handles[handle] + suffix
def parse_node(self, block=False, indentless_sequence=False):
# type: (bool, bool) -> Any
if self.scanner.check_token(AliasToken):
token = self.scanner.get_token()
event = AliasEvent(token.value, token.start_mark, token.end_mark) # type: Any
self.state = self.states.pop()
return event
anchor = None
tag = None
start_mark = end_mark = tag_mark = None
if self.scanner.check_token(AnchorToken):
token = self.scanner.get_token()
self.move_token_comment(token)
start_mark = token.start_mark
end_mark = token.end_mark
anchor = token.value
if self.scanner.check_token(TagToken):
token = self.scanner.get_token()
tag_mark = token.start_mark
end_mark = token.end_mark
tag = token.value
elif self.scanner.check_token(TagToken):
token = self.scanner.get_token()
start_mark = tag_mark = token.start_mark
end_mark = token.end_mark
tag = token.value
if self.scanner.check_token(AnchorToken):
token = self.scanner.get_token()
start_mark = tag_mark = token.start_mark
end_mark = token.end_mark
anchor = token.value
if tag is not None:
handle, suffix = tag
if handle is not None:
if handle not in self.tag_handles:
raise ParserError(
'while parsing a node',
start_mark,
_F('found undefined tag handle {handle!r}', handle=handle),
tag_mark,
)
tag = self.transform_tag(handle, suffix)
else:
tag = suffix
# if tag == '!':
# raise ParserError("while parsing a node", start_mark,
# "found non-specific tag '!'", tag_mark,
# "Please check 'http://pyyaml.org/wiki/YAMLNonSpecificTag'
# and share your opinion.")
if start_mark is None:
start_mark = end_mark = self.scanner.peek_token().start_mark
event = None
implicit = tag is None or tag == '!'
if indentless_sequence and self.scanner.check_token(BlockEntryToken):
comment = None
pt = self.scanner.peek_token()
if self.loader and self.loader.comment_handling is None:
if pt.comment and pt.comment[0]:
comment = [pt.comment[0], []]
pt.comment[0] = None
elif self.loader:
if pt.comment:
comment = pt.comment
end_mark = self.scanner.peek_token().end_mark
event = SequenceStartEvent(
anchor, tag, implicit, start_mark, end_mark, flow_style=False, comment=comment
)
self.state = self.parse_indentless_sequence_entry
return event
if self.scanner.check_token(ScalarToken):
token = self.scanner.get_token()
# self.scanner.peek_token_same_line_comment(token)
end_mark = token.end_mark
if (token.plain and tag is None) or tag == '!':
implicit = (True, False)
elif tag is None:
implicit = (False, True)
else:
implicit = (False, False)
# nprint('se', token.value, token.comment)
event = ScalarEvent(
anchor,
tag,
implicit,
token.value,
start_mark,
end_mark,
style=token.style,
comment=token.comment,
)
self.state = self.states.pop()
elif self.scanner.check_token(FlowSequenceStartToken):
pt = self.scanner.peek_token()
end_mark = pt.end_mark
event = SequenceStartEvent(
anchor,
tag,
implicit,
start_mark,
end_mark,
flow_style=True,
comment=pt.comment,
)
self.state = self.parse_flow_sequence_first_entry
elif self.scanner.check_token(FlowMappingStartToken):
pt = self.scanner.peek_token()
end_mark = pt.end_mark
event = MappingStartEvent(
anchor,
tag,
implicit,
start_mark,
end_mark,
flow_style=True,
comment=pt.comment,
)
self.state = self.parse_flow_mapping_first_key
elif block and self.scanner.check_token(BlockSequenceStartToken):
end_mark = self.scanner.peek_token().start_mark
# should inserting the comment be dependent on the
# indentation?
pt = self.scanner.peek_token()
comment = pt.comment
# nprint('pt0', type(pt))
if comment is None or comment[1] is None:
comment = pt.split_old_comment()
# nprint('pt1', comment)
event = SequenceStartEvent(
anchor, tag, implicit, start_mark, end_mark, flow_style=False, comment=comment
)
self.state = self.parse_block_sequence_first_entry
elif block and self.scanner.check_token(BlockMappingStartToken):
end_mark = self.scanner.peek_token().start_mark
comment = self.scanner.peek_token().comment
event = MappingStartEvent(
anchor, tag, implicit, start_mark, end_mark, flow_style=False, comment=comment
)
self.state = self.parse_block_mapping_first_key
elif anchor is not None or tag is not None:
# Empty scalars are allowed even if a tag or an anchor is
# specified.
event = ScalarEvent(anchor, tag, (implicit, False), "", start_mark, end_mark)
self.state = self.states.pop()
else:
if block:
node = 'block'
else:
node = 'flow'
token = self.scanner.peek_token()
raise ParserError(
_F('while parsing a {node!s} node', node=node),
start_mark,
_F('expected the node content, but found {token_id!r}', token_id=token.id),
token.start_mark,
)
return event
# block_sequence ::= BLOCK-SEQUENCE-START (BLOCK-ENTRY block_node?)*
# BLOCK-END
def parse_block_sequence_first_entry(self):
# type: () -> Any
token = self.scanner.get_token()
# move any comment from start token
# self.move_token_comment(token)
self.marks.append(token.start_mark)
return self.parse_block_sequence_entry()
def parse_block_sequence_entry(self):
# type: () -> Any
if self.scanner.check_token(BlockEntryToken):
token = self.scanner.get_token()
self.move_token_comment(token)
if not self.scanner.check_token(BlockEntryToken, BlockEndToken):
self.states.append(self.parse_block_sequence_entry)
return self.parse_block_node()
else:
self.state = self.parse_block_sequence_entry
return self.process_empty_scalar(token.end_mark)
if not self.scanner.check_token(BlockEndToken):
token = self.scanner.peek_token()
raise ParserError(
'while parsing a block collection',
self.marks[-1],
_F('expected <block end>, but found {token_id!r}', token_id=token.id),
token.start_mark,
)
token = self.scanner.get_token() # BlockEndToken
event = SequenceEndEvent(token.start_mark, token.end_mark, comment=token.comment)
self.state = self.states.pop()
self.marks.pop()
return event
# indentless_sequence ::= (BLOCK-ENTRY block_node?)+
# indentless_sequence?
# sequence:
# - entry
# - nested
def parse_indentless_sequence_entry(self):
# type: () -> Any
if self.scanner.check_token(BlockEntryToken):
token = self.scanner.get_token()
self.move_token_comment(token)
if not self.scanner.check_token(
BlockEntryToken, KeyToken, ValueToken, BlockEndToken
):
self.states.append(self.parse_indentless_sequence_entry)
return self.parse_block_node()
else:
self.state = self.parse_indentless_sequence_entry
return self.process_empty_scalar(token.end_mark)
token = self.scanner.peek_token()
c = None
if self.loader and self.loader.comment_handling is None:
c = token.comment
start_mark = token.start_mark
else:
start_mark = self.last_event.end_mark # type: ignore
c = self.distribute_comment(token.comment, start_mark.line) # type: ignore
event = SequenceEndEvent(start_mark, start_mark, comment=c)
self.state = self.states.pop()
return event
# block_mapping ::= BLOCK-MAPPING_START
# ((KEY block_node_or_indentless_sequence?)?
# (VALUE block_node_or_indentless_sequence?)?)*
# BLOCK-END
def parse_block_mapping_first_key(self):
# type: () -> Any
token = self.scanner.get_token()
self.marks.append(token.start_mark)
return self.parse_block_mapping_key()
def parse_block_mapping_key(self):
# type: () -> Any
if self.scanner.check_token(KeyToken):
token = self.scanner.get_token()
self.move_token_comment(token)
if not self.scanner.check_token(KeyToken, ValueToken, BlockEndToken):
self.states.append(self.parse_block_mapping_value)
return self.parse_block_node_or_indentless_sequence()
else:
self.state = self.parse_block_mapping_value
return self.process_empty_scalar(token.end_mark)
if self.resolver.processing_version > (1, 1) and self.scanner.check_token(ValueToken):
self.state = self.parse_block_mapping_value
return self.process_empty_scalar(self.scanner.peek_token().start_mark)
if not self.scanner.check_token(BlockEndToken):
token = self.scanner.peek_token()
raise ParserError(
'while parsing a block mapping',
self.marks[-1],
_F('expected <block end>, but found {token_id!r}', token_id=token.id),
token.start_mark,
)
token = self.scanner.get_token()
self.move_token_comment(token)
event = MappingEndEvent(token.start_mark, token.end_mark, comment=token.comment)
self.state = self.states.pop()
self.marks.pop()
return event
def parse_block_mapping_value(self):
# type: () -> Any
if self.scanner.check_token(ValueToken):
token = self.scanner.get_token()
# value token might have post comment move it to e.g. block
if self.scanner.check_token(ValueToken):
self.move_token_comment(token)
else:
if not self.scanner.check_token(KeyToken):
self.move_token_comment(token, empty=True)
# else: empty value for this key cannot move token.comment
if not self.scanner.check_token(KeyToken, ValueToken, BlockEndToken):
self.states.append(self.parse_block_mapping_key)
return self.parse_block_node_or_indentless_sequence()
else:
self.state = self.parse_block_mapping_key
comment = token.comment
if comment is None:
token = self.scanner.peek_token()
comment = token.comment
if comment:
token._comment = [None, comment[1]]
comment = [comment[0], None]
return self.process_empty_scalar(token.end_mark, comment=comment)
else:
self.state = self.parse_block_mapping_key
token = self.scanner.peek_token()
return self.process_empty_scalar(token.start_mark)
# flow_sequence ::= FLOW-SEQUENCE-START
# (flow_sequence_entry FLOW-ENTRY)*
# flow_sequence_entry?
# FLOW-SEQUENCE-END
# flow_sequence_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
#
# Note that while production rules for both flow_sequence_entry and
# flow_mapping_entry are equal, their interpretations are different.
# For `flow_sequence_entry`, the part `KEY flow_node? (VALUE flow_node?)?`
# generate an inline mapping (set syntax).
def parse_flow_sequence_first_entry(self):
# type: () -> Any
token = self.scanner.get_token()
self.marks.append(token.start_mark)
return self.parse_flow_sequence_entry(first=True)
def parse_flow_sequence_entry(self, first=False):
# type: (bool) -> Any
if not self.scanner.check_token(FlowSequenceEndToken):
if not first:
if self.scanner.check_token(FlowEntryToken):
self.scanner.get_token()
else:
token = self.scanner.peek_token()
raise ParserError(
'while parsing a flow sequence',
self.marks[-1],
_F("expected ',' or ']', but got {token_id!r}", token_id=token.id),
token.start_mark,
)
if self.scanner.check_token(KeyToken):
token = self.scanner.peek_token()
event = MappingStartEvent(
None, None, True, token.start_mark, token.end_mark, flow_style=True
) # type: Any
self.state = self.parse_flow_sequence_entry_mapping_key
return event
elif not self.scanner.check_token(FlowSequenceEndToken):
self.states.append(self.parse_flow_sequence_entry)
return self.parse_flow_node()
token = self.scanner.get_token()
event = SequenceEndEvent(token.start_mark, token.end_mark, comment=token.comment)
self.state = self.states.pop()
self.marks.pop()
return event
def parse_flow_sequence_entry_mapping_key(self):
# type: () -> Any
token = self.scanner.get_token()
if not self.scanner.check_token(ValueToken, FlowEntryToken, FlowSequenceEndToken):
self.states.append(self.parse_flow_sequence_entry_mapping_value)
return self.parse_flow_node()
else:
self.state = self.parse_flow_sequence_entry_mapping_value
return self.process_empty_scalar(token.end_mark)
def parse_flow_sequence_entry_mapping_value(self):
# type: () -> Any
if self.scanner.check_token(ValueToken):
token = self.scanner.get_token()
if not self.scanner.check_token(FlowEntryToken, FlowSequenceEndToken):
self.states.append(self.parse_flow_sequence_entry_mapping_end)
return self.parse_flow_node()
else:
self.state = self.parse_flow_sequence_entry_mapping_end
return self.process_empty_scalar(token.end_mark)
else:
self.state = self.parse_flow_sequence_entry_mapping_end
token = self.scanner.peek_token()
return self.process_empty_scalar(token.start_mark)
def parse_flow_sequence_entry_mapping_end(self):
# type: () -> Any
self.state = self.parse_flow_sequence_entry
token = self.scanner.peek_token()
return MappingEndEvent(token.start_mark, token.start_mark)
# flow_mapping ::= FLOW-MAPPING-START
# (flow_mapping_entry FLOW-ENTRY)*
# flow_mapping_entry?
# FLOW-MAPPING-END
# flow_mapping_entry ::= flow_node | KEY flow_node? (VALUE flow_node?)?
def parse_flow_mapping_first_key(self):
# type: () -> Any
token = self.scanner.get_token()
self.marks.append(token.start_mark)
return self.parse_flow_mapping_key(first=True)
def parse_flow_mapping_key(self, first=False):
# type: (Any) -> Any
if not self.scanner.check_token(FlowMappingEndToken):
if not first:
if self.scanner.check_token(FlowEntryToken):
self.scanner.get_token()
else:
token = self.scanner.peek_token()
raise ParserError(
'while parsing a flow mapping',
self.marks[-1],
_F("expected ',' or '}}', but got {token_id!r}", token_id=token.id),
token.start_mark,
)
if self.scanner.check_token(KeyToken):
token = self.scanner.get_token()
if not self.scanner.check_token(
ValueToken, FlowEntryToken, FlowMappingEndToken
):
self.states.append(self.parse_flow_mapping_value)
return self.parse_flow_node()
else:
self.state = self.parse_flow_mapping_value
return self.process_empty_scalar(token.end_mark)
elif self.resolver.processing_version > (1, 1) and self.scanner.check_token(
ValueToken
):
self.state = self.parse_flow_mapping_value
return self.process_empty_scalar(self.scanner.peek_token().end_mark)
elif not self.scanner.check_token(FlowMappingEndToken):
self.states.append(self.parse_flow_mapping_empty_value)
return self.parse_flow_node()
token = self.scanner.get_token()
event = MappingEndEvent(token.start_mark, token.end_mark, comment=token.comment)
self.state = self.states.pop()
self.marks.pop()
return event
def parse_flow_mapping_value(self):
# type: () -> Any
if self.scanner.check_token(ValueToken):
token = self.scanner.get_token()
if not self.scanner.check_token(FlowEntryToken, FlowMappingEndToken):
self.states.append(self.parse_flow_mapping_key)
return self.parse_flow_node()
else:
self.state = self.parse_flow_mapping_key
return self.process_empty_scalar(token.end_mark)
else:
self.state = self.parse_flow_mapping_key
token = self.scanner.peek_token()
return self.process_empty_scalar(token.start_mark)
def parse_flow_mapping_empty_value(self):
# type: () -> Any
self.state = self.parse_flow_mapping_key
return self.process_empty_scalar(self.scanner.peek_token().start_mark)
def process_empty_scalar(self, mark, comment=None):
# type: (Any, Any) -> Any
return ScalarEvent(None, None, (True, False), "", mark, mark, comment=comment)
def move_token_comment(self, token, nt=None, empty=False):
# type: (Any, Optional[Any], Optional[bool]) -> Any
pass
class RoundTripParser(Parser):
"""roundtrip is a safe loader, that wants to see the unmangled tag"""
def transform_tag(self, handle, suffix):
# type: (Any, Any) -> Any
# return self.tag_handles[handle]+suffix
if handle == '!!' and suffix in (
'null',
'bool',
'int',
'float',
'binary',
'timestamp',
'omap',
'pairs',
'set',
'str',
'seq',
'map',
):
return Parser.transform_tag(self, handle, suffix)
return handle + suffix
def move_token_comment(self, token, nt=None, empty=False):
# type: (Any, Optional[Any], Optional[bool]) -> Any
token.move_old_comment(self.scanner.peek_token() if nt is None else nt, empty=empty)
class RoundTripParserSC(RoundTripParser):
"""roundtrip is a safe loader, that wants to see the unmangled tag"""
# some of the differences are based on the superclass testing
# if self.loader.comment_handling is not None
def move_token_comment(self, token, nt=None, empty=False):
# type: (Any, Any, Any, Optional[bool]) -> None
token.move_new_comment(self.scanner.peek_token() if nt is None else nt, empty=empty)
def distribute_comment(self, comment, line):
# type: (Any, Any) -> Any
# ToDo, look at indentation of the comment to determine attachment
if comment is None:
return None
if not comment[0]:
return None
if comment[0][0] != line + 1:
nprintf('>>>dcxxx', comment, line)
assert comment[0][0] == line + 1
# if comment[0] - line > 1:
# return
typ = self.loader.comment_handling & 0b11
# nprintf('>>>dca', comment, line, typ)
if typ == C_POST:
return None
if typ == C_PRE:
c = [None, None, comment[0]]
comment[0] = None
return c
# nprintf('>>>dcb', comment[0])
for _idx, cmntidx in enumerate(comment[0]):
# nprintf('>>>dcb', cmntidx)
if isinstance(self.scanner.comments[cmntidx], BlankLineComment):
break
else:
return None # no space found
if _idx == 0:
return None # first line was blank
# nprintf('>>>dcc', idx)
if typ == C_SPLIT_ON_FIRST_BLANK:
c = [None, None, comment[0][:_idx]]
comment[0] = comment[0][_idx:]
return c
raise NotImplementedError # reserved
pipenv/vendor/ruamel/yaml/py.typed
Vendored
View File
pipenv/vendor/ruamel/yaml/reader.py
Vendored
+302
View File
@@ -0,0 +1,302 @@
# coding: utf-8
# This module contains abstractions for the input stream. You don't have to
# looks further, there are no pretty code.
#
# We define two classes here.
#
# Mark(source, line, column)
# It's just a record and its only use is producing nice error messages.
# Parser does not use it for any other purposes.
#
# Reader(source, data)
# Reader determines the encoding of `data` and converts it to unicode.
# Reader provides the following methods and attributes:
# reader.peek(length=1) - return the next `length` characters
# reader.forward(length=1) - move the current position to `length`
# characters.
# reader.index - the number of the current character.
# reader.line, stream.column - the line and the column of the current
# character.
import codecs
from pipenv.vendor.ruamel.yaml.error import YAMLError, FileMark, StringMark, YAMLStreamError
from pipenv.vendor.ruamel.yaml.compat import _F # NOQA
from pipenv.vendor.ruamel.yaml.util import RegExp
if False: # MYPY
from typing import Any, Dict, Optional, List, Union, Text, Tuple, Optional # NOQA
# from ruamel.compat import StreamTextType # NOQA
__all__ = ['Reader', 'ReaderError']
class ReaderError(YAMLError):
def __init__(self, name, position, character, encoding, reason):
# type: (Any, Any, Any, Any, Any) -> None
self.name = name
self.character = character
self.position = position
self.encoding = encoding
self.reason = reason
def __str__(self):
# type: () -> Any
if isinstance(self.character, bytes):
return _F(
"'{self_encoding!s}' codec can't decode byte #x{ord_self_character:02x}: "
'{self_reason!s}\n'
' in "{self_name!s}", position {self_position:d}',
self_encoding=self.encoding,
ord_self_character=ord(self.character),
self_reason=self.reason,
self_name=self.name,
self_position=self.position,
)
else:
return _F(
'unacceptable character #x{self_character:04x}: {self_reason!s}\n'
' in "{self_name!s}", position {self_position:d}',
self_character=self.character,
self_reason=self.reason,
self_name=self.name,
self_position=self.position,
)
class Reader:
# Reader:
# - determines the data encoding and converts it to a unicode string,
# - checks if characters are in allowed range,
# - adds '\0' to the end.
# Reader accepts
# - a `bytes` object,
# - a `str` object,
# - a file-like object with its `read` method returning `str`,
# - a file-like object with its `read` method returning `unicode`.
# Yeah, it's ugly and slow.
def __init__(self, stream, loader=None):
# type: (Any, Any) -> None
self.loader = loader
if self.loader is not None and getattr(self.loader, '_reader', None) is None:
self.loader._reader = self
self.reset_reader()
self.stream = stream # type: Any # as .read is called
def reset_reader(self):
# type: () -> None
self.name = None # type: Any
self.stream_pointer = 0
self.eof = True
self.buffer = ""
self.pointer = 0
self.raw_buffer = None # type: Any
self.raw_decode = None
self.encoding = None # type: Optional[Text]
self.index = 0
self.line = 0
self.column = 0
@property
def stream(self):
# type: () -> Any
try:
return self._stream
except AttributeError:
raise YAMLStreamError('input stream needs to specified')
@stream.setter
def stream(self, val):
# type: (Any) -> None
if val is None:
return
self._stream = None
if isinstance(val, str):
self.name = '<unicode string>'
self.check_printable(val)
self.buffer = val + '\0'
elif isinstance(val, bytes):
self.name = '<byte string>'
self.raw_buffer = val
self.determine_encoding()
else:
if not hasattr(val, 'read'):
raise YAMLStreamError('stream argument needs to have a read() method')
self._stream = val
self.name = getattr(self.stream, 'name', '<file>')
self.eof = False
self.raw_buffer = None
self.determine_encoding()
def peek(self, index=0):
# type: (int) -> Text
try:
return self.buffer[self.pointer + index]
except IndexError:
self.update(index + 1)
return self.buffer[self.pointer + index]
def prefix(self, length=1):
# type: (int) -> Any
if self.pointer + length >= len(self.buffer):
self.update(length)
return self.buffer[self.pointer : self.pointer + length]
def forward_1_1(self, length=1):
# type: (int) -> None
if self.pointer + length + 1 >= len(self.buffer):
self.update(length + 1)
while length != 0:
ch = self.buffer[self.pointer]
self.pointer += 1
self.index += 1
if ch in '\n\x85\u2028\u2029' or (
ch == '\r' and self.buffer[self.pointer] != '\n'
):
self.line += 1
self.column = 0
elif ch != '\uFEFF':
self.column += 1
length -= 1
def forward(self, length=1):
# type: (int) -> None
if self.pointer + length + 1 >= len(self.buffer):
self.update(length + 1)
while length != 0:
ch = self.buffer[self.pointer]
self.pointer += 1
self.index += 1
if ch == '\n' or (ch == '\r' and self.buffer[self.pointer] != '\n'):
self.line += 1
self.column = 0
elif ch != '\uFEFF':
self.column += 1
length -= 1
def get_mark(self):
# type: () -> Any
if self.stream is None:
return StringMark(
self.name, self.index, self.line, self.column, self.buffer, self.pointer
)
else:
return FileMark(self.name, self.index, self.line, self.column)
def determine_encoding(self):
# type: () -> None
while not self.eof and (self.raw_buffer is None or len(self.raw_buffer) < 2):
self.update_raw()
if isinstance(self.raw_buffer, bytes):
if self.raw_buffer.startswith(codecs.BOM_UTF16_LE):
self.raw_decode = codecs.utf_16_le_decode # type: ignore
self.encoding = 'utf-16-le'
elif self.raw_buffer.startswith(codecs.BOM_UTF16_BE):
self.raw_decode = codecs.utf_16_be_decode # type: ignore
self.encoding = 'utf-16-be'
else:
self.raw_decode = codecs.utf_8_decode # type: ignore
self.encoding = 'utf-8'
self.update(1)
NON_PRINTABLE = RegExp(
'[^\x09\x0A\x0D\x20-\x7E\x85' '\xA0-\uD7FF' '\uE000-\uFFFD' '\U00010000-\U0010FFFF' ']'
)
_printable_ascii = ('\x09\x0A\x0D' + "".join(map(chr, range(0x20, 0x7F)))).encode('ascii')
@classmethod
def _get_non_printable_ascii(cls, data): # type: ignore
# type: (Text, bytes) -> Optional[Tuple[int, Text]]
ascii_bytes = data.encode('ascii') # type: ignore
non_printables = ascii_bytes.translate(None, cls._printable_ascii) # type: ignore
if not non_printables:
return None
non_printable = non_printables[:1]
return ascii_bytes.index(non_printable), non_printable.decode('ascii')
@classmethod
def _get_non_printable_regex(cls, data):
# type: (Text) -> Optional[Tuple[int, Text]]
match = cls.NON_PRINTABLE.search(data)
if not bool(match):
return None
return match.start(), match.group()
@classmethod
def _get_non_printable(cls, data):
# type: (Text) -> Optional[Tuple[int, Text]]
try:
return cls._get_non_printable_ascii(data) # type: ignore
except UnicodeEncodeError:
return cls._get_non_printable_regex(data)
def check_printable(self, data):
# type: (Any) -> None
non_printable_match = self._get_non_printable(data)
if non_printable_match is not None:
start, character = non_printable_match
position = self.index + (len(self.buffer) - self.pointer) + start
raise ReaderError(
self.name,
position,
ord(character),
'unicode',
'special characters are not allowed',
)
def update(self, length):
# type: (int) -> None
if self.raw_buffer is None:
return
self.buffer = self.buffer[self.pointer :]
self.pointer = 0
while len(self.buffer) < length:
if not self.eof:
self.update_raw()
if self.raw_decode is not None:
try:
data, converted = self.raw_decode(self.raw_buffer, 'strict', self.eof)
except UnicodeDecodeError as exc:
character = self.raw_buffer[exc.start]
if self.stream is not None:
position = self.stream_pointer - len(self.raw_buffer) + exc.start
elif self.stream is not None:
position = self.stream_pointer - len(self.raw_buffer) + exc.start
else:
position = exc.start
raise ReaderError(self.name, position, character, exc.encoding, exc.reason)
else:
data = self.raw_buffer
converted = len(data)
self.check_printable(data)
self.buffer += data
self.raw_buffer = self.raw_buffer[converted:]
if self.eof:
self.buffer += '\0'
self.raw_buffer = None
break
def update_raw(self, size=None):
# type: (Optional[int]) -> None
if size is None:
size = 4096
data = self.stream.read(size)
if self.raw_buffer is None:
self.raw_buffer = data
else:
self.raw_buffer += data
self.stream_pointer += len(data)
if not data:
self.eof = True
# try:
# import psyco
# psyco.bind(Reader)
# except ImportError:
# pass
pipenv/vendor/ruamel/yaml/representer.py
Vendored
+1156
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/resolver.py
Vendored
+405
View File
@@ -0,0 +1,405 @@
# coding: utf-8
import re
if False: # MYPY
from typing import Any, Dict, List, Union, Text, Optional # NOQA
from pipenv.vendor.ruamel.yaml.compat import VersionType # NOQA
from pipenv.vendor.ruamel.yaml.compat import _DEFAULT_YAML_VERSION, _F # NOQA
from pipenv.vendor.ruamel.yaml.error import * # NOQA
from pipenv.vendor.ruamel.yaml.nodes import MappingNode, ScalarNode, SequenceNode # NOQA
from pipenv.vendor.ruamel.yaml.util import RegExp # NOQA
__all__ = ['BaseResolver', 'Resolver', 'VersionedResolver']
# fmt: off
# resolvers consist of
# - a list of applicable version
# - a tag
# - a regexp
# - a list of first characters to match
implicit_resolvers = [
([(1, 2)],
'tag:yaml.org,2002:bool',
RegExp('''^(?:true|True|TRUE|false|False|FALSE)$''', re.X),
list('tTfF')),
([(1, 1)],
'tag:yaml.org,2002:bool',
RegExp('''^(?:y|Y|yes|Yes|YES|n|N|no|No|NO
|true|True|TRUE|false|False|FALSE
|on|On|ON|off|Off|OFF)$''', re.X),
list('yYnNtTfFoO')),
([(1, 2)],
'tag:yaml.org,2002:float',
RegExp('''^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|[-+]?\\.[0-9_]+(?:[eE][-+][0-9]+)?
|[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$''', re.X),
list('-+0123456789.')),
([(1, 1)],
'tag:yaml.org,2002:float',
RegExp('''^(?:
[-+]?(?:[0-9][0-9_]*)\\.[0-9_]*(?:[eE][-+]?[0-9]+)?
|[-+]?(?:[0-9][0-9_]*)(?:[eE][-+]?[0-9]+)
|\\.[0-9_]+(?:[eE][-+][0-9]+)?
|[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+\\.[0-9_]* # sexagesimal float
|[-+]?\\.(?:inf|Inf|INF)
|\\.(?:nan|NaN|NAN))$''', re.X),
list('-+0123456789.')),
([(1, 2)],
'tag:yaml.org,2002:int',
RegExp('''^(?:[-+]?0b[0-1_]+
|[-+]?0o?[0-7_]+
|[-+]?[0-9_]+
|[-+]?0x[0-9a-fA-F_]+)$''', re.X),
list('-+0123456789')),
([(1, 1)],
'tag:yaml.org,2002:int',
RegExp('''^(?:[-+]?0b[0-1_]+
|[-+]?0?[0-7_]+
|[-+]?(?:0|[1-9][0-9_]*)
|[-+]?0x[0-9a-fA-F_]+
|[-+]?[1-9][0-9_]*(?::[0-5]?[0-9])+)$''', re.X), # sexagesimal int
list('-+0123456789')),
([(1, 2), (1, 1)],
'tag:yaml.org,2002:merge',
RegExp('^(?:<<)$'),
['<']),
([(1, 2), (1, 1)],
'tag:yaml.org,2002:null',
RegExp('''^(?: ~
|null|Null|NULL
| )$''', re.X),
['~', 'n', 'N', '']),
([(1, 2), (1, 1)],
'tag:yaml.org,2002:timestamp',
RegExp('''^(?:[0-9][0-9][0-9][0-9]-[0-9][0-9]-[0-9][0-9]
|[0-9][0-9][0-9][0-9] -[0-9][0-9]? -[0-9][0-9]?
(?:[Tt]|[ \\t]+)[0-9][0-9]?
:[0-9][0-9] :[0-9][0-9] (?:\\.[0-9]*)?
(?:[ \\t]*(?:Z|[-+][0-9][0-9]?(?::[0-9][0-9])?))?)$''', re.X),
list('0123456789')),
([(1, 2), (1, 1)],
'tag:yaml.org,2002:value',
RegExp('^(?:=)$'),
['=']),
# The following resolver is only for documentation purposes. It cannot work
# because plain scalars cannot start with '!', '&', or '*'.
([(1, 2), (1, 1)],
'tag:yaml.org,2002:yaml',
RegExp('^(?:!|&|\\*)$'),
list('!&*')),
]
# fmt: on
class ResolverError(YAMLError):
pass
class BaseResolver:
DEFAULT_SCALAR_TAG = 'tag:yaml.org,2002:str'
DEFAULT_SEQUENCE_TAG = 'tag:yaml.org,2002:seq'
DEFAULT_MAPPING_TAG = 'tag:yaml.org,2002:map'
yaml_implicit_resolvers = {} # type: Dict[Any, Any]
yaml_path_resolvers = {} # type: Dict[Any, Any]
def __init__(self, loadumper=None):
# type: (Any, Any) -> None
self.loadumper = loadumper
if self.loadumper is not None and getattr(self.loadumper, '_resolver', None) is None:
self.loadumper._resolver = self.loadumper
self._loader_version = None # type: Any
self.resolver_exact_paths = [] # type: List[Any]
self.resolver_prefix_paths = [] # type: List[Any]
@property
def parser(self):
# type: () -> Any
if self.loadumper is not None:
if hasattr(self.loadumper, 'typ'):
return self.loadumper.parser
return self.loadumper._parser
return None
@classmethod
def add_implicit_resolver_base(cls, tag, regexp, first):
# type: (Any, Any, Any) -> None
if 'yaml_implicit_resolvers' not in cls.__dict__:
# deepcopy doesn't work here
cls.yaml_implicit_resolvers = dict(
(k, cls.yaml_implicit_resolvers[k][:]) for k in cls.yaml_implicit_resolvers
)
if first is None:
first = [None]
for ch in first:
cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
@classmethod
def add_implicit_resolver(cls, tag, regexp, first):
# type: (Any, Any, Any) -> None
if 'yaml_implicit_resolvers' not in cls.__dict__:
# deepcopy doesn't work here
cls.yaml_implicit_resolvers = dict(
(k, cls.yaml_implicit_resolvers[k][:]) for k in cls.yaml_implicit_resolvers
)
if first is None:
first = [None]
for ch in first:
cls.yaml_implicit_resolvers.setdefault(ch, []).append((tag, regexp))
implicit_resolvers.append(([(1, 2), (1, 1)], tag, regexp, first))
# @classmethod
# def add_implicit_resolver(cls, tag, regexp, first):
@classmethod
def add_path_resolver(cls, tag, path, kind=None):
# type: (Any, Any, Any) -> None
# Note: `add_path_resolver` is experimental. The API could be changed.
# `new_path` is a pattern that is matched against the path from the
# root to the node that is being considered. `node_path` elements are
# tuples `(node_check, index_check)`. `node_check` is a node class:
# `ScalarNode`, `SequenceNode`, `MappingNode` or `None`. `None`
# matches any kind of a node. `index_check` could be `None`, a boolean
# value, a string value, or a number. `None` and `False` match against
# any _value_ of sequence and mapping nodes. `True` matches against
# any _key_ of a mapping node. A string `index_check` matches against
# a mapping value that corresponds to a scalar key which content is
# equal to the `index_check` value. An integer `index_check` matches
# against a sequence value with the index equal to `index_check`.
if 'yaml_path_resolvers' not in cls.__dict__:
cls.yaml_path_resolvers = cls.yaml_path_resolvers.copy()
new_path = [] # type: List[Any]
for element in path:
if isinstance(element, (list, tuple)):
if len(element) == 2:
node_check, index_check = element
elif len(element) == 1:
node_check = element[0]
index_check = True
else:
raise ResolverError(
_F('Invalid path element: {element!s}', element=element)
)
else:
node_check = None
index_check = element
if node_check is str:
node_check = ScalarNode
elif node_check is list:
node_check = SequenceNode
elif node_check is dict:
node_check = MappingNode
elif (
node_check not in [ScalarNode, SequenceNode, MappingNode]
and not isinstance(node_check, str)
and node_check is not None
):
raise ResolverError(
_F('Invalid node checker: {node_check!s}', node_check=node_check)
)
if not isinstance(index_check, (str, int)) and index_check is not None:
raise ResolverError(
_F('Invalid index checker: {index_check!s}', index_check=index_check)
)
new_path.append((node_check, index_check))
if kind is str:
kind = ScalarNode
elif kind is list:
kind = SequenceNode
elif kind is dict:
kind = MappingNode
elif kind not in [ScalarNode, SequenceNode, MappingNode] and kind is not None:
raise ResolverError(_F('Invalid node kind: {kind!s}', kind=kind))
cls.yaml_path_resolvers[tuple(new_path), kind] = tag
def descend_resolver(self, current_node, current_index):
# type: (Any, Any) -> None
if not self.yaml_path_resolvers:
return
exact_paths = {}
prefix_paths = []
if current_node:
depth = len(self.resolver_prefix_paths)
for path, kind in self.resolver_prefix_paths[-1]:
if self.check_resolver_prefix(depth, path, kind, current_node, current_index):
if len(path) > depth:
prefix_paths.append((path, kind))
else:
exact_paths[kind] = self.yaml_path_resolvers[path, kind]
else:
for path, kind in self.yaml_path_resolvers:
if not path:
exact_paths[kind] = self.yaml_path_resolvers[path, kind]
else:
prefix_paths.append((path, kind))
self.resolver_exact_paths.append(exact_paths)
self.resolver_prefix_paths.append(prefix_paths)
def ascend_resolver(self):
# type: () -> None
if not self.yaml_path_resolvers:
return
self.resolver_exact_paths.pop()
self.resolver_prefix_paths.pop()
def check_resolver_prefix(self, depth, path, kind, current_node, current_index):
# type: (int, Any, Any, Any, Any) -> bool
node_check, index_check = path[depth - 1]
if isinstance(node_check, str):
if current_node.tag != node_check:
return False
elif node_check is not None:
if not isinstance(current_node, node_check):
return False
if index_check is True and current_index is not None:
return False
if (index_check is False or index_check is None) and current_index is None:
return False
if isinstance(index_check, str):
if not (
isinstance(current_index, ScalarNode) and index_check == current_index.value
):
return False
elif isinstance(index_check, int) and not isinstance(index_check, bool):
if index_check != current_index:
return False
return True
def resolve(self, kind, value, implicit):
# type: (Any, Any, Any) -> Any
if kind is ScalarNode and implicit[0]:
if value == "":
resolvers = self.yaml_implicit_resolvers.get("", [])
else:
resolvers = self.yaml_implicit_resolvers.get(value[0], [])
resolvers += self.yaml_implicit_resolvers.get(None, [])
for tag, regexp in resolvers:
if regexp.match(value):
return tag
implicit = implicit[1]
if bool(self.yaml_path_resolvers):
exact_paths = self.resolver_exact_paths[-1]
if kind in exact_paths:
return exact_paths[kind]
if None in exact_paths:
return exact_paths[None]
if kind is ScalarNode:
return self.DEFAULT_SCALAR_TAG
elif kind is SequenceNode:
return self.DEFAULT_SEQUENCE_TAG
elif kind is MappingNode:
return self.DEFAULT_MAPPING_TAG
@property
def processing_version(self):
# type: () -> Any
return None
class Resolver(BaseResolver):
pass
for ir in implicit_resolvers:
if (1, 2) in ir[0]:
Resolver.add_implicit_resolver_base(*ir[1:])
class VersionedResolver(BaseResolver):
"""
contrary to the "normal" resolver, the smart resolver delays loading
the pattern matching rules. That way it can decide to load 1.1 rules
or the (default) 1.2 rules, that no longer support octal without 0o, sexagesimals
and Yes/No/On/Off booleans.
"""
def __init__(self, version=None, loader=None, loadumper=None):
# type: (Optional[VersionType], Any, Any) -> None
if loader is None and loadumper is not None:
loader = loadumper
BaseResolver.__init__(self, loader)
self._loader_version = self.get_loader_version(version)
self._version_implicit_resolver = {} # type: Dict[Any, Any]
def add_version_implicit_resolver(self, version, tag, regexp, first):
# type: (VersionType, Any, Any, Any) -> None
if first is None:
first = [None]
impl_resolver = self._version_implicit_resolver.setdefault(version, {})
for ch in first:
impl_resolver.setdefault(ch, []).append((tag, regexp))
def get_loader_version(self, version):
# type: (Optional[VersionType]) -> Any
if version is None or isinstance(version, tuple):
return version
if isinstance(version, list):
return tuple(version)
# assume string
return tuple(map(int, version.split('.')))
@property
def versioned_resolver(self):
# type: () -> Any
"""
select the resolver based on the version we are parsing
"""
version = self.processing_version
if isinstance(version, str):
version = tuple(map(int, version.split('.')))
if version not in self._version_implicit_resolver:
for x in implicit_resolvers:
if version in x[0]:
self.add_version_implicit_resolver(version, x[1], x[2], x[3])
return self._version_implicit_resolver[version]
def resolve(self, kind, value, implicit):
# type: (Any, Any, Any) -> Any
if kind is ScalarNode and implicit[0]:
if value == "":
resolvers = self.versioned_resolver.get("", [])
else:
resolvers = self.versioned_resolver.get(value[0], [])
resolvers += self.versioned_resolver.get(None, [])
for tag, regexp in resolvers:
if regexp.match(value):
return tag
implicit = implicit[1]
if bool(self.yaml_path_resolvers):
exact_paths = self.resolver_exact_paths[-1]
if kind in exact_paths:
return exact_paths[kind]
if None in exact_paths:
return exact_paths[None]
if kind is ScalarNode:
return self.DEFAULT_SCALAR_TAG
elif kind is SequenceNode:
return self.DEFAULT_SEQUENCE_TAG
elif kind is MappingNode:
return self.DEFAULT_MAPPING_TAG
@property
def processing_version(self):
# type: () -> Any
try:
version = self.loadumper._scanner.yaml_version
except AttributeError:
try:
if hasattr(self.loadumper, 'typ'):
version = self.loadumper.version
else:
version = self.loadumper._serializer.use_version # dumping
except AttributeError:
version = None
if version is None:
version = self._loader_version
if version is None:
version = _DEFAULT_YAML_VERSION
return version
pipenv/vendor/ruamel/yaml/scalarbool.py
Vendored
+47
View File
@@ -0,0 +1,47 @@
# coding: utf-8
"""
You cannot subclass bool, and this is necessary for round-tripping anchored
bool values (and also if you want to preserve the original way of writing)
bool.__bases__ is type 'int', so that is what is used as the basis for ScalarBoolean as well.
You can use these in an if statement, but not when testing equivalence
"""
from pipenv.vendor.ruamel.yaml.anchor import Anchor
if False: # MYPY
from typing import Text, Any, Dict, List # NOQA
__all__ = ['ScalarBoolean']
class ScalarBoolean(int):
def __new__(cls, *args, **kw):
# type: (Any, Any, Any) -> Any
anchor = kw.pop('anchor', None)
b = int.__new__(cls, *args, **kw)
if anchor is not None:
b.yaml_set_anchor(anchor, always_dump=True)
return b
@property
def anchor(self):
# type: () -> Any
if not hasattr(self, Anchor.attrib):
setattr(self, Anchor.attrib, Anchor())
return getattr(self, Anchor.attrib)
def yaml_anchor(self, any=False):
# type: (bool) -> Any
if not hasattr(self, Anchor.attrib):
return None
if any or self.anchor.always_dump:
return self.anchor
return None
def yaml_set_anchor(self, value, always_dump=False):
# type: (Any, bool) -> None
self.anchor.value = value
self.anchor.always_dump = always_dump
pipenv/vendor/ruamel/yaml/scalarfloat.py
Vendored
+124
View File
@@ -0,0 +1,124 @@
# coding: utf-8
import sys
from pipenv.vendor.ruamel.yaml.anchor import Anchor
if False: # MYPY
from typing import Text, Any, Dict, List # NOQA
__all__ = ['ScalarFloat', 'ExponentialFloat', 'ExponentialCapsFloat']
class ScalarFloat(float):
def __new__(cls, *args, **kw):
# type: (Any, Any, Any) -> Any
width = kw.pop('width', None)
prec = kw.pop('prec', None)
m_sign = kw.pop('m_sign', None)
m_lead0 = kw.pop('m_lead0', 0)
exp = kw.pop('exp', None)
e_width = kw.pop('e_width', None)
e_sign = kw.pop('e_sign', None)
underscore = kw.pop('underscore', None)
anchor = kw.pop('anchor', None)
v = float.__new__(cls, *args, **kw)
v._width = width
v._prec = prec
v._m_sign = m_sign
v._m_lead0 = m_lead0
v._exp = exp
v._e_width = e_width
v._e_sign = e_sign
v._underscore = underscore
if anchor is not None:
v.yaml_set_anchor(anchor, always_dump=True)
return v
def __iadd__(self, a): # type: ignore
# type: (Any) -> Any
return float(self) + a
x = type(self)(self + a)
x._width = self._width
x._underscore = self._underscore[:] if self._underscore is not None else None # NOQA
return x
def __ifloordiv__(self, a): # type: ignore
# type: (Any) -> Any
return float(self) // a
x = type(self)(self // a)
x._width = self._width
x._underscore = self._underscore[:] if self._underscore is not None else None # NOQA
return x
def __imul__(self, a): # type: ignore
# type: (Any) -> Any
return float(self) * a
x = type(self)(self * a)
x._width = self._width
x._underscore = self._underscore[:] if self._underscore is not None else None # NOQA
x._prec = self._prec # check for others
return x
def __ipow__(self, a): # type: ignore
# type: (Any) -> Any
return float(self) ** a
x = type(self)(self ** a)
x._width = self._width
x._underscore = self._underscore[:] if self._underscore is not None else None # NOQA
return x
def __isub__(self, a): # type: ignore
# type: (Any) -> Any
return float(self) - a
x = type(self)(self - a)
x._width = self._width
x._underscore = self._underscore[:] if self._underscore is not None else None # NOQA
return x
@property
def anchor(self):
# type: () -> Any
if not hasattr(self, Anchor.attrib):
setattr(self, Anchor.attrib, Anchor())
return getattr(self, Anchor.attrib)
def yaml_anchor(self, any=False):
# type: (bool) -> Any
if not hasattr(self, Anchor.attrib):
return None
if any or self.anchor.always_dump:
return self.anchor
return None
def yaml_set_anchor(self, value, always_dump=False):
# type: (Any, bool) -> None
self.anchor.value = value
self.anchor.always_dump = always_dump
def dump(self, out=sys.stdout):
# type: (Any) -> Any
out.write(
'ScalarFloat({}| w:{}, p:{}, s:{}, lz:{}, _:{}|{}, w:{}, s:{})\n'.format(
self,
self._width, # type: ignore
self._prec, # type: ignore
self._m_sign, # type: ignore
self._m_lead0, # type: ignore
self._underscore, # type: ignore
self._exp, # type: ignore
self._e_width, # type: ignore
self._e_sign, # type: ignore
)
)
class ExponentialFloat(ScalarFloat):
def __new__(cls, value, width=None, underscore=None):
# type: (Any, Any, Any) -> Any
return ScalarFloat.__new__(cls, value, width=width, underscore=underscore)
class ExponentialCapsFloat(ScalarFloat):
def __new__(cls, value, width=None, underscore=None):
# type: (Any, Any, Any) -> Any
return ScalarFloat.__new__(cls, value, width=width, underscore=underscore)
pipenv/vendor/ruamel/yaml/scalarint.py
Vendored
+127
View File
@@ -0,0 +1,127 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.anchor import Anchor
if False: # MYPY
from typing import Text, Any, Dict, List # NOQA
__all__ = ['ScalarInt', 'BinaryInt', 'OctalInt', 'HexInt', 'HexCapsInt', 'DecimalInt']
class ScalarInt(int):
def __new__(cls, *args, **kw):
# type: (Any, Any, Any) -> Any
width = kw.pop('width', None)
underscore = kw.pop('underscore', None)
anchor = kw.pop('anchor', None)
v = int.__new__(cls, *args, **kw)
v._width = width
v._underscore = underscore
if anchor is not None:
v.yaml_set_anchor(anchor, always_dump=True)
return v
def __iadd__(self, a): # type: ignore
# type: (Any) -> Any
x = type(self)(self + a)
x._width = self._width # type: ignore
x._underscore = ( # type: ignore
self._underscore[:] if self._underscore is not None else None # type: ignore
) # NOQA
return x
def __ifloordiv__(self, a): # type: ignore
# type: (Any) -> Any
x = type(self)(self // a)
x._width = self._width # type: ignore
x._underscore = ( # type: ignore
self._underscore[:] if self._underscore is not None else None # type: ignore
) # NOQA
return x
def __imul__(self, a): # type: ignore
# type: (Any) -> Any
x = type(self)(self * a)
x._width = self._width # type: ignore
x._underscore = ( # type: ignore
self._underscore[:] if self._underscore is not None else None # type: ignore
) # NOQA
return x
def __ipow__(self, a): # type: ignore
# type: (Any) -> Any
x = type(self)(self ** a)
x._width = self._width # type: ignore
x._underscore = ( # type: ignore
self._underscore[:] if self._underscore is not None else None # type: ignore
) # NOQA
return x
def __isub__(self, a): # type: ignore
# type: (Any) -> Any
x = type(self)(self - a)
x._width = self._width # type: ignore
x._underscore = ( # type: ignore
self._underscore[:] if self._underscore is not None else None # type: ignore
) # NOQA
return x
@property
def anchor(self):
# type: () -> Any
if not hasattr(self, Anchor.attrib):
setattr(self, Anchor.attrib, Anchor())
return getattr(self, Anchor.attrib)
def yaml_anchor(self, any=False):
# type: (bool) -> Any
if not hasattr(self, Anchor.attrib):
return None
if any or self.anchor.always_dump:
return self.anchor
return None
def yaml_set_anchor(self, value, always_dump=False):
# type: (Any, bool) -> None
self.anchor.value = value
self.anchor.always_dump = always_dump
class BinaryInt(ScalarInt):
def __new__(cls, value, width=None, underscore=None, anchor=None):
# type: (Any, Any, Any, Any) -> Any
return ScalarInt.__new__(cls, value, width=width, underscore=underscore, anchor=anchor)
class OctalInt(ScalarInt):
def __new__(cls, value, width=None, underscore=None, anchor=None):
# type: (Any, Any, Any, Any) -> Any
return ScalarInt.__new__(cls, value, width=width, underscore=underscore, anchor=anchor)
# mixed casing of A-F is not supported, when loading the first non digit
# determines the case
class HexInt(ScalarInt):
"""uses lower case (a-f)"""
def __new__(cls, value, width=None, underscore=None, anchor=None):
# type: (Any, Any, Any, Any) -> Any
return ScalarInt.__new__(cls, value, width=width, underscore=underscore, anchor=anchor)
class HexCapsInt(ScalarInt):
"""uses upper case (A-F)"""
def __new__(cls, value, width=None, underscore=None, anchor=None):
# type: (Any, Any, Any, Any) -> Any
return ScalarInt.__new__(cls, value, width=width, underscore=underscore, anchor=anchor)
class DecimalInt(ScalarInt):
"""needed if anchor"""
def __new__(cls, value, width=None, underscore=None, anchor=None):
# type: (Any, Any, Any, Any) -> Any
return ScalarInt.__new__(cls, value, width=width, underscore=underscore, anchor=anchor)
pipenv/vendor/ruamel/yaml/scalarstring.py
Vendored
+152
View File
@@ -0,0 +1,152 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.anchor import Anchor
if False: # MYPY
from typing import Text, Any, Dict, List # NOQA
__all__ = [
'ScalarString',
'LiteralScalarString',
'FoldedScalarString',
'SingleQuotedScalarString',
'DoubleQuotedScalarString',
'PlainScalarString',
# PreservedScalarString is the old name, as it was the first to be preserved on rt,
# use LiteralScalarString instead
'PreservedScalarString',
]
class ScalarString(str):
__slots__ = Anchor.attrib
def __new__(cls, *args, **kw):
# type: (Any, Any) -> Any
anchor = kw.pop('anchor', None)
ret_val = str.__new__(cls, *args, **kw)
if anchor is not None:
ret_val.yaml_set_anchor(anchor, always_dump=True)
return ret_val
def replace(self, old, new, maxreplace=-1):
# type: (Any, Any, int) -> Any
return type(self)((str.replace(self, old, new, maxreplace)))
@property
def anchor(self):
# type: () -> Any
if not hasattr(self, Anchor.attrib):
setattr(self, Anchor.attrib, Anchor())
return getattr(self, Anchor.attrib)
def yaml_anchor(self, any=False):
# type: (bool) -> Any
if not hasattr(self, Anchor.attrib):
return None
if any or self.anchor.always_dump:
return self.anchor
return None
def yaml_set_anchor(self, value, always_dump=False):
# type: (Any, bool) -> None
self.anchor.value = value
self.anchor.always_dump = always_dump
class LiteralScalarString(ScalarString):
__slots__ = 'comment' # the comment after the | on the first line
style = '|'
def __new__(cls, value, anchor=None):
# type: (Text, Any) -> Any
return ScalarString.__new__(cls, value, anchor=anchor)
PreservedScalarString = LiteralScalarString
class FoldedScalarString(ScalarString):
__slots__ = ('fold_pos', 'comment') # the comment after the > on the first line
style = '>'
def __new__(cls, value, anchor=None):
# type: (Text, Any) -> Any
return ScalarString.__new__(cls, value, anchor=anchor)
class SingleQuotedScalarString(ScalarString):
__slots__ = ()
style = "'"
def __new__(cls, value, anchor=None):
# type: (Text, Any) -> Any
return ScalarString.__new__(cls, value, anchor=anchor)
class DoubleQuotedScalarString(ScalarString):
__slots__ = ()
style = '"'
def __new__(cls, value, anchor=None):
# type: (Text, Any) -> Any
return ScalarString.__new__(cls, value, anchor=anchor)
class PlainScalarString(ScalarString):
__slots__ = ()
style = ''
def __new__(cls, value, anchor=None):
# type: (Text, Any) -> Any
return ScalarString.__new__(cls, value, anchor=anchor)
def preserve_literal(s):
# type: (Text) -> Text
return LiteralScalarString(s.replace('\r\n', '\n').replace('\r', '\n'))
def walk_tree(base, map=None):
# type: (Any, Any) -> None
"""
the routine here walks over a simple yaml tree (recursing in
dict values and list items) and converts strings that
have multiple lines to literal scalars
You can also provide an explicit (ordered) mapping for multiple transforms
(first of which is executed):
map = ruamel.compat.ordereddict
map['\n'] = preserve_literal
map[':'] = SingleQuotedScalarString
walk_tree(data, map=map)
"""
from collections.abc import MutableMapping, MutableSequence
if map is None:
map = {'\n': preserve_literal}
if isinstance(base, MutableMapping):
for k in base:
v = base[k] # type: Text
if isinstance(v, str):
for ch in map:
if ch in v:
base[k] = map[ch](v)
break
else:
walk_tree(v, map=map)
elif isinstance(base, MutableSequence):
for idx, elem in enumerate(base):
if isinstance(elem, str):
for ch in map:
if ch in elem:
base[idx] = map[ch](elem)
break
else:
walk_tree(elem, map=map)
pipenv/vendor/ruamel/yaml/scanner.py
Vendored
+2444
View File
File diff suppressed because it is too large Load Diff
pipenv/vendor/ruamel/yaml/serializer.py
Vendored
+241
View File
@@ -0,0 +1,241 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.error import YAMLError
from pipenv.vendor.ruamel.yaml.compat import nprint, DBG_NODE, dbg, nprintf # NOQA
from pipenv.vendor.ruamel.yaml.util import RegExp
from pipenv.vendor.ruamel.yaml.events import (
StreamStartEvent,
StreamEndEvent,
MappingStartEvent,
MappingEndEvent,
SequenceStartEvent,
SequenceEndEvent,
AliasEvent,
ScalarEvent,
DocumentStartEvent,
DocumentEndEvent,
)
from pipenv.vendor.ruamel.yaml.nodes import MappingNode, ScalarNode, SequenceNode
if False: # MYPY
from typing import Any, Dict, Union, Text, Optional # NOQA
from pipenv.vendor.ruamel.yaml.compat import VersionType # NOQA
__all__ = ['Serializer', 'SerializerError']
class SerializerError(YAMLError):
pass
class Serializer:
# 'id' and 3+ numbers, but not 000
ANCHOR_TEMPLATE = 'id%03d'
ANCHOR_RE = RegExp('id(?!000$)\\d{3,}')
def __init__(
self,
encoding=None,
explicit_start=None,
explicit_end=None,
version=None,
tags=None,
dumper=None,
):
# type: (Any, Optional[bool], Optional[bool], Optional[VersionType], Any, Any) -> None # NOQA
self.dumper = dumper
if self.dumper is not None:
self.dumper._serializer = self
self.use_encoding = encoding
self.use_explicit_start = explicit_start
self.use_explicit_end = explicit_end
if isinstance(version, str):
self.use_version = tuple(map(int, version.split('.')))
else:
self.use_version = version # type: ignore
self.use_tags = tags
self.serialized_nodes = {} # type: Dict[Any, Any]
self.anchors = {} # type: Dict[Any, Any]
self.last_anchor_id = 0
self.closed = None # type: Optional[bool]
self._templated_id = None
@property
def emitter(self):
# type: () -> Any
if hasattr(self.dumper, 'typ'):
return self.dumper.emitter
return self.dumper._emitter
@property
def resolver(self):
# type: () -> Any
if hasattr(self.dumper, 'typ'):
self.dumper.resolver
return self.dumper._resolver
def open(self):
# type: () -> None
if self.closed is None:
self.emitter.emit(StreamStartEvent(encoding=self.use_encoding))
self.closed = False
elif self.closed:
raise SerializerError('serializer is closed')
else:
raise SerializerError('serializer is already opened')
def close(self):
# type: () -> None
if self.closed is None:
raise SerializerError('serializer is not opened')
elif not self.closed:
self.emitter.emit(StreamEndEvent())
self.closed = True
# def __del__(self):
# self.close()
def serialize(self, node):
# type: (Any) -> None
if dbg(DBG_NODE):
nprint('Serializing nodes')
node.dump()
if self.closed is None:
raise SerializerError('serializer is not opened')
elif self.closed:
raise SerializerError('serializer is closed')
self.emitter.emit(
DocumentStartEvent(
explicit=self.use_explicit_start, version=self.use_version, tags=self.use_tags
)
)
self.anchor_node(node)
self.serialize_node(node, None, None)
self.emitter.emit(DocumentEndEvent(explicit=self.use_explicit_end))
self.serialized_nodes = {}
self.anchors = {}
self.last_anchor_id = 0
def anchor_node(self, node):
# type: (Any) -> None
if node in self.anchors:
if self.anchors[node] is None:
self.anchors[node] = self.generate_anchor(node)
else:
anchor = None
try:
if node.anchor.always_dump:
anchor = node.anchor.value
except: # NOQA
pass
self.anchors[node] = anchor
if isinstance(node, SequenceNode):
for item in node.value:
self.anchor_node(item)
elif isinstance(node, MappingNode):
for key, value in node.value:
self.anchor_node(key)
self.anchor_node(value)
def generate_anchor(self, node):
# type: (Any) -> Any
try:
anchor = node.anchor.value
except: # NOQA
anchor = None
if anchor is None:
self.last_anchor_id += 1
return self.ANCHOR_TEMPLATE % self.last_anchor_id
return anchor
def serialize_node(self, node, parent, index):
# type: (Any, Any, Any) -> None
alias = self.anchors[node]
if node in self.serialized_nodes:
node_style = getattr(node, 'style', None)
if node_style != '?':
node_style = None
self.emitter.emit(AliasEvent(alias, style=node_style))
else:
self.serialized_nodes[node] = True
self.resolver.descend_resolver(parent, index)
if isinstance(node, ScalarNode):
# here check if the node.tag equals the one that would result from parsing
# if not equal quoting is necessary for strings
detected_tag = self.resolver.resolve(ScalarNode, node.value, (True, False))
default_tag = self.resolver.resolve(ScalarNode, node.value, (False, True))
implicit = (
(node.tag == detected_tag),
(node.tag == default_tag),
node.tag.startswith('tag:yaml.org,2002:'),
)
self.emitter.emit(
ScalarEvent(
alias,
node.tag,
implicit,
node.value,
style=node.style,
comment=node.comment,
)
)
elif isinstance(node, SequenceNode):
implicit = node.tag == self.resolver.resolve(SequenceNode, node.value, True)
comment = node.comment
end_comment = None
seq_comment = None
if node.flow_style is True:
if comment: # eol comment on flow style sequence
seq_comment = comment[0]
# comment[0] = None
if comment and len(comment) > 2:
end_comment = comment[2]
else:
end_comment = None
self.emitter.emit(
SequenceStartEvent(
alias,
node.tag,
implicit,
flow_style=node.flow_style,
comment=node.comment,
)
)
index = 0
for item in node.value:
self.serialize_node(item, node, index)
index += 1
self.emitter.emit(SequenceEndEvent(comment=[seq_comment, end_comment]))
elif isinstance(node, MappingNode):
implicit = node.tag == self.resolver.resolve(MappingNode, node.value, True)
comment = node.comment
end_comment = None
map_comment = None
if node.flow_style is True:
if comment: # eol comment on flow style sequence
map_comment = comment[0]
# comment[0] = None
if comment and len(comment) > 2:
end_comment = comment[2]
self.emitter.emit(
MappingStartEvent(
alias,
node.tag,
implicit,
flow_style=node.flow_style,
comment=node.comment,
nr_items=len(node.value),
)
)
for key, value in node.value:
self.serialize_node(key, node, None)
self.serialize_node(value, node, key)
self.emitter.emit(MappingEndEvent(comment=[map_comment, end_comment]))
self.resolver.ascend_resolver()
def templated_id(s):
# type: (Text) -> Any
return Serializer.ANCHOR_RE.match(s)
pipenv/vendor/ruamel/yaml/timestamp.py
Vendored
+61
View File
@@ -0,0 +1,61 @@
# coding: utf-8
import datetime
import copy
# ToDo: at least on PY3 you could probably attach the tzinfo correctly to the object
# a more complete datetime might be used by safe loading as well
if False: # MYPY
from typing import Any, Dict, Optional, List # NOQA
class TimeStamp(datetime.datetime):
def __init__(self, *args, **kw):
# type: (Any, Any) -> None
self._yaml = dict(t=False, tz=None, delta=0) # type: Dict[Any, Any]
def __new__(cls, *args, **kw): # datetime is immutable
# type: (Any, Any) -> Any
return datetime.datetime.__new__(cls, *args, **kw)
def __deepcopy__(self, memo):
# type: (Any) -> Any
ts = TimeStamp(self.year, self.month, self.day, self.hour, self.minute, self.second)
ts._yaml = copy.deepcopy(self._yaml)
return ts
def replace(
self,
year=None,
month=None,
day=None,
hour=None,
minute=None,
second=None,
microsecond=None,
tzinfo=True,
fold=None,
):
# type: (Any, Any, Any, Any, Any, Any, Any, Any, Any) -> Any
if year is None:
year = self.year
if month is None:
month = self.month
if day is None:
day = self.day
if hour is None:
hour = self.hour
if minute is None:
minute = self.minute
if second is None:
second = self.second
if microsecond is None:
microsecond = self.microsecond
if tzinfo is True:
tzinfo = self.tzinfo
if fold is None:
fold = self.fold
ts = type(self)(year, month, day, hour, minute, second, microsecond, tzinfo, fold=fold)
ts._yaml = copy.deepcopy(self._yaml)
return ts
pipenv/vendor/ruamel/yaml/tokens.py
Vendored
+404
View File
@@ -0,0 +1,404 @@
# coding: utf-8
from pipenv.vendor.ruamel.yaml.compat import _F, nprintf # NOQA
if False: # MYPY
from typing import Text, Any, Dict, Optional, List # NOQA
from .error import StreamMark # NOQA
SHOW_LINES = True
class Token:
__slots__ = 'start_mark', 'end_mark', '_comment'
def __init__(self, start_mark, end_mark):
# type: (StreamMark, StreamMark) -> None
self.start_mark = start_mark
self.end_mark = end_mark
def __repr__(self):
# type: () -> Any
# attributes = [key for key in self.__slots__ if not key.endswith('_mark') and
# hasattr('self', key)]
attributes = [key for key in self.__slots__ if not key.endswith('_mark')]
attributes.sort()
# arguments = ', '.join(
# [_F('{key!s}={gattr!r})', key=key, gattr=getattr(self, key)) for key in attributes]
# )
arguments = [
_F('{key!s}={gattr!r}', key=key, gattr=getattr(self, key)) for key in attributes
]
if SHOW_LINES:
try:
arguments.append('line: ' + str(self.start_mark.line))
except: # NOQA
pass
try:
arguments.append('comment: ' + str(self._comment))
except: # NOQA
pass
return '{}({})'.format(self.__class__.__name__, ', '.join(arguments))
@property
def column(self):
# type: () -> int
return self.start_mark.column
@column.setter
def column(self, pos):
# type: (Any) -> None
self.start_mark.column = pos
# old style ( <= 0.17) is a TWO element list with first being the EOL
# comment concatenated with following FLC/BLNK; and second being a list of FLC/BLNK
# preceding the token
# new style ( >= 0.17 ) is a THREE element list with the first being a list of
# preceding FLC/BLNK, the second EOL and the third following FLC/BLNK
# note that new style has differing order, and does not consist of CommentToken(s)
# but of CommentInfo instances
# any non-assigned values in new style are None, but first and last can be empty list
# new style routines add one comment at a time
# going to be deprecated in favour of add_comment_eol/post
def add_post_comment(self, comment):
# type: (Any) -> None
if not hasattr(self, '_comment'):
self._comment = [None, None]
else:
assert len(self._comment) in [2, 5] # make sure it is version 0
# if isinstance(comment, CommentToken):
# if comment.value.startswith('# C09'):
# raise
self._comment[0] = comment
# going to be deprecated in favour of add_comment_pre
def add_pre_comments(self, comments):
# type: (Any) -> None
if not hasattr(self, '_comment'):
self._comment = [None, None]
else:
assert len(self._comment) == 2 # make sure it is version 0
assert self._comment[1] is None
self._comment[1] = comments
return
# new style
def add_comment_pre(self, comment):
# type: (Any) -> None
if not hasattr(self, '_comment'):
self._comment = [[], None, None] # type: ignore
else:
assert len(self._comment) == 3
if self._comment[0] is None:
self._comment[0] = [] # type: ignore
self._comment[0].append(comment) # type: ignore
def add_comment_eol(self, comment, comment_type):
# type: (Any, Any) -> None
if not hasattr(self, '_comment'):
self._comment = [None, None, None]
else:
assert len(self._comment) == 3
assert self._comment[1] is None
if self.comment[1] is None:
self._comment[1] = [] # type: ignore
self._comment[1].extend([None] * (comment_type + 1 - len(self.comment[1]))) # type: ignore # NOQA
# nprintf('commy', self.comment, comment_type)
self._comment[1][comment_type] = comment # type: ignore
def add_comment_post(self, comment):
# type: (Any) -> None
if not hasattr(self, '_comment'):
self._comment = [None, None, []] # type: ignore
else:
assert len(self._comment) == 3
if self._comment[2] is None:
self._comment[2] = [] # type: ignore
self._comment[2].append(comment) # type: ignore
# def get_comment(self):
# # type: () -> Any
# return getattr(self, '_comment', None)
@property
def comment(self):
# type: () -> Any
return getattr(self, '_comment', None)
def move_old_comment(self, target, empty=False):
# type: (Any, bool) -> Any
"""move a comment from this token to target (normally next token)
used to combine e.g. comments before a BlockEntryToken to the
ScalarToken that follows it
empty is a special for empty values -> comment after key
"""
c = self.comment
if c is None:
return
# don't push beyond last element
if isinstance(target, (StreamEndToken, DocumentStartToken)):
return
delattr(self, '_comment')
tc = target.comment
if not tc: # target comment, just insert
# special for empty value in key: value issue 25
if empty:
c = [c[0], c[1], None, None, c[0]]
target._comment = c
# nprint('mco2:', self, target, target.comment, empty)
return self
if c[0] and tc[0] or c[1] and tc[1]:
raise NotImplementedError(_F('overlap in comment {c!r} {tc!r}', c=c, tc=tc))
if c[0]:
tc[0] = c[0]
if c[1]:
tc[1] = c[1]
return self
def split_old_comment(self):
# type: () -> Any
""" split the post part of a comment, and return it
as comment to be added. Delete second part if [None, None]
abc: # this goes to sequence
# this goes to first element
- first element
"""
comment = self.comment
if comment is None or comment[0] is None:
return None # nothing to do
ret_val = [comment[0], None]
if comment[1] is None:
delattr(self, '_comment')
return ret_val
def move_new_comment(self, target, empty=False):
# type: (Any, bool) -> Any
"""move a comment from this token to target (normally next token)
used to combine e.g. comments before a BlockEntryToken to the
ScalarToken that follows it
empty is a special for empty values -> comment after key
"""
c = self.comment
if c is None:
return
# don't push beyond last element
if isinstance(target, (StreamEndToken, DocumentStartToken)):
return
delattr(self, '_comment')
tc = target.comment
if not tc: # target comment, just insert
# special for empty value in key: value issue 25
if empty:
c = [c[0], c[1], c[2]]
target._comment = c
# nprint('mco2:', self, target, target.comment, empty)
return self
# if self and target have both pre, eol or post comments, something seems wrong
for idx in range(3):
if c[idx] is not None and tc[idx] is not None:
raise NotImplementedError(_F('overlap in comment {c!r} {tc!r}', c=c, tc=tc))
# move the comment parts
for idx in range(3):
if c[idx]:
tc[idx] = c[idx]
return self
# class BOMToken(Token):
# id = '<byte order mark>'
class DirectiveToken(Token):
__slots__ = 'name', 'value'
id = '<directive>'
def __init__(self, name, value, start_mark, end_mark):
# type: (Any, Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.name = name
self.value = value
class DocumentStartToken(Token):
__slots__ = ()
id = '<document start>'
class DocumentEndToken(Token):
__slots__ = ()
id = '<document end>'
class StreamStartToken(Token):
__slots__ = ('encoding',)
id = '<stream start>'
def __init__(self, start_mark=None, end_mark=None, encoding=None):
# type: (Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.encoding = encoding
class StreamEndToken(Token):
__slots__ = ()
id = '<stream end>'
class BlockSequenceStartToken(Token):
__slots__ = ()
id = '<block sequence start>'
class BlockMappingStartToken(Token):
__slots__ = ()
id = '<block mapping start>'
class BlockEndToken(Token):
__slots__ = ()
id = '<block end>'
class FlowSequenceStartToken(Token):
__slots__ = ()
id = '['
class FlowMappingStartToken(Token):
__slots__ = ()
id = '{'
class FlowSequenceEndToken(Token):
__slots__ = ()
id = ']'
class FlowMappingEndToken(Token):
__slots__ = ()
id = '}'
class KeyToken(Token):
__slots__ = ()
id = '?'
# def x__repr__(self):
# return 'KeyToken({})'.format(
# self.start_mark.buffer[self.start_mark.index:].split(None, 1)[0])
class ValueToken(Token):
__slots__ = ()
id = ':'
class BlockEntryToken(Token):
__slots__ = ()
id = '-'
class FlowEntryToken(Token):
__slots__ = ()
id = ','
class AliasToken(Token):
__slots__ = ('value',)
id = '<alias>'
def __init__(self, value, start_mark, end_mark):
# type: (Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.value = value
class AnchorToken(Token):
__slots__ = ('value',)
id = '<anchor>'
def __init__(self, value, start_mark, end_mark):
# type: (Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.value = value
class TagToken(Token):
__slots__ = ('value',)
id = '<tag>'
def __init__(self, value, start_mark, end_mark):
# type: (Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.value = value
class ScalarToken(Token):
__slots__ = 'value', 'plain', 'style'
id = '<scalar>'
def __init__(self, value, plain, start_mark, end_mark, style=None):
# type: (Any, Any, Any, Any, Any) -> None
Token.__init__(self, start_mark, end_mark)
self.value = value
self.plain = plain
self.style = style
class CommentToken(Token):
__slots__ = '_value', 'pre_done'
id = '<comment>'
def __init__(self, value, start_mark=None, end_mark=None, column=None):
# type: (Any, Any, Any, Any) -> None
if start_mark is None:
assert column is not None
self._column = column
Token.__init__(self, start_mark, None) # type: ignore
self._value = value
@property
def value(self):
# type: () -> str
if isinstance(self._value, str):
return self._value
return "".join(self._value)
@value.setter
def value(self, val):
# type: (Any) -> None
self._value = val
def reset(self):
# type: () -> None
if hasattr(self, 'pre_done'):
delattr(self, 'pre_done')
def __repr__(self):
# type: () -> Any
v = '{!r}'.format(self.value)
if SHOW_LINES:
try:
v += ', line: ' + str(self.start_mark.line)
except: # NOQA
pass
try:
v += ', col: ' + str(self.start_mark.column)
except: # NOQA
pass
return 'CommentToken({})'.format(v)
def __eq__(self, other):
# type: (Any) -> bool
if self.start_mark != other.start_mark:
return False
if self.end_mark != other.end_mark:
return False
if self.value != other.value:
return False
return True
def __ne__(self, other):
# type: (Any) -> bool
return not self.__eq__(other)
pipenv/vendor/ruamel/yaml/util.py
Vendored
+256
View File
@@ -0,0 +1,256 @@
# coding: utf-8
"""
some helper functions that might be generally useful
"""
import datetime
from functools import partial
import re
if False: # MYPY
from typing import Any, Dict, Optional, List, Text # NOQA
from .compat import StreamTextType # NOQA
class LazyEval:
"""
Lightweight wrapper around lazily evaluated func(*args, **kwargs).
func is only evaluated when any attribute of its return value is accessed.
Every attribute access is passed through to the wrapped value.
(This only excludes special cases like method-wrappers, e.g., __hash__.)
The sole additional attribute is the lazy_self function which holds the
return value (or, prior to evaluation, func and arguments), in its closure.
"""
def __init__(self, func, *args, **kwargs):
# type: (Any, Any, Any) -> None
def lazy_self():
# type: () -> Any
return_value = func(*args, **kwargs)
object.__setattr__(self, 'lazy_self', lambda: return_value)
return return_value
object.__setattr__(self, 'lazy_self', lazy_self)
def __getattribute__(self, name):
# type: (Any) -> Any
lazy_self = object.__getattribute__(self, 'lazy_self')
if name == 'lazy_self':
return lazy_self
return getattr(lazy_self(), name)
def __setattr__(self, name, value):
# type: (Any, Any) -> None
setattr(self.lazy_self(), name, value)
RegExp = partial(LazyEval, re.compile)
timestamp_regexp = RegExp(
"""^(?P<year>[0-9][0-9][0-9][0-9])
-(?P<month>[0-9][0-9]?)
-(?P<day>[0-9][0-9]?)
(?:((?P<t>[Tt])|[ \\t]+) # explictly not retaining extra spaces
(?P<hour>[0-9][0-9]?)
:(?P<minute>[0-9][0-9])
:(?P<second>[0-9][0-9])
(?:\\.(?P<fraction>[0-9]*))?
(?:[ \\t]*(?P<tz>Z|(?P<tz_sign>[-+])(?P<tz_hour>[0-9][0-9]?)
(?::(?P<tz_minute>[0-9][0-9]))?))?)?$""",
re.X,
)
def create_timestamp(
year, month, day, t, hour, minute, second, fraction, tz, tz_sign, tz_hour, tz_minute
):
# type: (Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any, Any) -> Any
# create a timestamp from match against timestamp_regexp
MAX_FRAC = 999999
year = int(year)
month = int(month)
day = int(day)
if not hour:
return datetime.date(year, month, day)
hour = int(hour)
minute = int(minute)
second = int(second)
frac = 0
if fraction:
frac_s = fraction[:6]
while len(frac_s) < 6:
frac_s += '0'
frac = int(frac_s)
if len(fraction) > 6 and int(fraction[6]) > 4:
frac += 1
if frac > MAX_FRAC:
fraction = 0
else:
fraction = frac
else:
fraction = 0
delta = None
if tz_sign:
tz_hour = int(tz_hour)
tz_minute = int(tz_minute) if tz_minute else 0
delta = datetime.timedelta(
hours=tz_hour, minutes=tz_minute, seconds=1 if frac > MAX_FRAC else 0
)
if tz_sign == '-':
delta = -delta
elif frac > MAX_FRAC:
delta = -datetime.timedelta(seconds=1)
# should do something else instead (or hook this up to the preceding if statement
# in reverse
# if delta is None:
# return datetime.datetime(year, month, day, hour, minute, second, fraction)
# return datetime.datetime(year, month, day, hour, minute, second, fraction,
# datetime.timezone.utc)
# the above is not good enough though, should provide tzinfo. In Python3 that is easily
# doable drop that kind of support for Python2 as it has not native tzinfo
data = datetime.datetime(year, month, day, hour, minute, second, fraction)
if delta:
data -= delta
return data
# originally as comment
# https://github.com/pre-commit/pre-commit/pull/211#issuecomment-186466605
# if you use this in your code, I suggest adding a test in your test suite
# that check this routines output against a known piece of your YAML
# before upgrades to this code break your round-tripped YAML
def load_yaml_guess_indent(stream, **kw):
# type: (StreamTextType, Any) -> Any
"""guess the indent and block sequence indent of yaml stream/string
returns round_trip_loaded stream, indent level, block sequence indent
- block sequence indent is the number of spaces before a dash relative to previous indent
- if there are no block sequences, indent is taken from nested mappings, block sequence
indent is unset (None) in that case
"""
from .main import YAML
# load a YAML document, guess the indentation, if you use TABs you are on your own
def leading_spaces(line):
# type: (Any) -> int
idx = 0
while idx < len(line) and line[idx] == ' ':
idx += 1
return idx
if isinstance(stream, str):
yaml_str = stream # type: Any
elif isinstance(stream, bytes):
# most likely, but the Reader checks BOM for this
yaml_str = stream.decode('utf-8')
else:
yaml_str = stream.read()
map_indent = None
indent = None # default if not found for some reason
block_seq_indent = None
prev_line_key_only = None
key_indent = 0
for line in yaml_str.splitlines():
rline = line.rstrip()
lline = rline.lstrip()
if lline.startswith('- '):
l_s = leading_spaces(line)
block_seq_indent = l_s - key_indent
idx = l_s + 1
while line[idx] == ' ': # this will end as we rstripped
idx += 1
if line[idx] == '#': # comment after -
continue
indent = idx - key_indent
break
if map_indent is None and prev_line_key_only is not None and rline:
idx = 0
while line[idx] in ' -':
idx += 1
if idx > prev_line_key_only:
map_indent = idx - prev_line_key_only
if rline.endswith(':'):
key_indent = leading_spaces(line)
idx = 0
while line[idx] == ' ': # this will end on ':'
idx += 1
prev_line_key_only = idx
continue
prev_line_key_only = None
if indent is None and map_indent is not None:
indent = map_indent
yaml = YAML()
return yaml.load(yaml_str, **kw), indent, block_seq_indent # type: ignore
def configobj_walker(cfg):
# type: (Any) -> Any
"""
walks over a ConfigObj (INI file with comments) generating
corresponding YAML output (including comments
"""
from configobj import ConfigObj # type: ignore
assert isinstance(cfg, ConfigObj)
for c in cfg.initial_comment:
if c.strip():
yield c
for s in _walk_section(cfg):
if s.strip():
yield s
for c in cfg.final_comment:
if c.strip():
yield c
def _walk_section(s, level=0):
# type: (Any, int) -> Any
from configobj import Section
assert isinstance(s, Section)
indent = ' ' * level
for name in s.scalars:
for c in s.comments[name]:
yield indent + c.strip()
x = s[name]
if '\n' in x:
i = indent + ' '
x = '|\n' + i + x.strip().replace('\n', '\n' + i)
elif ':' in x:
x = "'" + x.replace("'", "''") + "'"
line = '{0}{1}: {2}'.format(indent, name, x)
c = s.inline_comments[name]
if c:
line += ' ' + c
yield line
for name in s.sections:
for c in s.comments[name]:
yield indent + c.strip()
line = '{0}{1}:'.format(indent, name)
c = s.inline_comments[name]
if c:
line += ' ' + c
yield line
for val in _walk_section(s[name], level=level + 1):
yield val
# def config_obj_2_rt_yaml(cfg):
# from .comments import CommentedMap, CommentedSeq
# from configobj import ConfigObj
# assert isinstance(cfg, ConfigObj)
# #for c in cfg.initial_comment:
# # if c.strip():
# # pass
# cm = CommentedMap()
# for name in s.sections:
# cm[name] = d = CommentedMap()
#
#
# #for c in cfg.final_comment:
# # if c.strip():
# # yield c
# return cm
pipenv/vendor/vendor.txt
Vendored
+1
View File
@@ -13,6 +13,7 @@ pyparsing==3.0.9
python-dotenv==0.19.0
pythonfinder==1.3.1
requirementslib==2.2.0
ruamel.yaml==0.17.21
shellingham==1.5.0
toml==0.10.2
tomlkit==0.9.2
pipenv/vendor/vistir/__init__.py
Vendored
+1
View File
@@ -1,4 +1,5 @@
# -*- coding=utf-8 -*-
from .contextmanagers import (
atomic_open_for_write,
cd,
pipenv/vendor/vistir/contextmanagers.py
Vendored
+1 -1
View File
@@ -177,7 +177,7 @@ def spinner(
has_yaspin = None
try:
import pipenv.vendor.yaspin as yaspin
import yaspin
except (ImportError, ModuleNotFoundError): # noqa
has_yaspin = False
if not nospin:
pipenv/vendor/vistir/spin.py
Vendored
+1 -1
View File
@@ -37,7 +37,7 @@ if typing.TYPE_CHECKING:
_T = TypeVar("_T", covariant=True)
try:
import pipenv.vendor.yaspin as yaspin
import yaspin
import yaspin.spinners
import yaspin.core
tasks/vendoring/__init__.py
+5 -3
View File
@@ -60,6 +60,7 @@ LIBRARY_RENAMES = {
"packaging": "pipenv.patched.pip._vendor.packaging",
"pep517": "pipenv.patched.pip._vendor.pep517",
"pkg_resources": "pipenv.patched.pip._vendor.pkg_resources",
"ruamel.yaml": "pipenv.vendor.ruamel.yaml",
"urllib3": "pipenv.patched.pip._vendor.urllib3",
}
@@ -81,6 +82,7 @@ GLOBAL_REPLACEMENT = [
),
(r"(?<!\.)pep517\.envbuild", r"envbuild"),
(r"(?<!\.)pep517\.wrappers", r"wrappers"),
(r" ruamel\.yaml", r" ruamel"),
(
"from platformdirs import user_cache_dir",
"from pipenv.patched.pip._vendor.platformdirs import user_cache_dir",
@@ -603,10 +605,10 @@ def license_destination(vendor_dir, libname, filename):
lowercase = vendor_dir / libname.lower().replace("-", "_")
if lowercase.is_dir():
return lowercase / filename
rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES
# rename_dict = LIBRARY_RENAMES if vendor_dir.name != "patched" else PATCHED_RENAMES
# Short circuit all logic if we are renaming the whole library
if libname in rename_dict:
return vendor_dir / rename_dict[libname] / filename
# if libname in rename_dict:
# return vendor_dir / rename_dict[libname] / filename
if libname in LIBRARY_DIRNAMES:
override = vendor_dir / LIBRARY_DIRNAMES[libname]
if not override.exists() and override.parent.exists():
tasks/vendoring/patches/patched/safety-main.patch
+3 -44
View File
@@ -1,54 +1,13 @@
diff --git a/pipenv/patched/safety/__main__.py b/pipenv/patched/safety/__main__.py
index d9a0bdab..f905408a 100644
index d9a0bdab..be36e88b 100644
--- a/pipenv/patched/safety/__main__.py
+++ b/pipenv/patched/safety/__main__.py
@@ -1,8 +1,48 @@
@@ -1,7 +1,7 @@
"""Allow safety to be executable through `python -m safety`."""
from __future__ import absolute_import
-from .cli import cli
+import os
+import sys
+import sysconfig
+
+
+PATCHED_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PIPENV_DIR = os.path.dirname(PATCHED_DIR)
+VENDORED_DIR = os.path.join("PIPENV_DIR", "vendor")
+
+
+def get_site_packages():
+ prefixes = {sys.prefix, sysconfig.get_config_var('prefix')}
+ try:
+ prefixes.add(sys.real_prefix)
+ except AttributeError:
+ pass
+ form = sysconfig.get_path('purelib', expand=False)
+ py_version_short = '{0[0]}.{0[1]}'.format(sys.version_info)
+ return {
+ form.format(base=prefix, py_version_short=py_version_short)
+ for prefix in prefixes
+ }
+
+
+def insert_before_site_packages(*paths):
+ site_packages = get_site_packages()
+ index = None
+ for i, path in enumerate(sys.path):
+ if path in site_packages:
+ index = i
+ break
+ if index is None:
+ sys.path += list(paths)
+ else:
+ sys.path = sys.path[:index] + list(paths) + sys.path[index:]
+
+
+def insert_pipenv_dirs():
+ insert_before_site_packages(os.path.dirname(PIPENV_DIR), PATCHED_DIR, VENDORED_DIR)
+from pipenv.patched.safety.cli import cli
if __name__ == "__main__": # pragma: no cover
+ insert_pipenv_dirs()
+ from safety.cli import cli
cli(prog_name="safety")
tests/integration/test_cli.py
+2 -2
View File
@@ -140,7 +140,7 @@ def test_pipenv_graph_reverse(pipenv_instance_private_pypi):
@pytest.mark.cli
@pytest.mark.needs_internet(reason='required by check')
@flaky
@pytest.mark.skip("Safety 2 ends up scanning the project virtualenv and not the instance created by this test.")
def test_pipenv_check(pipenv_instance_private_pypi):
with pipenv_instance_private_pypi() as p:
c = p.pipenv('install pyyaml')
@@ -154,7 +154,7 @@ def test_pipenv_check(pipenv_instance_private_pypi):
assert c.returncode == 0
# Note: added
# 51457: py <=1.11.0 resolved (1.11.0 installed)!
# this is install via pytest, and causes a false positive
# this is installed via pytest, and causes a false positive
# https://github.com/pytest-dev/py/issues/287
# the issue above is still not resolved.
# added also 51499