diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..4ceed0f --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,55 @@ +# Python CircleCI 2.0 configuration file +# +# Check https://circleci.com/docs/2.0/language-python/ for more details +# +version: 2 +jobs: + build: + docker: + # specify the version you desire here + # use `-browsers` prefix for selenium tests, e.g. `3.6.1-browsers` + - image: circleci/python:3.6.1 + + # Specify service dependencies here if necessary + # CircleCI maintains a library of pre-built images + # documented at https://circleci.com/docs/2.0/circleci-images/ + # - image: circleci/postgres:9.4 + + working_directory: ~/repo + + steps: + - checkout + + # Download and cache dependencies + - restore_cache: + keys: + - v1-dependencies-{{ checksum "requirements.txt" }} + # fallback to using the latest cache if no exact match is found + - v1-dependencies- + + - run: + name: install dependencies + command: | + python3 -m venv venv + . venv/bin/activate + pip install -r requirements.txt + + - save_cache: + paths: + - ./venv + key: v1-dependencies-{{ checksum "requirements.txt" }} + + # run tests! + # this example uses Django's built-in test-runner + # other common Python testing frameworks include pytest and nose + # https://pytest.org + # https://nose.readthedocs.io + - run: + name: run tests + command: | + . venv/bin/activate + python manage.py test + + - store_artifacts: + path: test-reports + destination: test-reports diff --git a/Bakefile b/Bakefile index 548ded7..6bc6931 100644 --- a/Bakefile +++ b/Bakefile @@ -1,17 +1,30 @@ -task: @confirm install-deps - echo "hi, $1" -install-deps: @skip:key=./Pipfile.lock system-deps +install: install/system install/python +install/python: install/system @skip:key=./Pipfile.lock pipenv install -system-deps: install-jq - set -ex +install/system: + lazy-brew jq docker-compose - brew install pipenv +docker/bash: docker/build install/system + docker-compose run --entrypoint=bash bake +docker/release: docker/build install/system + docker-compose push +docker/build: install/system + docker-compose build - if ! brew info --installed --json | jq 'map(.name) | index( "pipenv" )'; then - brew install pipenv +lazy-brew() { + set -e + # Install jq if it's not available. + + if ! which jq > /dev/null; then + set -ex && brew install jq > /dev/null fi -install-jq: - brew install jq + # Install requested packages, if they aren't installed. + for PACKAGE in "$@"; do + if ! brew info --installed --json | jq 'map(.name) | index( "$PACKAGE" )' > /dev/null; then + set -ex && brew install "$PACKAGE" > /dev/null + fi + done +} diff --git a/bake/bakefile.py b/bake/bakefile.py index a753073..7abb659 100644 --- a/bake/bakefile.py +++ b/bake/bakefile.py @@ -263,68 +263,69 @@ class TaskScript(BaseAction): return line - def prepare_init(self, sources=None, insert_source=None): - - tf = mkstemp(suffix=".sh", prefix="bashf-")[1] + def gen_source( + self, + sources=None, + insert_source=None, + remove_comments=False, + include_shebang=True, + ): stdlib_path = os.path.join(os.path.dirname(__file__), "scripts", "stdlib.sh") with open(stdlib_path, "r") as f: stdlib = f.read() + _sources = [stdlib, self.bashfile.funcs_source, self.bashfile.root_source] + if sources is None: - sources = (stdlib, self.bashfile.funcs_source, self.bashfile.root_source) + sources = [] - with open(tf, "w") as f: - if insert_source: - f.write(f"#!/usr/bin/env bash\nsource {insert_source}\n") - for source in sources: - f.write(source) - f.write("\n\n") + _sources.extend(sources) + sources = _sources - # Mark the temporary file as executable. - st = os.stat(tf) - os.chmod(tf, st.st_mode | stat.S_IEXEC) + source = "\n".join(sources) + first_natural_line = source.split("\n")[0] - return tf + if Bakefile._is_shebang_line(first_natural_line) and include_shebang: + yield first_natural_line + + if insert_source: + yield f". <(bake --source {insert_source})" + + for sourceline in source.split("\n"): + if not ( + remove_comments + and Bakefile._is_comment_line(sourceline, exclude_shebang=False) + ): + if sourceline: + yield sourceline + + yield "\n" def execute( self, *, blocking=False, debug=False, interactive=False, silent=False, **kwargs ): - init_tf = self.prepare_init() - if self.bashfile._is_shebang_line(self.source_lines[0]): - script_tf = self.prepare_init(sources=[self.source]) - if self.source_lines[0] == "#!/usr/bin/env bash": - with open(script_tf, "r") as f: - lines = f.readlines() - lines.insert(1, f"source {init_tf}") - with open(script_tf, "w") as f: - f.write("\n".join(lines)) - else: - script_tf = self.prepare_init(sources=[self.source], insert_source=init_tf) - args = " ".join([shlex_quote(a) for a in self.bashfile.args]) - if interactive: - script = f"source {shlex_quote(init_tf)}; {shlex_quote(script_tf)} {args}" - else: - script = f"source {shlex_quote(init_tf)}; {shlex_quote(script_tf)} {args} 2>&1 | bake:indent" - - cmd = f"bash -c {shlex_quote(script)}" + script_suffix = ( + "2>&1 | sed >&2 's/^/ | /' && exit \"${PIPESTATUS[0]}\"" + if not (interactive or silent) + else "" + ) + script_debug = "--verbose -x" if debug else "" + # --init-file <(bake --source __init__) + # --init-file <(bake --source __init__) + script = f"bash --noprofile {script_debug} <(bake --source {self.name}) {args} {script_suffix}" if debug: - click.echo(f" $ {cmd}", err=True) + click.echo(f" {click.style('$', fg='green')} {script}", err=True) - c = os.system(cmd) - - if not debug: - os.remove(script_tf) - os.remove(init_tf) - - return c + bash = Bash() + return bash.command(script, quote=False) def shellcheck(self, *, silent=False, debug=False, **kwargs): - tf = self.prepare_init(sources=[self.source]) + tf = self.gen_source(sources=[self.source]) cmd = f"shellcheck {shlex_quote(tf)} --external-sources --format=json" c = delegator.run(cmd) @@ -345,13 +346,13 @@ class TaskScript(BaseAction): return self.bashfile.chunks[self._chunk_index] def _iter_source(self): - try: - has_shebang = self._transform_line(self.chunk[1]).startswith("#!") - except IndexError: - has_shebang = False + # try: + # has_shebang = self._transform_line(self.chunk[1]).startswith("#!") + # except IndexError: + # has_shebang = False - if not has_shebang: - yield "#!/usr/bin/env bash" + # if not has_shebang: + # yield "#!/usr/bin/env bash" for line in self.chunk[1:]: line = self._transform_line(line) @@ -379,6 +380,7 @@ class Bakefile: os.environ["BAKEFILE_PATH"] = self.path os.environ["BAKE_SKIP_DONE"] = "1" + os.environ["PYTHONUNBUFFERED"] = "1" self.chunks self._tasks = None @@ -489,13 +491,20 @@ class Bakefile: line = line.replace("\t", " " * 4) return bool(len(line[:4].strip())) + def _is_task_line(self, line): + if line.startswith(INDENT_STYLES[0]) or line.startswith(INDENT_STYLES[1]): + return True + @staticmethod def _is_shebang_line(line): return line.startswith("#!") @staticmethod - def _is_comment_line(line): - return line.startswith("#") + def _is_comment_line(line, *, exclude_shebang=True): + if exclude_shebang: + return line.strip().startswith("#") and not line.startswith("#!") + else: + return line.strip().startswith("#") @staticmethod def _comment_line(line): @@ -523,6 +532,9 @@ class Bakefile: if self._is_declaration_line(line): task_active = True else: + if not self._is_task_line(line): + task_active = False + if not task_active: source_lines.append(line) @@ -536,10 +548,13 @@ class Bakefile: def funcs_source(self): source = [] - for task in self.tasks: - task = self[task] - source.append( - f"task:{task.name}()" + " { " + f"bake --silent {task.name} $@;" + "}" - ) + # for task in self.tasks: + # task = self[task] + # source.append( + # f"{task.name.replace('/', '_')}()" + # + " { " + # + f"bake --silent {task.name} $@" + # + "}" + # ) return "\n".join(source) diff --git a/bake/bash.py b/bake/bash.py index ca2daed..deef365 100644 --- a/bake/bash.py +++ b/bake/bash.py @@ -1,15 +1,15 @@ """ bash.py module """ -import re -import time import json as json_lib -import os -import stat -from tempfile import mkstemp +import re +import sys +import time from shlex import quote as shlex_quote - +import subprocess +import os import delegator +import click DELEGATOR_MINIMUM_TIMEOUT = 60 * 60 * 60 * 8 WHICH_BASH = "bash" @@ -21,35 +21,86 @@ if delegator.TIMEOUT < DELEGATOR_MINIMUM_TIMEOUT: __all__ = ["run", "Bash"] -class BashProcess: - """Bash process object.""" +def system_which(command, mult=False): + """Emulates the system's which. Returns None if not found.""" + _which = "which -a" if not os.name == "nt" else "where" + # os.environ = { + # vistir.compat.fs_str(k): vistir.compat.fs_str(val) + # for k, val in os.environ.items() + # } + result = None + try: + c = delegator.run("{0} {1}".format(_which, command)) + try: + # Which Not found… + if c.return_code == 127: + click.echo( + "{}: the {} system utility is required for bake to find bash properly." + "\n Please install it.".format( + click.style("Warning", bold=True), click.style(_which, fg="red") + ), + err=True, + ) + assert c.return_code == 0 + except AssertionError: + result = None + except TypeError: + if not result: + result = None + else: + if not result: + result = next(iter([c.out, c.err]), "").split("\n") + result = next(iter(result)) if not mult else result + return result + if not result: + result = None + result = [result] if mult else result + return result - def __init__(self, args, parent: "Bash", blocking: bool = True) -> None: - """constructor""" + +class BashProcess: + """bash process object""" + + def __init__( + self, + args, + parent: "bash", + interactive: bool = False, + blocking: bool = True, + **kwargs, + ) -> None: # Environ inherents from parent. # Remember passed-in arguments. self.parent = parent self.args = args - - # Run the subprocess. - args = " ".join(args) + self._return_code = None self.start_time = time.time() - self.sub = delegator.run( - f"{self.parent.path} {args}", env=self.parent.environ, block=blocking + self.elapsed_time = None + self.sub = None + + cmd = [system_which("bash"), *args] + + std_out = sys.stdout if interactive else subprocess.PIPE + std_in = sys.stdin if interactive else subprocess.PIPE + + self.sub = subprocess.Popen( + cmd, stdout=std_out, stdin=std_in, universal_newlines=True, **kwargs ) if blocking: - self.elapsed_time = time.time() - self.start_time + self._return_code = self.sub.wait() + + self.elapsed_time = time.time() - self.start_time @property def output(self) -> str: """stdout of the running process""" - return str(self.sub.out) + return str(self.sub.stdout) @property def err(self) -> str: """stderr of the running process""" - return str(self.sub.err) + return str(self.sub.stderr) @property def json(self) -> dict: @@ -59,12 +110,12 @@ class BashProcess: @property def ok(self) -> bool: """if the process exited with a 0 exit code""" - return self.sub.ok + return self.return_code == 0 @property def return_code(self) -> int: """the exit code of the process""" - return self.sub.return_code + return self._return_code or self.sub.returncode @property def pid(self) -> int: @@ -73,23 +124,18 @@ class BashProcess: def __repr__(self) -> str: """string representation of the bash process""" - return ( - f"" - ) + return f"" class Bash: - """An instance of Bash.""" + """an instance of bash""" - def __init__(self, *, path=WHICH_BASH, environ=None, interactive=False): + def __init__(self, *, path=WHICH_BASH, environ=None): """constructor""" self.path = path - self.interactive = interactive self.environ = environ or {} - ver_proc = self._exec("--version") - if not ver_proc.ok: - raise RuntimeError("bash is required.") + ver_proc = self("--version") self.about = ver_proc.output @property @@ -99,39 +145,14 @@ class Bash: # ...GNU Bash, version 4.4.19(1)-release ... --> 4.4.19(1)-release return matches.group(1) if matches else "version_unknown" - def _exec(self, *args, **kwargs) -> BashProcess: + def __call__(self, *args) -> BashProcess: """execute the bash process as a child of this process""" - return BashProcess(parent=self, args=args, **kwargs) + return BashProcess(parent=self, args=args) - def command(self, script: str, debug=False, **kwargs) -> BashProcess: + def command(self, script: str, quote=True) -> BashProcess: """form up the command with shlex and execute""" - - tf = mkstemp(suffix=".sh", prefix="bashf-")[1] - - with open(tf, "w") as f: - f.write(script) - - # Mark the temporary file as executable. - st = os.stat(tf) - os.chmod(tf, st.st_mode | stat.S_IEXEC) - - stdlib_path = os.path.join(os.path.dirname(__file__), "scripts", "stdlib.sh") - # print(stdlib_path) - - # cmd = f"bash -c {(script)}" - script = shlex_quote(f"unbuffer {tf} 2>&1 | bashf-indent") - cmd = f"bash --init-file {shlex_quote(stdlib_path)} -i -c {script} " - - if debug: - print(cmd) - - return_code = os.system(cmd) - - if not debug: - # Cleanup temporary file. - os.remove(tf) - - return return_code + maybe_quote = shlex_quote if quote else str + return self(f"-c", maybe_quote(script)) def run(script=None, **kwargs): diff --git a/bake/cli.py b/bake/cli.py index 9d13e48..97e7432 100644 --- a/bake/cli.py +++ b/bake/cli.py @@ -1,6 +1,7 @@ import sys import click import json +import random from .bakefile import Bakefile, TaskFilter, NoBakefileFound from .clint import eng_join @@ -20,6 +21,8 @@ SAFE_ENVIRONS = [ "TERM", "VIRTUAL_ENV", "BAKEFILE_PATH", + "PYTHONUNBUFFERED", + "PYTHONDONTWRITEBYTECODE", ] @@ -120,6 +123,7 @@ def echo_json(obj): hidden=False, help="Run shellcheck on Bakefile.", ) +@click.option("--source", default=False, nargs=1, hidden=True) @click.option( "--allow", default=False, @@ -198,6 +202,7 @@ def entrypoint( interactive, yes, help, + source, ): """bake — the strangely familiar task–runner.""" @@ -205,15 +210,18 @@ def entrypoint( do_help(0) # Default to list behavior, when no task is provided. - if _json: + if _json or source: silent = True + # Allow explicitly–passed environment variables. SAFE_ENVIRONS.extend(allow) + # Enable list functionality, by default. if task == "__LIST_ALL__": _list = True task = None + # Establish the Bakefile. try: if bakefile == "__BAKEFILE__": bakefile = Bakefile.find(root=".", filename="Bakefile") @@ -223,6 +231,29 @@ def entrypoint( click.echo(click.style("No Bakefile found!", fg="red"), err=True) do_help(1) + # --source (internal API) + if source: + + def echo_generator(g): + for g in g: + click.echo(g) + + if source == "__init__": + source = random.choice(list(bakefile.tasks.keys())) + source = bakefile.tasks[source].gen_source(remove_comments=True) + else: + task = bakefile.tasks[source] + source = task.gen_source( + sources=[task.source], + remove_comments=True, + insert_source="__init__", + include_shebang=True, + ) + for source_line in source: + # print(source_line) + click.echo(source_line) + sys.exit(0) + if not insecure: for key in bakefile.environ: if key not in SAFE_ENVIRONS: @@ -360,18 +391,25 @@ def entrypoint( + click.style(":", fg="white"), err=True, ) - return_code = task.execute( + usually_bash = task.execute( yes=yes, debug=debug, silent=silent, interactive=interactive ) if not _continue: - if (not return_code == 0) and (not isinstance(return_code, tuple)): - click.echo( - click.style(f"Task {task} failed!", fg="red"), err=True - ) - sys.exit(return_code) - if isinstance(return_code, tuple): - key, value = return_code + if hasattr(usually_bash, "ok"): + + if usually_bash.return_code > 0: + if not silent: + click.echo( + click.style(f"Task {task} failed!", fg="red"), + err=True, + ) + sys.exit(usually_bash.return_code) + + elif isinstance(usually_bash, tuple): + key, value = ( + usually_bash + ) # But, in this instance, clearly isn't. else: click.echo( click.style(" + ", fg="green") diff --git a/bake/scripts/stdlib.sh b/bake/scripts/stdlib.sh index 2ab8d68..e69de29 100644 --- a/bake/scripts/stdlib.sh +++ b/bake/scripts/stdlib.sh @@ -1,51 +0,0 @@ -#!/usr/bin/env bash - -if [ "$(uname)" == Darwin ]; then - bake:sed() { command sed -l "$@"; } -else - bake:sed() { command sed -u "$@"; } -fi - -# Syntax sugar. -bake:indent() { - bake:sed "s/^/ /" -} - -# --------------------- -# From: https://github.com/heroku/buildpack-stdlib/blob/master/stdlib.sh - -bake:step() { - if [[ "$*" == "-" ]]; then - read -r output - else - output=$* - fi - echo -e "\\e[1m\\e[36m=== $output\\e[0m" - unset output -} - -bake:error() { - if [[ "$*" == "-" ]]; then - read -r output - else - output=$* - fi - echo -e "\\e[1m\\e[31m=!= $output\\e[0m" -} - -bake:warn() { - if [[ "$*" == "-" ]]; then - read -r output - else - output=$* - fi - echo -e "\\e[1m\\e[33m=!= $output\\e[0m" -} - -# bake:eng_join() { -# for word in ${@}; do -# echo $word -# end - - -# }