From 7417ef33ed8f2cbd8a2c82f384fd287b980f91ee Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Tue, 2 Oct 2018 06:57:23 -0400 Subject: [PATCH] builds working --- Dockerfile | 11 +- bruce_operator/__main__.py | 8 ++ bruce_operator/apps.py | 13 +++ bruce_operator/buildpacks.py | 22 ++-- bruce_operator/builds.py | 201 +++++++++++++++++++++++++++++++++++ bruce_operator/env.py | 7 +- bruce_operator/operator.py | 25 ++++- bruce_operator/storage.py | 51 +++++---- deploy/operator.yml | 2 + 9 files changed, 296 insertions(+), 44 deletions(-) create mode 100644 bruce_operator/apps.py create mode 100644 bruce_operator/builds.py diff --git a/Dockerfile b/Dockerfile index a07d1b5..285b007 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,11 +24,6 @@ COPY Pipfile.lock Pipfile.lock # Install Docker. RUN apt install -y docker.io -# RUN apt-get update -qq && apt-get install -qq -y daemontools && apt-get -qq -y --allow-downgrades --allow-remove-essential --allow-change-held-packages dist-upgrade && apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /var/tmp/* - -# Install Herokuish. -# RUN curl --location --silent https://github.com/gliderlabs/herokuish/releases/download/v0.4.4/herokuish_0.4.4_linux_x86_64.tgz | tar -xzC /bin - # Instlall kube-ctl. RUN apt-get update && apt-get install -y apt-transport-https RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - @@ -37,6 +32,12 @@ RUN echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/ap RUN apt-get update RUN apt-get install -y kubectl +# Install daemontools +RUN apt-get update -qq && apt-get install -qq -y daemontools && apt-get -qq -y --allow-downgrades --allow-remove-essential --allow-change-held-packages dist-upgrade && apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /var/tmp/* + +# Install Herokuish. +RUN curl --location --silent https://github.com/gliderlabs/herokuish/releases/download/v0.4.4/herokuish_0.4.4_linux_x86_64.tgz | tar -xzC /bin + COPY . /bruce # -- Install dependencies: diff --git a/bruce_operator/__main__.py b/bruce_operator/__main__.py index c80645e..4e763e9 100644 --- a/bruce_operator/__main__.py +++ b/bruce_operator/__main__.py @@ -4,6 +4,7 @@ Usage: bruce-operator watch [--buildpacks|--apps] bruce-operator fetch-buildpacks bruce-operator http + bruce-operator build bruce-operator (-h | --help) Options: @@ -19,6 +20,7 @@ from docopt import docopt from .operator import Operator from .http import app from .env import IN_WINDOWS +from .builds import bootstrap_docker def main(): @@ -45,6 +47,12 @@ def main(): else: app.run(port=80) + if args["build"]: + app_name = args[""] + print(f"Building {app_name}") + bootstrap_docker() + operator.build_app(app_name=app_name) + if __name__ == "__main__": main() diff --git a/bruce_operator/apps.py b/bruce_operator/apps.py new file mode 100644 index 0000000..28c45ff --- /dev/null +++ b/bruce_operator/apps.py @@ -0,0 +1,13 @@ +class App: + def __init__(self, name): + self.name = name + self.auto_deploy_last_release = None + self.repo = None + + @classmethod + def from_info(kls, app_info): + self = kls(name=app_info["metadata"]["name"]) + self.auto_deploy_last_release = app_info["spec"].get("auto_deploy_last_release") + self.repo = app_info["spec"].get("repo") + + return self diff --git a/bruce_operator/buildpacks.py b/bruce_operator/buildpacks.py index a598e1b..566f4c4 100644 --- a/bruce_operator/buildpacks.py +++ b/bruce_operator/buildpacks.py @@ -2,12 +2,7 @@ import logme from requests import Session import os -from .env import ( - BUILDPACKS_DIR, - BUILDKIT_TEMPLATE, - BUILDPACKS_DOWNLOAD_DIR, - OPERATOR_HTTP_SERVICE_ADDRESS, -) +from .env import BUILDKIT_TEMPLATE, OPERATOR_HTTP_SERVICE_ADDRESS from . import storage @@ -27,10 +22,6 @@ class Buildpack: self.repo = None self.index = None self.meta = {} - self.minio = storage.get_minio() - - # Ensure the buildpacks directory exists. - os.makedirs(BUILDPACKS_DOWNLOAD_DIR, exist_ok=True) # Install buildpack into global dictionary. buildpacks.append(self) @@ -42,7 +33,7 @@ class Buildpack: def _download_url_to_minio(self, url, f_name): self.logger.info(f"Downloading {self.name!r} buildpack...") r = requests.get(url) - storage.set_buildpack(minio=self.minio, name=f_name, value=r.content) + storage.buildpacks.set(f_name, r.content) def _f_name(self, i): i = i = "%03d" % i @@ -51,7 +42,7 @@ class Buildpack: def fetch_repo(self, i=0): is_github = "github.com" in self.repo - cached_buildpack = storage.get_buildpack(minio=self.minio, name=self._f_name(i)) + cached_buildpack = storage.buildpacks.exists(self._f_name(i)) if not cached_buildpack: if is_github: url = f"{self.repo}/archive/master.tar.gz" @@ -60,7 +51,8 @@ class Buildpack: def fetch_buildkit(self, i=0): url = BUILDKIT_TEMPLATE.format(self.buildkit) - if not os.path.isfile(self._f_name(i)): + cached_buildpack = storage.buildpacks.exists(self._f_name(i)) + if not cached_buildpack: self._download_url_to_minio(url=url, f_name=self._f_name(i)) else: self.logger.info(f"Using cached {self.name!r} buildpack.") @@ -92,3 +84,7 @@ def fetch_buildpack(*, i=0, buildpack_info): bp = Buildpack.from_info(buildpack_info) bp_path = bp.fetch(i) # print(bp_path) + + +def extract_buildpacks(): + pass diff --git a/bruce_operator/builds.py b/bruce_operator/builds.py new file mode 100644 index 0000000..9bcc85a --- /dev/null +++ b/bruce_operator/builds.py @@ -0,0 +1,201 @@ +import os +import uuid +import tempfile +import json +import time +from shutil import rmtree +from pathlib import Path + +import delegator + +# import docker as docker_api +import logme + +# from .db import db +from .env import HEROKUISH_IMAGE, REGISTRY_URL + +OUR_HEROKUISH_IMAGE = f"{REGISTRY_URL}/herokuish" + + +# Run Docker service. +# delegator.run("service docker start") +@logme.log +def bootstrap_docker(logger=None, mirror_herokuish=True): + + # logger.debug("Configuring docker service to allow our insecure registry...") + # Configure our registry as insecure. + try: + with open("/etc/docker/daemon.json", "w") as f: + data = {"insecure-registries": [REGISTRY_URL]} + json.dump(data, f) + # This fails when running on Windows... + except FileNotFoundError: + pass + + # logger.info("Starting docker service...") + delegator.run("service docker start") + time.sleep(2) + + docker_running = delegator.run("docker ps").ok + + if not docker_running: + # logger.info("Assuming docker is not available...") + pass + + else: + logger.info("Docker started!") + if mirror_herokuish: + logger.info("Checking for mirrored Herokuish image...") + pull = delegator.run(f"docker pull {OUR_HEROKUISH_IMAGE}") + if not pull.ok: + logger.debug(pull.out) + + logger.info("Pulling official Herokuish image...") + delegator.run(f"docker pull {HEROKUISH_IMAGE}") + + logger.info("Pushing Herokuish image to our registry...") + tag = delegator.run( + f"docker tag {HEROKUISH_IMAGE} {OUR_HEROKUISH_IMAGE}" + ) + assert tag.ok + + push = delegator.run(f"docker push {OUR_HEROKUISH_IMAGE}") + assert push.ok + else: + logger.info("Herokuish is mirrored and up-to-date!") + + +@logme.log +class BaseBuild: + def __init__(self): + self.uuid = uuid.uuid4().hex + self.paths = {} + for name in ("cache", "import"): + self.paths[name] = tempfile.mkdtemp(prefix=f"build-{self.uuid}-{name}-") + self.repo_url = None + + @property + def build_name(self): + return f"build-{self.uuid}" + + @property + def service_name(self): + return self.uuid + + def clone(self, repo_url): + cmd = f"git clone {repo_url} {self.paths['import']}" + self.logger.debug(f"build {self.uuid!r}: Running $ {cmd}.") + + c = delegator.run(cmd) + if not c.ok: + self.logger.warning( + f"build {self.uuid!r}: The clone of {repo_url!r} failed!" + ) + self.logger.warning(f"build {self.uuid!r}: {c.err}") + raise RuntimeError("The clone failed!") + self.repo_url = repo_url + + +@logme.log +class Build(BaseBuild): + def __init__(self, repo_url, app_name, buildpacks_dir): + + super().__init__() + self.clone(repo_url=repo_url) + self.app_name = app_name + self.paths["buildpacks"] = buildpacks_dir + # self.timeout = HEROUISH_TIMEOUT + + @property + def has_dockerfile(self): + assert self.repo_url + return os.path.isfile((Path(self.paths["import"]) / "Dockerfile").resolve()) + + def docker(self, cmd, assert_ok=True, fail=True): + cmd = f"docker {cmd}" + self.logger.debug(f"$ {cmd}") + c = delegator.run(cmd) + try: + assert c.ok + except AssertionError as e: + self.logger.debug(c.out) + self.logger.debug(c.err) + + if fail: + raise e + + return c + + # Inspiration: + # https://raw.githubusercontent.com/gitlabhq/gitlabhq/04845fdeae75ba5de7c93992a5d55663edf647e0/vendor/gitlab-ci-yml/Auto-DevOps.gitlab-ci.yml + def build(self, push=True, promote=None): + + # Mark build as started in database. + # db.start_build(uuid=self.uuid, app_name=self.app_name, repo_url=self.repo_url) + + assert self.repo_url + docker_cmd = ( + f"run -i --name={self.build_name} -v {self.paths['import']}:/tmp/app -v {self.paths['buildpacks']}:/tmp/buildpacks" + f" {OUR_HEROKUISH_IMAGE} /bin/herokuish buildpack build" + ) + build = self.docker(docker_cmd, fail=False) + self.logger.debug(build.out) + if not build.ok: + self.logger.info(f"Build {self.uuid} failed!") + + # Mark build as failed in database. + # db.fail_build(uuid=self.uuid) + + return build + + # Commit to Docker. + docker_cmd = f"commit {self.build_name}" + commit = self.docker(docker_cmd) + commit_output = commit.out.strip() + + # Create runnable container in docker. + docker_cmd = ( + f"create --expose 80 --env PORT=80 " + f"--name={self.service_name} {commit_output} /bin/herokuish procfile start web" + ) + create = self.docker(docker_cmd) + create_output = create.out.strip() + self.logger.debug(create_output) + + # Commit to Docker. + docker_cmd = f"commit {self.service_name}" + commit = self.docker(docker_cmd) + commit_output = commit.out.strip() + + tag_name = f"{REGISTRY_URL}/{self.app_name}/{self.service_name}" + docker_cmd = f"tag {commit_output} {tag_name}" + + tag = self.docker(docker_cmd) + tag_output = tag.out.strip() + + if push: + self.logger.info(f"Pushing build {self.uuid!r} to registry...") + docker_cmd = f"push {tag_name}" + push = self.docker(docker_cmd) + pass + + # Mark build as finished in database. + # db.succeed_build(uuid=self.uuid) + if promote: + self.logger.info(f"Promoting {self.app_name}'s' {promote} to: {self.uuid}.") + # db.promote_build(app_name=self.app_name, uuid=self.uuid, target=promote) + return build + + def cleanup(self): + for name, path in self.paths.items(): + self.logger.info(f"Cleaning up {name}: {path!r}.") + try: + rmtree(path) + except FileNotFoundError: + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + self.cleanup() diff --git a/bruce_operator/env.py b/bruce_operator/env.py index 173cfb8..0fdedea 100644 --- a/bruce_operator/env.py +++ b/bruce_operator/env.py @@ -4,11 +4,8 @@ WATCH_NAMESPACE = os.environ.get("WATCH_NAMESPACE", "bruce") API_VERSION = "v1alpha1" API_GROUP = "bruce.kennethreitz.org" -# BUILDPACKS_DIR = "/tmp/buildpacks" -BUILDPACKS_DIR = os.path.expanduser("~/.bruce/buildpacks") -BUILDPACKS_DOWNLOAD_DIR = os.path.expanduser("~/.bruce/buildpacks/.dl") OPERATOR_HTTP_SERVICE_ADDRESS = "http://bruce.bruce-operator:80" - +REGISTRY_URL = os.environ.get("REGISTY_URL", "localhost:80") # APPCACHE_DIR = "/opt/caches" OPERATOR_IMAGE = "kennethreitz/bruce-operator:latest" TOKEN_LOCATION = "/var/run/secrets/kubernetes.io/serviceaccount/token" @@ -22,3 +19,5 @@ MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY") MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY") MINIO_SERVER = os.environ.get("MINIO_SERVER") BUILDPACKS_BUCKET = "buildpacks" + +HEROKUISH_IMAGE = "gliderlabs/herokuish" diff --git a/bruce_operator/operator.py b/bruce_operator/operator.py index de0ac95..c4d55db 100644 --- a/bruce_operator/operator.py +++ b/bruce_operator/operator.py @@ -22,7 +22,9 @@ from .env import ( TOKEN_LOCATION, ) from .kubectl import kubectl -from .buildpacks import fetch_buildpack +from .buildpacks import fetch_buildpack, extract_buildpacks +from .apps import App +from .builds import Build # https://github.com/kubernetes-client/python/blob/master/examples/create_thirdparty_resource.md @@ -71,6 +73,7 @@ class Operator: # Sort the buildpacks by their specified index. return sorted(items, key=lambda k: k["spec"]["index"]) + @property def installed_apps(self): group = "bruce.kennethreitz.org" # str | The custom resource's group name version = "v1alpha1" # str | The custom resource's version @@ -142,6 +145,23 @@ class Operator: for i, buildpack_info in enumerate(self.installed_buildpacks()): fetch_buildpack(i=i, buildpack_info=buildpack_info) + def build_app(self, app_name): + apps = [App.from_info(app) for app in self.installed_apps] + app = None + for _app in apps: + if _app.name == app_name: + app = _app + if not app: + self.logger.info(f"App {app_name!r} not found.") + return + + self.logger.info(f"Building {app_name}") + buildpacks_dir = extract_buildpacks() + build = Build( + repo_url=app.repo, app_name=app.name, buildpacks_dir=buildpacks_dir + ) + build.build() + def watch(self, fork=True, buildpacks=False, apps=False): if buildpacks and apps: raise RuntimeError("Can only watch one at a time: buildpacks and apps.") @@ -162,3 +182,6 @@ class Operator: self.logger.info(f"Blocking on subprocesses completion.") for subprocess in subprocesses: subprocess.block() + + if buildpacks: + pass diff --git a/bruce_operator/storage.py b/bruce_operator/storage.py index 2e43692..88fcdcb 100644 --- a/bruce_operator/storage.py +++ b/bruce_operator/storage.py @@ -9,37 +9,46 @@ os.environ["AWS_ACCESS_KEY_ID"] = MINIO_ACCESS_KEY os.environ["AWS_SECRET_ACCESS_KEY"] = MINIO_SECRET_KEY -def get_minio(): - try: - minio = boto3.resource( +class Buildpacks: + def __init__(self, mino=None): + self.bucket_name = BUILDPACKS_BUCKET + self.minio = boto3.resource( "s3", endpoint_url=f"http://{MINIO_SERVER}", config=boto3.session.Config(signature_version="s3v4"), ) - except ValueError: - minio = None - return minio + self.ensure_buckets() + self.bucket = self.minio.Bucket(self.bucket_name) - -def ensure_buckets(*, minio, buildpacks=True): - if buildpacks: + def ensure_buckets(self): try: - minio.create_bucket(Bucket=BUILDPACKS_BUCKET) + self.minio.create_bucket(Bucket=BUILDPACKS_BUCKET) except botocore.errorfactory.ClientError: pass + def list(self): + keys = [] + for key in self.bucket.objects.all(): + print(object) -def get_buildpack(*, minio, name): - ensure_buckets(minio=minio) - o = minio.Object(BUILDPACKS_BUCKET, name) - try: - return o.get()["Body"].read() - except botocore.errorfactory.ClientError: - return None + def exists(self, name): + o = self.minio.Object(self.bucket_name, name) + try: + return bool(o.get()) + except botocore.errorfactory.ClientError: + return None + + def get(self, name): + o = self.minio.Object(self.bucket_name, name) + try: + return o.get()["Body"].read() + except botocore.errorfactory.ClientError: + return None + + def set(self, name, value): + o = self.minio.Object(self.bucket_name, name) + return o.put(Body=value) -def set_buildpack(*, minio, name, value): - ensure_buckets(minio=minio) - o = minio.Object(BUILDPACKS_BUCKET, name) - return o.put(Body=value) +buildpacks = Buildpacks() diff --git a/deploy/operator.yml b/deploy/operator.yml index 5b7854e..ed20210 100644 --- a/deploy/operator.yml +++ b/deploy/operator.yml @@ -200,6 +200,8 @@ spec: image: kennethreitz/bruce-operator:latest name: bruce-operator resources: {} + securityContext: + privileged: true restartPolicy: Always status: {} ---