From a311bef677881cd01674f02b6130eaa2b550ffd3 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 1 Oct 2018 06:52:06 -0400 Subject: [PATCH] changes --- Pipfile | 4 ++ Pipfile.lock | 68 +++++++++++++++++++++++++++++- bruce_operator/__main__.py | 6 +++ bruce_operator/buildpacks.py | 82 ++++++++++++++++++++++++------------ bruce_operator/env.py | 6 ++- bruce_operator/http.py | 25 +++++++++++ bruce_operator/operator.py | 10 +++-- deploy/operator.yml | 24 ++++++++--- test.bat | 3 +- 9 files changed, 188 insertions(+), 40 deletions(-) create mode 100644 bruce_operator/http.py diff --git a/Pipfile b/Pipfile index 3f64d6b..1dd7b9a 100644 --- a/Pipfile +++ b/Pipfile @@ -10,6 +10,10 @@ click = "*" logme = "*" docopt = "*" kubeconfig = "*" +tzlocal = "==2.0.0b1" +gunicorn = "*" +flask = "*" +jsonpickle = "*" [dev-packages] bruce-operator = {editable = true, path = "."} diff --git a/Pipfile.lock b/Pipfile.lock index 8a0441e..8ca5b81 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "8e205985e9dce5cf161979b0748b6e7955234d382d67261417fc8da92370702b" + "sha256": "be1d282e2a46d2942349f35b72b9001b61da0726f54501b29bc9ef072d63f018" }, "pipfile-spec": 6, "requires": { @@ -144,6 +144,14 @@ "index": "pypi", "version": "==0.6.2" }, + "flask": { + "hashes": [ + "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", + "sha256:a080b744b7e345ccfcbc77954861cb05b3c63786e93f2b3875e0913d44b43f05" + ], + "index": "pypi", + "version": "==1.0.2" + }, "google-auth": { "hashes": [ "sha256:9ca363facbf2622d9ba828017536ccca2e0f58bd15e659b52f312172f8815530", @@ -151,6 +159,14 @@ ], "version": "==1.5.1" }, + "gunicorn": { + "hashes": [ + "sha256:aa8e0b40b4157b36a5df5e599f45c9c76d6af43845ba3b3b0efe2c70473c2471", + "sha256:fa2662097c66f920f53f70621c6c58ca4a3c4d3434205e608e121b5b3b71f4f3" + ], + "index": "pypi", + "version": "==19.9.0" + }, "idna": { "hashes": [ "sha256:156a6814fb5ac1fc6850fb002e0852d56c0c8d2531923a51032d1b70760e186e", @@ -158,6 +174,28 @@ ], "version": "==2.7" }, + "itsdangerous": { + "hashes": [ + "sha256:cbb3fcf8d3e33df861709ecaf89d9e6629cff0a217bc2848f1b41cd30d360519" + ], + "version": "==0.24" + }, + "jinja2": { + "hashes": [ + "sha256:74c935a1b8bb9a3947c50a54766a969d4846290e1e788ea44c1392163723c3bd", + "sha256:f84be1bb0040caca4cea721fcbbbbd61f9be9464ca236387158b0feea01914a4" + ], + "version": "==2.10" + }, + "jsonpickle": { + "hashes": [ + "sha256:8b6212f1155f43ce67fa945efae6d010ed059f3ca5ed377aa070e5903d45b722", + "sha256:d43ede55b3d9b5524a8e11566ea0b11c9c8109116ef6a509a1b619d2041e7397", + "sha256:ed4adf0d14564c56023862eabfac211cf01211a20c5271896c8ab6f80c68086c" + ], + "index": "pypi", + "version": "==1.0" + }, "kubeconfig": { "hashes": [ "sha256:393bdf6b03d4830c11c8d47e267fd65a3bed82518492b20b9fe554c7cd6c8111" @@ -181,6 +219,12 @@ "index": "pypi", "version": "==1.3.0" }, + "markupsafe": { + "hashes": [ + "sha256:a6be69091dac236ea9c6bc7d012beab42010fa914c459791d627dad4910eb665" + ], + "version": "==1.0" + }, "oauthlib": { "hashes": [ "sha256:ac35665a61c1685c56336bda97d5eefa246f1202618a1d6f34fccb1bdd404162", @@ -237,6 +281,13 @@ ], "version": "==2.7.3" }, + "pytz": { + "hashes": [ + "sha256:a061aa0a9e06881eb8b3b2b43f05b9439d6583c206d0a6c340ff72a7b6669053", + "sha256:ffb9ef1de172603304d9d2819af6f5ece76f2e85ec10692a524dd876e72bf277" + ], + "version": "==2018.5" + }, "pyyaml": { "hashes": [ "sha256:254bf6fda2b7c651837acb2c718e213df29d531eebf00edb54743d10bcb694eb", @@ -275,6 +326,14 @@ ], "version": "==1.11.0" }, + "tzlocal": { + "hashes": [ + "sha256:27d58a0958dc884d208cdaf45ef5892bf2a57d21d9611f2ac45e51f1973e8cab", + "sha256:f124f198e5d86b3538b140883472beaa82d2c0efc0cd9694dfdbe39079e22e69" + ], + "index": "pypi", + "version": "==2.0.0b1" + }, "urllib3": { "hashes": [ "sha256:a68ac5e15e76e7e5dd2b8f94007233e01effe3e50e8daddf69acfd81cb686baf", @@ -289,6 +348,13 @@ "sha256:f5889b1d0a994258cfcbc8f2dc3e457f6fc7b32a8d74873033d12e4eab4bdf63" ], "version": "==0.53.0" + }, + "werkzeug": { + "hashes": [ + "sha256:c3fd7a7d41976d9f44db327260e263132466836cef6f91512889ed60ad26557c", + "sha256:d5da73735293558eb1651ee2fddc4d0dedcfa06538b8813a2e20011583c9e49b" + ], + "version": "==0.14.1" } }, "develop": { diff --git a/bruce_operator/__main__.py b/bruce_operator/__main__.py index 17d190d..9758818 100644 --- a/bruce_operator/__main__.py +++ b/bruce_operator/__main__.py @@ -3,6 +3,7 @@ Usage: bruce-operator watch [--buildpacks|--apps] bruce-operator fetch-buildpacks + bruce-operator http bruce-operator (-h | --help) Options: @@ -13,6 +14,7 @@ import sys from docopt import docopt from .operator import Operator +from .http import app def main(): @@ -31,6 +33,10 @@ def main(): print("Fetching buildpacks...") operator.fetch_buildpacks() + if args["http"]: + print("Starting webapp...") + app.run() + if __name__ == "__main__": main() diff --git a/bruce_operator/buildpacks.py b/bruce_operator/buildpacks.py index 0f5c843..6077e03 100644 --- a/bruce_operator/buildpacks.py +++ b/bruce_operator/buildpacks.py @@ -2,60 +2,90 @@ import logme from requests import Session import os -from .env import BUILDPACKS_DIR, BUILDKIT_TEMPLATE +from .env import ( + BUILDPACKS_DIR, + BUILDKIT_TEMPLATE, + BUILDPACKS_DOWNLOAD_DIR, + OPERATOR_HTTP_SERVICE_ADDRESS, +) requests = Session() +# TODO: support builkit versions. + +buildpacks = [] + @logme.log class Buildpack: - def __init__(self): - self.name = None + def __init__(self, name): + global buildpacks + self.name = name self.buildkit = None self.repo = None + self.index = None self.meta = {} + # Ensure the buildpacks directory exists. + os.makedirs(BUILDPACKS_DOWNLOAD_DIR, exist_ok=True) + + # Install buildpack into global dictionary. + buildpacks.append(self) + @property def is_repo(self): return bool(self.repo) - def fetch_repo(self): - pass + def _download_url_to_fname(self, url, f_name): + self.logger.info(f"Downloading {self.name!r} buildpack...") + r = requests.get(url) + with open(f_name, "wb") as f: + f.write(r.content) - def fetch_buildkit(self): + def _f_name(self, i): + i = i = "%03d" % i + return f"{BUILDPACKS_DOWNLOAD_DIR}/{i}-{self.name}.tgz" + + def fetch_repo(self, i=0): + is_github = "github.com" in self.repo + + if not os.path.isfile(self._f_name(i)): + if is_github: + url = f"{self.repo}/archive/master.tar.gz" + self._download_url_to_fname(url=url, f_name=self._f_name(i)) + + def fetch_buildkit(self, i=0): url = BUILDKIT_TEMPLATE.format(self.buildkit) - f_name = f"{BUILDPACKS_DIR}/{self.name}.tgz" - if not os.path.isfile(f_name): - self.logger.info(f"Downloading {self.name!r} buildpack...") - - r = requests.get(url) - with open(f_name, "wb") as f: - f.write(r.content) - - return f_name - - def fetch(self): - if self.is_repo: - return self.fetch_repo() + if not os.path.isfile(self._f_name(i)): + self._download_url_to_fname(url=url, f_name=self._f_name(i)) else: - return self.fetch_buildkit() + self.logger.info(f"Using cached {self.name!r} buildpack.") + + def fetch(self, i=0): + if self.is_repo: + return self.fetch_repo(i) + else: + return self.fetch_buildkit(i) def __repr__(self): return f"" + @property + def url(self): + return f"{OPERATOR_HTTP_SERVICE_ADDRESS}/{self.name}.tgz" + @classmethod def from_info(kls, info): - self = kls() - - self.name = info["metadata"]["name"] + self = kls(name=info["metadata"]["name"]) self.buildkit = info["spec"].get("buildkit") self.repo = info["spec"].get("repo") + self.index = info["spec"].get("index") return self -def fetch_buildpack(buildpack_info): +def fetch_buildpack(*, i=0, buildpack_info): bp = Buildpack.from_info(buildpack_info) - bp_path = bp.fetch() - print(bp_path) + bp_path = bp.fetch(i) + # print(bp_path) diff --git a/bruce_operator/env.py b/bruce_operator/env.py index 40e3407..d671009 100644 --- a/bruce_operator/env.py +++ b/bruce_operator/env.py @@ -4,7 +4,11 @@ WATCH_NAMESPACE = os.environ.get("WATCH_NAMESPACE", "bruce") API_VERSION = "v1alpha1" API_GROUP = "bruce.kennethreitz.org" -BUILDPACKS_DIR = "/tmp/buildpacks" +# BUILDPACKS_DIR = "/tmp/buildpacks" +BUILDPACKS_DIR = os.path.expanduser("~/.bruce/buildpacks") +BUILDPACKS_DOWNLOAD_DIR = os.path.expanduser("~/.bruce/buildpacks/.dl") +OPERATOR_HTTP_SERVICE_ADDRESS = "http://bruce.bruce-operator:80" + # APPCACHE_DIR = "/opt/caches" OPERATOR_IMAGE = "kennethreitz/bruce-operator:latest" TOKEN_LOCATION = "/var/run/secrets/kubernetes.io/serviceaccount/token" diff --git a/bruce_operator/http.py b/bruce_operator/http.py new file mode 100644 index 0000000..cd7b32a --- /dev/null +++ b/bruce_operator/http.py @@ -0,0 +1,25 @@ +import jsonpickle +from flask import Flask, jsonify + +from .buildpacks import buildpacks +from .operator import Operator + +app = Flask(__name__) +operator = Operator() + + +@app.route("/") +def get_buildpacks(): + bps = {} + for buildpack in buildpacks: + bp = {} + + bp["name"] = buildpack.name + bp["index"] = buildpack.index + bp["buildkit"] = buildpack.buildkit + bp["repo"] = buildpack.repo + bp["url"] = buildpack.url + + bps[buildpack.name] = bp + + return jsonify({"buildpacks": bps}) diff --git a/bruce_operator/operator.py b/bruce_operator/operator.py index d8c11c9..339e151 100644 --- a/bruce_operator/operator.py +++ b/bruce_operator/operator.py @@ -64,11 +64,13 @@ class Operator: api_response = self.custom_client.list_namespaced_custom_object( group, version, namespace, plural, pretty=pretty, watch=watch ) - for item in api_response["items"]: - yield item + items = api_response["items"] except kubernetes.client.rest.ApiException: return None + # Sort the buildpacks by their specified index. + return sorted(items, key=lambda k: k["spec"]["index"]) + def installed_apps(self): group = "bruce.kennethreitz.org" # str | The custom resource's group name version = "v1alpha1" # str | The custom resource's version @@ -137,8 +139,8 @@ class Operator: kc.use_context("context") def fetch_buildpacks(self): - for buildpack_info in self.installed_buildpacks(): - fetch_buildpack(buildpack_info) + for i, buildpack_info in enumerate(self.installed_buildpacks()): + fetch_buildpack(i=i, buildpack_info=buildpack_info) def watch(self, fork=True, buildpacks=False, apps=False): if buildpacks and apps: diff --git a/deploy/operator.yml b/deploy/operator.yml index 04178f0..07d0aeb 100644 --- a/deploy/operator.yml +++ b/deploy/operator.yml @@ -58,78 +58,90 @@ spec: --- apiVersion: v1 items: +- apiVersion: bruce.kennethreitz.org/v1alpha1 + kind: Buildpack + metadata: + name: "multi" + spec: + repo: "https://github.com/heroku/heroku-buildpack-multi" + index: 0 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "ruby" spec: buildkit: "heroku/ruby" + index: 1 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "nodejs" spec: buildkit: "heroku/nodejs" + index: 2 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "clojure" spec: buildkit: "heroku/clojure" + index: 3 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "python" spec: buildkit: "heroku/python" + index: 4 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "java" spec: buildkit: "heroku/java" + index: 5 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "gradle" spec: buildkit: "heroku/gradle" + index: 6 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "scala" spec: buildkit: "heroku/scala" + index: 7 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "php" spec: buildkit: "heroku/php" + index: 8 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "go" spec: buildkit: "heroku/go" + index: 9 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "elixir" spec: buildkit: "hashnuke/elixir" + index: 10 - apiVersion: bruce.kennethreitz.org/v1alpha1 kind: Buildpack metadata: name: "static" spec: repo: "https://github.com/dokku/buildpack-nginx" -- apiVersion: bruce.kennethreitz.org/v1alpha1 - kind: Buildpack - metadata: - name: "multi" - spec: - repo: "https://github.com/heroku/heroku-buildpack-multi" + index: 11 kind: List metadata: resourceVersion: "" diff --git a/test.bat b/test.bat index fb6d9f8..6456321 100644 --- a/test.bat +++ b/test.bat @@ -1,4 +1,3 @@ docker build --tag kennethreitz/bruce-operator . docker push kennethreitz/bruce-operator -kubectl delete -f .\deploy\operator.yml -n bruce -kubectl create -f .\deploy\operator.yml -n bruce --validate=false +kubectl delete --all pods -n bruce