From 3e99b7cead443c888f9bb5d58e4f8f9935053b67 Mon Sep 17 00:00:00 2001 From: Kenneth Reitz Date: Mon, 1 Oct 2018 18:47:51 -0400 Subject: [PATCH] changes --- Dockerfile | 5 +- Pipfile | 1 + Pipfile.lock | 46 ++++++++++++++++-- bruce_operator/buildpacks.py | 19 +++++--- bruce_operator/env.py | 5 ++ bruce_operator/storage.py | 34 ++++++++++++++ deploy/operator.yml | 90 ++++++++++++++++++++++++++++++++++++ 7 files changed, 185 insertions(+), 15 deletions(-) create mode 100644 bruce_operator/storage.py diff --git a/Dockerfile b/Dockerfile index 4b1ddb5..a07d1b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,13 +25,10 @@ COPY Pipfile.lock Pipfile.lock RUN apt install -y docker.io # RUN apt-get update -qq && apt-get install -qq -y daemontools && apt-get -qq -y --allow-downgrades --allow-remove-essential --allow-change-held-packages dist-upgrade && apt-get clean && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* /var/tmp/* -COPY . /bruce - # Install Herokuish. # RUN curl --location --silent https://github.com/gliderlabs/herokuish/releases/download/v0.4.4/herokuish_0.4.4_linux_x86_64.tgz | tar -xzC /bin - # Instlall kube-ctl. RUN apt-get update && apt-get install -y apt-transport-https RUN curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add - @@ -40,7 +37,7 @@ RUN echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" | tee -a /etc/ap RUN apt-get update RUN apt-get install -y kubectl -COPY . /app +COPY . /bruce # -- Install dependencies: RUN set -ex && pipenv install --deploy --system diff --git a/Pipfile b/Pipfile index 1dd7b9a..2a84630 100644 --- a/Pipfile +++ b/Pipfile @@ -14,6 +14,7 @@ tzlocal = "==2.0.0b1" gunicorn = "*" flask = "*" jsonpickle = "*" +"boto3" = "*" [dev-packages] bruce-operator = {editable = true, path = "."} diff --git a/Pipfile.lock b/Pipfile.lock index 8ca5b81..2ac5e3e 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "be1d282e2a46d2942349f35b72b9001b61da0726f54501b29bc9ef072d63f018" + "sha256": "a2145d36f2b4b2a9e6cae30e71d2399faa9ef3b5712e110e305fcd0f84f4d58d" }, "pipfile-spec": 6, "requires": { @@ -39,6 +39,21 @@ "markers": "python_version >= '3'", "version": "==1.0.2" }, + "boto3": { + "hashes": [ + "sha256:1ae24968d87f5ee557814ee2980f262811bebaf747b57d08eaf8b8f12534fee7", + "sha256:60a3bbc6203536ff648672b17d9169ec6107b40899fe3a29d3d37bbffc5f2063" + ], + "index": "pypi", + "version": "==1.9.14" + }, + "botocore": { + "hashes": [ + "sha256:5317aa5cd5f66761d5366c5c80aa8ccd1d798f59a107ccc62437f9d88f5ecab0", + "sha256:55d9fb544039add87e8b070eda88f470114f1355df3b16978b90c714cdc7f0a9" + ], + "version": "==1.12.14" + }, "cachetools": { "hashes": [ "sha256:90f1d559512fc073483fe573ef5ceb39bf6ad3d39edc98dc55178a2b2b176fa3", @@ -144,6 +159,14 @@ "index": "pypi", "version": "==0.6.2" }, + "docutils": { + "hashes": [ + "sha256:02aec4bd92ab067f6ff27a38a38a41173bf01bed8f89157768c1573f53e474a6", + "sha256:51e64ef2ebfb29cae1faa133b3710143496eca21c530f3f71424d77687764274", + "sha256:7a4bd47eaf6596e1295ecb11361139febe29b084a87bf005bf899f9a42edc3c6" + ], + "version": "==0.14" + }, "flask": { "hashes": [ "sha256:2271c0070dbcb5275fad4a82e29f23ab92682dc45f9dfbc22c02ba9b9322ce48", @@ -187,6 +210,13 @@ ], "version": "==2.10" }, + "jmespath": { + "hashes": [ + "sha256:6a81d4c9aa62caf061cb517b4d9ad1dd300374cd4706997aff9cd6aedd61fc64", + "sha256:f11b4461f425740a1d908e9a3f7365c3d2e569f6ca68a2ff8bc5bcd9676edd63" + ], + "version": "==0.9.3" + }, "jsonpickle": { "hashes": [ "sha256:8b6212f1155f43ce67fa945efae6d010ed059f3ca5ed377aa070e5903d45b722", @@ -213,11 +243,11 @@ }, "logme": { "hashes": [ - "sha256:a1e66dab9375788b4a97c62c408195b60faf33a2a3af6bf8fab358cc3eb1e42e", - "sha256:d8e379368aebe80c822157c7c25325c25bb81731394f812186731c802df8838c" + "sha256:0a5b8415cd24d0d7d32f34082de471f000f0d93e5a26939f137536b9e38e7633", + "sha256:ead1fa94c612bc50914224b5da80821a5cd5af03247245e18dd4980ad8cdfdfc" ], "index": "pypi", - "version": "==1.3.0" + "version": "==1.3.1" }, "markupsafe": { "hashes": [ @@ -279,6 +309,7 @@ "sha256:1adb80e7a782c12e52ef9a8182bebeb73f1d7e24e374397af06fb4956c8dc5c0", "sha256:e27001de32f627c22380a688bcc43ce83504a7bc5da472209b4c70f02829f0b8" ], + "markers": "python_version >= '2.7'", "version": "==2.7.3" }, "pytz": { @@ -319,6 +350,13 @@ ], "version": "==4.0" }, + "s3transfer": { + "hashes": [ + "sha256:90dc18e028989c609146e241ea153250be451e05ecc0c2832565231dacdf59c1", + "sha256:c7a9ec356982d5e9ab2d4b46391a7d6a950e2b04c472419f5fdec70cc0ada72f" + ], + "version": "==0.1.13" + }, "six": { "hashes": [ "sha256:70e8a77beed4562e7f14fe23a786b54f6296e34344c23bc42f07b15018ff98e9", diff --git a/bruce_operator/buildpacks.py b/bruce_operator/buildpacks.py index 6077e03..ac3f673 100644 --- a/bruce_operator/buildpacks.py +++ b/bruce_operator/buildpacks.py @@ -9,6 +9,8 @@ from .env import ( OPERATOR_HTTP_SERVICE_ADDRESS, ) +from . import storage + requests = Session() # TODO: support builkit versions. @@ -25,6 +27,7 @@ class Buildpack: self.repo = None self.index = None self.meta = {} + self.minio = storage.get_minio() # Ensure the buildpacks directory exists. os.makedirs(BUILDPACKS_DOWNLOAD_DIR, exist_ok=True) @@ -36,29 +39,31 @@ class Buildpack: def is_repo(self): return bool(self.repo) - def _download_url_to_fname(self, url, f_name): + def _download_url_to_minio(self, url, f_name): self.logger.info(f"Downloading {self.name!r} buildpack...") r = requests.get(url) - with open(f_name, "wb") as f: - f.write(r.content) + storage.set_buildpack(minio=self.minio, name=f_name, value=r.content) def _f_name(self, i): i = i = "%03d" % i - return f"{BUILDPACKS_DOWNLOAD_DIR}/{i}-{self.name}.tgz" + return f"{i}-{self.name}.tgz" def fetch_repo(self, i=0): is_github = "github.com" in self.repo - if not os.path.isfile(self._f_name(i)): + cached_buildppack = storage.get_buildpack( + minio=self.minio, name=self._f_name(i) + ) + if not cached_buildpack: if is_github: url = f"{self.repo}/archive/master.tar.gz" - self._download_url_to_fname(url=url, f_name=self._f_name(i)) + self._download_url_to_minio(url=url, f_name=self._f_name(i)) def fetch_buildkit(self, i=0): url = BUILDKIT_TEMPLATE.format(self.buildkit) if not os.path.isfile(self._f_name(i)): - self._download_url_to_fname(url=url, f_name=self._f_name(i)) + self._download_url_to_minio(url=url, f_name=self._f_name(i)) else: self.logger.info(f"Using cached {self.name!r} buildpack.") diff --git a/bruce_operator/env.py b/bruce_operator/env.py index 7acc390..173cfb8 100644 --- a/bruce_operator/env.py +++ b/bruce_operator/env.py @@ -17,3 +17,8 @@ IN_KUBERNETES = os.path.isfile(TOKEN_LOCATION) IN_WINDOWS = os.name == "nt" KUBECONFIG_PATH = os.path.expanduser("~/.kube/config") BUILDKIT_TEMPLATE = "https://codon-buildpacks.s3.amazonaws.com/buildpacks/{}.tgz" + +MINIO_ACCESS_KEY = os.environ.get("MINIO_ACCESS_KEY") +MINIO_SECRET_KEY = os.environ.get("MINIO_SECRET_KEY") +MINIO_SERVER = os.environ.get("MINIO_SERVER") +BUILDPACKS_BUCKET = "buildpacks" diff --git a/bruce_operator/storage.py b/bruce_operator/storage.py new file mode 100644 index 0000000..63847b4 --- /dev/null +++ b/bruce_operator/storage.py @@ -0,0 +1,34 @@ +import boto3 + +from .env import MINIO_ACCESS_KEY, MINIO_SECRET_KEY, MINIO_SERVER, BUILDPACKS_BUCKET + + +def get_minio(): + try: + minio = boto3.resource( + "s3", + endpoint_url=f"http://{MINIO_SERVER}", + config=boto3.session.Config(signature_version="s3v4"), + ) + except ValueError: + minio = None + + return minio + + +def ensure_bucket(*minio, buildpacks=True): + if buildpacks: + try: + minio.create_bucket(Bucket=BUILDPACKS_BUCKET) + except ValueError: + pass + + +def get_buildpack(*, minio, name): + o = minio.Object(BUILDPACKS_BUCKET, name) + return o.get()["Body"].read() + + +def set_buildpack(*, minio, name, value): + o = minio.Object(BUILDPACKS_BUCKET, name) + return o.put(Body=value) diff --git a/deploy/operator.yml b/deploy/operator.yml index 112b7eb..5b7854e 100644 --- a/deploy/operator.yml +++ b/deploy/operator.yml @@ -181,12 +181,22 @@ spec: labels: io.kompose.service: bruce-operator spec: + initContainers: + - name: init-wait + image: alpine + command: ["sh", "-c", "for i in $(seq 1 300); do nc -zvw1 minio.bruce 9000 && exit 0 || sleep 3; done; exit 1"] containers: - env: - name: WATCH_NAMESPACE valueFrom: fieldRef: fieldPath: metadata.namespace + - name: MINIO_ACCESS_KEY + value: brucebruce + - name: MINIO_SECRET_KEY + value: brucebruce + - name: MINIO_SERVER + value: minio.bruce:9000 image: kennethreitz/bruce-operator:latest name: bruce-operator resources: {} @@ -288,3 +298,83 @@ spec: io.kompose.service: registry status: loadBalancer: {} +--- +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + creationTimestamp: null + labels: + io.kompose.service: bruce-minio-data + name: bruce-minio-data +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi +status: {} +--- +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + annotations: + kompose.cmd: C:\Users\me\OneDrive\system\bin\kompose.exe convert -f .\kubernetes-compose.yml + kompose.version: 1.1.0 (36652f6) + creationTimestamp: null + labels: + io.kompose.service: minio + name: minio +spec: + replicas: 1 + strategy: + type: Recreate + template: + metadata: + creationTimestamp: null + labels: + io.kompose.service: minio + spec: + containers: + - env: + - name: MINIO_ACCESS_KEY + value: brucebruce + - name: MINIO_SECRET_KEY + value: brucebruce + image: minio/minio:latest + ports: + - containerPort: 9000 + args: + - server + - /storage + name: minio + resources: {} + volumeMounts: + - mountPath: /storage + name: bruce-minio-data + restartPolicy: Always + volumes: + - name: bruce-minio-data + persistentVolumeClaim: + claimName: bruce-minio-data +status: {} +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + kompose.cmd: C:\Users\me\OneDrive\system\bin\kompose.exe convert -f .\kubernetes-compose.yml + kompose.version: 1.1.0 (36652f6) + creationTimestamp: null + labels: + io.kompose.service: minio + name: minio +spec: + clusterIP: None + ports: + - name: minio-http + port: 9000 + targetPort: 0 + selector: + io.kompose.service: minio +status: + loadBalancer: {}