From b8461ac4e68a1a8041d297039f340ad5978d2618 Mon Sep 17 00:00:00 2001 From: Adrien Bouvais Date: Mon, 17 Aug 2026 10:23:28 +0200 Subject: [PATCH] Test --- README.md | 260 ++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 205 insertions(+), 55 deletions(-) diff --git a/README.md b/README.md index dd2f41d..f7e026c 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,8 @@ First let's install everything that I will use using `brew`. ``` -brew install docker +% brew install docker +% brew install kind kubectl helm argocd k9s ``` # 2. Images @@ -16,66 +17,215 @@ brew install docker To created the needed dependency, I did a simple 2 images base + jupyter. The first image is a minimal python slim. I then add some -### 2.1. Base +Base is a simple `python:3.12-slim-bookworm` + tini using a non-root user. +Jupyter simply install python dependencies and start the jupyterlab server. -```images/base/dockerfile -FROM python:3.12-slim-bookworm AS base - -LABEL org.opencontainers.image.title="tenant-base" \ - org.opencontainers.image.description="Hardened base image for tenant workspaces" \ - org.opencontainers.image.source="https://git.bouvais.lu/adrien/" - -# System deps only — keep this layer stable so it's rarely rebuilt -RUN apt-get update && apt-get install -y --no-install-recommends \ - ca-certificates \ - curl \ - tini \ - && rm -rf /var/lib/apt/lists/* - -# Non-root user, fixed UID/GID for predictable K8s securityContext -RUN groupadd --gid 1000 appuser \ - && useradd --uid 1000 --gid appuser --shell /bin/bash --create-home appuser - -WORKDIR /home/appuser -USER appuser - -ENTRYPOINT ["tini", "--"] -``` - -Then build and push it to the registry +I then build and push them to my registry ``` -docker build -t registry.bouvais.lu/tenant-base:1.0.0 images/base -docker push registry.bouvais.lu/tenant-base:1.0.0 +% docker build -t registry.bouvais.lu/tenant-base:1.0.0 images/base +% docker push registry.bouvais.lu/tenant-base:1.0.0 + +% docker build -t registry.bouvais.lu/tenant-jupyter:1.0.0 images/jupyter +% docker push registry.bouvais.lu/tenant-jupyter:1.0.0 ``` -### 2.2. Jupyter +# 3. Cluster setup -Now I dp the same for a simple jupyter image. +In this section, will create the cluster, create namespaces and shared minio + argo cd namespace. + +### 3.1 Cluster + +Now for the actual deployment, I will make a simple local kubernetes cluser with `kind`. ``` -ARG BASE_IMAGE=registry.bouvais.lu/tenant-base:1.0.0 -FROM ${BASE_IMAGE} - -LABEL org.opencontainers.image.title="tenant-jupyter" \ - org.opencontainers.image.description="JupyterLab image built on tenant-base" \ - org.opencontainers.image.base.name="${BASE_IMAGE}" - -USER root -COPY --chown=appuser:appuser requirements.txt /tmp/requirements.txt -RUN pip install --no-cache-dir -r /tmp/requirements.txt \ - && rm /tmp/requirements.txt - -# No secrets baked in — S3 creds come from a mounted K8s Secret / env at runtime -USER appuser -WORKDIR /home/appuser/work - -EXPOSE 8888 - -CMD ["jupyter", "lab", \ - "--ip=0.0.0.0", \ - "--port=8888", \ - "--no-browser", \ - "--ServerApp.token=", \ - "--ServerApp.allow_remote_access=True"] +% kind create cluster --name ctie-exercice +% kubectl cluster-info ``` + +``` +% kubectl apply --server-side -k "https://github.com/argoproj/argo-cd/manifests/crds?ref=stable" +% kubectl apply -f https://github.com/emberstack/kubernetes-reflector/releases/latest/download/reflector.yaml +``` + +### 3.2 Minio + +I deploy a unique shared minio instance in a namespace named minio. +Obviously it can have its own scaling stategy later if needed but that's out of scope here. + +Create `minio/manifests.yaml` and deploying it: + +``` +% kubectl create ns minio +% kubectl apply -f minio/manifests.yaml +% kubectl -n minio get pods +NAME READY STATUS RESTARTS AGE +minio-688ffcdbbd-qm47b 1/1 Running 0 3m6s +``` + +Can check the connection by forwarding port and goinf to localhost:9001 + +``` +% kubectl -n minio port-forward svc/minio 9000:9000 9001:9001 +``` + +### 3.3 Argo CD + +Similar to minio, a single instance in a unique namespace. +First lets fetch and run an Argo CD insance. + +``` +% kubectl create ns argocd +% kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml +``` + +Similarly, can get admin password + forward to go to Argo CD webui. +Obviously getting and using admin credentials is not good, but onece again that's out of scope. + +``` +% kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo +% kubectl -n argocd port-forward svc/argocd-server 8080:443 +``` + +# 4. GitOps + +Now let's create Argo CD application set automated with a path in the repo. + +First let's add files in `gitops/` and run: + +``` +% kubectl apply -f gitops/bootstrap/root-app.yaml +``` + +At localhost:8080, we can see the `root` application. +Now when we add any directory in format `tenant-*` with a `config.yaml` file, +it will automatically create a namespace and a SA, deploy a jupyterlab server, add a new bucket to main monio. + +``` +[root-app.yaml] (Applied manually) + │ + ├──► Creates AppProject ("tenants") + └──► Creates ApplicationSet ("tenants") + │ + ├──► Scans Git for tenants/*/config.yaml + │ + └──► Generates Application: tenant-a + │ + ├──► Pulls Helm Chart from: charts/tenant + ├──► Applies Values from: tenants/tenant-a/config.yaml + └──► Create and deploy to Namespace: tenant-a +``` + +### 3.4 Secrets + +Now that we have working automated tenants, they need secrets. +I will also use the addon `reflector` to automatically add secrets to tenants. + +#### Docker Registry + +First let's add docker registry credentials so it can pull the built jupyter image. + +``` +kubectl create secret docker-registry registry-credentials \ + --docker-server=registry.bouvais.lu \ + --docker-username="" \ + --docker-password="" + --namespace=argocd +``` + +And to automatically make them available to tenant namespaces using reflector. +It will propagate `registry-credentials` secret to namespace in format `tenant-something`. + +``` +kubectl annotate secret registry-credentials -n argocd --overwrite \ + reflector.v1.k8s.emberstack.com/reflection-allowed="true" \ + reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \ + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+" \ + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+" +``` + +From here we can access the notebook with: + +``` +kubectl port-forward -n tenant-a deployment/jupyter 8888:8888 +``` + +For this demo, I will stop here. Meaning accessing the notebook using a manual port forwarding. +But in reality, this would need a route, automated forward, CA, ect. But that's out of scope again. + +#### Minio Admin + +Let's add Minio Admin credentials as a secret too. +Similarly propagating them to tenant namespaces. + +``` +kubectl create secret generic minio-admin-credentials \ + --from-literal=MINIO_ROOT_USER=something \ + --from-literal=MINIO_ROOT_PASSWORD=something \ + -n argocd + +kubectl annotate secret minio-admin-credentials -n argocd --overwrite \ + reflector.v1.k8s.emberstack.com/reflection-allowed="true" \ + reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \ + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+" \ + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+" +``` + +#### Minio User + +The tenant chart also automatically create the 2 new buckets and a random credential saved only in the +tenant namespace that will use it. + +``` +% kubectl get events -n tenant-a --field-selector reason=Completed +LAST SEEN TYPE REASON OBJECT MESSAGE +7m6s Normal Completed job/tenant-a-minio-setup Job completed +% kubectl get secrets -n tenant-a +NAME TYPE DATA AGE +s3-credentials Opaque 5 90s +% kubectl get secret s3-credentials -n tenant-a -o jsonpath='{.data.AWS_SECRET_ACCESS_KEY}' | base64 --decode +echo "" +xXjvsoRXaEI1GtvVfUrMZOkR +``` + +# Use Jupyter + +Now we should have everything to use the notebook. + +Let's go to localhost:8888 and run `images/jupyter/test_script.py`. + +``` +SUCCESS: List items in ref bucket: 1 hello tenant-a +SUCCESS: Write to reference bucket blocked: An error occurred (AccessDenied) when calling the PutObject operation: Access Denied. +SUCCESS: Read back from work bucket: hello tenant +SUCCESS: Access to tenant-b blocked: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied. +``` + +We are succesfully: +- Reading but not writing from bucket ref. +- Read and write in work bucket +- Cant read nor write in bucket tenant-b + +# Docker build + +Now let's automate docker images. +I will go with a simple Action. Gitea has Action like Github. + +It is trigger only on tag `*.*.*`. It then build the base and then jupyter image and push it to the registry +using the tag. Jupyter image use the just previously build base. + +TODO: +une NetworkPolicy limite les communications du notebook aux services nécessaires ; +Une pipeline doit : + • valider les fichiers de configuration ; + • construire les images dans le bon ordre ; + • exécuter au moins un test ; + • publier les images dans un registre ou simuler clairement cette étape ; + • mettre à jour la version utilisée par le déploiement. + +# Configs + +I kept configs minimal, but in real here a list of things that could be added: +- A storage limit for Work bucket +- A GPU option for the Jupyter +-