Removed tenant in path

Reviewed-on: #5

.gitea/workflows/docker-build.yml

@@ -16,7 +16,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v4
+        uses: actions/checkout@v6
 
       - name: Log in to the Container registry
         run: |
@@ -45,11 +45,6 @@ jobs:
           docker tag ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.ref_name }} \
             ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:latest
 
-      - name: Push Base image
-        run: |
-          docker push ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.ref_name }}
-          docker push ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:latest
-
       # --- 2. BUILD & TEST JUPYTER IMAGE ---
       - name: Build Jupyter image
         run: |
@@ -72,7 +67,9 @@ jobs:
           docker tag ${{ env.REGISTRY }}/${{ env.JUPYTER_IMAGE_NAME }}:${{ github.ref_name }} \
             ${{ env.REGISTRY }}/${{ env.JUPYTER_IMAGE_NAME }}:latest
 
-      - name: Push Jupyter image
+      - name: Push images
         run: |
           docker push ${{ env.REGISTRY }}/${{ env.JUPYTER_IMAGE_NAME }}:${{ github.ref_name }}
           docker push ${{ env.REGISTRY }}/${{ env.JUPYTER_IMAGE_NAME }}:latest
+          docker push ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:${{ github.ref_name }}
+          docker push ${{ env.REGISTRY }}/${{ env.BASE_IMAGE_NAME }}:latest

.gitea/workflows/trufflehog.yml

@@ -0,0 +1,21 @@
+name: Build and Push Docker Images
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run TruffleHog CLI
+        run: |
+          curl -sSfL https://raw.githubusercontent.com/trufflesecurity/trufflehog/main/scripts/install.sh | sudo sh -s -- -b /usr/local/bin
+          trufflehog git file://$(pwd) --results=verified

.gitea/workflows/validate.yaml

@@ -0,0 +1,43 @@
+name: Validate Kubernetes & YAML
+
+on:
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  validate-all:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@v4
+
+      - name: Install Validation Tools
+        run: |
+          # Install yamllint
+          sudo apt-get update && sudo apt-get install -y yamllint
+          
+          # Install Helm
+          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
+          
+          # Install Kubeconform
+          wget https://github.com/yannh/kubeconform/releases/download/v0.6.4/kubeconform-linux-amd64.tar.gz
+          tar xf kubeconform-linux-amd64.tar.gz
+          sudo mv kubeconform /usr/local/bin/
+
+      - name: Lint Pure YAML Files
+        run: yamllint ./tenants
+
+      - name: Validate Tenant Configurations against Chart
+        run: |
+          for tenant_dir in tenants/*; do
+            if [ -d "$tenant_dir" ]; then
+              tenant_name=$(basename "$tenant_dir")
+              echo "------------------------------------------------"
+              echo "Validating rendered output for: $tenant_name"
+              echo "------------------------------------------------"
+              
+              helm template "$tenant_name" charts/tenant/ -f "$tenant_dir/config.yaml" | \
+              kubeconform -strict -summary -schema-location default
+            fi
+          done

README.md

@@ -1,81 +1,292 @@
-# 0. Introduction
+# Technical Assessment: Multi-Tenant Jupyter Platform
 
-- Did it on a macbook air as I am away from my Linux workstation at home
-- I will use some of my selfhosted services in examples, e.g. `registry.bouvais.lu` for docker registry and `git.bouvais.lu` for gitea.
+## 1. Introduction & Architecture
 
-# 1. Create env
+This repository contains the declarative configuration, infrastructure code,
+and CI/CD pipelines to deploy and manage a multi-tenant Jupyter platform on Kubernetes.
 
-First let's install everything that I will use using `brew`.
+The goal is to provide isolated workspaces for different teams (tenants) using a GitOps methodology,
+meaning no manual manifest duplication or imperative `kubectl` commands are required to onboard new projects.
+
+### High-Level Architecture
+
+The solution relies on **Argo CD** for GitOps synchronization, **Helm** for templating
+the tenant resources, and a shared **MinIO** instance for S3-compatible storage.
+
+```mermaid
+graph TD
+    subgraph Git Repository
+        A[GitOps: tenants/] -->|Contains| B(tenant-a/config.yaml)
+        A -->|Contains| C(tenant-b/config.yaml)
+    end
+
+    subgraph Kubernetes Cluster
+        D[Argo CD] -->|Scans Git & Generates| E{ApplicationSet}
+        
+        E -->|Deploys via Helm| F[Namespace: tenant-a]
+        E -->|Deploys via Helm| G[Namespace: tenant-b]
+
+        subgraph argo [Namespace: argocd]
+            D(Argo CD)
+            E(ApplicationSet)
+        end
+        
+        subgraph F [Namespace: tenant-a]
+            F1(Jupyter Pod)
+            F2(ServiceAccount)
+            F3(NetworkPolicy)
+        end
+        
+        subgraph MinIO [Namespace: minio]
+            M1[(Shared MinIO S3)]
+            M2(MinIO Setup Job)
+        end
+    end
+
+    B -.-> D
+    C -.-> D
+    M2 -->|Creates Buckets & Credentials| M1
+    M2 -->|Provisions Scoped Secret| F
+    F1 -->|Reads/Writes| M1
 
-```
-brew install docker
 ```
 
-# 2. Images
+---
 
-To created the needed dependency, I did a simple 2 images base + jupyter.
-The first image is a minimal python slim. I then add some 
+## 2. Requirements
 
-### 2.1. Base
+This section details how the technical choices address the specific requirements of the assessment.
 
-```images/base/dockerfile
-FROM python:3.12-slim-bookworm AS base
+### 2.1 Multi-Tenant Management & GitOps
 
-LABEL org.opencontainers.image.title="tenant-base" \
-      org.opencontainers.image.description="Hardened base image for tenant workspaces" \
-      org.opencontainers.image.source="https://git.bouvais.lu/adrien/"
+* **Zero Duplication:** Tenant configurations are fully abstracted. The 
+`charts/tenant` Helm chart acts as the single source of truth for the Kubernetes manifests.
+* **Declarative Onboarding:** Adding a new workspace simply requires creating a
+new folder named `tenant-*` with a `config.yaml` file inside `tenants/`.
+* **Argo CD ApplicationSet:** A root ApplicationSet monitors the Git repository
+path `tenants/*/config.yaml`. Upon detecting a new folder, it automatically
+generates an Argo CD `Application`, pulls the Helm chart, injects the specific
+values, and provisions the new namespace.
 
-# System deps only — keep this layer stable so it's rarely rebuilt
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        ca-certificates \
-        curl \
-        tini \
-    && rm -rf /var/lib/apt/lists/*
+### 2.2 Kubernetes & S3 Isolation
 
-# Non-root user, fixed UID/GID for predictable K8s securityContext
-RUN groupadd --gid 1000 appuser \
-    && useradd --uid 1000 --gid appuser --shell /bin/bash --create-home appuser
+* **Namespace & Identity:** Each tenant is strictly deployed in its own namespace
+(`tenant-a`, `tenant-b`) and runs under a dedicated Kubernetes `ServiceAccount`.
+* **Network Security:** A `NetworkPolicy` is deployed within each tenant namespace
+to restrict ingress/egress traffic. The Jupyter notebook can only communicate
+with necessary services (like MinIO) and the internet, preventing cross-namespace communication.
+**Automated & Isolated S3 Setup:** To prevent exposing admin credentials outside the storage
+infrastructure, a Kubernetes Job runs directly within the minio namespace during tenant provisioning.
+This job connects to the MinIO admin API locally, creates a reference bucket (Read-Only), a work bucket (Read/Write),
+and generates dedicated, scoped S3 credentials. It then creates the resulting secret containing only
+tenant-scoped credentials directly in the target tenant namespace.
+**dedicated, scoped S3 credentials**. These credentials are saved as a Kubernetes
+Secret locally in the tenant's namespace.
+* **Security Context:** The Docker images use the `tini` init system and execute
+application containers as a non-root user.
 
-WORKDIR /home/appuser
-USER appuser
+### 2.3 Container Images & Dependencies
 
-ENTRYPOINT ["tini", "--"]
-```
+* **Base & Dependent Images:** The project uses two custom images located in
+`images/`. The `tenant/base` image contains the core OS (`python:3.12-slim-bookworm`),
+security contexts, and user setups. The `tenant/jupyter` image builds on top
+of the base image and installs the application requirements.
+* **No Hardcoded Secrets:** Images are completely stateless and free of secrets. They
+rely solely on environment variables injected at runtime via Kubernetes Secrets.
 
-Then build and push it to the registry
+### 2.4 CI/CD & Secret Management
+
+* **Gitea Actions Pipeline:** The CI/CD workflow (`.gitea/workflows/`) automatically
+manages the image lifecycle and code quality.
+* **Validation:** Helm charts and tenant configs are validated before deployment.
+* **Ordered Build Strategy:** The pipeline respects the image hierarchy. It builds the base image first,
+then uses it to build the Jupyter image. Triggered on `*.*.*` tags, it pushes the versioned
+artifacts to a private Docker registry.
+* **Testing:** Basic testing is executed during the CI phase to ensure the Jupyter environment boots correctly.
+* **Branch Protection & PR Flow:** The `main` branch is protected against direct pushes.
+All infrastructure and application changes must be proposed via Pull Requests.
+The CI pipelines act as mandatory status checks, ensuring that no code can be merged
+into `main` unless the configurations are validated, no secrets are leaked, and all tests pass successfully.
+* **Secret Scanning:** `TruffleHog` runs as a CI step to scan the repository and prevent any accidental commit of sensitive information (AWS keys, passwords).
+* **Secret Injection:** Docker registry credentials are injected into the argocd namespace and
+propagated to tenant namespaces dynamically matching tenant-[a-z0-9-]+ using
+the Reflector Kubernetes addon. MinIO admin credentials remain strictly contained
+within the minio namespace and are never reflected to tenant environments.
+
+---
+
+## 3. Deployment Guide (Quickstart)
+
+This section provides the step-by-step instructions to set up the local Kubernetes cluster and
+bootstrap the GitOps environment.
+
+*Note: You will not be able to use this repo directly as it require credentials for
+my selfhosted gitea and docker registry.*
+
+### 3.1 Prerequisites
+
+Ensure the following tools are installed on your machine (e.g., via Homebrew on macOS):
+
+```bash
+brew install docker kind kubectl helm argocd k9s
 
-```
-docker build -t registry.bouvais.lu/tenant-base:1.0.0 images/base
-docker push registry.bouvais.lu/tenant-base:1.0.0
 ```
 
-### 2.2. Jupyter
+### 3.2 Cluster Creation
 
-Now I dp the same for a simple jupyter image.
+Spin up a local Kubernetes cluster using `kind`:
+
+```bash
+kind create cluster --name ctie-exercice
+kubectl cluster-info
 
 ```
-ARG BASE_IMAGE=registry.bouvais.lu/tenant-base:1.0.0
-FROM ${BASE_IMAGE}
 
-LABEL org.opencontainers.image.title="tenant-jupyter" \
-      org.opencontainers.image.description="JupyterLab image built on tenant-base" \
-      org.opencontainers.image.base.name="${BASE_IMAGE}"
+### 3.3 Install Cluster Dependencies (Argo CD & Reflector)
 
-USER root
-COPY --chown=appuser:appuser requirements.txt /tmp/requirements.txt
-RUN pip install --no-cache-dir -r /tmp/requirements.txt \
-    && rm /tmp/requirements.txt
+Install the Argo CD CRDs, the Reflector addon (used to propagate secrets to dynamically created tenant namespaces),
+and the Argo CD engine itself:
 
-# No secrets baked in — S3 creds come from a mounted K8s Secret / env at runtime
-USER appuser
-WORKDIR /home/appuser/work
+```bash
+# Reflector & Argo CD CRDs
+kubectl apply --server-side -k "https://github.com/argoproj/argo-cd/manifests/crds?ref=stable"
+kubectl apply -f https://github.com/emberstack/kubernetes-reflector/releases/latest/download/reflector.yaml
 
-EXPOSE 8888
+# Argo CD in its dedicated namespace
+kubectl create ns argocd
+kubectl apply -n argocd -f https://raw.githubusercontent.com/argoproj/argo-cd/stable/manifests/install.yaml
 
-CMD ["jupyter", "lab", \
-     "--ip=0.0.0.0", \
-     "--port=8888", \
-     "--no-browser", \
-     "--ServerApp.token=", \
-     "--ServerApp.allow_remote_access=True"]
 ```
+
+### 3.4 Install MinIO (Shared Storage)
+
+Deploy the shared MinIO instance which will act as the S3 backend for all tenants:
+
+```bash
+kubectl create ns minio
+kubectl apply -f minio/manifests.yaml
+
+# Verify MinIO is running
+kubectl -n minio get pods -w
+
+```
+
+### 3.5 Global Secrets Setup
+
+Before launching the GitOps automated deployments, we need to provide the base credentials.
+Reflector will automatically replicate these to any namespace matching `tenant-[a-z0-9-]+`.
+
+**1. Docker Registry Credentials:**
+
+```bash
+kubectl create secret docker-registry registry-credentials \
+  --docker-server=registry.bouvais.lu \
+  --docker-username="<YOUR_USER>" \
+  --docker-password="<YOUR_TOKEN>" \
+  --namespace=argocd
+
+kubectl annotate secret registry-credentials -n argocd --overwrite \
+  reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
+  reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
+  reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+,minio" \
+  reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+,minio"
+
+```
+
+**2. MinIO Admin Credentials:**
+
+```bash
+kubectl create secret generic minio-admin-credentials \
+  --from-literal=MINIO_ROOT_USER=admin \
+  --from-literal=MINIO_ROOT_PASSWORD=password123 \
+  -n minio
+
+```
+
+### 3.6 GitOps Bootstrap
+
+Finally, apply the root Argo CD ApplicationSet. This will instruct Argo CD to monitor
+the repository and automatically provision `tenant-a` and `tenant-b`.
+
+```bash
+kubectl apply -f gitops/bootstrap/root-app.yaml
+
+```
+
+*(Optional)* You can access the Argo CD UI to watch the automated deployment:
+
+```bash
+kubectl -n argocd get secret argocd-initial-admin-secret -o jsonpath="{.data.password}" | base64 -d; echo
+kubectl -n argocd port-forward svc/argocd-server 8080:443
+# Access at https://localhost:8080 using username 'admin'
+
+```
+
+---
+
+## 4. Tests and Validation
+
+Once the tenants are fully deployed by Argo CD, we can verify that data access and workspace
+isolation behave exactly as expected.
+
+### 4.1 Accessing the Jupyter Workspace
+
+Because Ingress routing is out-of-scope for this exercise, we will connect to `tenant-a`'s
+notebook using a manual port-forward:
+
+```bash
+kubectl port-forward -n tenant-a deployment/jupyter 8888:8888
+
+```
+
+Open a browser and navigate to `http://localhost:8888`.
+
+### 4.2 Running the Isolation Tests
+
+Inside the JupyterLab interface, open a terminal or a notebook and run the provided Python
+validation script (`test_buckets.py`). The output will look like this:
+
+```text
+SUCCESS: List items in ref bucket: 1 hello tenant-a
+SUCCESS: Write to reference bucket blocked: An error occurred (AccessDenied) when calling the PutObject operation: Access Denied.
+SUCCESS: Read back from work bucket: hello tenant
+SUCCESS: Access to tenant-b blocked: An error occurred (AccessDenied) when calling the ListObjectsV2 operation: Access Denied.
+
+```
+
+**What this proves:**
+
+1. **Read-only Reference:** The tenant can read the reference bucket, but writing to
+it is successfully blocked by MinIO policies.
+2. **Read/Write Work Bucket:** The tenant has full capability in its designated working bucket.
+3. **Strict Isolation:** Any attempt to list, read, or write to `tenant-b`'s buckets from
+`tenant-a`'s credentials results in an `AccessDenied` error.
+
+---
+
+## 5. Teardown
+
+To clean up your local environment, you simply need to delete the `kind` cluster.
+This will remove all associated resources, namespaces, and volumes.
+
+```bash
+kind delete cluster --name ctie-exercice
+
+```
+
+---
+
+## 6. Future Improvements
+
+While this setup successfully demonstrates a GitOps-driven multi-tenant architecture, deploying this to a production environment would require several additions:
+
+* **Ingress & TLS:** Replacing manual `kubectl port-forward` commands with a proper Ingress Controller
+(e.g., NGINX or Traefik) and integrating `cert-manager` for automated SSL/TLS certificates.
+* **Storage Quotas:** Implementing strict S3 storage limits (quotas) on the work buckets to prevent
+a single tenant from exhausting shared storage resources.
+* **Kubernetes Resource Limits:** Adding Kubernetes `ResourceQuotas` per tenant namespace and
+`LimitRanges` to cap CPU, Memory, and optionally GPU consumption per Jupyter pod.
+* **SSO / Authentication:** Replacing token-based Jupyter authentication with OIDC integration
+(e.g., Keycloak, Dex, or Azure AD) for seamless enterprise user login.
+* **Automated DNS:** Integrating `ExternalDNS` to automatically map newly created tenant
+workspaces to human-readable URLs (e.g., `tenant-a.jupyter.internal.domain`).

charts/tenant/templates/deployment.yaml

@@ -6,8 +6,6 @@ metadata:
   labels:
     app: jupyter
     tenant: 'tenant-{{.Values.name}}'
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
   selector:
@@ -25,11 +23,11 @@ spec:
         - name: registry-credentials
       containers:
         - name: jupyter
-          image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
+          image: "registry.bouvais.lu/tenant/jupyter:{{ .Values.version }}"
           imagePullPolicy: IfNotPresent
           envFrom:
             - secretRef:
-                name: s3-credentials
+                name: 'tenant-{{.Values.name}}-s3-credentials'
           resources:
             requests:
               cpu: {{ .Values.resources.requests.cpu }}

charts/tenant/templates/minio-setup.yaml

@@ -2,99 +2,94 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: 'tenant-{{ .Values.name }}-minio-setup'
-  namespace: {{ .Release.Namespace }}
+  namespace: minio
   annotations:
+    argocd.argoproj.io/sync-wave: "-1"
     argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded
 spec:
   ttlSecondsAfterFinished: 100
   template:
     spec:
+      serviceAccountName: minio-provisioner
       restartPolicy: OnFailure
+      imagePullSecrets:
+        - name: registry-credentials
       containers:
-        - name: mc
-          image: minio/mc:RELEASE.2024-03-03T00-13-08Z
-          env:
-            # 1. MinIO Admin Credentials (Replicated from argocd namespace)
-            - name: MINIO_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: minio-admin-credentials
-                  key: MINIO_ROOT_USER
-            - name: MINIO_ADMIN_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: minio-admin-credentials
-                  key: MINIO_ROOT_PASSWORD
-
-            # 2. Randomly Generated Tenant Credentials
-            - name: TENANT_USER
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: MINIO_ACCESS_KEY_ID
-            - name: TENANT_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: MINIO_SECRET_ACCESS_KEY
-            - name: BUCKET_REF
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: BUCKET_REF
-            - name: BUCKET_WORK
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: BUCKET_WORK
-          command:
-            - /bin/sh
-            - -c
+        - name: setup
+          image: registry.bouvais.lu/minio-runner:1.0.0
+          command: ["/bin/sh", "-c"]
+          args:
             - |
-              # Fail immediately if any command fails
-              set -e 
+              set -e
 
-              # Connect to MinIO using quotes to protect special characters in passwords
-              mc alias set minioadmin http://minio.minio.svc.cluster.local:9000 "$MINIO_ADMIN_USER" "$MINIO_ADMIN_PASS"
+              # Generate password
+              SCOPED_SECRET=$(tr -dc 'a-zA-Z0-9' < /dev/urandom | head -c 32)
+
+              # Configure MinIO alias
+              mc alias set local http://minio.minio.svc:9000 \
+                "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
 
               # Create buckets
-              mc mb --ignore-existing minioadmin/$BUCKET_REF
-              mc mb --ignore-existing minioadmin/$BUCKET_WORK
+              mc mb -p local/ref-tenant-{{ .Values.name }} local/work-tenant-{{ .Values.name }}
 
-              echo "hello tenant-{{ .Values.name }}" | mc pipe minioadmin/$BUCKET_REF/hello.txt
+              # Create user with generated credentials
+              mc admin user add local tenant-{{ .Values.name }}-user "$SCOPED_SECRET"
 
-              # Create tenant user (REMOVED || true)
-              mc admin user add minioadmin "$TENANT_USER" "$TENANT_PASS"
+              # Add demo file
+              echo "hello tenant-{{ .Values.name }}" | mc pipe local/ref-tenant-{{ .Values.name }}/hello.txt
 
-              # Apply IAM isolation policy
+              # Inline policy generation to avoid missing mount issues
               cat <<EOF > /tmp/policy.json
               {
                 "Version": "2012-10-17",
                 "Statement": [
                   {
+                    "Sid": "ReadOnlyRefBucket",
                     "Effect": "Allow",
-                    "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                    "Action": [
+                      "s3:GetBucketLocation",
+                      "s3:ListBucket",
+                      "s3:GetObject"
+                    ],
                     "Resource": [
-                      "arn:aws:s3:::$BUCKET_REF",
-                      "arn:aws:s3:::$BUCKET_WORK"
+                      "arn:aws:s3:::ref-tenant-{{ .Values.name }}",
+                      "arn:aws:s3:::ref-tenant-{{ .Values.name }}/*"
                     ]
                   },
                   {
+                    "Sid": "ReadWriteWorkBucket",
                     "Effect": "Allow",
-                    "Action": ["s3:GetObject"],
-                    "Resource": ["arn:aws:s3:::$BUCKET_REF/*"]
-                  },
-                  {
-                    "Effect": "Allow",
-                    "Action": ["s3:*"],
-                    "Resource": ["arn:aws:s3:::$BUCKET_WORK/*"]
+                    "Action": [
+                      "s3:*"
+                    ],
+                    "Resource": [
+                      "arn:aws:s3:::work-tenant-{{ .Values.name }}",
+                      "arn:aws:s3:::work-tenant-{{ .Values.name }}/*"
+                    ]
                   }
                 ]
               }
               EOF
+              # Apply policy and attach to user
+              mc admin policy create local tenant-{{ .Values.name }}-policy /tmp/policy.json
+              mc admin policy attach local tenant-{{ .Values.name }}-policy --user tenant-{{ .Values.name }}-user
 
-              mc admin policy create minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json || mc admin policy update minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json
-              # Good enough for now, would need better error handeling
-              mc admin policy attach minioadmin tenant-{{ .Values.name }}-policy --user="$TENANT_USER" || echo "Policy already attached, skipping."
-              echo "MinIO setup completed for tenant-{{ .Values.name }}"
+              # Create target secret with matching credentials
+              kubectl create secret generic tenant-{{ .Values.name }}-s3-credentials \
+                --from-literal=S3_ENDPOINT="http://minio.minio.svc.cluster.local:9000" \
+                --from-literal=MINIO_ACCESS_KEY_ID=tenant-{{ .Values.name }}-user \
+                --from-literal=MINIO_SECRET_ACCESS_KEY="$SCOPED_SECRET" \
+                --from-literal=BUCKET_REF="ref-tenant-{{ .Values.name }}" \
+                --from-literal=BUCKET_WORK="work-tenant-{{ .Values.name }}" \
+                -n minio --dry-run=client -o yaml | kubectl apply -f -
+
+              # Annotate for Emberstack Reflector
+              kubectl annotate secret tenant-{{ .Values.name }}-s3-credentials -n minio --overwrite \
+                reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
+                reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
+                reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \
+                reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-{{ .Values.name }}"
+          envFrom:
+            - secretRef:
+                name: minio-admin-credentials

charts/tenant/templates/networkpolicy.yaml

@@ -0,0 +1,43 @@
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+  name: jupyter-netpol
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app: jupyter
+    tenant: 'tenant-{{ .Values.name }}'
+spec:
+  podSelector:
+    matchLabels:
+      app: jupyter
+      tenant: 'tenant-{{ .Values.name }}'
+  policyTypes:
+    - Ingress
+    - Egress
+
+  # Entrée
+  ingress:
+    - ports:
+        - protocol: TCP
+          port: 8888
+
+  # Sortie
+  egress:
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: kube-system
+      ports:
+        - protocol: UDP
+          port: 53
+        - protocol: TCP
+          port: 53
+
+    # 2. Vers MinIO
+    - to:
+        - namespaceSelector:
+            matchLabels:
+              kubernetes.io/metadata.name: minio
+      ports:
+        - protocol: TCP
+          port: 9000

charts/tenant/templates/secret-s3.yaml

@@ -10,7 +10,8 @@ type: Opaque
 stringData:
   MINIO_ACCESS_KEY_ID: 'tenant-{{ .Values.name }}-user'
   # If the secret exist, keep. Otherwise create a new random one. (To prevent a new password at each sync).
-  MINIO_SECRET_ACCESS_KEY: {{ if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "AWS_SECRET_ACCESS_KEY") }}{{ index $existingSecret.data "AWS_SECRET_ACCESS_KEY" | b64dec }}{{ else }}{{ randAlphaNum 24 }}{{ end }}
+  MINIO_SECRET_ACCESS_KEY: {{ if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "MINIO_SECRET_ACCESS_KEY") }}{{ index $existingSecret.data "MINIO_SECRET_ACCESS_KEY" | b64dec }}{{ else }}{{ randAlphaNum 24 }}{{ end }}
   S3_ENDPOINT: "http://minio.minio.svc.cluster.local:9000"
   BUCKET_REF: "ref-tenant-{{ .Values.name }}"
   BUCKET_WORK: "work-tenant-{{ .Values.name }}"
+

charts/tenant/templates/serviceaccount.yaml

@@ -1,3 +1,4 @@
+# TODO: Attach IAM to the SA
 apiVersion: v1
 kind: ServiceAccount
 metadata:

gitops/bootstrap/root-app.yaml

@@ -8,7 +8,7 @@ spec:
   source:
     repoURL: https://git.bouvais.lu/adrien/ctie-exercice
     targetRevision: main
-    path: gitops/tenants
+    path: gitops
     directory:
       recurse: true
   destination:

gitops/tenants/appset.yaml

@@ -11,7 +11,7 @@ spec:
         repoURL: https://git.bouvais.lu/adrien/ctie-exercice
         revision: main
         files:
-          - path: "tenants/*/config.yaml"
+          - path: "tenants/tenant-*/config.yaml"
   template:
     metadata:
       name: 'tenant-{{ trimPrefix "tenant-" .path.basename }}'

gitops/tenants/project.yaml

@@ -4,12 +4,18 @@ metadata:
   name: tenants
   namespace: argocd
 spec:
+  # Ensure only pull from here
   sourceRepos:
     - https://git.bouvais.lu/adrien/ctie-exercice
+
+  # So this namespace can only affect other tenant and minio (not kube-system or argocd)
   destinations:
     - server: https://kubernetes.default.svc
       namespace: 'tenant-*'
+    - server: https://kubernetes.default.svc
+      namespace: 'minio'
 
+  # This is necessary to create namespace
   clusterResourceWhitelist:
     - group: ''
       kind: Namespace

images/minio-setup-runner/dockerfile

@@ -0,0 +1,13 @@
+FROM alpine:3.20
+
+RUN apk add --no-cache curl bash openssl \
+    && curl -fsSL -o /usr/local/bin/mc https://dl.min.io/client/mc/release/linux-amd64/mc \
+    && chmod +x /usr/local/bin/mc \
+    && curl -fsSL -o /usr/local/bin/kubectl "https://dl.k8s.io/release/$(curl -fsSL https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" \
+    && chmod +x /usr/local/bin/kubectl \
+    && adduser -D -u 1000 runner
+
+USER runner
+WORKDIR /home/runner
+
+ENTRYPOINT ["/bin/bash"]

minio/manifests.yaml

@@ -1,3 +1,33 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: minio-provisioner
+  namespace: minio
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-writer
+  namespace: minio
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: minio-provisioner-secret-writer
+  namespace: minio
+subjects:
+  - kind: ServiceAccount
+    name: minio-provisioner
+    namespace: minio
+roleRef:
+  kind: Role
+  name: secret-writer
+  apiGroup: rbac.authorization.k8s.io
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
@@ -13,7 +43,7 @@ spec:
     spec:
       containers:
         - name: minio
-          image: minio/minio:latest
+          image: quay.io/minio/minio:latest
           args: ["server", "/data", "--console-address", ":9001"]
           env:
             - name: MINIO_ROOT_USER

tenants/tenant-a/config.yaml

@@ -1,6 +1,4 @@
-image:
-  repository: registry.bouvais.lu/tenant/jupyter
-  tag: "1.0.1"
+version: "1.0.0"
 
 resources:
   requests:
@@ -9,4 +7,3 @@ resources:
   limits:
     cpu: "1"
     memory: 2Gi
-

tenants/tenant-b/config.yaml

@@ -1,11 +1,9 @@
-image:
-  repository: registry.bouvais.lu/tenant/jupyter
-  tag: "1.0.1"
+version: "1.0.0"
 
 resources:
   requests:
-    cpu: 250m
+    cpu: 100m
     memory: 512Mi
   limits:
-    cpu: "1"
-    memory: 2Gi
+    cpu: 250m
+    memory: 1Gi