From 174c5faf6b3de64a9f8cc3dc2ab62de12076dc46 Mon Sep 17 00:00:00 2001 From: Adrien Bouvais Date: Mon, 17 Aug 2026 11:29:00 +0200 Subject: [PATCH] Stop propagating minio admin credentials to each tenants --- charts/tenant/templates/deployment.yaml | 2 - charts/tenant/templates/minio-setup.yaml | 109 ++++++----------------- gitops/tenants/project.yaml | 2 + minio/manifests.yaml | 30 +++++++ 4 files changed, 58 insertions(+), 85 deletions(-) diff --git a/charts/tenant/templates/deployment.yaml b/charts/tenant/templates/deployment.yaml index 553dc0c..1c39aac 100644 --- a/charts/tenant/templates/deployment.yaml +++ b/charts/tenant/templates/deployment.yaml @@ -6,8 +6,6 @@ metadata: labels: app: jupyter tenant: 'tenant-{{.Values.name}}' - annotations: - argocd.argoproj.io/sync-wave: "1" spec: replicas: 1 selector: diff --git a/charts/tenant/templates/minio-setup.yaml b/charts/tenant/templates/minio-setup.yaml index c025efe..80bed3b 100644 --- a/charts/tenant/templates/minio-setup.yaml +++ b/charts/tenant/templates/minio-setup.yaml @@ -2,99 +2,42 @@ apiVersion: batch/v1 kind: Job metadata: name: 'tenant-{{ .Values.name }}-minio-setup' - namespace: {{ .Release.Namespace }} + namespace: minio annotations: + argocd.argoproj.io/sync-wave: "-1" argocd.argoproj.io/hook: Sync argocd.argoproj.io/hook-delete-policy: HookSucceeded spec: ttlSecondsAfterFinished: 100 template: spec: + serviceAccountName: minio-provisioner restartPolicy: OnFailure containers: - - name: mc - image: minio/mc:RELEASE.2024-03-03T00-13-08Z - env: - # 1. MinIO Admin Credentials (Replicated from argocd namespace) - - name: MINIO_ADMIN_USER - valueFrom: - secretKeyRef: - name: minio-admin-credentials - key: MINIO_ROOT_USER - - name: MINIO_ADMIN_PASS - valueFrom: - secretKeyRef: - name: minio-admin-credentials - key: MINIO_ROOT_PASSWORD - - # 2. Randomly Generated Tenant Credentials - - name: TENANT_USER - valueFrom: - secretKeyRef: - name: s3-credentials - key: MINIO_ACCESS_KEY_ID - - name: TENANT_PASS - valueFrom: - secretKeyRef: - name: s3-credentials - key: MINIO_SECRET_ACCESS_KEY - - name: BUCKET_REF - valueFrom: - secretKeyRef: - name: s3-credentials - key: BUCKET_REF - - name: BUCKET_WORK - valueFrom: - secretKeyRef: - name: s3-credentials - key: BUCKET_WORK - command: - - /bin/sh - - -c + - name: setup + image: minio/mc:latest + command: ["/bin/sh", "-c"] + args: - | - # Fail immediately if any command fails - set -e + mc alias set local http://minio.minio.svc:9000 \ + "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD" - # Connect to MinIO using quotes to protect special characters in passwords - mc alias set minioadmin http://minio.minio.svc.cluster.local:9000 "$MINIO_ADMIN_USER" "$MINIO_ADMIN_PASS" + mc mb -p local/{{ .Values.name }}-ref local/{{ .Values.name }}-work + mc admin user add local {{ .Values.name }}-user "$(openssl rand -hex 16)" || true + mc admin policy create local {{ .Values.name }}-policy /policies/{{ .Values.name }}.json + mc admin policy attach local {{ .Values.name }}-policy --user {{ .Values.name }}-user - # Create buckets - mc mb --ignore-existing minioadmin/$BUCKET_REF - mc mb --ignore-existing minioadmin/$BUCKET_WORK + # write the SCOPED secret in this same namespace + kubectl create secret generic {{ .Values.name }}-s3-credentials \ + --from-literal=AWS_ACCESS_KEY_ID={{ .Values.name }}-user \ + --from-literal=AWS_SECRET_ACCESS_KEY="$SCOPED_SECRET" \ + -n minio --dry-run=client -o yaml | kubectl apply -f - - echo "hello tenant-{{ .Values.name }}" | mc pipe minioadmin/$BUCKET_REF/hello.txt - - # Create tenant user (REMOVED || true) - mc admin user add minioadmin "$TENANT_USER" "$TENANT_PASS" - - # Apply IAM isolation policy - cat < /tmp/policy.json - { - "Version": "2012-10-17", - "Statement": [ - { - "Effect": "Allow", - "Action": ["s3:GetBucketLocation", "s3:ListBucket"], - "Resource": [ - "arn:aws:s3:::$BUCKET_REF", - "arn:aws:s3:::$BUCKET_WORK" - ] - }, - { - "Effect": "Allow", - "Action": ["s3:GetObject"], - "Resource": ["arn:aws:s3:::$BUCKET_REF/*"] - }, - { - "Effect": "Allow", - "Action": ["s3:*"], - "Resource": ["arn:aws:s3:::$BUCKET_WORK/*"] - } - ] - } - EOF - - mc admin policy create minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json || mc admin policy update minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json - # Good enough for now, would need better error handeling - mc admin policy attach minioadmin tenant-{{ .Values.name }}-policy --user="$TENANT_USER" || echo "Policy already attached, skipping." - echo "MinIO setup completed for tenant-{{ .Values.name }}" + kubectl annotate secret {{ .Values.name }}-s3-credentials -n minio --overwrite \ + reflector.v1.k8s.emberstack.com/reflection-allowed="true" \ + reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \ + reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \ + reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-{{ .Values.name }}" + envFrom: + - secretRef: + name: minio-admin-credentials diff --git a/gitops/tenants/project.yaml b/gitops/tenants/project.yaml index 81f9cfc..1bc8d1e 100644 --- a/gitops/tenants/project.yaml +++ b/gitops/tenants/project.yaml @@ -9,6 +9,8 @@ spec: destinations: - server: https://kubernetes.default.svc namespace: 'tenant-*' + - server: https://kubernetes.default.svc + namespace: 'minio' clusterResourceWhitelist: - group: '' diff --git a/minio/manifests.yaml b/minio/manifests.yaml index 638976c..20d6cc1 100644 --- a/minio/manifests.yaml +++ b/minio/manifests.yaml @@ -1,3 +1,33 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: minio-provisioner + namespace: minio +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: secret-writer + namespace: minio +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["create", "get", "update", "patch"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: minio-provisioner-secret-writer + namespace: minio +subjects: + - kind: ServiceAccount + name: minio-provisioner + namespace: minio +roleRef: + kind: Role + name: secret-writer + apiGroup: rbac.authorization.k8s.io +--- apiVersion: apps/v1 kind: Deployment metadata: