Stop propagating minio admin credentials to each tenants

charts/tenant/templates/deployment.yaml

@@ -6,8 +6,6 @@ metadata:
   labels:
     app: jupyter
     tenant: 'tenant-{{.Values.name}}'
-  annotations:
-    argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
   selector:

charts/tenant/templates/minio-setup.yaml

@@ -2,99 +2,42 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: 'tenant-{{ .Values.name }}-minio-setup'
-  namespace: {{ .Release.Namespace }}
+  namespace: minio
   annotations:
+    argocd.argoproj.io/sync-wave: "-1"
     argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded
 spec:
   ttlSecondsAfterFinished: 100
   template:
     spec:
+      serviceAccountName: minio-provisioner
       restartPolicy: OnFailure
       containers:
-        - name: mc
+        - name: setup
-          image: minio/mc:RELEASE.2024-03-03T00-13-08Z
+          image: minio/mc:latest
-          env:
+          command: ["/bin/sh", "-c"]
-            # 1. MinIO Admin Credentials (Replicated from argocd namespace)
+          args:
-            - name: MINIO_ADMIN_USER
-              valueFrom:
-                secretKeyRef:
-                  name: minio-admin-credentials
-                  key: MINIO_ROOT_USER
-            - name: MINIO_ADMIN_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: minio-admin-credentials
-                  key: MINIO_ROOT_PASSWORD
-
-            # 2. Randomly Generated Tenant Credentials
-            - name: TENANT_USER
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: MINIO_ACCESS_KEY_ID
-            - name: TENANT_PASS
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: MINIO_SECRET_ACCESS_KEY
-            - name: BUCKET_REF
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: BUCKET_REF
-            - name: BUCKET_WORK
-              valueFrom:
-                secretKeyRef:
-                  name: s3-credentials
-                  key: BUCKET_WORK
-          command:
-            - /bin/sh
-            - -c
             - |
-              # Fail immediately if any command fails
+              mc alias set local http://minio.minio.svc:9000 \
-              set -e 
+                "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
 
-              # Connect to MinIO using quotes to protect special characters in passwords
+              mc mb -p local/{{ .Values.name }}-ref local/{{ .Values.name }}-work
-              mc alias set minioadmin http://minio.minio.svc.cluster.local:9000 "$MINIO_ADMIN_USER" "$MINIO_ADMIN_PASS"
+              mc admin user add local {{ .Values.name }}-user "$(openssl rand -hex 16)" || true
+              mc admin policy create local {{ .Values.name }}-policy /policies/{{ .Values.name }}.json
+              mc admin policy attach local {{ .Values.name }}-policy --user {{ .Values.name }}-user
 
-              # Create buckets
+              # write the SCOPED secret in this same namespace
-              mc mb --ignore-existing minioadmin/$BUCKET_REF
+              kubectl create secret generic {{ .Values.name }}-s3-credentials \
-              mc mb --ignore-existing minioadmin/$BUCKET_WORK
+                --from-literal=AWS_ACCESS_KEY_ID={{ .Values.name }}-user \
+                --from-literal=AWS_SECRET_ACCESS_KEY="$SCOPED_SECRET" \
+                -n minio --dry-run=client -o yaml | kubectl apply -f -
 
-              echo "hello tenant-{{ .Values.name }}" | mc pipe minioadmin/$BUCKET_REF/hello.txt
+              kubectl annotate secret {{ .Values.name }}-s3-credentials -n minio --overwrite \
-
+                reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
-              # Create tenant user (REMOVED || true)
+                reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
-              mc admin user add minioadmin "$TENANT_USER" "$TENANT_PASS"
+                reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \
-
+                reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-{{ .Values.name }}"
-              # Apply IAM isolation policy
+          envFrom:
-              cat <<EOF > /tmp/policy.json
+            - secretRef:
-              {
+                name: minio-admin-credentials
-                "Version": "2012-10-17",
-                "Statement": [
-                  {
-                    "Effect": "Allow",
-                    "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
-                    "Resource": [
-                      "arn:aws:s3:::$BUCKET_REF",
-                      "arn:aws:s3:::$BUCKET_WORK"
-                    ]
-                  },
-                  {
-                    "Effect": "Allow",
-                    "Action": ["s3:GetObject"],
-                    "Resource": ["arn:aws:s3:::$BUCKET_REF/*"]
-                  },
-                  {
-                    "Effect": "Allow",
-                    "Action": ["s3:*"],
-                    "Resource": ["arn:aws:s3:::$BUCKET_WORK/*"]
-                  }
-                ]
-              }
-              EOF
-
-              mc admin policy create minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json || mc admin policy update minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json
-              # Good enough for now, would need better error handeling
-              mc admin policy attach minioadmin tenant-{{ .Values.name }}-policy --user="$TENANT_USER" || echo "Policy already attached, skipping."
-              echo "MinIO setup completed for tenant-{{ .Values.name }}"

gitops/tenants/project.yaml

@@ -9,6 +9,8 @@ spec:
   destinations:
     - server: https://kubernetes.default.svc
       namespace: 'tenant-*'
+    - server: https://kubernetes.default.svc
+      namespace: 'minio'
 
   clusterResourceWhitelist:
     - group: ''

minio/manifests.yaml

@@ -1,3 +1,33 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: minio-provisioner
+  namespace: minio
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: secret-writer
+  namespace: minio
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    verbs: ["create", "get", "update", "patch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: minio-provisioner-secret-writer
+  namespace: minio
+subjects:
+  - kind: ServiceAccount
+    name: minio-provisioner
+    namespace: minio
+roleRef:
+  kind: Role
+  name: secret-writer
+  apiGroup: rbac.authorization.k8s.io
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata: