charts/tenant/templates/deployment.yaml

@@ -6,6 +6,8 @@ metadata:
   labels:
     app: jupyter
     tenant: 'tenant-{{.Values.name}}'
+  annotations:
+    argocd.argoproj.io/sync-wave: "1"
 spec:
   replicas: 1
   selector:

charts/tenant/templates/minio-setup.yaml

@@ -2,42 +2,99 @@ apiVersion: batch/v1
 kind: Job
 metadata:
   name: 'tenant-{{ .Values.name }}-minio-setup'
-  namespace: minio
+  namespace: {{ .Release.Namespace }}
   annotations:
-    argocd.argoproj.io/sync-wave: "-1"
     argocd.argoproj.io/hook: Sync
     argocd.argoproj.io/hook-delete-policy: HookSucceeded
 spec:
   ttlSecondsAfterFinished: 100
   template:
     spec:
-      serviceAccountName: minio-provisioner
       restartPolicy: OnFailure
       containers:
-        - name: setup
+        - name: mc
-          image: minio/mc:latest
+          image: minio/mc:RELEASE.2024-03-03T00-13-08Z
-          command: ["/bin/sh", "-c"]
+          env:
-          args:
+            # 1. MinIO Admin Credentials (Replicated from argocd namespace)
+            - name: MINIO_ADMIN_USER
+              valueFrom:
+                secretKeyRef:
+                  name: minio-admin-credentials
+                  key: MINIO_ROOT_USER
+            - name: MINIO_ADMIN_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: minio-admin-credentials
+                  key: MINIO_ROOT_PASSWORD
+
+            # 2. Randomly Generated Tenant Credentials
+            - name: TENANT_USER
+              valueFrom:
+                secretKeyRef:
+                  name: s3-credentials
+                  key: MINIO_ACCESS_KEY_ID
+            - name: TENANT_PASS
+              valueFrom:
+                secretKeyRef:
+                  name: s3-credentials
+                  key: MINIO_SECRET_ACCESS_KEY
+            - name: BUCKET_REF
+              valueFrom:
+                secretKeyRef:
+                  name: s3-credentials
+                  key: BUCKET_REF
+            - name: BUCKET_WORK
+              valueFrom:
+                secretKeyRef:
+                  name: s3-credentials
+                  key: BUCKET_WORK
+          command:
+            - /bin/sh
+            - -c
             - |
-              mc alias set local http://minio.minio.svc:9000 \
+              # Fail immediately if any command fails
-                "$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
+              set -e 
 
-              mc mb -p local/{{ .Values.name }}-ref local/{{ .Values.name }}-work
+              # Connect to MinIO using quotes to protect special characters in passwords
-              mc admin user add local {{ .Values.name }}-user "$(openssl rand -hex 16)" || true
+              mc alias set minioadmin http://minio.minio.svc.cluster.local:9000 "$MINIO_ADMIN_USER" "$MINIO_ADMIN_PASS"
-              mc admin policy create local {{ .Values.name }}-policy /policies/{{ .Values.name }}.json
-              mc admin policy attach local {{ .Values.name }}-policy --user {{ .Values.name }}-user
 
-              # write the SCOPED secret in this same namespace
+              # Create buckets
-              kubectl create secret generic {{ .Values.name }}-s3-credentials \
+              mc mb --ignore-existing minioadmin/$BUCKET_REF
-                --from-literal=AWS_ACCESS_KEY_ID={{ .Values.name }}-user \
+              mc mb --ignore-existing minioadmin/$BUCKET_WORK
-                --from-literal=AWS_SECRET_ACCESS_KEY="$SCOPED_SECRET" \
-                -n minio --dry-run=client -o yaml | kubectl apply -f -
 
-              kubectl annotate secret {{ .Values.name }}-s3-credentials -n minio --overwrite \
+              echo "hello tenant-{{ .Values.name }}" | mc pipe minioadmin/$BUCKET_REF/hello.txt
-                reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
+
-                reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
+              # Create tenant user (REMOVED || true)
-                reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \
+              mc admin user add minioadmin "$TENANT_USER" "$TENANT_PASS"
-                reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-{{ .Values.name }}"
+
-          envFrom:
+              # Apply IAM isolation policy
-            - secretRef:
+              cat <<EOF > /tmp/policy.json
-                name: minio-admin-credentials
+              {
+                "Version": "2012-10-17",
+                "Statement": [
+                  {
+                    "Effect": "Allow",
+                    "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
+                    "Resource": [
+                      "arn:aws:s3:::$BUCKET_REF",
+                      "arn:aws:s3:::$BUCKET_WORK"
+                    ]
+                  },
+                  {
+                    "Effect": "Allow",
+                    "Action": ["s3:GetObject"],
+                    "Resource": ["arn:aws:s3:::$BUCKET_REF/*"]
+                  },
+                  {
+                    "Effect": "Allow",
+                    "Action": ["s3:*"],
+                    "Resource": ["arn:aws:s3:::$BUCKET_WORK/*"]
+                  }
+                ]
+              }
+              EOF
+
+              mc admin policy create minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json || mc admin policy update minioadmin tenant-{{ .Values.name }}-policy /tmp/policy.json
+              # Good enough for now, would need better error handeling
+              mc admin policy attach minioadmin tenant-{{ .Values.name }}-policy --user="$TENANT_USER" || echo "Policy already attached, skipping."
+              echo "MinIO setup completed for tenant-{{ .Values.name }}"

gitops/tenants/project.yaml

@@ -9,8 +9,6 @@ spec:
   destinations:
     - server: https://kubernetes.default.svc
       namespace: 'tenant-*'
-    - server: https://kubernetes.default.svc
-      namespace: 'minio'
 
   clusterResourceWhitelist:
     - group: ''

minio/manifests.yaml

@@ -1,33 +1,3 @@
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  name: minio-provisioner
-  namespace: minio
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: secret-writer
-  namespace: minio
-rules:
-  - apiGroups: [""]
-    resources: ["secrets"]
-    verbs: ["create", "get", "update", "patch"]
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: minio-provisioner-secret-writer
-  namespace: minio
-subjects:
-  - kind: ServiceAccount
-    name: minio-provisioner
-    namespace: minio
-roleRef:
-  kind: Role
-  name: secret-writer
-  apiGroup: rbac.authorization.k8s.io
----
 apiVersion: apps/v1
 kind: Deployment
 metadata: