Compare commits
15
Commits
28cd5c00aa
..
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4de69e5f5f | ||
|
|
3939d3e541 | ||
|
|
cb08ae231e | ||
|
|
5758410bef | ||
|
|
b7be87b668 | ||
|
|
70a67c7009 | ||
|
|
df08869b8b | ||
|
|
ea28a0dccd | ||
|
|
f3b3af1feb | ||
|
|
77b588c450 | ||
|
|
45d9ffdc77 | ||
|
|
adfa295255 | ||
|
|
2710c63b0e | ||
|
|
4507b46853 | ||
|
|
efd8dab852 |
@@ -35,17 +35,18 @@ graph TD
|
|||||||
F1(Jupyter Pod)
|
F1(Jupyter Pod)
|
||||||
F2(ServiceAccount)
|
F2(ServiceAccount)
|
||||||
F3(NetworkPolicy)
|
F3(NetworkPolicy)
|
||||||
F4(MinIO Setup Job)
|
|
||||||
end
|
end
|
||||||
|
|
||||||
subgraph MinIO [Namespace: minio]
|
subgraph MinIO [Namespace: minio]
|
||||||
M1[(Shared MinIO S3)]
|
M1[(Shared MinIO S3)]
|
||||||
|
M2(MinIO Setup Job)
|
||||||
end
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
B -.-> D
|
B -.-> D
|
||||||
C -.-> D
|
C -.-> D
|
||||||
F4 -->|Creates Buckets & Credentials| M1
|
M2 -->|Creates Buckets & Credentials| M1
|
||||||
|
M2 -->|Provisions Scoped Secret| F
|
||||||
F1 -->|Reads/Writes| M1
|
F1 -->|Reads/Writes| M1
|
||||||
|
|
||||||
```
|
```
|
||||||
@@ -74,9 +75,11 @@ values, and provisions the new namespace.
|
|||||||
* **Network Security:** A `NetworkPolicy` is deployed within each tenant namespace
|
* **Network Security:** A `NetworkPolicy` is deployed within each tenant namespace
|
||||||
to restrict ingress/egress traffic. The Jupyter notebook can only communicate
|
to restrict ingress/egress traffic. The Jupyter notebook can only communicate
|
||||||
with necessary services (like MinIO) and the internet, preventing cross-namespace communication.
|
with necessary services (like MinIO) and the internet, preventing cross-namespace communication.
|
||||||
* **Automated & Isolated S3 Setup:** Instead of sharing credentials, a one-off Kubernetes `Job`
|
**Automated & Isolated S3 Setup:** To prevent exposing admin credentials outside the storage
|
||||||
runs during the Helm deployment for each tenant. This job connects to the MinIO admin API,
|
infrastructure, a Kubernetes Job runs directly within the minio namespace during tenant provisioning.
|
||||||
creates a reference bucket (Read-Only), a work bucket (Read/Write), and generates
|
This job connects to the MinIO admin API locally, creates a reference bucket (Read-Only), a work bucket (Read/Write),
|
||||||
|
and generates dedicated, scoped S3 credentials. It then creates the resulting secret containing only
|
||||||
|
tenant-scoped credentials directly in the target tenant namespace.
|
||||||
**dedicated, scoped S3 credentials**. These credentials are saved as a Kubernetes
|
**dedicated, scoped S3 credentials**. These credentials are saved as a Kubernetes
|
||||||
Secret locally in the tenant's namespace.
|
Secret locally in the tenant's namespace.
|
||||||
* **Security Context:** The Docker images use the `tini` init system and execute
|
* **Security Context:** The Docker images use the `tini` init system and execute
|
||||||
@@ -105,9 +108,10 @@ All infrastructure and application changes must be proposed via Pull Requests.
|
|||||||
The CI pipelines act as mandatory status checks, ensuring that no code can be merged
|
The CI pipelines act as mandatory status checks, ensuring that no code can be merged
|
||||||
into `main` unless the configurations are validated, no secrets are leaked, and all tests pass successfully.
|
into `main` unless the configurations are validated, no secrets are leaked, and all tests pass successfully.
|
||||||
* **Secret Scanning:** `TruffleHog` runs as a CI step to scan the repository and prevent any accidental commit of sensitive information (AWS keys, passwords).
|
* **Secret Scanning:** `TruffleHog` runs as a CI step to scan the repository and prevent any accidental commit of sensitive information (AWS keys, passwords).
|
||||||
* **Secret Injection:** Admin credentials (Docker registry, MinIO admin) are injected into the Argo CD
|
* **Secret Injection:** Docker registry credentials are injected into the argocd namespace and
|
||||||
namespace and propagated to tenant namespaces dynamically matching `tenant-[a-z0-9-]+`
|
propagated to tenant namespaces dynamically matching tenant-[a-z0-9-]+ using
|
||||||
using the `Reflector` Kubernetes addon.
|
the Reflector Kubernetes addon. MinIO admin credentials remain strictly contained
|
||||||
|
within the minio namespace and are never reflected to tenant environments.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -184,8 +188,8 @@ kubectl create secret docker-registry registry-credentials \
|
|||||||
kubectl annotate secret registry-credentials -n argocd --overwrite \
|
kubectl annotate secret registry-credentials -n argocd --overwrite \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
|
reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
|
reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+" \
|
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+,minio" \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+"
|
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+,minio"
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
@@ -195,13 +199,7 @@ kubectl annotate secret registry-credentials -n argocd --overwrite \
|
|||||||
kubectl create secret generic minio-admin-credentials \
|
kubectl create secret generic minio-admin-credentials \
|
||||||
--from-literal=MINIO_ROOT_USER=admin \
|
--from-literal=MINIO_ROOT_USER=admin \
|
||||||
--from-literal=MINIO_ROOT_PASSWORD=password123 \
|
--from-literal=MINIO_ROOT_PASSWORD=password123 \
|
||||||
-n argocd
|
-n minio
|
||||||
|
|
||||||
kubectl annotate secret minio-admin-credentials -n argocd --overwrite \
|
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
|
|
||||||
reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
|
|
||||||
reflector.v1.k8s.emberstack.com/reflection-auto-namespaces="tenant-[a-z0-9-]+" \
|
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-[a-z0-9-]+"
|
|
||||||
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
|||||||
@@ -27,7 +27,7 @@ spec:
|
|||||||
imagePullPolicy: IfNotPresent
|
imagePullPolicy: IfNotPresent
|
||||||
envFrom:
|
envFrom:
|
||||||
- secretRef:
|
- secretRef:
|
||||||
name: s3-credentials
|
name: 'tenant-{{.Values.name}}-s3-credentials'
|
||||||
resources:
|
resources:
|
||||||
requests:
|
requests:
|
||||||
cpu: {{ .Values.resources.requests.cpu }}
|
cpu: {{ .Values.resources.requests.cpu }}
|
||||||
|
|||||||
@@ -31,10 +31,13 @@ spec:
|
|||||||
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
"$MINIO_ROOT_USER" "$MINIO_ROOT_PASSWORD"
|
||||||
|
|
||||||
# Create buckets
|
# Create buckets
|
||||||
mc mb -p local/{{ .Values.name }}-ref local/{{ .Values.name }}-work
|
mc mb -p local/ref-tenant-{{ .Values.name }} local/work-tenant-{{ .Values.name }}
|
||||||
|
|
||||||
# Create user with generated credentials
|
# Create user with generated credentials
|
||||||
mc admin user add local {{ .Values.name }}-user "$SCOPED_SECRET"
|
mc admin user add local tenant-{{ .Values.name }}-user "$SCOPED_SECRET"
|
||||||
|
|
||||||
|
# Add demo file
|
||||||
|
echo "hello tenant-{{ .Values.name }}" | mc pipe local/ref-tenant-{{ .Values.name }}/hello.txt
|
||||||
|
|
||||||
# Inline policy generation to avoid missing mount issues
|
# Inline policy generation to avoid missing mount issues
|
||||||
cat <<EOF > /tmp/policy.json
|
cat <<EOF > /tmp/policy.json
|
||||||
@@ -50,8 +53,8 @@ spec:
|
|||||||
"s3:GetObject"
|
"s3:GetObject"
|
||||||
],
|
],
|
||||||
"Resource": [
|
"Resource": [
|
||||||
"arn:aws:s3:::{{ .Values.name }}-ref",
|
"arn:aws:s3:::ref-tenant-{{ .Values.name }}",
|
||||||
"arn:aws:s3:::{{ .Values.name }}-ref/*"
|
"arn:aws:s3:::ref-tenant-{{ .Values.name }}/*"
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
@@ -61,25 +64,28 @@ spec:
|
|||||||
"s3:*"
|
"s3:*"
|
||||||
],
|
],
|
||||||
"Resource": [
|
"Resource": [
|
||||||
"arn:aws:s3:::{{ .Values.name }}-work",
|
"arn:aws:s3:::work-tenant-{{ .Values.name }}",
|
||||||
"arn:aws:s3:::{{ .Values.name }}-work/*"
|
"arn:aws:s3:::work-tenant-{{ .Values.name }}/*"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
EOF
|
EOF
|
||||||
# Apply policy and attach to user
|
# Apply policy and attach to user
|
||||||
mc admin policy create local {{ .Values.name }}-policy /tmp/policy.json
|
mc admin policy create local tenant-{{ .Values.name }}-policy /tmp/policy.json
|
||||||
mc admin policy attach local {{ .Values.name }}-policy --user {{ .Values.name }}-user
|
mc admin policy attach local tenant-{{ .Values.name }}-policy --user tenant-{{ .Values.name }}-user
|
||||||
|
|
||||||
# Create target secret with matching credentials
|
# Create target secret with matching credentials
|
||||||
kubectl create secret generic {{ .Values.name }}-s3-credentials \
|
kubectl create secret generic tenant-{{ .Values.name }}-s3-credentials \
|
||||||
--from-literal=AWS_ACCESS_KEY_ID={{ .Values.name }}-user \
|
--from-literal=S3_ENDPOINT="http://minio.minio.svc.cluster.local:9000" \
|
||||||
--from-literal=AWS_SECRET_ACCESS_KEY="$SCOPED_SECRET" \
|
--from-literal=MINIO_ACCESS_KEY_ID=tenant-{{ .Values.name }}-user \
|
||||||
|
--from-literal=MINIO_SECRET_ACCESS_KEY="$SCOPED_SECRET" \
|
||||||
|
--from-literal=BUCKET_REF="ref-tenant-{{ .Values.name }}" \
|
||||||
|
--from-literal=BUCKET_WORK="work-tenant-{{ .Values.name }}" \
|
||||||
-n minio --dry-run=client -o yaml | kubectl apply -f -
|
-n minio --dry-run=client -o yaml | kubectl apply -f -
|
||||||
|
|
||||||
# Annotate for Emberstack Reflector
|
# Annotate for Emberstack Reflector
|
||||||
kubectl annotate secret {{ .Values.name }}-s3-credentials -n minio --overwrite \
|
kubectl annotate secret tenant-{{ .Values.name }}-s3-credentials -n minio --overwrite \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
|
reflector.v1.k8s.emberstack.com/reflection-allowed="true" \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
|
reflector.v1.k8s.emberstack.com/reflection-auto-enabled="true" \
|
||||||
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \
|
reflector.v1.k8s.emberstack.com/reflection-allowed-namespaces="tenant-{{ .Values.name }}" \
|
||||||
|
|||||||
@@ -15,15 +15,14 @@ spec:
|
|||||||
- Ingress
|
- Ingress
|
||||||
- Egress
|
- Egress
|
||||||
|
|
||||||
# Entrée : Autorise le trafic vers le Notebook Jupyter sur le port 8888
|
# Entrée
|
||||||
ingress:
|
ingress:
|
||||||
- ports:
|
- ports:
|
||||||
- protocol: TCP
|
- protocol: TCP
|
||||||
port: 8888
|
port: 8888
|
||||||
|
|
||||||
# Sortie : Restreint les communications aux composants strictement nécessaires
|
# Sortie
|
||||||
egress:
|
egress:
|
||||||
# 1. Résolution DNS dans kube-system
|
|
||||||
- to:
|
- to:
|
||||||
- namespaceSelector:
|
- namespaceSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
@@ -34,7 +33,7 @@ spec:
|
|||||||
- protocol: TCP
|
- protocol: TCP
|
||||||
port: 53
|
port: 53
|
||||||
|
|
||||||
# 2. Flux vers l'API MinIO (Stockage S3)
|
# 2. Vers MinIO
|
||||||
- to:
|
- to:
|
||||||
- namespaceSelector:
|
- namespaceSelector:
|
||||||
matchLabels:
|
matchLabels:
|
||||||
|
|||||||
@@ -10,7 +10,8 @@ type: Opaque
|
|||||||
stringData:
|
stringData:
|
||||||
MINIO_ACCESS_KEY_ID: 'tenant-{{ .Values.name }}-user'
|
MINIO_ACCESS_KEY_ID: 'tenant-{{ .Values.name }}-user'
|
||||||
# If the secret exist, keep. Otherwise create a new random one. (To prevent a new password at each sync).
|
# If the secret exist, keep. Otherwise create a new random one. (To prevent a new password at each sync).
|
||||||
MINIO_SECRET_ACCESS_KEY: {{ if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "AWS_SECRET_ACCESS_KEY") }}{{ index $existingSecret.data "AWS_SECRET_ACCESS_KEY" | b64dec }}{{ else }}{{ randAlphaNum 24 }}{{ end }}
|
MINIO_SECRET_ACCESS_KEY: {{ if and $existingSecret $existingSecret.data (hasKey $existingSecret.data "MINIO_SECRET_ACCESS_KEY") }}{{ index $existingSecret.data "MINIO_SECRET_ACCESS_KEY" | b64dec }}{{ else }}{{ randAlphaNum 24 }}{{ end }}
|
||||||
S3_ENDPOINT: "http://minio.minio.svc.cluster.local:9000"
|
S3_ENDPOINT: "http://minio.minio.svc.cluster.local:9000"
|
||||||
BUCKET_REF: "ref-tenant-{{ .Values.name }}"
|
BUCKET_REF: "ref-tenant-{{ .Values.name }}"
|
||||||
BUCKET_WORK: "work-tenant-{{ .Values.name }}"
|
BUCKET_WORK: "work-tenant-{{ .Values.name }}"
|
||||||
|
|
||||||
|
|||||||
@@ -8,7 +8,7 @@ spec:
|
|||||||
source:
|
source:
|
||||||
repoURL: https://git.bouvais.lu/adrien/ctie-exercice
|
repoURL: https://git.bouvais.lu/adrien/ctie-exercice
|
||||||
targetRevision: main
|
targetRevision: main
|
||||||
path: gitops/tenants
|
path: gitops
|
||||||
directory:
|
directory:
|
||||||
recurse: true
|
recurse: true
|
||||||
destination:
|
destination:
|
||||||
|
|||||||
@@ -4,14 +4,18 @@ metadata:
|
|||||||
name: tenants
|
name: tenants
|
||||||
namespace: argocd
|
namespace: argocd
|
||||||
spec:
|
spec:
|
||||||
|
# Ensure only pull from here
|
||||||
sourceRepos:
|
sourceRepos:
|
||||||
- https://git.bouvais.lu/adrien/ctie-exercice
|
- https://git.bouvais.lu/adrien/ctie-exercice
|
||||||
|
|
||||||
|
# So this namespace can only affect other tenant and minio (not kube-system or argocd)
|
||||||
destinations:
|
destinations:
|
||||||
- server: https://kubernetes.default.svc
|
- server: https://kubernetes.default.svc
|
||||||
namespace: 'tenant-*'
|
namespace: 'tenant-*'
|
||||||
- server: https://kubernetes.default.svc
|
- server: https://kubernetes.default.svc
|
||||||
namespace: 'minio'
|
namespace: 'minio'
|
||||||
|
|
||||||
|
# This is necessary to create namespace
|
||||||
clusterResourceWhitelist:
|
clusterResourceWhitelist:
|
||||||
- group: ''
|
- group: ''
|
||||||
kind: Namespace
|
kind: Namespace
|
||||||
|
|||||||
@@ -43,7 +43,7 @@ spec:
|
|||||||
spec:
|
spec:
|
||||||
containers:
|
containers:
|
||||||
- name: minio
|
- name: minio
|
||||||
image: minio/minio:latest
|
image: quay.io/minio/minio:latest
|
||||||
args: ["server", "/data", "--console-address", ":9001"]
|
args: ["server", "/data", "--console-address", ":9001"]
|
||||||
env:
|
env:
|
||||||
- name: MINIO_ROOT_USER
|
- name: MINIO_ROOT_USER
|
||||||
|
|||||||
@@ -1,9 +0,0 @@
|
|||||||
version: "1.0.0"
|
|
||||||
|
|
||||||
resources:
|
|
||||||
requests:
|
|
||||||
cpu: 250m
|
|
||||||
memory: 512Mi
|
|
||||||
limits:
|
|
||||||
cpu: "1"
|
|
||||||
memory: 2Gi
|
|
||||||
Reference in New Issue
Block a user