Add Garage S3-compatible object store to platform
Cluster-local object store for build artifacts (CLI binaries etc.) so Docker builds don't depend on flaky external downloads. - Single-node Garage v1.0.1 StatefulSet (LMDB, replication=1) - Metadata on longhorn-nvme (1Gi), data on longhorn HDD (20Gi) - S3 API at garage.platform.svc:3900 - External ingress at s3.coreworlds.io (internal-only) - SealedSecret for admin token and RPC secret
This commit is contained in:
31
infra/kubernetes/platform/garage/configmap.yaml
Normal file
31
infra/kubernetes/platform/garage/configmap.yaml
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: ConfigMap
|
||||||
|
metadata:
|
||||||
|
name: garage-config
|
||||||
|
namespace: platform
|
||||||
|
data:
|
||||||
|
garage.toml: |
|
||||||
|
db_engine = "lmdb"
|
||||||
|
replication_factor = 1
|
||||||
|
compression_level = 1
|
||||||
|
|
||||||
|
metadata_dir = "/mnt/meta"
|
||||||
|
data_dir = "/mnt/data"
|
||||||
|
|
||||||
|
[rpc]
|
||||||
|
bind_addr = "[::]:3901"
|
||||||
|
# Single-node: rpc_secret is only used for inter-node auth (N/A here)
|
||||||
|
rpc_secret = "PLACEHOLDER"
|
||||||
|
|
||||||
|
[s3_api]
|
||||||
|
api_bind_addr = "[::]:3900"
|
||||||
|
s3_region = "garage"
|
||||||
|
root_domain = ".s3.garage.svc"
|
||||||
|
|
||||||
|
[s3_web]
|
||||||
|
bind_addr = "[::]:3902"
|
||||||
|
root_domain = ".web.garage.svc"
|
||||||
|
index = "index.html"
|
||||||
|
|
||||||
|
[admin]
|
||||||
|
api_bind_addr = "[::]:3903"
|
||||||
@@ -0,0 +1,14 @@
|
|||||||
|
---
|
||||||
|
apiVersion: bitnami.com/v1alpha1
|
||||||
|
kind: SealedSecret
|
||||||
|
metadata:
|
||||||
|
name: garage-credentials
|
||||||
|
namespace: platform
|
||||||
|
spec:
|
||||||
|
encryptedData:
|
||||||
|
admin-token: AgB+RSg4mNJKHOUXUi/ODlmOiAvYa1cAytc2pxVTsNlQZ5gJgF/stOOmMF3upKRwCmHbBIcPry0LlPxqkZ5Gs6tXi8YsxfqzmvFONHkRFNtUWyxoo2esIlIyvFZ2TfKuiaeP1XCgYW3yoFQOnrEqNBTuIzs2Ss09TtfcbjA2AgNWPo5UtRrU2ms7wn6puEFOATWGYqzQgzq958tW8OCHhyWDwlHEzfqrDm2lXSSoTKfeSVrY9lT+7y0EUAwNDB4NFkRQRmVJYIUbGc3aAdNhSMMC88UGJVVGTy511Dq1uUmkhB9ChOMRwKxYTHjiinQ1cKYQJ+8EgV98EqKHQoC0d0p04SGKCr/TBLdVhWF6h76ynnI+EoBDY0f/bZPnt/6AuELtjpcHXiQMO4WiFOPnj9nVXMu+WAfYuKmEaG0DgkOIsGXYlhOrkXrZtymKr/xWvWKfZZtwImhHyCjBiwlFNxDBv5z7bAz7bxgQKVJtu/mlh+o7UrQdozRUKhQuvqp838Z4m1kXscF5HZqJ1N8rpDnjh7wPmkc317phpmQ9dM/Fbc3x9ZCNNPKrhtJYzGbcU8QhL+ZSxe8bevCdVYZ6/hNJy8kIqTiWDjCDPPrG8jYPPBTFhAwtVtCXLxgztvNxv9FOK7ILGK3E0wgV+luU3NynoXSGNkFMYoRn8xAboSplkYVOcRC2LTyWggBa1xvea/mnc8FWOKhPdFOaY7IuZ01NGbun+C2++jzALqEokh49TL3soLqP7TsMwZo2NxyYbuqGNtFKmFqSh7PpgeXVyg5C
|
||||||
|
rpc-secret: AgB95o6wk/trqbwpY2KG2exkpFwLmE8Ic2JgF2Rnj+SVOLMzs6e/Fd8ukhsqGn+t+YXZn28B2EpkPrfkuvwr3mrcLwPR4ZussmlQvRW7bzH/n8yDehmh+4/IzAZ/SxlBRUMadOrbW+BhlhRctVjJ1GhO8jGqpsb8nbrfSFWw4TVICJome6et4v1FTphNbp89ajjNkgXKGT3prVYeGsWXUENvZih2IkY5qqW6YGhLOln5SMVCI2itztnrO3QFWb1O0sDD2KoESlLT39F1zZL9x42ZhlQ9aIYBNQk444PXDQhvvSURvXOqnX1c/2I+3V1vR3ErKmP3IGYAmebu0S0Pa6XM6hOQAcgBJaStFEXg0JYMxCXOrAQ9AdSsm1nKE279AnuLgVvVM//6eC51cHOaA5gQiUjxq5GvPKGZqD5gA3yzBNgy33w4OPf7celD/HDbm23sHoCxDTilbzebve1AK6/IDdKJc+Fvg2JwaiyxnK6Wlp0wen9+/xlfK0Xw6H2aAN2wxI/ib77ec4VA/Y6tYHNn/BMSf5FQBqSamr1OuwFbCRam0+eKsIbo3FqbnoUeTU1GTNZq1FYbh3STb2sp3BMi8VtUAsnOhi3CnfWxzo2FRbtkvzAuoBuxTesb2QPiW0FJBNAbafS3Gxf13gwvzc3HrYdNjyn9ca2yHep8gc2QeUcdVpDkEyT47ozYaAuBSFgl1twAn9YceM40tuHFF0lW07XSx5JHjKPp+5I9zrr3/g07OfymC1XDATBpMFgD3MkYN7iBlGXa+koPCqH6lCgV
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
name: garage-credentials
|
||||||
|
namespace: platform
|
||||||
7
infra/kubernetes/platform/garage/kustomization.yaml
Normal file
7
infra/kubernetes/platform/garage/kustomization.yaml
Normal file
@@ -0,0 +1,7 @@
|
|||||||
|
apiVersion: kustomize.config.k8s.io/v1beta1
|
||||||
|
kind: Kustomization
|
||||||
|
resources:
|
||||||
|
- configmap.yaml
|
||||||
|
- statefulset.yaml
|
||||||
|
- service.yaml
|
||||||
|
- garage-credentials-sealed.yaml
|
||||||
20
infra/kubernetes/platform/garage/service.yaml
Normal file
20
infra/kubernetes/platform/garage/service.yaml
Normal file
@@ -0,0 +1,20 @@
|
|||||||
|
apiVersion: v1
|
||||||
|
kind: Service
|
||||||
|
metadata:
|
||||||
|
name: garage
|
||||||
|
namespace: platform
|
||||||
|
labels:
|
||||||
|
app: garage
|
||||||
|
spec:
|
||||||
|
type: ClusterIP
|
||||||
|
ports:
|
||||||
|
- port: 3900
|
||||||
|
targetPort: 3900
|
||||||
|
protocol: TCP
|
||||||
|
name: s3-api
|
||||||
|
- port: 3903
|
||||||
|
targetPort: 3903
|
||||||
|
protocol: TCP
|
||||||
|
name: admin
|
||||||
|
selector:
|
||||||
|
app: garage
|
||||||
103
infra/kubernetes/platform/garage/statefulset.yaml
Normal file
103
infra/kubernetes/platform/garage/statefulset.yaml
Normal file
@@ -0,0 +1,103 @@
|
|||||||
|
apiVersion: apps/v1
|
||||||
|
kind: StatefulSet
|
||||||
|
metadata:
|
||||||
|
name: garage
|
||||||
|
namespace: platform
|
||||||
|
labels:
|
||||||
|
app: garage
|
||||||
|
spec:
|
||||||
|
serviceName: garage
|
||||||
|
replicas: 1
|
||||||
|
selector:
|
||||||
|
matchLabels:
|
||||||
|
app: garage
|
||||||
|
template:
|
||||||
|
metadata:
|
||||||
|
labels:
|
||||||
|
app: garage
|
||||||
|
spec:
|
||||||
|
initContainers:
|
||||||
|
- name: config
|
||||||
|
image: busybox:1.37
|
||||||
|
command: ["sh", "-c"]
|
||||||
|
args:
|
||||||
|
- |
|
||||||
|
sed "s/PLACEHOLDER/$RPC_SECRET/" /config-tmpl/garage.toml > /config/garage.toml
|
||||||
|
env:
|
||||||
|
- name: RPC_SECRET
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: garage-credentials
|
||||||
|
key: rpc-secret
|
||||||
|
volumeMounts:
|
||||||
|
- name: config-tmpl
|
||||||
|
mountPath: /config-tmpl
|
||||||
|
- name: config
|
||||||
|
mountPath: /config
|
||||||
|
containers:
|
||||||
|
- name: garage
|
||||||
|
image: dxflrs/garage:v1.0.1
|
||||||
|
ports:
|
||||||
|
- containerPort: 3900
|
||||||
|
name: s3-api
|
||||||
|
- containerPort: 3901
|
||||||
|
name: rpc
|
||||||
|
- containerPort: 3902
|
||||||
|
name: web
|
||||||
|
- containerPort: 3903
|
||||||
|
name: admin
|
||||||
|
env:
|
||||||
|
- name: GARAGE_ADMIN_TOKEN
|
||||||
|
valueFrom:
|
||||||
|
secretKeyRef:
|
||||||
|
name: garage-credentials
|
||||||
|
key: admin-token
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
memory: 256Mi
|
||||||
|
cpu: 100m
|
||||||
|
limits:
|
||||||
|
memory: 512Mi
|
||||||
|
volumeMounts:
|
||||||
|
- name: config
|
||||||
|
mountPath: /etc/garage.toml
|
||||||
|
subPath: garage.toml
|
||||||
|
- name: meta
|
||||||
|
mountPath: /mnt/meta
|
||||||
|
- name: data
|
||||||
|
mountPath: /mnt/data
|
||||||
|
readinessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 3903
|
||||||
|
initialDelaySeconds: 5
|
||||||
|
periodSeconds: 10
|
||||||
|
livenessProbe:
|
||||||
|
httpGet:
|
||||||
|
path: /health
|
||||||
|
port: 3903
|
||||||
|
initialDelaySeconds: 15
|
||||||
|
periodSeconds: 20
|
||||||
|
volumes:
|
||||||
|
- name: config-tmpl
|
||||||
|
configMap:
|
||||||
|
name: garage-config
|
||||||
|
- name: config
|
||||||
|
emptyDir: {}
|
||||||
|
volumeClaimTemplates:
|
||||||
|
- metadata:
|
||||||
|
name: meta
|
||||||
|
spec:
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
storageClassName: longhorn-nvme
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 1Gi
|
||||||
|
- metadata:
|
||||||
|
name: data
|
||||||
|
spec:
|
||||||
|
accessModes: ["ReadWriteOnce"]
|
||||||
|
storageClassName: longhorn
|
||||||
|
resources:
|
||||||
|
requests:
|
||||||
|
storage: 20Gi
|
||||||
@@ -62,3 +62,16 @@ spec:
|
|||||||
kind: ClusterIssuer
|
kind: ClusterIssuer
|
||||||
dnsNames:
|
dnsNames:
|
||||||
- gitea.coreworlds.io
|
- gitea.coreworlds.io
|
||||||
|
---
|
||||||
|
apiVersion: cert-manager.io/v1
|
||||||
|
kind: Certificate
|
||||||
|
metadata:
|
||||||
|
name: garage-s3-tls
|
||||||
|
namespace: platform
|
||||||
|
spec:
|
||||||
|
secretName: garage-s3-tls
|
||||||
|
issuerRef:
|
||||||
|
name: letsencrypt-production
|
||||||
|
kind: ClusterIssuer
|
||||||
|
dnsNames:
|
||||||
|
- s3.coreworlds.io
|
||||||
|
|||||||
22
infra/kubernetes/platform/traefik/ingressroute-garage.yaml
Normal file
22
infra/kubernetes/platform/traefik/ingressroute-garage.yaml
Normal file
@@ -0,0 +1,22 @@
|
|||||||
|
apiVersion: traefik.io/v1alpha1
|
||||||
|
kind: IngressRoute
|
||||||
|
metadata:
|
||||||
|
name: garage-s3
|
||||||
|
namespace: platform
|
||||||
|
annotations:
|
||||||
|
cert-manager.io/cluster-issuer: letsencrypt-production
|
||||||
|
spec:
|
||||||
|
entryPoints:
|
||||||
|
- websecure
|
||||||
|
routes:
|
||||||
|
- match: Host(`s3.coreworlds.io`)
|
||||||
|
kind: Rule
|
||||||
|
middlewares:
|
||||||
|
- name: internal-only
|
||||||
|
namespace: platform
|
||||||
|
services:
|
||||||
|
- name: garage
|
||||||
|
namespace: platform
|
||||||
|
port: 3900
|
||||||
|
tls:
|
||||||
|
secretName: garage-s3-tls
|
||||||
@@ -9,5 +9,6 @@ resources:
|
|||||||
- ingressroute-longhorn.yaml
|
- ingressroute-longhorn.yaml
|
||||||
- ingressroute-harness.yaml
|
- ingressroute-harness.yaml
|
||||||
- ingressroute-gitea.yaml
|
- ingressroute-gitea.yaml
|
||||||
|
- ingressroute-garage.yaml
|
||||||
- certificate-internal.yaml
|
- certificate-internal.yaml
|
||||||
- servicemonitor.yaml
|
- servicemonitor.yaml
|
||||||
|
|||||||
Reference in New Issue
Block a user