From 3c4ff6fb9f0afc453e30cffb4c48b1ca36da7b48 Mon Sep 17 00:00:00 2001 From: Julia McGhee Date: Sun, 22 Mar 2026 09:38:47 +0000 Subject: [PATCH] Add Garage S3-compatible object store to platform Cluster-local object store for build artifacts (CLI binaries etc.) so Docker builds don't depend on flaky external downloads. - Single-node Garage v1.0.1 StatefulSet (LMDB, replication=1) - Metadata on longhorn-nvme (1Gi), data on longhorn HDD (20Gi) - S3 API at garage.platform.svc:3900 - External ingress at s3.coreworlds.io (internal-only) - SealedSecret for admin token and RPC secret --- .../kubernetes/platform/garage/configmap.yaml | 31 ++++++ .../garage/garage-credentials-sealed.yaml | 14 +++ .../platform/garage/kustomization.yaml | 7 ++ infra/kubernetes/platform/garage/service.yaml | 20 ++++ .../platform/garage/statefulset.yaml | 103 ++++++++++++++++++ .../traefik/certificate-internal.yaml | 13 +++ .../platform/traefik/ingressroute-garage.yaml | 22 ++++ .../platform/traefik/kustomization.yaml | 1 + 8 files changed, 211 insertions(+) create mode 100644 infra/kubernetes/platform/garage/configmap.yaml create mode 100644 infra/kubernetes/platform/garage/garage-credentials-sealed.yaml create mode 100644 infra/kubernetes/platform/garage/kustomization.yaml create mode 100644 infra/kubernetes/platform/garage/service.yaml create mode 100644 infra/kubernetes/platform/garage/statefulset.yaml create mode 100644 infra/kubernetes/platform/traefik/ingressroute-garage.yaml diff --git a/infra/kubernetes/platform/garage/configmap.yaml b/infra/kubernetes/platform/garage/configmap.yaml new file mode 100644 index 0000000..ca6bc3b --- /dev/null +++ b/infra/kubernetes/platform/garage/configmap.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: garage-config + namespace: platform +data: + garage.toml: | + db_engine = "lmdb" + replication_factor = 1 + compression_level = 1 + + metadata_dir = "/mnt/meta" + data_dir = "/mnt/data" + + [rpc] + bind_addr = "[::]:3901" + # Single-node: rpc_secret is only used for inter-node auth (N/A here) + rpc_secret = "PLACEHOLDER" + + [s3_api] + api_bind_addr = "[::]:3900" + s3_region = "garage" + root_domain = ".s3.garage.svc" + + [s3_web] + bind_addr = "[::]:3902" + root_domain = ".web.garage.svc" + index = "index.html" + + [admin] + api_bind_addr = "[::]:3903" diff --git a/infra/kubernetes/platform/garage/garage-credentials-sealed.yaml b/infra/kubernetes/platform/garage/garage-credentials-sealed.yaml new file mode 100644 index 0000000..e2279dd --- /dev/null +++ b/infra/kubernetes/platform/garage/garage-credentials-sealed.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: bitnami.com/v1alpha1 +kind: SealedSecret +metadata: + name: garage-credentials + namespace: platform +spec: + encryptedData: + admin-token: AgB+RSg4mNJKHOUXUi/ODlmOiAvYa1cAytc2pxVTsNlQZ5gJgF/stOOmMF3upKRwCmHbBIcPry0LlPxqkZ5Gs6tXi8YsxfqzmvFONHkRFNtUWyxoo2esIlIyvFZ2TfKuiaeP1XCgYW3yoFQOnrEqNBTuIzs2Ss09TtfcbjA2AgNWPo5UtRrU2ms7wn6puEFOATWGYqzQgzq958tW8OCHhyWDwlHEzfqrDm2lXSSoTKfeSVrY9lT+7y0EUAwNDB4NFkRQRmVJYIUbGc3aAdNhSMMC88UGJVVGTy511Dq1uUmkhB9ChOMRwKxYTHjiinQ1cKYQJ+8EgV98EqKHQoC0d0p04SGKCr/TBLdVhWF6h76ynnI+EoBDY0f/bZPnt/6AuELtjpcHXiQMO4WiFOPnj9nVXMu+WAfYuKmEaG0DgkOIsGXYlhOrkXrZtymKr/xWvWKfZZtwImhHyCjBiwlFNxDBv5z7bAz7bxgQKVJtu/mlh+o7UrQdozRUKhQuvqp838Z4m1kXscF5HZqJ1N8rpDnjh7wPmkc317phpmQ9dM/Fbc3x9ZCNNPKrhtJYzGbcU8QhL+ZSxe8bevCdVYZ6/hNJy8kIqTiWDjCDPPrG8jYPPBTFhAwtVtCXLxgztvNxv9FOK7ILGK3E0wgV+luU3NynoXSGNkFMYoRn8xAboSplkYVOcRC2LTyWggBa1xvea/mnc8FWOKhPdFOaY7IuZ01NGbun+C2++jzALqEokh49TL3soLqP7TsMwZo2NxyYbuqGNtFKmFqSh7PpgeXVyg5C + rpc-secret: AgB95o6wk/trqbwpY2KG2exkpFwLmE8Ic2JgF2Rnj+SVOLMzs6e/Fd8ukhsqGn+t+YXZn28B2EpkPrfkuvwr3mrcLwPR4ZussmlQvRW7bzH/n8yDehmh+4/IzAZ/SxlBRUMadOrbW+BhlhRctVjJ1GhO8jGqpsb8nbrfSFWw4TVICJome6et4v1FTphNbp89ajjNkgXKGT3prVYeGsWXUENvZih2IkY5qqW6YGhLOln5SMVCI2itztnrO3QFWb1O0sDD2KoESlLT39F1zZL9x42ZhlQ9aIYBNQk444PXDQhvvSURvXOqnX1c/2I+3V1vR3ErKmP3IGYAmebu0S0Pa6XM6hOQAcgBJaStFEXg0JYMxCXOrAQ9AdSsm1nKE279AnuLgVvVM//6eC51cHOaA5gQiUjxq5GvPKGZqD5gA3yzBNgy33w4OPf7celD/HDbm23sHoCxDTilbzebve1AK6/IDdKJc+Fvg2JwaiyxnK6Wlp0wen9+/xlfK0Xw6H2aAN2wxI/ib77ec4VA/Y6tYHNn/BMSf5FQBqSamr1OuwFbCRam0+eKsIbo3FqbnoUeTU1GTNZq1FYbh3STb2sp3BMi8VtUAsnOhi3CnfWxzo2FRbtkvzAuoBuxTesb2QPiW0FJBNAbafS3Gxf13gwvzc3HrYdNjyn9ca2yHep8gc2QeUcdVpDkEyT47ozYaAuBSFgl1twAn9YceM40tuHFF0lW07XSx5JHjKPp+5I9zrr3/g07OfymC1XDATBpMFgD3MkYN7iBlGXa+koPCqH6lCgV + template: + metadata: + name: garage-credentials + namespace: platform diff --git a/infra/kubernetes/platform/garage/kustomization.yaml b/infra/kubernetes/platform/garage/kustomization.yaml new file mode 100644 index 0000000..b1328ba --- /dev/null +++ b/infra/kubernetes/platform/garage/kustomization.yaml @@ -0,0 +1,7 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - configmap.yaml + - statefulset.yaml + - service.yaml + - garage-credentials-sealed.yaml diff --git a/infra/kubernetes/platform/garage/service.yaml b/infra/kubernetes/platform/garage/service.yaml new file mode 100644 index 0000000..8fa5cca --- /dev/null +++ b/infra/kubernetes/platform/garage/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: garage + namespace: platform + labels: + app: garage +spec: + type: ClusterIP + ports: + - port: 3900 + targetPort: 3900 + protocol: TCP + name: s3-api + - port: 3903 + targetPort: 3903 + protocol: TCP + name: admin + selector: + app: garage diff --git a/infra/kubernetes/platform/garage/statefulset.yaml b/infra/kubernetes/platform/garage/statefulset.yaml new file mode 100644 index 0000000..3976218 --- /dev/null +++ b/infra/kubernetes/platform/garage/statefulset.yaml @@ -0,0 +1,103 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: garage + namespace: platform + labels: + app: garage +spec: + serviceName: garage + replicas: 1 + selector: + matchLabels: + app: garage + template: + metadata: + labels: + app: garage + spec: + initContainers: + - name: config + image: busybox:1.37 + command: ["sh", "-c"] + args: + - | + sed "s/PLACEHOLDER/$RPC_SECRET/" /config-tmpl/garage.toml > /config/garage.toml + env: + - name: RPC_SECRET + valueFrom: + secretKeyRef: + name: garage-credentials + key: rpc-secret + volumeMounts: + - name: config-tmpl + mountPath: /config-tmpl + - name: config + mountPath: /config + containers: + - name: garage + image: dxflrs/garage:v1.0.1 + ports: + - containerPort: 3900 + name: s3-api + - containerPort: 3901 + name: rpc + - containerPort: 3902 + name: web + - containerPort: 3903 + name: admin + env: + - name: GARAGE_ADMIN_TOKEN + valueFrom: + secretKeyRef: + name: garage-credentials + key: admin-token + resources: + requests: + memory: 256Mi + cpu: 100m + limits: + memory: 512Mi + volumeMounts: + - name: config + mountPath: /etc/garage.toml + subPath: garage.toml + - name: meta + mountPath: /mnt/meta + - name: data + mountPath: /mnt/data + readinessProbe: + httpGet: + path: /health + port: 3903 + initialDelaySeconds: 5 + periodSeconds: 10 + livenessProbe: + httpGet: + path: /health + port: 3903 + initialDelaySeconds: 15 + periodSeconds: 20 + volumes: + - name: config-tmpl + configMap: + name: garage-config + - name: config + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: meta + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: longhorn-nvme + resources: + requests: + storage: 1Gi + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: longhorn + resources: + requests: + storage: 20Gi diff --git a/infra/kubernetes/platform/traefik/certificate-internal.yaml b/infra/kubernetes/platform/traefik/certificate-internal.yaml index d4c49ac..737c954 100644 --- a/infra/kubernetes/platform/traefik/certificate-internal.yaml +++ b/infra/kubernetes/platform/traefik/certificate-internal.yaml @@ -62,3 +62,16 @@ spec: kind: ClusterIssuer dnsNames: - gitea.coreworlds.io +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: garage-s3-tls + namespace: platform +spec: + secretName: garage-s3-tls + issuerRef: + name: letsencrypt-production + kind: ClusterIssuer + dnsNames: + - s3.coreworlds.io diff --git a/infra/kubernetes/platform/traefik/ingressroute-garage.yaml b/infra/kubernetes/platform/traefik/ingressroute-garage.yaml new file mode 100644 index 0000000..f0ccc28 --- /dev/null +++ b/infra/kubernetes/platform/traefik/ingressroute-garage.yaml @@ -0,0 +1,22 @@ +apiVersion: traefik.io/v1alpha1 +kind: IngressRoute +metadata: + name: garage-s3 + namespace: platform + annotations: + cert-manager.io/cluster-issuer: letsencrypt-production +spec: + entryPoints: + - websecure + routes: + - match: Host(`s3.coreworlds.io`) + kind: Rule + middlewares: + - name: internal-only + namespace: platform + services: + - name: garage + namespace: platform + port: 3900 + tls: + secretName: garage-s3-tls diff --git a/infra/kubernetes/platform/traefik/kustomization.yaml b/infra/kubernetes/platform/traefik/kustomization.yaml index 411efe8..b224196 100644 --- a/infra/kubernetes/platform/traefik/kustomization.yaml +++ b/infra/kubernetes/platform/traefik/kustomization.yaml @@ -9,5 +9,6 @@ resources: - ingressroute-longhorn.yaml - ingressroute-harness.yaml - ingressroute-gitea.yaml + - ingressroute-garage.yaml - certificate-internal.yaml - servicemonitor.yaml