Add observability stack: ServiceMonitors, Tempo, OTel API instrumentation, dashboards

- Add ServiceMonitors for Traefik, ArgoCD, and Longhorn
- Enable cert-manager ServiceMonitor via helm values
- Deploy Grafana Tempo for distributed tracing (single-binary, Longhorn PVC)
- Add Tempo datasource with trace-to-logs and trace-to-metrics correlation
- Instrument API with OpenTelemetry SDK (Prometheus metrics + OTLP traces)
- Replace console.log with pino structured logging + pino-http middleware
- Add Grafana dashboards for Traefik, API overview, and PostgreSQL (CNPG)

apps/api/k8s/base/deployment.yaml

@@ -21,7 +21,12 @@ spec:
           image: ghcr.io/lazorgurl/homelab-api:latest
           ports:
             - containerPort: 4000
+              name: http
+            - containerPort: 9464
+              name: metrics
           env:
+            - name: OTEL_EXPORTER_OTLP_ENDPOINT
+              value: http://tempo.observability.svc:4318
             - name: DATABASE_URL
               valueFrom:
                 secretKeyRef:

apps/api/k8s/base/kustomization.yaml

@@ -6,3 +6,4 @@ resources:
   - ingress.yaml
   - ghcr-pull-secret-sealed.yaml
   - api-secrets-sealed.yaml
+  - servicemonitor.yaml

apps/api/k8s/base/service.yaml

@@ -8,5 +8,10 @@ spec:
     - port: 80
       targetPort: 4000
       protocol: TCP
+      name: http
+    - port: 9464
+      targetPort: 9464
+      protocol: TCP
+      name: metrics
   selector:
     app: api

apps/api/k8s/base/servicemonitor.yaml

@@ -0,0 +1,14 @@
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  name: api
+  labels:
+    app: api
+spec:
+  selector:
+    matchLabels:
+      app: api
+  endpoints:
+    - port: metrics
+      interval: 30s
+      path: /metrics