Add observability stack: ServiceMonitors, Tempo, OTel API instrumentation, dashboards

- Add ServiceMonitors for Traefik, ArgoCD, and Longhorn
- Enable cert-manager ServiceMonitor via helm values
- Deploy Grafana Tempo for distributed tracing (single-binary, Longhorn PVC)
- Add Tempo datasource with trace-to-logs and trace-to-metrics correlation
- Instrument API with OpenTelemetry SDK (Prometheus metrics + OTLP traces)
- Replace console.log with pino structured logging + pino-http middleware
- Add Grafana dashboards for Traefik, API overview, and PostgreSQL (CNPG)

apps/api/src/index.ts

@@ -1,11 +1,17 @@
+import "./instrumentation";
+
 import express from "express";
 import cors from "cors";
+import pino from "pino";
+import pinoHttp from "pino-http";
 
+const logger = pino({ name: "api" });
 const app = express();
 const port = process.env.PORT || 4000;
 
 app.use(cors());
 app.use(express.json());
+app.use(pinoHttp({ logger }));
 
 app.get("/health", (_req, res) => {
   res.json({ status: "ok", timestamp: new Date().toISOString() });
@@ -16,5 +22,5 @@ app.get("/api", (_req, res) => {
 });
 
 app.listen(port, () => {
-  console.log(`API server running on port ${port}`);
+  logger.info(`API server running on port ${port}`);
 });

apps/api/src/instrumentation.ts

@@ -0,0 +1,36 @@
+import { NodeSDK } from "@opentelemetry/sdk-node";
+import { getNodeAutoInstrumentations } from "@opentelemetry/auto-instrumentations-node";
+import { PrometheusExporter } from "@opentelemetry/exporter-prometheus";
+import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-http";
+import { resourceFromAttributes } from "@opentelemetry/resources";
+import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
+
+const prometheusExporter = new PrometheusExporter({ port: 9464 });
+
+const traceExporter = new OTLPTraceExporter({
+  url:
+    process.env.OTEL_EXPORTER_OTLP_ENDPOINT ??
+    "http://tempo.observability.svc:4318/v1/traces",
+});
+
+const sdk = new NodeSDK({
+  resource: resourceFromAttributes({
+    [ATTR_SERVICE_NAME]: "api",
+  }),
+  metricReader: prometheusExporter,
+  traceExporter,
+  instrumentations: [
+    getNodeAutoInstrumentations({
+      "@opentelemetry/instrumentation-fs": { enabled: false },
+    }),
+  ],
+});
+
+sdk.start();
+
+process.on("SIGTERM", () => {
+  sdk.shutdown().then(
+    () => process.exit(0),
+    () => process.exit(1),
+  );
+});