# Request rate per service (req/s)
sum(rate(http_requests_total[1m])) by (service)

# Error rate %
sum(rate(http_requests_total{status_code=~"5.."}[5m])) by (service)
/ sum(rate(http_requests_total[5m])) by (service) * 100

# P95 latency per service
histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket[5m])) by (le, service))

# CPU per container (cores)
sum(rate(container_cpu_usage_seconds_total{name=~"obs-.*"}[5m])) by (name)

# Memory per container
container_memory_usage_bytes{name=~"obs-.*"}

# Network I/O
sum(rate(container_network_receive_bytes_total{name=~"obs-.*"}[5m])) by (name)

# All logs from api and worker
{service=~"api|worker"}

# Only errors
{service=~"api|worker"} | json | level = "ERROR"

# Logs for a specific trace
{service=~"api|worker"} | trace_id = "<paste-trace-id>"

# Error rate from logs (metric query)
sum(rate({service=~"api|worker"} | json | level="ERROR" [5m])) by (service)
/ sum(rate({service=~"api|worker"} [5m])) by (service)

# All traces for a service
{ resource.service.name = "api" }

# Traces with errors only
{ status = error }

# Slow requests with errors
{ resource.service.name = "api" && status = error && duration > 50ms }

# Traces involving both services (cross-service calls)
{ resource.service.name =~ "api|worker" } | by(rootServiceName)

docs/observability-mastery/
├── README.md                          ← start here — overview + quick lookup
├── alloy/
│   ├── 01-core-concepts.md            component model, River syntax, CLI
│   ├── 02-metrics-pipeline.md         discovery, relabeling, scrape, remote_write
│   ├── 03-logs-pipeline.md            loki.source, all processing stages, loki.write
│   ├── 04-traces-pipeline.md          otelcol components, tail sampling, Tempo export
│   └── 05-production-advanced.md      K8s, hot reload, clustering, debugging
└── loki/
    ├── 01-architecture.md             mental model, write/read path, deployment modes
    ├── 02-labels-design.md            cardinality, labels vs structured metadata
    ├── 03-logql-reference.md          full query language with examples
    ├── 04-production-config.md        loki.yaml, schema, storage, ruler, Helm
    └── 05-operations-cost.md          troubleshooting, cost reduction, sizing

.
├── src/
│   ├── api/                        # Payments API — FastAPI + OTel + Prometheus
│   │   ├── main.py
│   │   ├── requirements.txt
│   │   └── Dockerfile
│   └── worker/                     # Background processor — same instrumentation
│       ├── main.py
│       ├── requirements.txt
│       └── Dockerfile
├── observability/
│   ├── alloy/
│   │   └── config.alloy            # Metrics + logs (Docker) + OTLP traces pipeline
│   ├── prometheus/
│   │   ├── prometheus.yml
│   │   └── rules/
│   │       ├── alerts.yml          # ServiceDown, HighErrorRate, CPU, Memory
│   │       └── slo.yml             # SLI recording rules + error budget + burn-rate
│   ├── alertmanager/
│   │   └── config.yml
│   ├── tempo/
│   │   └── config.yml              # Tracing + span metrics generator
│   ├── loki/
│   │   └── config.yml
│   ├── grafana/
│   │   └── provisioning/
│   │       ├── datasources/        # Prometheus + Loki + Tempo with cross-links
│   │       └── dashboards/
│   │           ├── hello-api.json  # Application Overview dashboard
│   │           └── infra.json      # Infrastructure Overview dashboard
│   └── k6/
│       └── load.js                 # Load test: 4 scenarios across both services
├── docs/
│   ├── images/                     # Screenshots of the running stack
│   ├── decisions.md                # Architecture Decision Records (ADRs)
│   └── observability-mastery/      # Reference cards for the stack
│       ├── README.md
│       ├── alloy/                  # 5 reference cards for Grafana Alloy
│       └── loki/                   # 5 reference cards for Grafana Loki
├── docker-compose.app.yml          # Server 1 — applications
├── docker-compose.observability.yml # Server 2 — observability stack
├── omd.yaml                        # Observability Metadata Definition — label standards
├── Makefile
└── README.md