Monitoring

kubectl rollout status deployment/bloodraven -n bloodraven
kubectl logs -n bloodraven deploy/bloodraven
kubectl port-forward -n bloodraven deploy/bloodraven 8080:8080
curl http://localhost:8080/metrics | grep '^bloodraven_'

# prometheus.yml
scrape_configs:
  - job_name: bloodraven
    kubernetes_sd_configs:
      - role: pod
        namespaces:
          names: [bloodraven]
    relabel_configs:
      - source_labels: [__meta_kubernetes_pod_label_app_kubernetes_io_name]
        regex: bloodraven
        action: keep
      - source_labels: [__meta_kubernetes_pod_container_port_number]
        regex: "8080"
        action: keep

apiVersion: monitoring.coreos.com/v1
kind: PodMonitor
metadata:
  name: bloodraven
  namespace: bloodraven
spec:
  selector:
    matchLabels:
      app.kubernetes.io/name: bloodraven
  podMetricsEndpoints:
    - port: metrics
      interval: 15s

groups:
  - name: bloodraven
    rules:
      # Site has no healthy primary
      - alert: BloodravenNoPrimary
        expr: |
          count by (site) (
            bloodraven_state_transitions_total{to="writable"}
          ) == 0
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Site {{ $labels.site }} has no writable primary"

      # High poll latency may indicate network issues
      - alert: BloodravenHighPollLatency
        expr: |
          histogram_quantile(0.99, rate(bloodraven_poll_latency_seconds_bucket[5m])) > 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Poll latency to {{ $labels.site }} exceeds 1 second (p99)"

      # Failover occurred
      - alert: BloodravenFailoverOccurred
        expr: increase(bloodraven_failovers_total[5m]) > 0
        labels:
          severity: warning
        annotations:
          summary: "Failover occurred: {{ $labels.target_site }} promoted as new primary"

      # Replication lag exceeds threshold on a data-bearing replica.
      # role=~"primary-candidate|disaster-recovery" pages RPO risk and stays
      # silent on role="read-only" readers, whose lag is designed (they are shed
      # from mysql-<group>-replicas). For promotable replicas only, use
      # role="primary-candidate".
      - alert: BloodravenReplicationLagging
        expr: bloodraven_replication_lag_seconds{role=~"primary-candidate|disaster-recovery"} > 300
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Replication lag on {{ $labels.group }}/{{ $labels.site }} ({{ $labels.role }}) is {{ $value }}s"

      # A site holds transactions the primary lacks (old primary after an
      # emergency failover, or a replica found by the GTID audit)
      - alert: BloodravenDivergentTransactions
        expr: bloodraven_divergent_transactions > 0
        labels:
          severity: critical
        annotations:
          summary: "{{ $labels.site }} has {{ $value }} transactions the primary lacks — reclone it or make the primary contain them; ordered updates and planned failovers will not promote it"

      # Replication thread down on a data-bearing replica. Sites awaiting
      # their post-restore reclone have their receiver stopped by design.
      - alert: BloodravenReplicationDown
        expr: |
          bloodraven_replication_running{role=~"primary-candidate|disaster-recovery"} == 0
          unless on(namespace, group, site) bloodraven_restore_reclone_pending == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Replication {{ $labels.thread }} thread stopped on {{ $labels.group }}/{{ $labels.site }} ({{ $labels.role }})"

      # A failed full-instance restore left peers held with their
      # replication stopped; an administrator must decide what to keep.
      - alert: BloodravenRestoreReclonesHeld
        expr: bloodraven_restore_reclone_pending{phase="held"} == 1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "{{ $labels.group }}/{{ $labels.site }} is held after a failed in-place restore: no replication, not promotable"

      # No writable site in a failover group
      - alert: BloodravenNoWritableSite
        expr: |
          max by (namespace, group) (bloodraven_site_state{state="writable"}) == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "No site is currently writable in {{ $labels.namespace }}/{{ $labels.group }}"

      # Dragonfly site unavailable; MySQL remains authoritative, but planned
      # session/cache continuity is at risk.
      - alert: BloodravenDragonflySiteDown
        expr: bloodraven_dragonfly_site_up == 0
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Dragonfly site {{ $labels.site }} is unreachable for {{ $labels.group }}"

      # Failed Dragonfly promotion means the MySQL failover may have succeeded
      # with cache unavailable.
      - alert: BloodravenDragonflyPromotionFailed
        expr: increase(bloodraven_dragonfly_promotions_total{result="failed"}[5m]) > 0
        labels:
          severity: warning
        annotations:
          summary: "Dragonfly promotion failed for {{ $labels.group }} target {{ $labels.target_site }}"

      # Empty-master fallback: target is writable but REPLTAKEOVER did not run.
      - alert: BloodravenDragonflySessionsLost
        expr: increase(bloodraven_dragonfly_promotions_total{result="sessions_lost"}[5m]) > 0
        labels:
          severity: warning
        annotations:
          summary: "Dragonfly promoted {{ $labels.target_site }} via REPLICAOF NO ONE for {{ $labels.group }}; session continuity is not guaranteed"

      # A recovered panic keeps the manager alive, but it is still a bug signal.
      - alert: BloodravenDragonflyManagerPanic
        expr: increase(bloodraven_dragonfly_manager_panics_total[5m]) > 0
        labels:
          severity: critical
        annotations:
          summary: "Dragonfly manager recovered a panic for {{ $labels.namespace }}/{{ $labels.name }}"

helm upgrade --install bloodraven bloodraven/bloodraven \
  --namespace bloodraven --create-namespace \
  --set grafanaDashboards.enabled=true

grafanaDashboards:
  enabled: true
  # Sidecar is often scoped to the monitoring namespace
  namespace: monitoring
  # Change folder name
  folder: MySQL / Bloodraven
  # If your sidecar watches a non-default label
  label: grafana_dashboard
  labelValue: "1"

# One-time copy (re-run on upgrade to get dashboard updates)
kubectl -n monitoring cp -c grafana \
  bloodraven/charts/bloodraven/dashboards \
  grafana-pod:/var/lib/grafana/dashboards/bloodraven

# /etc/grafana/provisioning/dashboards/bloodraven.yaml
apiVersion: 1
providers:
  - name: bloodraven
    folder: Bloodraven
    type: file
    disableDeletion: false
    updateIntervalSeconds: 30
    allowUiUpdates: true
    options:
      path: /var/lib/grafana/dashboards/bloodraven

apiVersion: v1
kind: ConfigMap
metadata:
  name: kubewatch
data:
  .kubewatch.yaml: |
    handler:
      slack:
        channel: "#mysql-alerts"
    resource:
      event: true
    namespaces:
      - bloodraven
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: kubewatch
spec:
  template:
    spec:
      containers:
        - name: kubewatch
          env:
            - name: KW_SLACK_TOKEN
              valueFrom:
                secretKeyRef:
                  name: kubewatch-secrets
                  key: slack-token

curl http://localhost:8082/status

{
  "default/orders": {
    "activeSite": "iad",
    "sites": [
      {
        "name": "iad",
        "role": "PrimaryCandidate",
        "state": "Writable"
      },
      {
        "name": "pdx",
        "role": "PrimaryCandidate",
        "state": "ReadOnly"
      }
    ],
    "pollTime": "2025-01-01T00:00:00Z"
  }
}

curl "http://localhost:8082/active-site?namespace=default&group=orders"

{
  "namespace": "default",
  "group": "orders",
  "activeSite": "iad"
}

{
  "namespace": "default",
  "group": "orders",
  "activeSite": "iad",
  "sites": [
    {
      "name": "iad",
      "role": "PrimaryCandidate",
      "state": "Writable",
      "lastSeenTime": "2026-04-10T00:00:00Z",
      "replication": "NotApplicable"
    },
    {
      "name": "pdx",
      "role": "PrimaryCandidate",
      "state": "ReadOnly",
      "lastSeenTime": "2026-04-10T00:00:00Z",
      "replication": "Healthy",
      "secondsBehindSource": 0,
      "gtidExecuted": "3e11fa47-71ca-11e1-9e33-c80aa9429562:1-45839"
    }
  ],
  "lastFailover": {
    "time": "2026-04-09T22:10:00Z",
    "target": "iad",
    "promotionGtidExecuted": "3e11fa47-71ca-11e1-9e33-c80aa9429562:1-45839"
  },
  "pollTime": "2026-04-10T00:00:00Z"
}

{
  "name": "pdx",
  "role": "PrimaryCandidate",
  "state": "ReadOnly",
  "lastSeenTime": "2026-04-10T00:00:00Z",
  "replication": "Broken",
  "recoveryState": "RecoveryBlocked",
  "divergentGtid": "a1b2c3d4-0000-0000-0000-000000000000:11-15",
  "divergentTransactionCount": 5
}

const ws = new WebSocket("ws://localhost:8082/ws/status");
ws.onmessage = (event) => {
  const msg = JSON.parse(event.data);
  console.log(`${msg.group}: active=${msg.activeSite}`);
};

kubectl logs -n bloodraven deploy/bloodraven | jq 'select(.msg == "failover")'
kubectl logs -n bloodraven deploy/bloodraven | jq 'select(.level == "ERROR")'

IAD_POD=$(kubectl get pod -n orders \
  -l app.kubernetes.io/name=mysql,bloodraven.shipstream.io/failover-group=orders,bloodraven.shipstream.io/site=iad \
  -o jsonpath='{.items[0].metadata.name}')
kubectl logs -n orders "$IAD_POD" -c sidecar | jq 'select(.msg | contains("self-fence"))'