# Prometheus Alert Rules for Mercury Device Middleware - Phase 4.1
# Critical alerts for reversal system monitoring

groups:
  - name: reversal_system_alerts
    rules:
      # High-priority alerts for production issues
      
      - alert: ReversalFailureRateHigh
        expr: |
          (
            sum(rate(reversal_failures_total[5m])) / 
            (sum(rate(reversal_successes_total[5m])) + sum(rate(reversal_failures_total[5m])))
          ) * 100 > 10
        for: 2m
        labels:
          severity: critical
          component: reversal_system
        annotations:
          summary: "Reversal failure rate is above 10%"
          description: "Reversal failure rate is {{ $value }}% over the last 5 minutes"
          
      - alert: ReversalQueueBacklog
        expr: reversal_queue_size > 100
        for: 5m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "Reversal queue has high backlog"
          description: "Reversal queue size is {{ $value }} items"
          
      - alert: ReversalProcessingTimeHigh
        expr: |
          histogram_quantile(0.95, 
            sum(rate(reversal_duration_seconds_bucket[5m])) by (le)
          ) > 30
        for: 3m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "95th percentile reversal processing time is high"
          description: "95% of reversals take longer than {{ $value }} seconds"
          
      - alert: NetworkTimeoutRateHigh
        expr: |
          (
            sum(rate(network_transmission_failures_total{reason="timeout"}[5m])) /
            sum(rate(network_transmission_failures_total[5m]))
          ) * 100 > 5
        for: 2m
        labels:
          severity: critical
          component: network
        annotations:
          summary: "Network timeout rate is above 5%"
          description: "Network timeout rate is {{ $value }}% over the last 5 minutes"
          
      - alert: ReversalSystemHealthLow
        expr: system_health_score < 0.8
        for: 1m
        labels:
          severity: critical
          component: reversal_system
        annotations:
          summary: "Reversal system health score is low"
          description: "System health score is {{ $value }} (threshold: 0.8)"
          
      - alert: PacketGenerationFailureRate
        expr: |
          # Alert based on packet generation performance degradation
          histogram_quantile(0.95,
            sum(rate(packet_generation_duration_seconds_bucket[5m])) by (le)
          ) > 10
        for: 1m
        labels:
          severity: warning
          component: packet_builder
        annotations:
          summary: "Packet generation performance degraded"
          description: "95% of packet generation operations take longer than {{ $value }} seconds"
          
      - alert: StanBatchServiceSlow
        expr: |
          histogram_quantile(0.95,
            sum(rate(stan_batch_retrieval_duration_seconds_bucket[5m])) by (le)
          ) > 5
        for: 3m
        labels:
          severity: warning
          component: stan_batch_service
        annotations:
          summary: "STAN/Batch service is responding slowly"
          description: "95% of STAN/Batch operations take longer than {{ $value }} seconds"

  - name: reversal_system_business_rules
    rules:
      # Business rule violations
      
      - alert: ExcessiveReversalRetries
        expr: increase(reversal_failures_total{reason="max_retries_exceeded"}[1h]) > 5
        for: 5m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "Excessive reversal retry failures"
          description: "{{ $value }} reversals failed due to max retries in the last hour"

      - alert: DuplicateReversalAttempts
        expr: increase(reversal_failures_total{reason="already_reversed"}[1h]) > 10
        for: 5m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "High number of duplicate reversal attempts"
          description: "{{ $value }} duplicate reversal attempts in the last hour"

  - name: reversal_system_performance
    rules:
      # Performance monitoring using actual metrics
      
      - alert: CleanupWorkerDown
        expr: cleanup_worker_health < 1
        for: 1m
        labels:
          severity: critical
          component: reversal_system
        annotations:
          summary: "Cleanup worker is not healthy"
          description: "Cleanup worker health status is {{ $value }}"

      - alert: HighReversalQueueSize
        expr: reversal_queue_size > 50
        for: 2m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "Reversal queue size is high"
          description: "Queue size is {{ $value }} (threshold: 50)"

      - alert: ManualReviewQueueBacklog
        expr: manual_review_queue_size > 20
        for: 5m
        labels:
          severity: warning
          component: reversal_system
        annotations:
          summary: "Manual review queue has backlog"
          description: "Manual review queue size is {{ $value }} (threshold: 20)"

  - name: reversal_system_recovery
    rules:
      # Recovery and data integrity alerts
      
      - alert: StuckTransactionsDetected
        expr: increase(stuck_transactions_total[1h]) > 0
        for: 0m
        labels:
          severity: info
          component: data_integrity
        annotations:
          summary: "Stuck transactions detected"
          description: "{{ $value }} stuck transactions found and processed"
          
      - alert: DatabaseOperationFailures
        expr: increase(db_operation_failures_total[1h]) > 5
        for: 5m
        labels:
          severity: warning
          component: data_integrity
        annotations:
          summary: "Database operation failures detected"
          description: "{{ $value }} database operation failures in the last hour"

      - alert: StanBatchServiceHealthCheck
        expr: rate(stan_batch_operations_total[5m]) == 0
        for: 10m
        labels:
          severity: warning
          component: stan_batch_service
        annotations:
          summary: "STAN/Batch service appears inactive"
          description: "No STAN/batch operations detected in the last 10 minutes"