defmodule DaProductApp.Telemetry.SwitchMetrics do @moduledoc """ Production metrics collector for the ISO 8583 switch core. Runs as a supervised GenServer and collects three metric families: 1. **ISO 8583 transaction metrics** — counters, TPS gauge, and latency histograms per MTI and upstream network. Driven by SwitchTelemetry events. 2. **Network channel metrics** — per-network connection counts, request counters, and response-time histograms. Pulled from AsyncCorrelatorTelemetry ETS on a 10-second timer so SwitchMetrics never blocks the correlator. 3. **BEAM VM metrics** — memory breakdown, process count, scheduler utilization, run queue, GC counters. Sampled on a 15-second timer. Access the current snapshot via `get_metrics_snapshot/0` for the Prometheus exporter or any other consumer. """ use GenServer require Logger alias DaProductApp.MercuryISO8583.{SwitchTelemetry, AsyncCorrelatorTelemetry} @network_poll_interval 10_000 @beam_poll_interval 15_000 @tps_window_seconds 60 # Keep last 1 000 samples for histogram percentile calculations @histogram_max_samples 1_000 # --------------------------------------------------------------------------- # Public API # --------------------------------------------------------------------------- def start_link(opts \\ []) do GenServer.start_link(__MODULE__, opts, name: __MODULE__) end @doc "Return a snapshot of all current metric values for the Prometheus exporter." def get_metrics_snapshot do GenServer.call(__MODULE__, :get_metrics_snapshot) end # --------------------------------------------------------------------------- # GenServer init # --------------------------------------------------------------------------- @impl true def init(_opts) do :telemetry.attach_many( "switch-metrics-collector", SwitchTelemetry.event_names(), &handle_telemetry_event/4, nil ) state = %{ # --- transaction metrics --- tx_received: %{}, # {mti, network} => count tx_processed: %{}, # {mti, network, outcome} => count tx_routed: %{}, # {mti, target_network, rule} => count echo_tests: %{}, # {network, success} => count active_tx: %{}, # network => count (in-flight) tx_timestamps: %{}, # network => [monotonic_ms, ...] (for TPS) tps: %{}, # network => float duration_samples: %{}, # {mti, network} => [ms, ...] (histogram) routing_samples: %{}, # network => [ms, ...] # --- network metrics (polled from AsyncCorrelatorTelemetry) --- network_stats: %{}, # network_id => %{active: int, sent: int, ...} # --- BEAM VM metrics (polled) --- beam_vm: initial_beam_vm_snapshot(), last_updated: DateTime.utc_now() } schedule_network_poll() schedule_beam_poll() schedule_tps_calculation() Logger.info("SwitchMetrics collector started") {:ok, state} end # --------------------------------------------------------------------------- # Telemetry event handlers (called from :telemetry, must be fast) # --------------------------------------------------------------------------- @doc false def handle_telemetry_event( [:da_product_app, :switch, :transaction_received], _measurements, %{mti: mti, network: network} = _meta, _config ) do GenServer.cast(__MODULE__, {:tx_received, mti, network}) end def handle_telemetry_event( [:da_product_app, :switch, :transaction_processed], %{duration_ms: duration_ms}, %{mti: mti, network: network, outcome: outcome} = _meta, _config ) do GenServer.cast(__MODULE__, {:tx_processed, mti, network, outcome, duration_ms}) end def handle_telemetry_event( [:da_product_app, :switch, :transaction_routed], %{routing_ms: routing_ms}, %{mti: mti, target_network: target_network, rule: rule} = _meta, _config ) do GenServer.cast(__MODULE__, {:tx_routed, mti, target_network, rule, routing_ms}) end def handle_telemetry_event( [:da_product_app, :switch, :echo_test_sent], _measurements, %{network: network} = _meta, _config ) do GenServer.cast(__MODULE__, {:echo_sent, network}) end def handle_telemetry_event( [:da_product_app, :switch, :echo_test_received], %{rtt_ms: rtt_ms}, %{network: network, success: success} = _meta, _config ) do GenServer.cast(__MODULE__, {:echo_received, network, success, rtt_ms}) end def handle_telemetry_event(_event, _measurements, _meta, _config), do: :ok # --------------------------------------------------------------------------- # GenServer handle_call # --------------------------------------------------------------------------- @impl true def handle_call(:get_metrics_snapshot, _from, state) do snapshot = %{ transactions: build_transaction_snapshot(state), networks: build_network_snapshot(state), beam_vm: state.beam_vm, last_updated: state.last_updated } {:reply, snapshot, state} end # --------------------------------------------------------------------------- # GenServer handle_cast — transaction events # --------------------------------------------------------------------------- @impl true def handle_cast({:tx_received, mti, network}, state) do now_ms = System.monotonic_time(:millisecond) # increment received counter key = {mti, to_string(network)} new_received = Map.update(state.tx_received, key, 1, &(&1 + 1)) # increment active (in-flight) gauge net = to_string(network) new_active = Map.update(state.active_tx, net, 1, &(&1 + 1)) # record timestamp for TPS window new_timestamps = Map.update(state.tx_timestamps, net, [now_ms], fn ts -> cutoff = now_ms - @tps_window_seconds * 1_000 [now_ms | Enum.filter(ts, &(&1 >= cutoff))] end) {:noreply, %{state | tx_received: new_received, active_tx: new_active, tx_timestamps: new_timestamps, last_updated: DateTime.utc_now() }} end @impl true def handle_cast({:tx_processed, mti, network, outcome, duration_ms}, state) do net = to_string(network) key = {mti, net, to_string(outcome)} new_processed = Map.update(state.tx_processed, key, 1, &(&1 + 1)) # decrement active gauge (floor at 0) new_active = Map.update(state.active_tx, net, 0, &max(0, &1 - 1)) # record duration histogram sample hist_key = {mti, net} new_durations = append_sample(state.duration_samples, hist_key, duration_ms / 1.0) {:noreply, %{state | tx_processed: new_processed, active_tx: new_active, duration_samples: new_durations, last_updated: DateTime.utc_now() }} end @impl true def handle_cast({:tx_routed, mti, target_network, rule, routing_ms}, state) do key = {mti, to_string(target_network), to_string(rule)} new_routed = Map.update(state.tx_routed, key, 1, &(&1 + 1)) net = to_string(target_network) new_routing = append_sample(state.routing_samples, net, routing_ms / 1.0) {:noreply, %{state | tx_routed: new_routed, routing_samples: new_routing, last_updated: DateTime.utc_now() }} end @impl true def handle_cast({:echo_sent, _network}, state) do # echo_sent contributes to sent counter; success is tracked on receipt {:noreply, state} end @impl true def handle_cast({:echo_received, network, success, _rtt_ms}, state) do key = {to_string(network), to_string(success)} new_echo = Map.update(state.echo_tests, key, 1, &(&1 + 1)) {:noreply, %{state | echo_tests: new_echo, last_updated: DateTime.utc_now()}} end # --------------------------------------------------------------------------- # GenServer handle_info — timers # --------------------------------------------------------------------------- @impl true def handle_info(:poll_network_stats, state) do new_network_stats = fetch_network_stats() schedule_network_poll() {:noreply, %{state | network_stats: new_network_stats, last_updated: DateTime.utc_now()}} end @impl true def handle_info(:poll_beam_vm, state) do new_beam = sample_beam_vm() schedule_beam_poll() {:noreply, %{state | beam_vm: new_beam, last_updated: DateTime.utc_now()}} end @impl true def handle_info(:calculate_tps, state) do now_ms = System.monotonic_time(:millisecond) cutoff = now_ms - @tps_window_seconds * 1_000 # Prune old timestamps and recalculate TPS per network {new_tps, new_timestamps} = Enum.reduce(state.tx_timestamps, {%{}, %{}}, fn {net, ts}, {tps_acc, ts_acc} -> recent = Enum.filter(ts, &(&1 >= cutoff)) tps = length(recent) / @tps_window_seconds {Map.put(tps_acc, net, Float.round(tps, 2)), Map.put(ts_acc, net, recent)} end) schedule_tps_calculation() {:noreply, %{state | tps: new_tps, tx_timestamps: new_timestamps}} end # --------------------------------------------------------------------------- # Snapshot builders # --------------------------------------------------------------------------- defp build_transaction_snapshot(state) do %{ received: state.tx_received, processed: state.tx_processed, routed: state.tx_routed, echo_tests: state.echo_tests, active: state.active_tx, tps: state.tps, duration_hist: histogram_summaries(state.duration_samples), routing_hist: histogram_summaries(state.routing_samples) } end defp build_network_snapshot(state) do state.network_stats end # --------------------------------------------------------------------------- # Network stats — pull from AsyncCorrelatorTelemetry ETS # --------------------------------------------------------------------------- defp fetch_network_stats do try do stats = AsyncCorrelatorTelemetry.get_statistics() case Map.get(stats, :networks, %{}) do networks when is_map(networks) -> Enum.reduce(networks, %{}, fn {network_id, net_stats}, acc -> Map.put(acc, to_string(network_id), %{ requests_sent: Map.get(net_stats, :total_requests, 0), responses_success: Map.get(net_stats, :successful_responses, 0), responses_timeout: Map.get(net_stats, :timeout_responses, 0), responses_failed: Map.get(net_stats, :failed_correlations, 0), avg_response_time_ms: Map.get(net_stats, :average_response_time, 0.0), instances: extract_instance_stats(Map.get(net_stats, :instances, %{})) }) end) _ -> %{} end rescue _ -> %{} end end defp extract_instance_stats(instances) when is_map(instances) do Enum.reduce(instances, %{}, fn {iid, s}, acc -> Map.put(acc, to_string(iid), %{ sent: Map.get(s, :total_requests, 0), success: Map.get(s, :successful_responses, 0), timeout: Map.get(s, :timeout_responses, 0), avg_ms: Map.get(s, :average_response_time, 0.0) }) end) end defp extract_instance_stats(_), do: %{} # --------------------------------------------------------------------------- # BEAM VM sampling # --------------------------------------------------------------------------- defp initial_beam_vm_snapshot do %{ memory_total_bytes: 0, memory_processes_bytes: 0, memory_binary_bytes: 0, memory_ets_bytes: 0, memory_atom_bytes: 0, process_count: 0, process_limit: 0, run_queue_length: 0, scheduler_count: 0, port_count: 0, uptime_seconds: 0, gc_count_total: 0, gc_words_reclaimed: 0, scheduler_utilization: %{} } end defp sample_beam_vm do mem = :erlang.memory() {wall_ms, _} = :erlang.statistics(:wall_clock) {gc_count, gc_words, _} = :erlang.statistics(:garbage_collection) %{ memory_total_bytes: Keyword.get(mem, :total, 0), memory_processes_bytes: Keyword.get(mem, :processes, 0), memory_binary_bytes: Keyword.get(mem, :binary, 0), memory_ets_bytes: Keyword.get(mem, :ets, 0), memory_atom_bytes: Keyword.get(mem, :atom, 0), process_count: :erlang.system_info(:process_count), process_limit: :erlang.system_info(:process_limit), run_queue_length: :erlang.statistics(:run_queue), scheduler_count: :erlang.system_info(:schedulers_online), port_count: :erlang.system_info(:port_count), uptime_seconds: div(wall_ms, 1_000), gc_count_total: gc_count, gc_words_reclaimed: gc_words, scheduler_utilization: sample_scheduler_utilization() } end defp sample_scheduler_utilization do try do :erlang.system_flag(:scheduler_wall_time, true) t1 = :erlang.statistics(:scheduler_wall_time) Process.sleep(100) t2 = :erlang.statistics(:scheduler_wall_time) Enum.reduce(Enum.zip(Enum.sort(t1), Enum.sort(t2)), %{}, fn {{id, a1, t1_val}, {id, a2, t2_val}}, acc -> util = if t2_val - t1_val > 0, do: Float.round((a2 - a1) / (t2_val - t1_val), 4), else: 0.0 Map.put(acc, id, util) end) rescue _ -> %{} end end # --------------------------------------------------------------------------- # Histogram helpers # --------------------------------------------------------------------------- defp append_sample(samples_map, key, value) do Map.update(samples_map, key, [value], fn existing -> [value | existing] |> Enum.take(@histogram_max_samples) end) end defp histogram_summaries(samples_map) do Enum.reduce(samples_map, %{}, fn {key, values}, acc -> Map.put(acc, key, summarize(values)) end) end defp summarize([]) do %{count: 0, sum: 0.0, p50: 0.0, p95: 0.0, p99: 0.0} end defp summarize(values) do sorted = Enum.sort(values) count = length(sorted) sum = Enum.sum(sorted) %{ count: count, sum: Float.round(sum, 3), p50: percentile(sorted, count, 50), p95: percentile(sorted, count, 95), p99: percentile(sorted, count, 99) } end defp percentile(sorted, count, p) do idx = max(0, round(count * p / 100) - 1) Float.round(Enum.at(sorted, min(idx, count - 1), 0.0) / 1.0, 3) end # --------------------------------------------------------------------------- # Timers # --------------------------------------------------------------------------- defp schedule_network_poll, do: Process.send_after(self(), :poll_network_stats, @network_poll_interval) defp schedule_beam_poll, do: Process.send_after(self(), :poll_beam_vm, @beam_poll_interval) defp schedule_tps_calculation, do: Process.send_after(self(), :calculate_tps, 5_000) end