defmodule DaProductApp.Acquirer.ReversalTelemetry do @moduledoc """ Telemetry events for reversal operations. Provides comprehensive observability for the reversal lifecycle through standard telemetry events. All events follow the pattern: `[:da_product_app, :reversal, :event_name]` ## Event Categories - **Reversal Lifecycle** - attempt, success, failure events - **Detection** - stuck transaction detection - **Retry** - retry scheduling and execution - **Cleanup** - cleanup worker operations ## Usage Attach telemetry handlers in your application supervision tree: :telemetry.attach_many( "reversal-metrics", [ [:da_product_app, :reversal, :attempt], [:da_product_app, :reversal, :success], [:da_product_app, :reversal, :failure] ], &MyApp.handle_reversal_event/4, nil ) ## Example Handler def handle_reversal_event(event, measurements, metadata, _config) do Logger.info("Reversal event: \#{inspect(event)}") # Update Prometheus metrics, send to monitoring system, etc. end """ require Logger @doc """ Emit when reversal attempt starts. ## Measurements - `:count` - Always 1 - `:attempt_number` - Current retry attempt (1-based) ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID (may be nil if creation failed) - `:reason` - Reason for reversal (e.g., "TIMEOUT", "CONNECTION_LOST") """ @spec emit_reversal_attempt(integer(), integer() | nil, non_neg_integer(), String.t()) :: :ok def emit_reversal_attempt(temp_txn_id, reversal_id, attempt_number, reason) do Logger.info( "Reversal attempt ##{attempt_number} for temp_txn=#{temp_txn_id}, reversal=#{inspect(reversal_id)}, reason=#{reason}" ) :telemetry.execute( [:da_product_app, :reversal, :attempt], %{count: 1, attempt_number: attempt_number}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id, reason: reason} ) end @doc """ Emit when reversal completes successfully. ## Measurements - `:count` - Always 1 - `:duration_ms` - Time taken for reversal operation (milliseconds) ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID """ @spec emit_reversal_success(integer(), integer(), non_neg_integer()) :: :ok def emit_reversal_success(temp_txn_id, reversal_id, duration_ms) do Logger.info( "Reversal succeeded: temp_txn=#{temp_txn_id}, reversal=#{reversal_id}, duration=#{duration_ms}ms" ) :telemetry.execute( [:da_product_app, :reversal, :success], %{count: 1, duration_ms: duration_ms}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id} ) end @doc """ Emit when reversal fails. ## Measurements - `:count` - Always 1 - `:attempt_number` - Attempt number that failed ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID (may be nil) - `:reason` - Failure reason (inspected term) """ @spec emit_reversal_failure(integer(), integer() | nil, term(), non_neg_integer()) :: :ok def emit_reversal_failure(temp_txn_id, reversal_id, reason, attempt_number) do Logger.error( "Reversal failed: temp_txn=#{temp_txn_id}, reversal=#{inspect(reversal_id)}, attempt=#{attempt_number}, reason=#{inspect(reason)}" ) :telemetry.execute( [:da_product_app, :reversal, :failure], %{count: 1, attempt_number: attempt_number}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id, reason: inspect(reason)} ) end @doc """ Emit when stuck transaction is detected. A stuck transaction is one that has been in processing state for longer than expected and requires reversal. ## Measurements - `:count` - Always 1 - `:age_seconds` - How long transaction has been stuck ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:status` - Current transaction status """ @spec emit_stuck_transaction_detected(integer(), non_neg_integer(), String.t()) :: :ok def emit_stuck_transaction_detected(temp_txn_id, age_seconds, status) do Logger.warning( "Stuck transaction detected: temp_txn=#{temp_txn_id}, age=#{age_seconds}s, status=#{status}" ) :telemetry.execute( [:da_product_app, :reversal, :stuck_transaction], %{count: 1, age_seconds: age_seconds}, %{temp_txn_id: temp_txn_id, status: status} ) end @doc """ Emit when reversal retry is scheduled. ## Measurements - `:count` - Always 1 - `:delay_seconds` - How long until next retry attempt ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID """ @spec emit_reversal_retry_scheduled(integer(), integer(), non_neg_integer()) :: :ok def emit_reversal_retry_scheduled(temp_txn_id, reversal_id, delay_seconds) do Logger.info( "Reversal retry scheduled: temp_txn=#{temp_txn_id}, reversal=#{reversal_id}, delay=#{delay_seconds}s" ) :telemetry.execute( [:da_product_app, :reversal, :retry_scheduled], %{count: 1, delay_seconds: delay_seconds}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id} ) end @doc """ Emit when max retries exceeded and transaction marked for manual review. ## Measurements - `:count` - Always 1 - `:retry_count` - Total retry attempts made ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID (may be nil) - `:reason` - Last failure reason """ @spec emit_max_retries_exceeded(integer(), integer() | nil, non_neg_integer(), String.t()) :: :ok def emit_max_retries_exceeded(temp_txn_id, reversal_id, retry_count, reason) do Logger.error( "Max retries exceeded: temp_txn=#{temp_txn_id}, reversal=#{inspect(reversal_id)}, retries=#{retry_count}, reason=#{reason}" ) :telemetry.execute( [:da_product_app, :reversal, :max_retries_exceeded], %{count: 1}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id, reason: reason, retry_count: retry_count} ) end @doc """ Emit cleanup worker cycle start. ## Measurements - `:count` - Always 1 ## Metadata - `:worker` - Worker identifier (e.g., "ReversalCleanupWorker") """ @spec emit_cleanup_cycle_start(String.t()) :: :ok def emit_cleanup_cycle_start(worker) do Logger.debug("Cleanup cycle starting: worker=#{worker}") :telemetry.execute( [:da_product_app, :reversal, :cleanup_cycle_start], %{count: 1}, %{worker: worker} ) end @doc """ Emit cleanup worker cycle completion. ## Measurements - `:count` - Always 1 - `:duration_ms` - Cycle duration in milliseconds - `:candidates_found` - Number of stuck transactions found - `:reversals_created` - Number of reversals successfully created - `:errors` - Number of errors encountered ## Metadata - `:worker` - Worker identifier """ @spec emit_cleanup_cycle_complete(String.t(), map()) :: :ok def emit_cleanup_cycle_complete(worker, stats) do Logger.info("Cleanup cycle complete: worker=#{worker}, stats=#{inspect(stats)}") :telemetry.execute( [:da_product_app, :reversal, :cleanup_cycle_complete], %{ count: 1, duration_ms: Map.get(stats, :duration_ms, 0), candidates_found: Map.get(stats, :total_candidates, 0), reversals_created: Map.get(stats, :reversals_created, 0), errors: Map.get(stats, :errors, 0) }, %{worker: worker, stats: stats} ) end @doc """ Emit connection loss event. ## Measurements - `:count` - Always 1 - `:affected_count` - Number of transactions affected ## Metadata - `:connection_id` - Connection identifier - `:reason` - Reason for connection loss """ @spec emit_connection_loss(String.t(), term(), non_neg_integer()) :: :ok def emit_connection_loss(connection_id, reason, affected_count) do Logger.error( "Connection lost: connection=#{connection_id}, affected=#{affected_count}, reason=#{inspect(reason)}" ) :telemetry.execute( [:da_product_app, :reversal, :connection_loss], %{count: 1, affected_count: affected_count}, %{connection_id: connection_id, reason: inspect(reason)} ) end @doc """ Emit database operation failure during reversal processing. ## Measurements - `:count` - Always 1 ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:operation` - Failed operation (e.g., "finalize", "create_reversal") - `:reason` - Failure reason """ @spec emit_db_operation_failure(integer(), String.t(), term()) :: :ok def emit_db_operation_failure(temp_txn_id, operation, reason) do Logger.error( "DB operation failed: temp_txn=#{temp_txn_id}, operation=#{operation}, reason=#{inspect(reason)}" ) :telemetry.execute( [:da_product_app, :reversal, :db_operation_failure], %{count: 1}, %{temp_txn_id: temp_txn_id, operation: operation, reason: inspect(reason)} ) end @doc """ Emit startup cleanup initiated. ## Measurements - `:count` - Always 1 - `:candidates_found` - Number of old transactions found ## Metadata - `:age_threshold_minutes` - Cleanup threshold in minutes """ @spec emit_startup_cleanup_initiated(non_neg_integer(), non_neg_integer()) :: :ok def emit_startup_cleanup_initiated(candidates_found, age_threshold_minutes) do Logger.info( "Startup cleanup initiated: found=#{candidates_found} transactions, threshold=#{age_threshold_minutes}min" ) :telemetry.execute( [:da_product_app, :reversal, :startup_cleanup_initiated], %{count: 1, candidates_found: candidates_found}, %{age_threshold_minutes: age_threshold_minutes} ) end @doc """ Emit Phase 3 packet generation performance metrics. ## Measurements - `:count` - Always 1 - `:generation_time_ms` - Time to generate MTI 0400 packet - `:packet_size_bytes` - Size of generated packet ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:packet_type` - Type of packet generated (MTI 0400) - `:field_count` - Number of fields in packet """ @spec emit_packet_generation_metrics(integer(), non_neg_integer(), non_neg_integer(), non_neg_integer()) :: :ok def emit_packet_generation_metrics(temp_txn_id, generation_time_ms, packet_size_bytes, field_count) do Logger.debug( "Packet generated: temp_txn=#{temp_txn_id}, time=#{generation_time_ms}ms, size=#{packet_size_bytes}bytes, fields=#{field_count}" ) :telemetry.execute( [:da_product_app, :reversal, :packet_generation], %{count: 1, generation_time_ms: generation_time_ms, packet_size_bytes: packet_size_bytes}, %{temp_txn_id: temp_txn_id, packet_type: "MTI_0400", field_count: field_count} ) end @doc """ Emit Phase 3 network transmission metrics. ## Measurements - `:count` - Always 1 - `:transmission_time_ms` - Time for upstream transmission - `:response_time_ms` - Time to receive MTI 0410 response ## Metadata - `:temp_txn_id` - Temp transaction database ID - `:reversal_id` - Reversal record ID - `:network_type` - Network type (YSP, VISA, etc.) - `:response_code` - Response code from upstream """ @spec emit_network_transmission_metrics(integer(), integer(), non_neg_integer(), non_neg_integer(), String.t(), String.t()) :: :ok def emit_network_transmission_metrics(temp_txn_id, reversal_id, transmission_time_ms, response_time_ms, network_type, response_code) do Logger.info( "Network transmission: temp_txn=#{temp_txn_id}, reversal=#{reversal_id}, tx_time=#{transmission_time_ms}ms, resp_time=#{response_time_ms}ms, network=#{network_type}, code=#{response_code}" ) :telemetry.execute( [:da_product_app, :reversal, :network_transmission], %{count: 1, transmission_time_ms: transmission_time_ms, response_time_ms: response_time_ms}, %{temp_txn_id: temp_txn_id, reversal_id: reversal_id, network_type: network_type, response_code: response_code} ) end @doc """ Emit STAN/batch service operation metrics. ## Measurements - `:count` - Always 1 - `:operation_time_ms` - Time for database operation - `:stan_number` - Assigned STAN number - `:batch_number` - Assigned batch number ## Metadata - `:terminal_id` - Terminal ID - `:operation_type` - Type of operation (get_stan, get_batch, get_both) """ @spec emit_stan_batch_operation_metrics(String.t(), non_neg_integer(), String.t(), String.t(), String.t()) :: :ok def emit_stan_batch_operation_metrics(terminal_id, operation_time_ms, stan_number, batch_number, operation_type) do Logger.debug( "STAN/Batch operation: terminal=#{terminal_id}, time=#{operation_time_ms}ms, stan=#{stan_number}, batch=#{batch_number}, op=#{operation_type}" ) :telemetry.execute( [:da_product_app, :reversal, :stan_batch_operation], %{count: 1, operation_time_ms: operation_time_ms, stan_number: String.to_integer(stan_number), batch_number: String.to_integer(batch_number)}, %{terminal_id: terminal_id, operation_type: operation_type} ) end @doc """ Get list of all reversal telemetry event names. Useful for attaching telemetry handlers. """ @spec event_names :: [list(atom())] def event_names do [ [:da_product_app, :reversal, :attempt], [:da_product_app, :reversal, :success], [:da_product_app, :reversal, :failure], [:da_product_app, :reversal, :stuck_transaction], [:da_product_app, :reversal, :retry_scheduled], [:da_product_app, :reversal, :max_retries_exceeded], [:da_product_app, :reversal, :cleanup_cycle_start], [:da_product_app, :reversal, :cleanup_cycle_complete], [:da_product_app, :reversal, :connection_loss], [:da_product_app, :reversal, :db_operation_failure], [:da_product_app, :reversal, :startup_cleanup_initiated] ] end end