defmodule DaProductApp.Telemetry.PrometheusExporter do @moduledoc """ Prometheus metrics exporter for CloudLayer. Exposes HTTP, Ecto, BEAM VM, application, payment and external-provider (Alipay/AANI) metrics in Prometheus text format for scraping. Follows the same hand-rolled exporter pattern used by Mercury (`mercury_device_middlelayer`'s `DaProductApp.Telemetry.PrometheusExporter`) rather than a hex Prometheus client library — CloudLayer already ships every dependency this needs (`Plug`, `:telemetry`, `telemetry_metrics`). ## Usage # router.ex get "/monitoring/metrics", DaProductApp.Telemetry.PrometheusExporter, :metrics """ import Plug.Conn require Logger alias DaProductApp.Telemetry.MetricsCollector @content_type "text/plain; version=0.0.4; charset=utf-8" @doc "Plug interface for the metrics endpoint." def init(opts), do: opts def call(conn, _opts) do metrics(conn, %{}) end @doc "Phoenix controller action for the metrics endpoint." def metrics(conn, _params) do try do body = generate_prometheus_metrics() conn |> put_resp_content_type(@content_type, nil) |> send_resp(200, body) rescue error -> Logger.error("Failed to generate Prometheus metrics: #{inspect(error)}") conn |> put_resp_content_type("text/plain") |> send_resp(500, "# Error generating metrics\n") end end @doc "Generate the Prometheus-formatted metrics string." def generate_prometheus_metrics do snapshot = MetricsCollector.get_metrics_snapshot() timestamp = DateTime.utc_now() |> DateTime.to_unix(:millisecond) [ app_section(snapshot.app, timestamp), payment_section(snapshot.payment, timestamp), external_api_section(snapshot.external_api, timestamp), http_section(snapshot.http, timestamp), db_section(snapshot.db, timestamp), beam_vm_section(snapshot.beam_vm, timestamp), merchant_section(snapshot.merchants, timestamp) ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # Application info # --------------------------------------------------------------------------- defp app_section(app, timestamp) do [ """ # HELP application_info CloudLayer application information # TYPE application_info gauge # HELP application_build_info CloudLayer application build information # TYPE application_build_info gauge # HELP application_uptime_seconds Time since the application supervisor started # TYPE application_uptime_seconds gauge # HELP application_start_time_seconds Unix timestamp (seconds) when this instance started, constant for its lifetime # TYPE application_start_time_seconds gauge # HELP application_config_loaded Whether required configuration (main repo, ShukriaMms repo, endpoint HTTP) is present # TYPE application_config_loaded gauge # HELP application_health Whether critical supervised processes are alive # TYPE application_health gauge # HELP application_ready Whether the instance is healthy, configured, and global settings have loaded # TYPE application_ready gauge """, "application_info{version=\"#{app.version}\"} 1 #{timestamp}", "application_build_info{version=\"#{app.version}\"} 1 #{timestamp}", "application_uptime_seconds #{app.uptime_seconds} #{timestamp}", "application_start_time_seconds #{app.start_time_unix_seconds} #{timestamp}", "application_config_loaded #{bool_to_gauge(config_loaded?())} #{timestamp}", "application_health #{bool_to_gauge(core_processes_alive?())} #{timestamp}", "application_ready #{bool_to_gauge(ready?())} #{timestamp}" ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # Application health / readiness / config checks # # All checks below are local, synchronous, and make no DB or external API # calls - safe to run on every /monitoring/metrics scrape. # --------------------------------------------------------------------------- @critical_processes [ DaProductApp.Repo, DaProductApp.Repos.ShukriaMmsRepo, DaProductAppWeb.Endpoint, DaProductApp.Telemetry.MetricsCollector ] defp core_processes_alive? do Enum.all?(@critical_processes, fn name -> case Process.whereis(name) do nil -> false pid -> Process.alive?(pid) end end) end defp config_loaded? do endpoint_config = Application.get_env(:da_product_app, DaProductAppWeb.Endpoint) || [] not is_nil(Application.get_env(:da_product_app, DaProductApp.Repo)) and not is_nil(Application.get_env(:da_product_app, DaProductApp.Repos.ShukriaMmsRepo)) and match?([_ | _], Keyword.get(endpoint_config, :http)) end defp ready? do core_processes_alive?() and config_loaded?() and DaProductApp.GlobalSettings.loaded?() end defp bool_to_gauge(true), do: 1 defp bool_to_gauge(false), do: 0 # --------------------------------------------------------------------------- # Payment metrics # --------------------------------------------------------------------------- defp payment_section(payment, timestamp) do [ """ # HELP payment_requests_total Total payment/QR provider flows completed # TYPE payment_requests_total counter # HELP payment_duration_seconds Time taken for a payment/QR provider flow to complete # TYPE payment_duration_seconds histogram """, generate_labeled_counters( "payment_requests_total", payment.totals, [:provider, :transaction_type, :result], timestamp ), generate_histogram( "payment_duration_seconds", payment.duration_seconds, [:provider, :transaction_type], timestamp ), """ # HELP payment_started_total Total payment/QR provider flows started (purchase, cancel, refund) # TYPE payment_started_total counter # HELP payment_finalized_total Total transactions that genuinely transitioned from pending to a final status (success/failed), confirmed customer payment completion - currently Alipay webhook only # TYPE payment_finalized_total counter # HELP payment_failure_reason_total Total payment failures by fixed low-cardinality reason # TYPE payment_failure_reason_total counter # HELP payment_amount_total Cumulative purchase/refund amount (major currency units) with a terminal result, NOT confirmed revenue - purchase success means QR/provider initiation succeeded, not settlement # TYPE payment_amount_total counter # HELP payment_amount_success_total Cumulative purchase/refund amount (major currency units) where QR/provider initiation succeeded - NOT confirmed revenue # TYPE payment_amount_success_total counter # HELP payment_amount_failed_total Cumulative purchase/refund amount (major currency units) where the flow failed # TYPE payment_amount_failed_total counter """, generate_labeled_counters( "payment_started_total", payment.started_totals, [:provider, :transaction_type], timestamp ), generate_labeled_counters( "payment_finalized_total", payment.finalized_totals, [:provider, :result, :transaction_type], timestamp ), generate_labeled_counters( "payment_failure_reason_total", payment.failure_reason_totals, [:provider, :transaction_type, :reason], timestamp ), generate_labeled_counters( "payment_amount_total", payment.amount_totals, [:provider, :transaction_type, :currency], timestamp ), generate_labeled_counters( "payment_amount_success_total", payment.amount_success_totals, [:provider, :transaction_type, :currency], timestamp ), generate_labeled_counters( "payment_amount_failed_total", payment.amount_failed_totals, [:provider, :transaction_type, :currency], timestamp ) ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # External provider (Alipay/AANI) metrics # --------------------------------------------------------------------------- defp external_api_section(external_api, timestamp) do [ """ # HELP external_api_requests_total Total calls made to external provider APIs (Alipay/AANI) # TYPE external_api_requests_total counter # HELP external_api_duration_seconds Time taken for an external provider API call to complete # TYPE external_api_duration_seconds histogram # HELP external_api_timeout_total Total external provider API calls that timed out (Alipay/AANI) # TYPE external_api_timeout_total counter # HELP external_api_error_reason_total Total failed external provider API calls by fixed low-cardinality reason # TYPE external_api_error_reason_total counter """, generate_labeled_counters( "external_api_requests_total", external_api.totals, [:provider, :operation, :result], timestamp ), generate_histogram( "external_api_duration_seconds", external_api.duration_seconds, [:provider, :operation], timestamp ), generate_labeled_counters( "external_api_timeout_total", external_api.timeout_totals, [:provider, :operation], timestamp ), generate_labeled_counters( "external_api_error_reason_total", external_api.error_reason_totals, [:provider, :operation, :reason], timestamp ) ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # HTTP metrics # --------------------------------------------------------------------------- defp http_section(http, timestamp) do [ """ # HELP http_requests_total Total HTTP requests dispatched, by route/method/status # TYPE http_requests_total counter # HELP http_request_duration_seconds HTTP request duration, by route/method # TYPE http_request_duration_seconds histogram # HELP http_requests_active Number of HTTP requests currently being processed # TYPE http_requests_active gauge # HELP http_request_size_bytes HTTP request body size in bytes, by route/method # TYPE http_request_size_bytes histogram # HELP http_response_size_bytes HTTP response body size in bytes, by route/method # TYPE http_response_size_bytes histogram """, generate_labeled_counters( "http_requests_total", http.totals, [:route, :method, :status], timestamp ), generate_histogram( "http_request_duration_seconds", http.duration_seconds, [:route, :method], timestamp ), "http_requests_active #{http.active_requests} #{timestamp}", generate_histogram( "http_request_size_bytes", http.request_size_bytes, [:route, :method], timestamp ), generate_histogram( "http_response_size_bytes", http.response_size_bytes, [:route, :method], timestamp ) ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # Ecto / DB metrics # --------------------------------------------------------------------------- defp db_section(db, timestamp) do [ """ # HELP db_queries_total Total Ecto queries executed, by repo # TYPE db_queries_total counter # HELP db_query_errors_total Total Ecto queries that returned an error, by repo # TYPE db_query_errors_total counter # HELP db_query_duration_seconds Ecto query duration, by repo # TYPE db_query_duration_seconds histogram # HELP db_slow_queries_total Total Ecto queries at or above the configured slow-query threshold, by repo # TYPE db_slow_queries_total counter # HELP db_pool_size Configured DBConnection pool size, by repo # TYPE db_pool_size gauge # HELP db_pool_available Idle/ready DBConnection pool connections not currently checked out, by repo # TYPE db_pool_available gauge # HELP db_pool_checkout_queue_length Processes currently waiting to check out a DBConnection pool connection, by repo # TYPE db_pool_checkout_queue_length gauge """, generate_labeled_counters("db_queries_total", db.queries_total, [:repo], timestamp), generate_labeled_counters("db_query_errors_total", db.errors_total, [:repo], timestamp), generate_histogram("db_query_duration_seconds", db.duration_seconds, [:repo], timestamp), generate_labeled_counters( "db_slow_queries_total", db.slow_queries_total, [:repo], timestamp ), generate_labeled_counters("db_pool_size", db.pool_size, [:repo], timestamp), generate_labeled_counters("db_pool_available", db.pool_available, [:repo], timestamp), generate_labeled_counters( "db_pool_checkout_queue_length", db.pool_checkout_queue_length, [:repo], timestamp ) ] |> Enum.join("\n") end # --------------------------------------------------------------------------- # BEAM VM metrics # --------------------------------------------------------------------------- defp beam_vm_section(beam, timestamp) do [ """ # HELP beam_memory_total_bytes Total BEAM VM memory usage in bytes # TYPE beam_memory_total_bytes gauge # HELP beam_memory_processes_bytes Memory used by Erlang processes # TYPE beam_memory_processes_bytes gauge # HELP beam_memory_binary_bytes Memory used by binary data # TYPE beam_memory_binary_bytes gauge # HELP beam_memory_ets_bytes Memory used by ETS tables # TYPE beam_memory_ets_bytes gauge # HELP beam_memory_atom_bytes Memory used by atoms # TYPE beam_memory_atom_bytes gauge # HELP beam_process_count Number of currently running Erlang processes # TYPE beam_process_count gauge # HELP beam_process_limit Maximum number of allowed Erlang processes # TYPE beam_process_limit gauge # HELP beam_run_queue_length Total run queue length across all schedulers # TYPE beam_run_queue_length gauge # HELP beam_scheduler_count Number of online schedulers # TYPE beam_scheduler_count gauge # HELP beam_port_count Number of open ports # TYPE beam_port_count gauge # HELP beam_uptime_seconds Node uptime in seconds # TYPE beam_uptime_seconds gauge # HELP beam_scheduler_utilization Scheduler wall-time utilization ratio (0-1) # TYPE beam_scheduler_utilization gauge # HELP beam_gc_total Total VM-wide garbage collections since node start # TYPE beam_gc_total counter # HELP beam_reductions_total Total VM-wide reductions since node start # TYPE beam_reductions_total counter # HELP beam_io_bytes_total Total bytes transferred through ports, by direction, since node start # TYPE beam_io_bytes_total counter # HELP beam_mailbox_queue_length Mailbox length of selected critical named processes # TYPE beam_mailbox_queue_length gauge # HELP beam_mailbox_queue_length_max Largest mailbox length among selected critical named processes at the latest poll (not a maximum since start) # TYPE beam_mailbox_queue_length_max gauge """, "beam_memory_total_bytes #{beam.memory_total_bytes} #{timestamp}", "beam_memory_processes_bytes #{beam.memory_processes_bytes} #{timestamp}", "beam_memory_binary_bytes #{beam.memory_binary_bytes} #{timestamp}", "beam_memory_ets_bytes #{beam.memory_ets_bytes} #{timestamp}", "beam_memory_atom_bytes #{beam.memory_atom_bytes} #{timestamp}", "beam_process_count #{beam.process_count} #{timestamp}", "beam_process_limit #{beam.process_limit} #{timestamp}", "beam_run_queue_length #{beam.run_queue_length} #{timestamp}", "beam_scheduler_count #{beam.scheduler_count} #{timestamp}", "beam_port_count #{beam.port_count} #{timestamp}", "beam_uptime_seconds #{beam.uptime_seconds} #{timestamp}", generate_scheduler_utilization(beam.scheduler_utilization, timestamp), "beam_gc_total #{beam.gc_total} #{timestamp}", "beam_reductions_total #{beam.reductions_total} #{timestamp}", "beam_io_bytes_total{direction=\"input\"} #{beam.io_input_bytes} #{timestamp}", "beam_io_bytes_total{direction=\"output\"} #{beam.io_output_bytes} #{timestamp}", generate_labeled_counters( "beam_mailbox_queue_length", beam.mailbox_lengths, [:name], timestamp ), generate_mailbox_max(beam.mailbox_max, timestamp) ] |> Enum.join("\n") end # beam_mailbox_queue_length_max has no labels and must be omitted entirely # (not published as a guessed 0) when no allowlisted process could be # sampled this poll. defp generate_mailbox_max(nil, _timestamp), do: "" defp generate_mailbox_max(max, timestamp), do: "beam_mailbox_queue_length_max #{max} #{timestamp}" defp generate_scheduler_utilization(utilization, _timestamp) when map_size(utilization) == 0, do: "" defp generate_scheduler_utilization(utilization, timestamp) do Enum.map_join(utilization, "\n", fn {scheduler_id, util} -> "beam_scheduler_utilization{scheduler_id=\"#{scheduler_id}\"} #{util} #{timestamp}" end) end # --------------------------------------------------------------------------- # Merchant metrics (cached gauges, polled every 5 minutes by MetricsCollector # - never computed from the database on the scrape path itself) # --------------------------------------------------------------------------- defp merchant_section(merchants, timestamp) do [ """ # HELP merchant_total Total number of merchants (groups), polled periodically # TYPE merchant_total gauge # HELP merchant_active Number of merchants (groups) with status="active", polled periodically # TYPE merchant_active gauge """, optional_gauge_line("merchant_total", merchants.total, timestamp), optional_gauge_line("merchant_active", merchants.active, timestamp) ] |> Enum.join("\n") end # Omit the sample line entirely (not a guessed 0) when no successful poll # has happened yet. defp optional_gauge_line(_metric_name, nil, _timestamp), do: "" defp optional_gauge_line(metric_name, value, timestamp), do: "#{metric_name} #{value} #{timestamp}" # --------------------------------------------------------------------------- # Generic formatting helpers # --------------------------------------------------------------------------- # counters keyed either by a bare value (single label) or a tuple (multiple labels) defp generate_labeled_counters(metric_name, totals, label_names, timestamp) do Enum.map_join(totals, "\n", fn {key, count} -> label_values = case key do tuple when is_tuple(tuple) -> Tuple.to_list(tuple) value -> [value] end labels = build_labels(label_names, label_values) "#{metric_name}{#{labels}} #{count} #{timestamp}" end) end # histograms keyed either by a bare value (single label) or a tuple (multiple labels) defp generate_histogram(metric_name, histograms, label_names, timestamp) do Enum.map_join(histograms, "\n", fn {key, summary} -> label_values = case key do tuple when is_tuple(tuple) -> Tuple.to_list(tuple) value -> [value] end labels = build_labels(label_names, label_values) generate_histogram_line(metric_name, labels, summary, timestamp) end) end defp generate_histogram_line(metric_name, "", summary, timestamp) do bucket_lines = Enum.map_join(summary.buckets, "\n", fn {le, count} -> "#{metric_name}_bucket{le=\"#{bucket_label(le)}\"} #{count} #{timestamp}" end) [ bucket_lines, "#{metric_name}_sum #{summary.sum} #{timestamp}", "#{metric_name}_count #{summary.count} #{timestamp}" ] |> Enum.join("\n") end defp generate_histogram_line(metric_name, labels, summary, timestamp) do bucket_lines = Enum.map_join(summary.buckets, "\n", fn {le, count} -> "#{metric_name}_bucket{#{labels},le=\"#{bucket_label(le)}\"} #{count} #{timestamp}" end) [ bucket_lines, "#{metric_name}_sum{#{labels}} #{summary.sum} #{timestamp}", "#{metric_name}_count{#{labels}} #{summary.count} #{timestamp}" ] |> Enum.join("\n") end defp bucket_label(:infinity), do: "+Inf" defp bucket_label(le), do: to_string(le) defp build_labels(label_names, label_values) do label_names |> Enum.zip(label_values) |> Enum.map_join(",", fn {name, value} -> "#{name}=\"#{value}\"" end) end end