defmodule Mix.Tasks.LoadInterchangeRates do @shortdoc "Load interchange rates from a YSP-supplied CSV file" @moduledoc """ Bulk-loads interchange rates from a YSP-supplied CSV into the `interchange_rates` database table. ## Usage mix load_interchange_rates --file=path/to/interchange.csv ## Options * `--file` — (required) path to the interchange rates CSV file from YSP. * `--dry-run` — validate and count rows without writing to the database. * `--loaded-by` — integer user ID to stamp on each row (default: 0). ## Expected CSV Format The file must be UTF-8 encoded with a header row. Required columns: scheme_name, card_type_code, interchange_percentage, effective_date Optional columns: card_category, interchange_fixed_fee, expiry_date, notes `effective_date` and `expiry_date` must be in `YYYY-MM-DD` format. `interchange_percentage` must be a decimal between 0 and 100. ## Behavior - Each row inserts as a new versioned rate record. - Duplicate rows (same scheme + card_type_code + effective_date) are ignored via ON CONFLICT DO NOTHING — making the task safe to re-run. - The `InterchangeRate.rate_query/3` always selects the most recent active rate for a (scheme, card_type, date), so adding a new row for the same scheme/card_type with a newer effective_date is how you update rates. ## Example mix load_interchange_rates --file=/data/ysp_interchange_20260306.csv mix load_interchange_rates --file=/data/ysp_interchange_20260306.csv --dry-run """ use Mix.Task require Logger alias PlatformCore.Repo alias SettlementCore.InterchangeRate @chunk_size 200 @impl Mix.Task def run(argv) do {opts, _rest, _invalid} = OptionParser.parse(argv, strict: [file: :string, dry_run: :boolean, loaded_by: :integer] ) file_path = Keyword.get(opts, :file) dry_run = Keyword.get(opts, :dry_run, false) loaded_by = Keyword.get(opts, :loaded_by, 0) unless file_path do Mix.raise("--file is required. Usage: mix load_interchange_rates --file=path/to/interchange.csv") end unless File.exists?(file_path) do Mix.raise("File not found: #{file_path}") end Mix.Task.run("app.start") Mix.shell().info("Loading interchange rates from: #{file_path}") if dry_run, do: Mix.shell().info("DRY RUN — no records will be written.") {valid_rows, error_count} = parse_csv(file_path, loaded_by) Mix.shell().info("Parsed #{length(valid_rows)} valid rows, #{error_count} skipped.") if dry_run do Mix.shell().info("Dry run complete. #{length(valid_rows)} rows would be inserted.") else inserted = bulk_insert(valid_rows) Mix.shell().info("Done. Inserted #{inserted} new interchange rate records (duplicates skipped).") end end # ─── CSV Parsing ───────────────────────────────────────────────────────────── defp parse_csv(file_path, loaded_by) do now = NaiveDateTime.utc_now() |> NaiveDateTime.truncate(:second) lines = File.read!(file_path) |> String.split(["\r\n", "\n"], trim: true) case lines do [] -> Mix.shell().info("Warning: empty file.") {[], 0} [_ | _] = all_lines -> [header_line | data_lines] = all_lines headers = header_line |> String.split(",") |> Enum.map(&(String.trim(&1) |> String.downcase())) data_lines |> Enum.with_index(2) |> Enum.reduce({[], 0}, fn {line, line_no}, {valid, errors} -> values = String.split(line, ",") row = headers |> Enum.zip(values) |> Map.new(fn {k, v} -> {k, String.trim(v)} end) case build_row(row, loaded_by, now) do {:ok, attrs} -> {[attrs | valid], errors} {:skip, reason} -> Logger.warning("[LoadInterchangeRates] Line #{line_no} skipped: #{reason}") {valid, errors + 1} end end) |> then(fn {valid, errs} -> {Enum.reverse(valid), errs} end) end end defp build_row(row, loaded_by, loaded_at) do scheme = upcase(get_field(row, ~w(scheme_name scheme))) card_type_code = get_field(row, ~w(card_type_code card_type)) pct_str = get_field(row, ~w(interchange_percentage pct percentage)) fixed_str = get_field(row, ~w(interchange_fixed_fee fixed_fee)) || "0" eff_date_str = get_field(row, ~w(effective_date eff_date)) exp_date_str = get_field(row, ~w(expiry_date exp_date)) category = upcase(get_field(row, ~w(card_category category))) notes = get_field(row, ~w(notes note)) with :ok <- require_field(scheme, "scheme_name"), :ok <- require_field(card_type_code, "card_type_code"), :ok <- require_field(pct_str, "interchange_percentage"), :ok <- require_field(eff_date_str, "effective_date"), {:ok, pct} <- parse_decimal(pct_str, "interchange_percentage"), {:ok, fixed} <- parse_decimal(fixed_str, "interchange_fixed_fee"), {:ok, eff} <- parse_date(eff_date_str, "effective_date") do exp = parse_optional_date(exp_date_str) attrs = %{ scheme_name: scheme, card_type_code: card_type_code, card_category: category, interchange_percentage: pct, interchange_fixed_fee: fixed, effective_date: eff, expiry_date: exp, is_active: true, loaded_at: loaded_at, loaded_by: loaded_by, notes: notes, inserted_at: loaded_at, updated_at: loaded_at } {:ok, attrs} end end defp require_field(nil, field), do: {:skip, "missing #{field}"} defp require_field("", field), do: {:skip, "empty #{field}"} defp require_field(_, _), do: :ok defp parse_decimal(str, field) do case Decimal.parse(str) do {dec, ""} -> {:ok, dec} _ -> {:skip, "invalid decimal for #{field}: '#{str}'"} end end defp parse_date(str, field) do case Date.from_iso8601(str) do {:ok, date} -> {:ok, date} {:error, _} -> {:skip, "invalid date for #{field}: '#{str}' (expected YYYY-MM-DD)"} end end defp parse_optional_date(nil), do: nil defp parse_optional_date(""), do: nil defp parse_optional_date(str) do case Date.from_iso8601(str) do {:ok, date} -> date {:error, _} -> nil end end defp get_field(row, candidates) do Enum.find_value(candidates, fn k -> v = Map.get(row, k) if v && v != "", do: v, else: nil end) end defp upcase(nil), do: nil defp upcase(v), do: String.upcase(v) # ─── Bulk Insert ───────────────────────────────────────────────────────────── defp bulk_insert(rows) do rows |> Enum.chunk_every(@chunk_size) |> Enum.with_index() |> Enum.reduce(0, fn {chunk, idx}, acc -> {count, _} = Repo.insert_all( InterchangeRate, chunk, on_conflict: :nothing, conflict_target: [:scheme_name, :card_type_code, :effective_date] ) inserted_so_far = acc + count Mix.shell().info(" Chunk #{idx + 1}: #{inserted_so_far} total inserted so far...") inserted_so_far end) end end