defmodule Contex.Dataset do @moduledoc """ `Dataset` is a simple wrapper around a datasource for plotting charts. Dataset marshalls a couple of different data structures into a consistent form for consumption by the chart plotting functions. It allows a list of maps, list of lists or a list of tuples to be treated the same. The most sensible way to work with a dataset is to provide column headers - it makes code elsewhere readable. When the provided data is a list of maps, headers are inferred from the map keys. If you don't want to, you can also refer to columns by index. Dataset provides a few convenience functions for calculating data extents for a column, extracting unique values from columns, calculating combined extents for multiple columns (handy when plotting bar charts) and guessing column type (handy when determining whether to use a `Contex.TimeScale` or a `Contex.ContinuousLinearScale`). Datasets can be created from a list of maps: iex> data = [ ...> %{x: 0.0, y: 0.0, category: "Hippo"}, ...> %{x: 0.2, y: 0.3, category: "Rabbit"} ...> ] # Wherever your data comes from (e.g. could be straight from Ecto) ...> dataset = Dataset.new(data) %Contex.Dataset{ data: [ %{category: "Hippo", x: 0.0, y: 0.0}, %{category: "Rabbit", x: 0.2, y: 0.3} ], headers: nil, title: nil } iex> Dataset.column_names(dataset) [:category, :x, :y] # Note ordering of column names from map data is not guaranteed or from a list of tuples (or lists): iex> data = [ ...> {0.0, 0.0, "Hippo"}, ...> {0.5, 0.3, "Turtle"}, ...> {0.4, 0.3, "Turtle"}, ...> {0.2, 0.3, "Rabbit"} ...> ] ...> dataset = Dataset.new(data, ["x", "y", "category"]) # Attach descriptive headers iex> Dataset.column_names(dataset) ["x", "y", "category"] ...> Dataset.column_extents(dataset, "x") # Get extents for a named column {0.0, 0.5} iex> Dataset.column_index(dataset, "x") # Get index of column by name 0 iex> category_col = Dataset.column_name(dataset, 2) # Get name of column by index "category" iex> Enum.map(dataset.data, fn row -> # Enumerate values in a column ...> accessor = Dataset.value_fn(dataset, category_col) ...> accessor.(row) ...> end) ["Hippo", "Turtle", "Turtle", "Rabbit"] iex> Dataset.unique_values(dataset, "category") # Extract unique values for legends etc. ["Hippo", "Turtle", "Rabbit"] Dataset gives facilities to map between names and column indexes. Where headers are not supplied (either directly or via map keys), the column index is treated as the column name internally. Data values are retrieved by column name using accessor functions, in order to avoid expensive mappings in tight loops. **Note** There are very few validation checks when a dataset is created (for example, to checks that number of headers supplied matches) the size of each array or tuple in the data. If there are any issues finding a value, nil is returned. """ alias __MODULE__ alias Contex.Utils defstruct [:headers, :data, :title, :meta] @type column_name() :: String.t() | integer() | atom() @type column_type() :: :datetime | :number | :string | :unknown | nil @type row() :: list() | tuple() | map() @type t() :: %__MODULE__{} @doc """ Creates a new Dataset wrapper around some data. Data is expected to be a list of tuples of the same size, a list of lists of same size, or a list of maps with the same keys. Columns in map data are accessed by key. For lists of lists or tuples, if no headers are specified, columns are access by index. """ @spec new(list(row())) :: Contex.Dataset.t() def new(data) when is_list(data) do %Dataset{headers: nil, data: data} end @doc """ Creates a new Dataset wrapper around some data with headers. Data is expected to be a list of tuples of the same size or list of lists of same size. Headers provided with a list of maps are ignored; column names from map data are inferred from the maps' keys. """ @spec new(list(row()), list(String.t())) :: Contex.Dataset.t() def new(data, headers) when is_list(data) and is_list(headers) do %Dataset{headers: headers, data: data} end @doc """ Optionally sets a title. Not really used at the moment to be honest, but seemed like a good idea at the time. Might come in handy when overlaying plots. """ @spec title(Contex.Dataset.t(), String.t()) :: Contex.Dataset.t() def title(%Dataset{} = dataset, title) do %{dataset | title: title} end @doc """ Optionally sets some metadata. Allows you to attach whatever you want to the dataset for later retrieval - e.g. information about where the data came from. """ @spec meta(Contex.Dataset.t(), String.t()) :: Contex.Dataset.t() def meta(%Dataset{} = dataset, meta) do %{dataset | meta: meta} end @doc """ Looks up the index for a given column name. Returns nil if not found. """ @spec column_index(Contex.Dataset.t(), column_name()) :: nil | column_name() def column_index(%Dataset{data: [first_row | _rest]}, column_name) when is_map(first_row) do if Map.has_key?(first_row, column_name) do column_name else nil end end def column_index(%Dataset{headers: headers}, column_name) when is_list(headers) do Enum.find_index(headers, fn col -> col == column_name end) end def column_index(_, column_name) when is_integer(column_name) do column_name end def column_index(_, _), do: nil # TODO: Should this be column_ids - they are essentially the internal column names @doc """ Returns a list of the names of all of the columns in the dataset data (irrespective of whether the column names are mapped to plot elements). """ @spec column_names(Contex.Dataset.t()) :: list(column_name()) def column_names(%Dataset{headers: headers}) when not is_nil(headers), do: headers def column_names(%Dataset{data: [first_row | _]}) when is_map(first_row) do Map.keys(first_row) end def column_names(%Dataset{data: [first_row | _]}) when is_tuple(first_row) do max = tuple_size(first_row) - 1 0..max |> Enum.into([]) end def column_names(%Dataset{data: [first_row | _]}) when is_list(first_row) do max = length(first_row) - 1 0..max |> Enum.into([]) end def column_names(%Dataset{headers: headers}), do: headers @doc """ Looks up the column name for a given index. If there are no headers, or the index is outside the range of the headers the requested index is returned. """ @spec column_name(Contex.Dataset.t(), integer() | any) :: column_name() def column_name(%Dataset{headers: headers} = _dataset, column_index) when is_list(headers) and is_integer(column_index) and column_index < length(headers) do # Maybe drop the length guard above and have it throw an exception Enum.at(headers, column_index) end def column_name(_, column_index), do: column_index @doc """ Returns a function that retrieves the value for a given column in given row, accessed by the column name. ## Examples iex> data = [ ...> %{x: 0.0, y: 0.0, category: "Hippo"}, ...> %{x: 0.2, y: 0.3, category: "Rabbit"} ...> ] iex> dataset = Dataset.new(data) iex> category_accessor = Dataset.value_fn(dataset, :category) iex> category_accessor.(hd(data)) "Hippo" """ @spec value_fn(Contex.Dataset.t(), column_name()) :: (row() -> any) def value_fn(%Dataset{data: [first_row | _]}, column_name) when is_map(first_row) and is_binary(column_name) do fn row -> row[column_name] end end def value_fn(%Dataset{data: [first_row | _]}, column_name) when is_map(first_row) and is_atom(column_name) do fn row -> row[column_name] end end def value_fn(%Dataset{data: [first_row | _]} = dataset, column_name) when is_list(first_row) do column_index = column_index(dataset, column_name) fn row -> Enum.at(row, column_index, nil) end end def value_fn(%Dataset{data: [first_row | _]} = dataset, column_name) when is_tuple(first_row) do column_index = column_index(dataset, column_name) if column_index < tuple_size(first_row) do fn row -> elem(row, column_index) end else fn _ -> nil end end end def value_fn(_dataset, _column_name), do: fn _ -> nil end @doc """ Calculates the min and max value in the specified column """ @spec column_extents(Contex.Dataset.t(), column_name()) :: {any, any} def column_extents(%Dataset{data: data} = dataset, column_name) do accessor = Dataset.value_fn(dataset, column_name) Enum.reduce(data, {nil, nil}, fn row, {min, max} -> val = accessor.(row) {Utils.safe_min(val, min), Utils.safe_max(val, max)} end) end @doc """ Tries to guess the data type for a column based on contained data. Looks through the rows and returns the first match it can find. """ @spec guess_column_type(Contex.Dataset.t(), column_name()) :: column_type() def guess_column_type(%Dataset{data: data} = dataset, column_name) do accessor = Dataset.value_fn(dataset, column_name) Enum.reduce_while(data, nil, fn row, _result -> val = accessor.(row) case evaluate_type(val) do {:ok, type} -> {:halt, type} _ -> {:cont, nil} end end) end defp evaluate_type(%DateTime{}), do: {:ok, :datetime} defp evaluate_type(%NaiveDateTime{}), do: {:ok, :datetime} defp evaluate_type(v) when is_number(v), do: {:ok, :number} defp evaluate_type(v) when is_binary(v), do: {:ok, :string} defp evaluate_type(_), do: {:unknown} @doc """ Calculates the data extents for the sum of the columns supplied. It is the equivalent of evaluating the extents of a calculated row where the calculating is the sum of the values identified by column_names. """ @spec combined_column_extents(Contex.Dataset.t(), list(column_name())) :: {any(), any()} def combined_column_extents(%Dataset{data: data} = dataset, column_names) do accessors = Enum.map(column_names, fn column_name -> Dataset.value_fn(dataset, column_name) end) Enum.reduce(data, {nil, nil}, fn row, {min, max} -> val = sum_row_values(row, accessors) {Utils.safe_min(val, min), Utils.safe_max(val, max)} end) end defp sum_row_values(row, accessors) do Enum.reduce(accessors, 0, fn accessor, acc -> val = accessor.(row) Utils.safe_add(acc, val) end) end @doc """ Extracts a list of unique values in the given column. Note that the unique values will maintain order of first detection in the data. """ @spec unique_values(Contex.Dataset.t(), String.t() | integer()) :: [any] def unique_values(%Dataset{data: data} = dataset, column_name) do accessor = Dataset.value_fn(dataset, column_name) {result, _found} = Enum.reduce(data, {[], MapSet.new()}, fn row, {result, found} -> val = accessor.(row) case MapSet.member?(found, val) do true -> {result, found} _ -> {[val | result], MapSet.put(found, val)} end end) # Maintain order they are found in Enum.reverse(result) end end