feat: adds header header normalization
This commit is contained in:
parent
0673684cc1
commit
67072f0c52
2 changed files with 720 additions and 15 deletions
|
|
@ -2,6 +2,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
@moduledoc """
|
||||
Service module for importing members from CSV files.
|
||||
|
||||
require Ash.Query
|
||||
|
||||
This module provides the core API for CSV member import functionality:
|
||||
- `prepare/2` - Parses and validates CSV content, returns import state
|
||||
- `process_chunk/3` - Processes a chunk of rows and creates members
|
||||
|
|
@ -70,6 +72,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
errors: list(Error.t())
|
||||
}
|
||||
|
||||
alias Mv.Membership.Import.CsvParser
|
||||
alias Mv.Membership.Import.HeaderMapper
|
||||
|
||||
@doc """
|
||||
Prepares CSV content for import by parsing, mapping headers, and validating limits.
|
||||
|
||||
|
|
@ -104,12 +109,120 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
"""
|
||||
@spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()}
|
||||
def prepare(file_content, opts \\ []) do
|
||||
# TODO: Implement in Issue #3 (CSV Parsing)
|
||||
# This is a skeleton implementation that will be filled in later
|
||||
_ = {file_content, opts}
|
||||
max_rows = Keyword.get(opts, :max_rows, 1000)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, 200)
|
||||
|
||||
# Placeholder return - will be replaced with actual implementation
|
||||
{:error, "Not yet implemented"}
|
||||
with {:ok, headers, rows} <- CsvParser.parse(file_content),
|
||||
{:ok, custom_fields} <- load_custom_fields(),
|
||||
{:ok, maps, warnings} <- build_header_maps(headers, custom_fields),
|
||||
:ok <- validate_row_count(rows, max_rows) do
|
||||
chunks = chunk_rows(rows, maps, chunk_size)
|
||||
|
||||
{:ok,
|
||||
%{
|
||||
chunks: chunks,
|
||||
column_map: maps.member,
|
||||
custom_field_map: maps.custom,
|
||||
warnings: warnings
|
||||
}}
|
||||
end
|
||||
end
|
||||
|
||||
# Loads all custom fields from the database
|
||||
defp load_custom_fields do
|
||||
custom_fields =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.read!()
|
||||
|
||||
{:ok, custom_fields}
|
||||
rescue
|
||||
e ->
|
||||
{:error, "Failed to load custom fields: #{Exception.message(e)}"}
|
||||
end
|
||||
|
||||
# Builds header maps using HeaderMapper and collects warnings for unknown custom fields
|
||||
defp build_header_maps(headers, custom_fields) do
|
||||
# Convert custom fields to maps with id and name
|
||||
custom_field_maps =
|
||||
Enum.map(custom_fields, fn cf ->
|
||||
%{id: to_string(cf.id), name: cf.name}
|
||||
end)
|
||||
|
||||
case HeaderMapper.build_maps(headers, custom_field_maps) do
|
||||
{:ok, %{member: member_map, custom: custom_map, unknown: unknown}} ->
|
||||
# Build warnings for unknown custom field columns
|
||||
warnings =
|
||||
unknown
|
||||
|> Enum.filter(fn header ->
|
||||
# Check if it could be a custom field (not a known member field)
|
||||
normalized = HeaderMapper.normalize_header(header)
|
||||
# If it's not empty and not a member field, it might be a custom field
|
||||
normalized != "" && not member_field?(normalized)
|
||||
end)
|
||||
|> Enum.map(fn header ->
|
||||
"Unknown column '#{header}' will be ignored. " <>
|
||||
"If this is a custom field, create it in Mila before importing."
|
||||
end)
|
||||
|
||||
{:ok, %{member: member_map, custom: custom_map}, warnings}
|
||||
|
||||
{:error, reason} ->
|
||||
{:error, reason}
|
||||
end
|
||||
end
|
||||
|
||||
# Checks if a normalized header matches a member field
|
||||
# Uses HeaderMapper's internal logic to check if header would map to a member field
|
||||
defp member_field?(normalized) do
|
||||
# Try to build maps with just this header - if it maps to a member field, it's a member field
|
||||
case HeaderMapper.build_maps([normalized], []) do
|
||||
{:ok, %{member: member_map}} ->
|
||||
# If member_map is not empty, it's a member field
|
||||
map_size(member_map) > 0
|
||||
|
||||
_ ->
|
||||
false
|
||||
end
|
||||
end
|
||||
|
||||
# Validates that row count doesn't exceed limit
|
||||
defp validate_row_count(rows, max_rows) do
|
||||
if length(rows) > max_rows do
|
||||
{:error, "CSV file exceeds maximum row limit of #{max_rows} rows"}
|
||||
else
|
||||
:ok
|
||||
end
|
||||
end
|
||||
|
||||
# Chunks rows and converts them to row maps using column maps
|
||||
defp chunk_rows(rows, maps, chunk_size) do
|
||||
rows
|
||||
|> Enum.chunk_every(chunk_size)
|
||||
|> Enum.map(fn chunk ->
|
||||
Enum.map(chunk, fn {line_number, row_values} ->
|
||||
row_map = build_row_map(row_values, maps)
|
||||
{line_number, row_map}
|
||||
end)
|
||||
end)
|
||||
end
|
||||
|
||||
# Builds a row map from raw row values using column maps
|
||||
defp build_row_map(row_values, maps) do
|
||||
member_map =
|
||||
maps.member
|
||||
|> Enum.reduce(%{}, fn {field, index}, acc ->
|
||||
value = Enum.at(row_values, index, "")
|
||||
Map.put(acc, field, value)
|
||||
end)
|
||||
|
||||
custom_map =
|
||||
maps.custom
|
||||
|> Enum.reduce(%{}, fn {custom_field_id, index}, acc ->
|
||||
value = Enum.at(row_values, index, "")
|
||||
Map.put(acc, custom_field_id, value)
|
||||
end)
|
||||
|
||||
%{member: member_map, custom: custom_map}
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
@ -126,8 +239,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
- `chunk_rows_with_lines` - List of tuples `{csv_line_number, row_map}` where:
|
||||
- `csv_line_number` - Physical line number in CSV (1-based)
|
||||
- `row_map` - Map of column names to values
|
||||
- `column_map` - Map of canonical field names (atoms) to column indices
|
||||
- `row_map` - Map with `:member` and `:custom` keys containing field values
|
||||
- `column_map` - Map of canonical field names (atoms) to column indices (for reference)
|
||||
- `custom_field_map` - Map of custom field IDs (strings) to column indices (for reference)
|
||||
- `opts` - Optional keyword list for processing options
|
||||
|
||||
## Returns
|
||||
|
|
@ -137,22 +251,233 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
## Examples
|
||||
|
||||
iex> chunk = [{2, %{"email" => "john@example.com"}}]
|
||||
iex> chunk = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}]
|
||||
iex> column_map = %{email: 0}
|
||||
iex> MemberCSV.process_chunk(chunk, column_map)
|
||||
iex> custom_field_map = %{}
|
||||
iex> MemberCSV.process_chunk(chunk, column_map, custom_field_map)
|
||||
{:ok, %{inserted: 1, failed: 0, errors: []}}
|
||||
"""
|
||||
@spec process_chunk(
|
||||
list({pos_integer(), map()}),
|
||||
%{atom() => non_neg_integer()},
|
||||
%{String.t() => non_neg_integer()},
|
||||
keyword()
|
||||
) :: {:ok, chunk_result()} | {:error, String.t()}
|
||||
def process_chunk(chunk_rows_with_lines, column_map, opts \\ []) do
|
||||
# TODO: Implement in Issue #6 (Persistence)
|
||||
# This is a skeleton implementation that will be filled in later
|
||||
_ = {chunk_rows_with_lines, column_map, opts}
|
||||
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, _opts \\ []) do
|
||||
{inserted, failed, errors} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, []}, fn {line_number, row_map},
|
||||
{acc_inserted, acc_failed, acc_errors} ->
|
||||
case process_row(row_map, line_number) do
|
||||
{:ok, _member} ->
|
||||
{acc_inserted + 1, acc_failed, acc_errors}
|
||||
|
||||
# Placeholder return - will be replaced with actual implementation
|
||||
{:ok, %{inserted: 0, failed: 0, errors: []}}
|
||||
{:error, error} ->
|
||||
{acc_inserted, acc_failed + 1, [error | acc_errors]}
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, %{inserted: inserted, failed: failed, errors: Enum.reverse(errors)}}
|
||||
end
|
||||
|
||||
# Processes a single row and creates member with custom field values
|
||||
defp process_row(%{member: member_attrs, custom: custom_attrs}, line_number) do
|
||||
# Prepare custom field values for Ash
|
||||
custom_field_values = prepare_custom_field_values(custom_attrs)
|
||||
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
member_attrs
|
||||
|> Map.put(:custom_field_values, custom_field_values)
|
||||
|> trim_string_values()
|
||||
|
||||
# Only include custom_field_values if not empty
|
||||
final_attrs =
|
||||
if Enum.empty?(custom_field_values) do
|
||||
Map.delete(member_attrs_with_cf, :custom_field_values)
|
||||
else
|
||||
member_attrs_with_cf
|
||||
end
|
||||
|
||||
case Mv.Membership.create_member(final_attrs) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
|
||||
{:error, %Ash.Error.Invalid{} = error} ->
|
||||
{:error, format_ash_error(error, line_number)}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
end
|
||||
rescue
|
||||
e ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
|
||||
end
|
||||
|
||||
# Prepares custom field values from row map for Ash
|
||||
defp prepare_custom_field_values(custom_attrs) when is_map(custom_attrs) do
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.map(fn {custom_field_id_str, value} ->
|
||||
# Load custom field to get value_type and ensure ID is correct
|
||||
case load_custom_field_by_id(custom_field_id_str) do
|
||||
{:ok, custom_field} ->
|
||||
# Use the actual custom_field.id (UUID) from the database
|
||||
%{
|
||||
"custom_field_id" => to_string(custom_field.id),
|
||||
"value" => format_custom_field_value(value, custom_field.value_type)
|
||||
}
|
||||
|
||||
{:error, _} ->
|
||||
# Skip if custom field not found
|
||||
nil
|
||||
end
|
||||
end)
|
||||
|> Enum.filter(&(&1 != nil))
|
||||
end
|
||||
|
||||
defp prepare_custom_field_values(_), do: []
|
||||
|
||||
# Loads a custom field by ID (string or UUID)
|
||||
defp load_custom_field_by_id(id) when is_binary(id) do
|
||||
require Ash.Query
|
||||
|
||||
try do
|
||||
# Try to parse as UUID first
|
||||
uuid_id =
|
||||
case Ecto.UUID.cast(id) do
|
||||
{:ok, uuid} -> uuid
|
||||
:error -> id
|
||||
end
|
||||
|
||||
custom_field =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.Query.filter(id == ^uuid_id)
|
||||
|> Ash.read_one!()
|
||||
|
||||
{:ok, custom_field}
|
||||
rescue
|
||||
_ -> {:error, :not_found}
|
||||
end
|
||||
end
|
||||
|
||||
defp load_custom_field_by_id(_), do: {:error, :invalid_id}
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
defp format_custom_field_value(value, :string) when is_binary(value) do
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :integer) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value}
|
||||
:error -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean) when is_binary(value) do
|
||||
bool_value =
|
||||
value
|
||||
|> String.trim()
|
||||
|> String.downcase()
|
||||
|> case do
|
||||
"true" -> true
|
||||
"1" -> true
|
||||
"yes" -> true
|
||||
"ja" -> true
|
||||
_ -> false
|
||||
end
|
||||
|
||||
%{"_union_type" => "boolean", "_union_value" => bool_value}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :date) when is_binary(value) do
|
||||
case Date.from_iso8601(String.trim(value)) do
|
||||
{:ok, date} -> %{"_union_type" => "date", "_union_value" => date}
|
||||
{:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :email) when is_binary(value) do
|
||||
%{"_union_type" => "email", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type) when is_binary(value) do
|
||||
# Default to string if type is unknown
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
|
||||
# Trims all string values in member attributes
|
||||
defp trim_string_values(attrs) do
|
||||
Enum.reduce(attrs, %{}, fn {key, value}, acc ->
|
||||
trimmed_value =
|
||||
if is_binary(value) do
|
||||
String.trim(value)
|
||||
else
|
||||
value
|
||||
end
|
||||
|
||||
Map.put(acc, key, trimmed_value)
|
||||
end)
|
||||
end
|
||||
|
||||
# Formats Ash errors into MemberCSV.Error structs
|
||||
defp format_ash_error(%Ash.Error.Invalid{errors: errors}, line_number) do
|
||||
# Try to find email-related errors first (for better error messages)
|
||||
email_error =
|
||||
Enum.find(errors, fn error ->
|
||||
case error do
|
||||
%{field: :email} -> true
|
||||
_ -> false
|
||||
end
|
||||
end)
|
||||
|
||||
case email_error || List.first(errors) do
|
||||
%{field: field, message: message} when is_atom(field) ->
|
||||
%Error{
|
||||
csv_line_number: line_number,
|
||||
field: field,
|
||||
message: format_error_message(message, field)
|
||||
}
|
||||
|
||||
%{message: message} ->
|
||||
%Error{
|
||||
csv_line_number: line_number,
|
||||
field: nil,
|
||||
message: format_error_message(message, nil)
|
||||
}
|
||||
|
||||
_ ->
|
||||
%Error{
|
||||
csv_line_number: line_number,
|
||||
field: nil,
|
||||
message: "Validation failed"
|
||||
}
|
||||
end
|
||||
end
|
||||
|
||||
# Formats error messages, handling common cases like email uniqueness
|
||||
defp format_error_message(message, field) when is_binary(message) do
|
||||
if email_uniqueness_error?(message, field) do
|
||||
"email has already been taken"
|
||||
else
|
||||
message
|
||||
end
|
||||
end
|
||||
|
||||
defp format_error_message(message, _field), do: to_string(message)
|
||||
|
||||
# Checks if error message indicates email uniqueness constraint violation
|
||||
defp email_uniqueness_error?(message, :email) do
|
||||
message_lower = String.downcase(message)
|
||||
|
||||
String.contains?(message_lower, "unique") or
|
||||
String.contains?(message_lower, "constraint") or
|
||||
String.contains?(message_lower, "duplicate") or
|
||||
String.contains?(message_lower, "already been taken") or
|
||||
String.contains?(message_lower, "already exists") or
|
||||
String.contains?(message_lower, "violates unique constraint")
|
||||
end
|
||||
|
||||
defp email_uniqueness_error?(_message, _field), do: false
|
||||
end
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue