feat: import custom fields via CSV
This commit is contained in:
parent
b9dd990f52
commit
3f8797c356
3 changed files with 251 additions and 132 deletions
|
|
@ -63,7 +63,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
chunks: list(list({pos_integer(), map()})),
|
||||
column_map: %{atom() => non_neg_integer()},
|
||||
custom_field_map: %{String.t() => non_neg_integer()},
|
||||
custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}},
|
||||
custom_field_lookup: %{
|
||||
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
|
||||
},
|
||||
warnings: list(String.t())
|
||||
}
|
||||
|
||||
|
|
@ -79,6 +81,11 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
use Gettext, backend: MvWeb.Gettext
|
||||
|
||||
alias Mv.Helpers.SystemActor
|
||||
|
||||
# Import FieldTypes for human-readable type labels
|
||||
alias MvWeb.Translations.FieldTypes
|
||||
|
||||
# Configuration constants
|
||||
@default_max_errors 50
|
||||
@default_chunk_size 200
|
||||
|
|
@ -102,6 +109,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
- `opts` - Optional keyword list:
|
||||
- `:max_rows` - Maximum number of data rows allowed (default: 1000)
|
||||
- `:chunk_size` - Number of rows per chunk (default: 200)
|
||||
- `:actor` - Actor for authorization (default: system actor for systemic operations)
|
||||
|
||||
## Returns
|
||||
|
||||
|
|
@ -120,9 +128,10 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
def prepare(file_content, opts \\ []) do
|
||||
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
|
||||
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
|
||||
|
||||
with {:ok, headers, rows} <- CsvParser.parse(file_content),
|
||||
{:ok, custom_fields} <- load_custom_fields(),
|
||||
{:ok, custom_fields} <- load_custom_fields(actor),
|
||||
{:ok, maps, warnings} <- build_header_maps(headers, custom_fields),
|
||||
:ok <- validate_row_count(rows, max_rows) do
|
||||
chunks = chunk_rows(rows, maps, chunk_size)
|
||||
|
|
@ -142,10 +151,10 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Loads all custom fields from the database
|
||||
defp load_custom_fields do
|
||||
defp load_custom_fields(actor) do
|
||||
custom_fields =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.read!()
|
||||
|> Ash.read!(actor: actor)
|
||||
|
||||
{:ok, custom_fields}
|
||||
rescue
|
||||
|
|
@ -158,7 +167,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
custom_fields
|
||||
|> Enum.reduce(%{}, fn cf, acc ->
|
||||
id_str = to_string(cf.id)
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type})
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type, name: cf.name})
|
||||
end)
|
||||
end
|
||||
|
||||
|
|
@ -508,32 +517,39 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
|
||||
# Prepare custom field values for Ash
|
||||
custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup)
|
||||
case prepare_custom_field_values(custom_attrs, custom_field_lookup) do
|
||||
{:error, validation_errors} ->
|
||||
# Custom field validation errors - return first error
|
||||
first_error = List.first(validation_errors)
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
|
||||
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
trimmed_member_attrs
|
||||
|> Map.put(:custom_field_values, custom_field_values)
|
||||
{:ok, custom_field_values} ->
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
trimmed_member_attrs
|
||||
|> Map.put(:custom_field_values, custom_field_values)
|
||||
|
||||
# Only include custom_field_values if not empty
|
||||
final_attrs =
|
||||
if Enum.empty?(custom_field_values) do
|
||||
Map.delete(member_attrs_with_cf, :custom_field_values)
|
||||
else
|
||||
member_attrs_with_cf
|
||||
end
|
||||
# Only include custom_field_values if not empty
|
||||
final_attrs =
|
||||
if Enum.empty?(custom_field_values) do
|
||||
Map.delete(member_attrs_with_cf, :custom_field_values)
|
||||
else
|
||||
member_attrs_with_cf
|
||||
end
|
||||
|
||||
case Mv.Membership.create_member(final_attrs, actor: actor) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
case Mv.Membership.create_member(final_attrs, actor: actor) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
|
||||
{:error, %Ash.Error.Invalid{} = error} ->
|
||||
# Extract email from final_attrs for better error messages
|
||||
email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email)
|
||||
{:error, format_ash_error(error, line_number, email)}
|
||||
{:error, %Ash.Error.Invalid{} = error} ->
|
||||
# Extract email from final_attrs for better error messages
|
||||
email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email)
|
||||
{:error, format_ash_error(error, line_number, email)}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
{:error, error} ->
|
||||
{:error,
|
||||
%Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue
|
||||
|
|
@ -542,70 +558,160 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Prepares custom field values from row map for Ash
|
||||
# Returns {:ok, [custom_field_value_maps]} or {:error, [validation_errors]}
|
||||
defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.map(fn {custom_field_id_str, value} ->
|
||||
case Map.get(custom_field_lookup, custom_field_id_str) do
|
||||
nil ->
|
||||
# Custom field not found, skip
|
||||
nil
|
||||
{values, errors} =
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.reduce({[], []}, fn {custom_field_id_str, value}, {acc_values, acc_errors} ->
|
||||
case Map.get(custom_field_lookup, custom_field_id_str) do
|
||||
nil ->
|
||||
# Custom field not found, skip
|
||||
{acc_values, acc_errors}
|
||||
|
||||
%{id: custom_field_id, value_type: value_type} ->
|
||||
%{
|
||||
"custom_field_id" => to_string(custom_field_id),
|
||||
"value" => format_custom_field_value(value, value_type)
|
||||
}
|
||||
end
|
||||
end)
|
||||
|> Enum.filter(&(&1 != nil))
|
||||
end
|
||||
%{id: custom_field_id, value_type: value_type, name: custom_field_name} ->
|
||||
case format_custom_field_value(value, value_type, custom_field_name) do
|
||||
{:ok, formatted_value} ->
|
||||
value_map = %{
|
||||
"custom_field_id" => to_string(custom_field_id),
|
||||
"value" => formatted_value
|
||||
}
|
||||
|
||||
defp prepare_custom_field_values(_, _), do: []
|
||||
{[value_map | acc_values], acc_errors}
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
defp format_custom_field_value(value, :string) when is_binary(value) do
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
{:error, reason} ->
|
||||
{acc_values, [reason | acc_errors]}
|
||||
end
|
||||
end
|
||||
end)
|
||||
|
||||
defp format_custom_field_value(value, :integer) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value}
|
||||
:error -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
if Enum.empty?(errors) do
|
||||
{:ok, Enum.reverse(values)}
|
||||
else
|
||||
{:error, Enum.reverse(errors)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean) when is_binary(value) do
|
||||
defp prepare_custom_field_values(_, _), do: {:ok, []}
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
# Returns {:ok, formatted_value} or {:error, error_message}
|
||||
defp format_custom_field_value(value, :string, _custom_field_name) when is_binary(value) do
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :integer, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Integer.parse(trimmed) do
|
||||
{int_value, ""} ->
|
||||
# Fully consumed - valid integer
|
||||
{:ok, %{"_union_type" => "integer", "_union_value" => int_value}}
|
||||
|
||||
{_int_value, _remaining} ->
|
||||
# Not fully consumed - invalid
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
|
||||
:error ->
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
lower = String.downcase(trimmed)
|
||||
|
||||
bool_value =
|
||||
value
|
||||
|> String.trim()
|
||||
|> String.downcase()
|
||||
|> case do
|
||||
case lower do
|
||||
"true" -> true
|
||||
"1" -> true
|
||||
"yes" -> true
|
||||
"ja" -> true
|
||||
_ -> false
|
||||
"false" -> false
|
||||
"0" -> false
|
||||
"no" -> false
|
||||
"nein" -> false
|
||||
_ -> nil
|
||||
end
|
||||
|
||||
%{"_union_type" => "boolean", "_union_value" => bool_value}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :date) when is_binary(value) do
|
||||
case Date.from_iso8601(String.trim(value)) do
|
||||
{:ok, date} -> %{"_union_type" => "date", "_union_value" => date}
|
||||
{:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
if bool_value != nil do
|
||||
{:ok, %{"_union_type" => "boolean", "_union_value" => bool_value}}
|
||||
else
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:boolean,
|
||||
trimmed,
|
||||
gettext("(true/false/1/0/yes/no/ja/nein)")
|
||||
)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :email) when is_binary(value) do
|
||||
%{"_union_type" => "email", "_union_value" => String.trim(value)}
|
||||
defp format_custom_field_value(value, :date, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Date.from_iso8601(trimmed) do
|
||||
{:ok, date} ->
|
||||
{:ok, %{"_union_type" => "date", "_union_value" => date}}
|
||||
|
||||
{:error, _} ->
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:date,
|
||||
trimmed,
|
||||
gettext("(ISO-8601 format: YYYY-MM-DD)")
|
||||
)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type) when is_binary(value) do
|
||||
defp format_custom_field_value(value, :email, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
# Use simple validation: must contain @ and have valid format
|
||||
# For CSV import, we use a simpler check than EctoCommons.EmailValidator
|
||||
# to avoid dependencies and keep it fast
|
||||
if String.contains?(trimmed, "@") and String.length(trimmed) >= 5 and
|
||||
String.length(trimmed) <= 254 do
|
||||
# Basic format check: username@domain.tld
|
||||
if Regex.match?(~r/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/, trimmed) do
|
||||
{:ok, %{"_union_type" => "email", "_union_value" => trimmed}}
|
||||
else
|
||||
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
|
||||
end
|
||||
else
|
||||
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type, _custom_field_name) when is_binary(value) do
|
||||
# Default to string if type is unknown
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
end
|
||||
|
||||
# Generates a consistent error message for custom field validation failures
|
||||
# Uses human-readable field type labels (e.g., "Number" instead of "integer")
|
||||
defp format_custom_field_error(custom_field_name, value_type, value) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
# Generates an error message with additional details (e.g., format hints)
|
||||
defp format_custom_field_error_with_details(custom_field_name, value_type, value, details) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type} %{details}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
details: details,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
# Trims all string values in member attributes
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue