feat: import custom fields via CSV
This commit is contained in:
parent
b9dd990f52
commit
3f8797c356
3 changed files with 251 additions and 132 deletions
|
|
@ -63,7 +63,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
chunks: list(list({pos_integer(), map()})),
|
||||
column_map: %{atom() => non_neg_integer()},
|
||||
custom_field_map: %{String.t() => non_neg_integer()},
|
||||
custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}},
|
||||
custom_field_lookup: %{
|
||||
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
|
||||
},
|
||||
warnings: list(String.t())
|
||||
}
|
||||
|
||||
|
|
@ -79,6 +81,11 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
use Gettext, backend: MvWeb.Gettext
|
||||
|
||||
alias Mv.Helpers.SystemActor
|
||||
|
||||
# Import FieldTypes for human-readable type labels
|
||||
alias MvWeb.Translations.FieldTypes
|
||||
|
||||
# Configuration constants
|
||||
@default_max_errors 50
|
||||
@default_chunk_size 200
|
||||
|
|
@ -102,6 +109,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
- `opts` - Optional keyword list:
|
||||
- `:max_rows` - Maximum number of data rows allowed (default: 1000)
|
||||
- `:chunk_size` - Number of rows per chunk (default: 200)
|
||||
- `:actor` - Actor for authorization (default: system actor for systemic operations)
|
||||
|
||||
## Returns
|
||||
|
||||
|
|
@ -120,9 +128,10 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
def prepare(file_content, opts \\ []) do
|
||||
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
|
||||
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
|
||||
|
||||
with {:ok, headers, rows} <- CsvParser.parse(file_content),
|
||||
{:ok, custom_fields} <- load_custom_fields(),
|
||||
{:ok, custom_fields} <- load_custom_fields(actor),
|
||||
{:ok, maps, warnings} <- build_header_maps(headers, custom_fields),
|
||||
:ok <- validate_row_count(rows, max_rows) do
|
||||
chunks = chunk_rows(rows, maps, chunk_size)
|
||||
|
|
@ -142,10 +151,10 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Loads all custom fields from the database
|
||||
defp load_custom_fields do
|
||||
defp load_custom_fields(actor) do
|
||||
custom_fields =
|
||||
Mv.Membership.CustomField
|
||||
|> Ash.read!()
|
||||
|> Ash.read!(actor: actor)
|
||||
|
||||
{:ok, custom_fields}
|
||||
rescue
|
||||
|
|
@ -158,7 +167,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
custom_fields
|
||||
|> Enum.reduce(%{}, fn cf, acc ->
|
||||
id_str = to_string(cf.id)
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type})
|
||||
Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type, name: cf.name})
|
||||
end)
|
||||
end
|
||||
|
||||
|
|
@ -508,32 +517,39 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
|
||||
# Prepare custom field values for Ash
|
||||
custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup)
|
||||
case prepare_custom_field_values(custom_attrs, custom_field_lookup) do
|
||||
{:error, validation_errors} ->
|
||||
# Custom field validation errors - return first error
|
||||
first_error = List.first(validation_errors)
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
|
||||
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
trimmed_member_attrs
|
||||
|> Map.put(:custom_field_values, custom_field_values)
|
||||
{:ok, custom_field_values} ->
|
||||
# Create member with custom field values
|
||||
member_attrs_with_cf =
|
||||
trimmed_member_attrs
|
||||
|> Map.put(:custom_field_values, custom_field_values)
|
||||
|
||||
# Only include custom_field_values if not empty
|
||||
final_attrs =
|
||||
if Enum.empty?(custom_field_values) do
|
||||
Map.delete(member_attrs_with_cf, :custom_field_values)
|
||||
else
|
||||
member_attrs_with_cf
|
||||
end
|
||||
# Only include custom_field_values if not empty
|
||||
final_attrs =
|
||||
if Enum.empty?(custom_field_values) do
|
||||
Map.delete(member_attrs_with_cf, :custom_field_values)
|
||||
else
|
||||
member_attrs_with_cf
|
||||
end
|
||||
|
||||
case Mv.Membership.create_member(final_attrs, actor: actor) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
case Mv.Membership.create_member(final_attrs, actor: actor) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
|
||||
{:error, %Ash.Error.Invalid{} = error} ->
|
||||
# Extract email from final_attrs for better error messages
|
||||
email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email)
|
||||
{:error, format_ash_error(error, line_number, email)}
|
||||
{:error, %Ash.Error.Invalid{} = error} ->
|
||||
# Extract email from final_attrs for better error messages
|
||||
email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email)
|
||||
{:error, format_ash_error(error, line_number, email)}
|
||||
|
||||
{:error, error} ->
|
||||
{:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
{:error, error} ->
|
||||
{:error,
|
||||
%Error{csv_line_number: line_number, field: nil, message: inspect(error)}}
|
||||
end
|
||||
end
|
||||
end
|
||||
rescue
|
||||
|
|
@ -542,70 +558,160 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Prepares custom field values from row map for Ash
|
||||
# Returns {:ok, [custom_field_value_maps]} or {:error, [validation_errors]}
|
||||
defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.map(fn {custom_field_id_str, value} ->
|
||||
case Map.get(custom_field_lookup, custom_field_id_str) do
|
||||
nil ->
|
||||
# Custom field not found, skip
|
||||
nil
|
||||
{values, errors} =
|
||||
custom_attrs
|
||||
|> Enum.filter(fn {_id, value} -> value != nil && value != "" end)
|
||||
|> Enum.reduce({[], []}, fn {custom_field_id_str, value}, {acc_values, acc_errors} ->
|
||||
case Map.get(custom_field_lookup, custom_field_id_str) do
|
||||
nil ->
|
||||
# Custom field not found, skip
|
||||
{acc_values, acc_errors}
|
||||
|
||||
%{id: custom_field_id, value_type: value_type} ->
|
||||
%{
|
||||
"custom_field_id" => to_string(custom_field_id),
|
||||
"value" => format_custom_field_value(value, value_type)
|
||||
}
|
||||
end
|
||||
end)
|
||||
|> Enum.filter(&(&1 != nil))
|
||||
end
|
||||
%{id: custom_field_id, value_type: value_type, name: custom_field_name} ->
|
||||
case format_custom_field_value(value, value_type, custom_field_name) do
|
||||
{:ok, formatted_value} ->
|
||||
value_map = %{
|
||||
"custom_field_id" => to_string(custom_field_id),
|
||||
"value" => formatted_value
|
||||
}
|
||||
|
||||
defp prepare_custom_field_values(_, _), do: []
|
||||
{[value_map | acc_values], acc_errors}
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
defp format_custom_field_value(value, :string) when is_binary(value) do
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
end
|
||||
{:error, reason} ->
|
||||
{acc_values, [reason | acc_errors]}
|
||||
end
|
||||
end
|
||||
end)
|
||||
|
||||
defp format_custom_field_value(value, :integer) when is_binary(value) do
|
||||
case Integer.parse(value) do
|
||||
{int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value}
|
||||
:error -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
if Enum.empty?(errors) do
|
||||
{:ok, Enum.reverse(values)}
|
||||
else
|
||||
{:error, Enum.reverse(errors)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean) when is_binary(value) do
|
||||
defp prepare_custom_field_values(_, _), do: {:ok, []}
|
||||
|
||||
# Formats a custom field value according to its type
|
||||
# Uses _union_type and _union_value format as expected by Ash
|
||||
# Returns {:ok, formatted_value} or {:error, error_message}
|
||||
defp format_custom_field_value(value, :string, _custom_field_name) when is_binary(value) do
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :integer, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Integer.parse(trimmed) do
|
||||
{int_value, ""} ->
|
||||
# Fully consumed - valid integer
|
||||
{:ok, %{"_union_type" => "integer", "_union_value" => int_value}}
|
||||
|
||||
{_int_value, _remaining} ->
|
||||
# Not fully consumed - invalid
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
|
||||
:error ->
|
||||
{:error, format_custom_field_error(custom_field_name, :integer, trimmed)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :boolean, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
lower = String.downcase(trimmed)
|
||||
|
||||
bool_value =
|
||||
value
|
||||
|> String.trim()
|
||||
|> String.downcase()
|
||||
|> case do
|
||||
case lower do
|
||||
"true" -> true
|
||||
"1" -> true
|
||||
"yes" -> true
|
||||
"ja" -> true
|
||||
_ -> false
|
||||
"false" -> false
|
||||
"0" -> false
|
||||
"no" -> false
|
||||
"nein" -> false
|
||||
_ -> nil
|
||||
end
|
||||
|
||||
%{"_union_type" => "boolean", "_union_value" => bool_value}
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :date) when is_binary(value) do
|
||||
case Date.from_iso8601(String.trim(value)) do
|
||||
{:ok, date} -> %{"_union_type" => "date", "_union_value" => date}
|
||||
{:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
if bool_value != nil do
|
||||
{:ok, %{"_union_type" => "boolean", "_union_value" => bool_value}}
|
||||
else
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:boolean,
|
||||
trimmed,
|
||||
gettext("(true/false/1/0/yes/no/ja/nein)")
|
||||
)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, :email) when is_binary(value) do
|
||||
%{"_union_type" => "email", "_union_value" => String.trim(value)}
|
||||
defp format_custom_field_value(value, :date, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
case Date.from_iso8601(trimmed) do
|
||||
{:ok, date} ->
|
||||
{:ok, %{"_union_type" => "date", "_union_value" => date}}
|
||||
|
||||
{:error, _} ->
|
||||
{:error,
|
||||
format_custom_field_error_with_details(
|
||||
custom_field_name,
|
||||
:date,
|
||||
trimmed,
|
||||
gettext("(ISO-8601 format: YYYY-MM-DD)")
|
||||
)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type) when is_binary(value) do
|
||||
defp format_custom_field_value(value, :email, custom_field_name) when is_binary(value) do
|
||||
trimmed = String.trim(value)
|
||||
|
||||
# Use simple validation: must contain @ and have valid format
|
||||
# For CSV import, we use a simpler check than EctoCommons.EmailValidator
|
||||
# to avoid dependencies and keep it fast
|
||||
if String.contains?(trimmed, "@") and String.length(trimmed) >= 5 and
|
||||
String.length(trimmed) <= 254 do
|
||||
# Basic format check: username@domain.tld
|
||||
if Regex.match?(~r/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/, trimmed) do
|
||||
{:ok, %{"_union_type" => "email", "_union_value" => trimmed}}
|
||||
else
|
||||
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
|
||||
end
|
||||
else
|
||||
{:error, format_custom_field_error(custom_field_name, :email, trimmed)}
|
||||
end
|
||||
end
|
||||
|
||||
defp format_custom_field_value(value, _type, _custom_field_name) when is_binary(value) do
|
||||
# Default to string if type is unknown
|
||||
%{"_union_type" => "string", "_union_value" => String.trim(value)}
|
||||
{:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}}
|
||||
end
|
||||
|
||||
# Generates a consistent error message for custom field validation failures
|
||||
# Uses human-readable field type labels (e.g., "Number" instead of "integer")
|
||||
defp format_custom_field_error(custom_field_name, value_type, value) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
# Generates an error message with additional details (e.g., format hints)
|
||||
defp format_custom_field_error_with_details(custom_field_name, value_type, value, details) do
|
||||
type_label = FieldTypes.label(value_type)
|
||||
|
||||
gettext("custom_field: %{name} – expected %{type} %{details}, got: %{value}",
|
||||
name: custom_field_name,
|
||||
type: type_label,
|
||||
details: details,
|
||||
value: value
|
||||
)
|
||||
end
|
||||
|
||||
# Trims all string values in member attributes
|
||||
|
|
|
|||
|
|
@ -50,66 +50,69 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do
|
|||
</div>
|
||||
|
||||
<%!-- Hide table when form is visible --%>
|
||||
<.table
|
||||
:if={!@show_form}
|
||||
id="custom_fields"
|
||||
rows={@streams.custom_fields}
|
||||
row_click={
|
||||
fn {_id, custom_field} ->
|
||||
JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself)
|
||||
end
|
||||
}
|
||||
>
|
||||
<:col :let={{_id, custom_field}} label={gettext("Name")}>{custom_field.name}</:col>
|
||||
|
||||
<:col :let={{_id, custom_field}} label={gettext("Value Type")}>
|
||||
{@field_type_label.(custom_field.value_type)}
|
||||
</:col>
|
||||
|
||||
<:col :let={{_id, custom_field}} label={gettext("Description")}>
|
||||
{custom_field.description}
|
||||
</:col>
|
||||
|
||||
<:col
|
||||
:let={{_id, custom_field}}
|
||||
label={gettext("Required")}
|
||||
class="max-w-[9.375rem] text-center"
|
||||
<div :if={!@show_form} id="custom_fields">
|
||||
<.table
|
||||
id="custom_fields_table"
|
||||
rows={@streams.custom_fields}
|
||||
row_click={
|
||||
fn {_id, custom_field} ->
|
||||
JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself)
|
||||
end
|
||||
}
|
||||
>
|
||||
<span :if={custom_field.required} class="text-base-content font-semibold">
|
||||
{gettext("Required")}
|
||||
</span>
|
||||
<span :if={!custom_field.required} class="text-base-content/70">
|
||||
{gettext("Optional")}
|
||||
</span>
|
||||
</:col>
|
||||
<:col :let={{_id, custom_field}} label={gettext("Name")}>{custom_field.name}</:col>
|
||||
|
||||
<:col
|
||||
:let={{_id, custom_field}}
|
||||
label={gettext("Show in overview")}
|
||||
class="max-w-[9.375rem] text-center"
|
||||
>
|
||||
<span :if={custom_field.show_in_overview} class="badge badge-success">
|
||||
{gettext("Yes")}
|
||||
</span>
|
||||
<span :if={!custom_field.show_in_overview} class="badge badge-ghost">
|
||||
{gettext("No")}
|
||||
</span>
|
||||
</:col>
|
||||
<:col :let={{_id, custom_field}} label={gettext("Value Type")}>
|
||||
{@field_type_label.(custom_field.value_type)}
|
||||
</:col>
|
||||
|
||||
<:action :let={{_id, custom_field}}>
|
||||
<.link phx-click={
|
||||
JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself)
|
||||
}>
|
||||
{gettext("Edit")}
|
||||
</.link>
|
||||
</:action>
|
||||
<:col :let={{_id, custom_field}} label={gettext("Description")}>
|
||||
{custom_field.description}
|
||||
</:col>
|
||||
|
||||
<:action :let={{_id, custom_field}}>
|
||||
<.link phx-click={JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)}>
|
||||
{gettext("Delete")}
|
||||
</.link>
|
||||
</:action>
|
||||
</.table>
|
||||
<:col
|
||||
:let={{_id, custom_field}}
|
||||
label={gettext("Required")}
|
||||
class="max-w-[9.375rem] text-center"
|
||||
>
|
||||
<span :if={custom_field.required} class="text-base-content font-semibold">
|
||||
{gettext("Required")}
|
||||
</span>
|
||||
<span :if={!custom_field.required} class="text-base-content/70">
|
||||
{gettext("Optional")}
|
||||
</span>
|
||||
</:col>
|
||||
|
||||
<:col
|
||||
:let={{_id, custom_field}}
|
||||
label={gettext("Show in overview")}
|
||||
class="max-w-[9.375rem] text-center"
|
||||
>
|
||||
<span :if={custom_field.show_in_overview} class="badge badge-success">
|
||||
{gettext("Yes")}
|
||||
</span>
|
||||
<span :if={!custom_field.show_in_overview} class="badge badge-ghost">
|
||||
{gettext("No")}
|
||||
</span>
|
||||
</:col>
|
||||
|
||||
<:action :let={{_id, custom_field}}>
|
||||
<.link phx-click={
|
||||
JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself)
|
||||
}>
|
||||
{gettext("Edit")}
|
||||
</.link>
|
||||
</:action>
|
||||
|
||||
<:action :let={{_id, custom_field}}>
|
||||
<.link phx-click={
|
||||
JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)
|
||||
}>
|
||||
{gettext("Delete")}
|
||||
</.link>
|
||||
</:action>
|
||||
</.table>
|
||||
</div>
|
||||
|
||||
<%!-- Delete Confirmation Modal --%>
|
||||
<dialog :if={@show_delete_modal} id="delete-custom-field-modal" class="modal modal-open">
|
||||
|
|
|
|||
|
|
@ -138,16 +138,24 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
<%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %>
|
||||
<.form_section title={gettext("Import Members (CSV)")}>
|
||||
<div role="note" class="alert alert-info mb-4">
|
||||
<.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
|
||||
<div>
|
||||
<p class="font-semibold">
|
||||
<p class="font-semibold mb-1">
|
||||
{gettext("Custom Fields in CSV Import")}
|
||||
</p>
|
||||
<p class="text-sm mb-2">
|
||||
{gettext(
|
||||
"Custom fields must be created in Mila before importing CSV files with custom field columns"
|
||||
"Custom fields must be created in Mila before importing. Use the custom field name as the CSV column header. Unknown custom field columns will be ignored with a warning."
|
||||
)}
|
||||
</p>
|
||||
<p class="text-sm mt-2">
|
||||
{gettext(
|
||||
"Use the custom field name as the CSV column header (same normalization as member fields applies)"
|
||||
)}
|
||||
<p class="text-sm">
|
||||
<.link
|
||||
href="#custom_fields"
|
||||
class="link link-primary"
|
||||
data-testid="custom-fields-link"
|
||||
>
|
||||
{gettext("Manage Custom Fields")}
|
||||
</.link>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -408,8 +416,10 @@ defmodule MvWeb.GlobalSettingsLive do
|
|||
|
||||
# Processes CSV upload and starts import
|
||||
defp process_csv_upload(socket) do
|
||||
actor = MvWeb.LiveHelpers.current_actor(socket)
|
||||
|
||||
with {:ok, content} <- consume_and_read_csv(socket),
|
||||
{:ok, import_state} <- MemberCSV.prepare(content) do
|
||||
{:ok, import_state} <- MemberCSV.prepare(content, actor: actor) do
|
||||
start_import(socket, import_state)
|
||||
else
|
||||
{:error, reason} when is_binary(reason) ->
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue