diff --git a/lib/mv/membership/import/member_csv.ex b/lib/mv/membership/import/member_csv.ex index e351d68..5924001 100644 --- a/lib/mv/membership/import/member_csv.ex +++ b/lib/mv/membership/import/member_csv.ex @@ -63,7 +63,9 @@ defmodule Mv.Membership.Import.MemberCSV do chunks: list(list({pos_integer(), map()})), column_map: %{atom() => non_neg_integer()}, custom_field_map: %{String.t() => non_neg_integer()}, - custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}}, + custom_field_lookup: %{ + String.t() => %{id: String.t(), value_type: atom(), name: String.t()} + }, warnings: list(String.t()) } @@ -79,6 +81,11 @@ defmodule Mv.Membership.Import.MemberCSV do use Gettext, backend: MvWeb.Gettext + alias Mv.Helpers.SystemActor + + # Import FieldTypes for human-readable type labels + alias MvWeb.Translations.FieldTypes + # Configuration constants @default_max_errors 50 @default_chunk_size 200 @@ -102,6 +109,7 @@ defmodule Mv.Membership.Import.MemberCSV do - `opts` - Optional keyword list: - `:max_rows` - Maximum number of data rows allowed (default: 1000) - `:chunk_size` - Number of rows per chunk (default: 200) + - `:actor` - Actor for authorization (default: system actor for systemic operations) ## Returns @@ -120,9 +128,10 @@ defmodule Mv.Membership.Import.MemberCSV do def prepare(file_content, opts \\ []) do max_rows = Keyword.get(opts, :max_rows, @default_max_rows) chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size) + actor = Keyword.get(opts, :actor, SystemActor.get_system_actor()) with {:ok, headers, rows} <- CsvParser.parse(file_content), - {:ok, custom_fields} <- load_custom_fields(), + {:ok, custom_fields} <- load_custom_fields(actor), {:ok, maps, warnings} <- build_header_maps(headers, custom_fields), :ok <- validate_row_count(rows, max_rows) do chunks = chunk_rows(rows, maps, chunk_size) @@ -142,10 +151,10 @@ defmodule Mv.Membership.Import.MemberCSV do end # Loads all custom fields from the database - defp load_custom_fields do + defp load_custom_fields(actor) do custom_fields = Mv.Membership.CustomField - |> Ash.read!() + |> Ash.read!(actor: actor) {:ok, custom_fields} rescue @@ -158,7 +167,7 @@ defmodule Mv.Membership.Import.MemberCSV do custom_fields |> Enum.reduce(%{}, fn cf, acc -> id_str = to_string(cf.id) - Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type}) + Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type, name: cf.name}) end) end @@ -508,32 +517,39 @@ defmodule Mv.Membership.Import.MemberCSV do {:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} -> # Prepare custom field values for Ash - custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup) + case prepare_custom_field_values(custom_attrs, custom_field_lookup) do + {:error, validation_errors} -> + # Custom field validation errors - return first error + first_error = List.first(validation_errors) + {:error, %Error{csv_line_number: line_number, field: nil, message: first_error}} - # Create member with custom field values - member_attrs_with_cf = - trimmed_member_attrs - |> Map.put(:custom_field_values, custom_field_values) + {:ok, custom_field_values} -> + # Create member with custom field values + member_attrs_with_cf = + trimmed_member_attrs + |> Map.put(:custom_field_values, custom_field_values) - # Only include custom_field_values if not empty - final_attrs = - if Enum.empty?(custom_field_values) do - Map.delete(member_attrs_with_cf, :custom_field_values) - else - member_attrs_with_cf - end + # Only include custom_field_values if not empty + final_attrs = + if Enum.empty?(custom_field_values) do + Map.delete(member_attrs_with_cf, :custom_field_values) + else + member_attrs_with_cf + end - case Mv.Membership.create_member(final_attrs, actor: actor) do - {:ok, member} -> - {:ok, member} + case Mv.Membership.create_member(final_attrs, actor: actor) do + {:ok, member} -> + {:ok, member} - {:error, %Ash.Error.Invalid{} = error} -> - # Extract email from final_attrs for better error messages - email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email) - {:error, format_ash_error(error, line_number, email)} + {:error, %Ash.Error.Invalid{} = error} -> + # Extract email from final_attrs for better error messages + email = Map.get(final_attrs, :email) || Map.get(trimmed_member_attrs, :email) + {:error, format_ash_error(error, line_number, email)} - {:error, error} -> - {:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}} + {:error, error} -> + {:error, + %Error{csv_line_number: line_number, field: nil, message: inspect(error)}} + end end end rescue @@ -542,70 +558,160 @@ defmodule Mv.Membership.Import.MemberCSV do end # Prepares custom field values from row map for Ash + # Returns {:ok, [custom_field_value_maps]} or {:error, [validation_errors]} defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do - custom_attrs - |> Enum.filter(fn {_id, value} -> value != nil && value != "" end) - |> Enum.map(fn {custom_field_id_str, value} -> - case Map.get(custom_field_lookup, custom_field_id_str) do - nil -> - # Custom field not found, skip - nil + {values, errors} = + custom_attrs + |> Enum.filter(fn {_id, value} -> value != nil && value != "" end) + |> Enum.reduce({[], []}, fn {custom_field_id_str, value}, {acc_values, acc_errors} -> + case Map.get(custom_field_lookup, custom_field_id_str) do + nil -> + # Custom field not found, skip + {acc_values, acc_errors} - %{id: custom_field_id, value_type: value_type} -> - %{ - "custom_field_id" => to_string(custom_field_id), - "value" => format_custom_field_value(value, value_type) - } - end - end) - |> Enum.filter(&(&1 != nil)) - end + %{id: custom_field_id, value_type: value_type, name: custom_field_name} -> + case format_custom_field_value(value, value_type, custom_field_name) do + {:ok, formatted_value} -> + value_map = %{ + "custom_field_id" => to_string(custom_field_id), + "value" => formatted_value + } - defp prepare_custom_field_values(_, _), do: [] + {[value_map | acc_values], acc_errors} - # Formats a custom field value according to its type - # Uses _union_type and _union_value format as expected by Ash - defp format_custom_field_value(value, :string) when is_binary(value) do - %{"_union_type" => "string", "_union_value" => String.trim(value)} - end + {:error, reason} -> + {acc_values, [reason | acc_errors]} + end + end + end) - defp format_custom_field_value(value, :integer) when is_binary(value) do - case Integer.parse(value) do - {int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value} - :error -> %{"_union_type" => "string", "_union_value" => String.trim(value)} + if Enum.empty?(errors) do + {:ok, Enum.reverse(values)} + else + {:error, Enum.reverse(errors)} end end - defp format_custom_field_value(value, :boolean) when is_binary(value) do + defp prepare_custom_field_values(_, _), do: {:ok, []} + + # Formats a custom field value according to its type + # Uses _union_type and _union_value format as expected by Ash + # Returns {:ok, formatted_value} or {:error, error_message} + defp format_custom_field_value(value, :string, _custom_field_name) when is_binary(value) do + {:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}} + end + + defp format_custom_field_value(value, :integer, custom_field_name) when is_binary(value) do + trimmed = String.trim(value) + + case Integer.parse(trimmed) do + {int_value, ""} -> + # Fully consumed - valid integer + {:ok, %{"_union_type" => "integer", "_union_value" => int_value}} + + {_int_value, _remaining} -> + # Not fully consumed - invalid + {:error, format_custom_field_error(custom_field_name, :integer, trimmed)} + + :error -> + {:error, format_custom_field_error(custom_field_name, :integer, trimmed)} + end + end + + defp format_custom_field_value(value, :boolean, custom_field_name) when is_binary(value) do + trimmed = String.trim(value) + lower = String.downcase(trimmed) + bool_value = - value - |> String.trim() - |> String.downcase() - |> case do + case lower do "true" -> true "1" -> true "yes" -> true "ja" -> true - _ -> false + "false" -> false + "0" -> false + "no" -> false + "nein" -> false + _ -> nil end - %{"_union_type" => "boolean", "_union_value" => bool_value} - end - - defp format_custom_field_value(value, :date) when is_binary(value) do - case Date.from_iso8601(String.trim(value)) do - {:ok, date} -> %{"_union_type" => "date", "_union_value" => date} - {:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)} + if bool_value != nil do + {:ok, %{"_union_type" => "boolean", "_union_value" => bool_value}} + else + {:error, + format_custom_field_error_with_details( + custom_field_name, + :boolean, + trimmed, + gettext("(true/false/1/0/yes/no/ja/nein)") + )} end end - defp format_custom_field_value(value, :email) when is_binary(value) do - %{"_union_type" => "email", "_union_value" => String.trim(value)} + defp format_custom_field_value(value, :date, custom_field_name) when is_binary(value) do + trimmed = String.trim(value) + + case Date.from_iso8601(trimmed) do + {:ok, date} -> + {:ok, %{"_union_type" => "date", "_union_value" => date}} + + {:error, _} -> + {:error, + format_custom_field_error_with_details( + custom_field_name, + :date, + trimmed, + gettext("(ISO-8601 format: YYYY-MM-DD)") + )} + end end - defp format_custom_field_value(value, _type) when is_binary(value) do + defp format_custom_field_value(value, :email, custom_field_name) when is_binary(value) do + trimmed = String.trim(value) + + # Use simple validation: must contain @ and have valid format + # For CSV import, we use a simpler check than EctoCommons.EmailValidator + # to avoid dependencies and keep it fast + if String.contains?(trimmed, "@") and String.length(trimmed) >= 5 and + String.length(trimmed) <= 254 do + # Basic format check: username@domain.tld + if Regex.match?(~r/^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}$/, trimmed) do + {:ok, %{"_union_type" => "email", "_union_value" => trimmed}} + else + {:error, format_custom_field_error(custom_field_name, :email, trimmed)} + end + else + {:error, format_custom_field_error(custom_field_name, :email, trimmed)} + end + end + + defp format_custom_field_value(value, _type, _custom_field_name) when is_binary(value) do # Default to string if type is unknown - %{"_union_type" => "string", "_union_value" => String.trim(value)} + {:ok, %{"_union_type" => "string", "_union_value" => String.trim(value)}} + end + + # Generates a consistent error message for custom field validation failures + # Uses human-readable field type labels (e.g., "Number" instead of "integer") + defp format_custom_field_error(custom_field_name, value_type, value) do + type_label = FieldTypes.label(value_type) + + gettext("custom_field: %{name} – expected %{type}, got: %{value}", + name: custom_field_name, + type: type_label, + value: value + ) + end + + # Generates an error message with additional details (e.g., format hints) + defp format_custom_field_error_with_details(custom_field_name, value_type, value, details) do + type_label = FieldTypes.label(value_type) + + gettext("custom_field: %{name} – expected %{type} %{details}, got: %{value}", + name: custom_field_name, + type: type_label, + details: details, + value: value + ) end # Trims all string values in member attributes diff --git a/lib/mv_web/live/custom_field_live/index_component.ex b/lib/mv_web/live/custom_field_live/index_component.ex index 784d1ef..5cf0f6b 100644 --- a/lib/mv_web/live/custom_field_live/index_component.ex +++ b/lib/mv_web/live/custom_field_live/index_component.ex @@ -50,66 +50,69 @@ defmodule MvWeb.CustomFieldLive.IndexComponent do <%!-- Hide table when form is visible --%> - <.table - :if={!@show_form} - id="custom_fields" - rows={@streams.custom_fields} - row_click={ - fn {_id, custom_field} -> - JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself) - end - } - > - <:col :let={{_id, custom_field}} label={gettext("Name")}>{custom_field.name} - - <:col :let={{_id, custom_field}} label={gettext("Value Type")}> - {@field_type_label.(custom_field.value_type)} - - - <:col :let={{_id, custom_field}} label={gettext("Description")}> - {custom_field.description} - - - <:col - :let={{_id, custom_field}} - label={gettext("Required")} - class="max-w-[9.375rem] text-center" +
+ <.table + id="custom_fields_table" + rows={@streams.custom_fields} + row_click={ + fn {_id, custom_field} -> + JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself) + end + } > - - {gettext("Required")} - - - {gettext("Optional")} - - + <:col :let={{_id, custom_field}} label={gettext("Name")}>{custom_field.name} - <:col - :let={{_id, custom_field}} - label={gettext("Show in overview")} - class="max-w-[9.375rem] text-center" - > - - {gettext("Yes")} - - - {gettext("No")} - - + <:col :let={{_id, custom_field}} label={gettext("Value Type")}> + {@field_type_label.(custom_field.value_type)} + - <:action :let={{_id, custom_field}}> - <.link phx-click={ - JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself) - }> - {gettext("Edit")} - - + <:col :let={{_id, custom_field}} label={gettext("Description")}> + {custom_field.description} + - <:action :let={{_id, custom_field}}> - <.link phx-click={JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself)}> - {gettext("Delete")} - - - + <:col + :let={{_id, custom_field}} + label={gettext("Required")} + class="max-w-[9.375rem] text-center" + > + + {gettext("Required")} + + + {gettext("Optional")} + + + + <:col + :let={{_id, custom_field}} + label={gettext("Show in overview")} + class="max-w-[9.375rem] text-center" + > + + {gettext("Yes")} + + + {gettext("No")} + + + + <:action :let={{_id, custom_field}}> + <.link phx-click={ + JS.push("edit_custom_field", value: %{id: custom_field.id}, target: @myself) + }> + {gettext("Edit")} + + + + <:action :let={{_id, custom_field}}> + <.link phx-click={ + JS.push("prepare_delete", value: %{id: custom_field.id}, target: @myself) + }> + {gettext("Delete")} + + + +
<%!-- Delete Confirmation Modal --%> diff --git a/lib/mv_web/live/global_settings_live.ex b/lib/mv_web/live/global_settings_live.ex index bd0036b..9d3bec9 100644 --- a/lib/mv_web/live/global_settings_live.ex +++ b/lib/mv_web/live/global_settings_live.ex @@ -138,16 +138,24 @@ defmodule MvWeb.GlobalSettingsLive do <%= if Authorization.can?(@current_user, :create, Mv.Membership.Member) do %> <.form_section title={gettext("Import Members (CSV)")}>
+ <.icon name="hero-information-circle" class="size-5" aria-hidden="true" />
-

+

+ {gettext("Custom Fields in CSV Import")} +

+

{gettext( - "Custom fields must be created in Mila before importing CSV files with custom field columns" + "Custom fields must be created in Mila before importing. Use the custom field name as the CSV column header. Unknown custom field columns will be ignored with a warning." )}

-

- {gettext( - "Use the custom field name as the CSV column header (same normalization as member fields applies)" - )} +

+ <.link + href="#custom_fields" + class="link link-primary" + data-testid="custom-fields-link" + > + {gettext("Manage Custom Fields")} +

@@ -408,8 +416,10 @@ defmodule MvWeb.GlobalSettingsLive do # Processes CSV upload and starts import defp process_csv_upload(socket) do + actor = MvWeb.LiveHelpers.current_actor(socket) + with {:ok, content} <- consume_and_read_csv(socket), - {:ok, import_state} <- MemberCSV.prepare(content) do + {:ok, import_state} <- MemberCSV.prepare(content, actor: actor) do start_import(socket, import_state) else {:error, reason} when is_binary(reason) ->