diff --git a/lib/mv/membership/import/header_mapper.ex b/lib/mv/membership/import/header_mapper.ex index caf7245..4e4a77d 100644 --- a/lib/mv/membership/import/header_mapper.ex +++ b/lib/mv/membership/import/header_mapper.ex @@ -97,17 +97,33 @@ defmodule Mv.Membership.Import.HeaderMapper do } # Build reverse map: normalized_variant -> canonical_field - # This is computed at runtime on first access and cached + # Cached on first access for performance defp normalized_to_canonical do + cached = Process.get({__MODULE__, :normalized_to_canonical}) + + if cached do + cached + else + map = build_normalized_to_canonical_map() + Process.put({__MODULE__, :normalized_to_canonical}, map) + map + end + end + + # Builds the normalized variant -> canonical field map + defp build_normalized_to_canonical_map do @member_field_variants_raw - |> Enum.flat_map(fn {canonical, variants} -> - Enum.map(variants, fn variant -> - {normalize_header(variant), canonical} - end) - end) + |> Enum.flat_map(&map_variants_to_normalized/1) |> Map.new() end + # Maps a canonical field and its variants to normalized tuples + defp map_variants_to_normalized({canonical, variants}) do + Enum.map(variants, fn variant -> + {normalize_header(variant), canonical} + end) + end + @doc """ Normalizes a CSV header string for comparison. @@ -208,9 +224,10 @@ defmodule Mv.Membership.Import.HeaderMapper do |> String.replace(<<0x2212::utf8>>, "-") end - # Normalizes punctuation: parentheses, slashes become spaces + # Normalizes punctuation: parentheses, slashes, underscores become spaces defp normalize_punctuation(str) do str + |> String.replace("_", " ") |> String.replace(~r/[()\[\]{}]/, " ") |> String.replace(~r/[\/\\]/, " ") end @@ -228,18 +245,18 @@ defmodule Mv.Membership.Import.HeaderMapper do result = headers |> Enum.with_index() - |> Enum.reduce_while({%{}, [], %{}}, fn {header, index}, {acc_map, acc_unknown, acc_seen} -> + |> Enum.reduce_while({%{}, []}, fn {header, index}, {acc_map, acc_unknown} -> normalized = normalize_header(header) - case process_member_header(header, index, normalized, acc_map, acc_seen) do + case process_member_header(header, index, normalized, acc_map, %{}) do {:error, reason} -> {:halt, {:error, reason}} - {:ok, new_map, new_seen} -> - {:cont, {new_map, acc_unknown, new_seen}} + {:ok, new_map, _} -> + {:cont, {new_map, acc_unknown}} {:unknown} -> - {:cont, {acc_map, [index | acc_unknown], acc_seen}} + {:cont, {acc_map, [index | acc_unknown]}} end end) @@ -247,7 +264,7 @@ defmodule Mv.Membership.Import.HeaderMapper do {:error, reason} -> {:error, reason} - {member_map, unknown_indices, _normalized_seen} -> + {member_map, unknown_indices} -> validate_required_fields(member_map, unknown_indices) end end @@ -258,17 +275,17 @@ defmodule Mv.Membership.Import.HeaderMapper do {:ok, acc_map, acc_seen} end - defp process_member_header(_header, index, normalized, acc_map, acc_seen) do - if Map.has_key?(normalized_to_canonical(), normalized) do - canonical = normalized_to_canonical()[normalized] + defp process_member_header(_header, index, normalized, acc_map, _acc_seen) do + case Map.get(normalized_to_canonical(), normalized) do + nil -> + {:unknown} - if Map.has_key?(acc_map, canonical) do - {:error, "duplicate header for #{canonical} (normalized: #{normalized})"} - else - {:ok, Map.put(acc_map, canonical, index), Map.put(acc_seen, normalized, canonical)} - end - else - {:unknown} + canonical -> + if Map.has_key?(acc_map, canonical) do + {:error, "duplicate header for #{canonical} (normalized: #{normalized})"} + else + {:ok, Map.put(acc_map, canonical, index), %{}} + end end end @@ -295,7 +312,7 @@ defmodule Mv.Membership.Import.HeaderMapper do result = unknown_indices - |> Enum.reduce_while({%{}, [], %{}}, fn index, {acc_map, acc_unknown, acc_seen} -> + |> Enum.reduce_while({%{}, []}, fn index, {acc_map, acc_unknown} -> header = Enum.at(headers, index) normalized = normalize_header(header) @@ -305,16 +322,16 @@ defmodule Mv.Membership.Import.HeaderMapper do normalized, custom_field_lookup, acc_map, - acc_seen + %{} ) do {:error, reason} -> {:halt, {:error, reason}} - {:ok, new_map, new_seen} -> - {:cont, {new_map, acc_unknown, new_seen}} + {:ok, new_map, _} -> + {:cont, {new_map, acc_unknown}} {:unknown} -> - {:cont, {acc_map, [index | acc_unknown], acc_seen}} + {:cont, {acc_map, [index | acc_unknown]}} end end) @@ -322,7 +339,7 @@ defmodule Mv.Membership.Import.HeaderMapper do {:error, reason} -> {:error, reason} - {custom_map, remaining_unknown, _normalized_seen} -> + {custom_map, remaining_unknown} -> {:ok, custom_map, Enum.reverse(remaining_unknown)} end end @@ -350,10 +367,10 @@ defmodule Mv.Membership.Import.HeaderMapper do normalized, _custom_field_lookup, acc_map, - acc_seen + _acc_seen ) when normalized == "" do - {:ok, acc_map, acc_seen} + {:ok, acc_map, %{}} end defp process_custom_field_header( @@ -362,7 +379,7 @@ defmodule Mv.Membership.Import.HeaderMapper do normalized, custom_field_lookup, acc_map, - acc_seen + _acc_seen ) do if Map.has_key?(custom_field_lookup, normalized) do custom_field_id = custom_field_lookup[normalized] @@ -370,8 +387,7 @@ defmodule Mv.Membership.Import.HeaderMapper do if Map.has_key?(acc_map, custom_field_id) do {:error, "duplicate custom field header (normalized: #{normalized})"} else - {:ok, Map.put(acc_map, custom_field_id, index), - Map.put(acc_seen, normalized, custom_field_id)} + {:ok, Map.put(acc_map, custom_field_id, index), %{}} end else {:unknown} diff --git a/lib/mv/membership/import/member_csv.ex b/lib/mv/membership/import/member_csv.ex index 7790bff..26756b4 100644 --- a/lib/mv/membership/import/member_csv.ex +++ b/lib/mv/membership/import/member_csv.ex @@ -63,6 +63,7 @@ defmodule Mv.Membership.Import.MemberCSV do chunks: list(list({pos_integer(), map()})), column_map: %{atom() => non_neg_integer()}, custom_field_map: %{String.t() => non_neg_integer()}, + custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}}, warnings: list(String.t()) } @@ -118,11 +119,15 @@ defmodule Mv.Membership.Import.MemberCSV do :ok <- validate_row_count(rows, max_rows) do chunks = chunk_rows(rows, maps, chunk_size) + # Build custom field lookup for efficient value processing + custom_field_lookup = build_custom_field_lookup(custom_fields) + {:ok, %{ chunks: chunks, column_map: maps.member, custom_field_map: maps.custom, + custom_field_lookup: custom_field_lookup, warnings: warnings }} end @@ -140,6 +145,15 @@ defmodule Mv.Membership.Import.MemberCSV do {:error, "Failed to load custom fields: #{Exception.message(e)}"} end + # Builds custom field lookup map for efficient value processing + defp build_custom_field_lookup(custom_fields) do + custom_fields + |> Enum.reduce(%{}, fn cf, acc -> + id_str = to_string(cf.id) + Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type}) + end) + end + # Builds header maps using HeaderMapper and collects warnings for unknown custom fields defp build_header_maps(headers, custom_fields) do # Convert custom fields to maps with id and name @@ -263,11 +277,13 @@ defmodule Mv.Membership.Import.MemberCSV do %{String.t() => non_neg_integer()}, keyword() ) :: {:ok, chunk_result()} | {:error, String.t()} - def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, _opts \\ []) do + def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do + custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{}) + {inserted, failed, errors} = Enum.reduce(chunk_rows_with_lines, {0, 0, []}, fn {line_number, row_map}, {acc_inserted, acc_failed, acc_errors} -> - case process_row(row_map, line_number) do + case process_row(row_map, line_number, custom_field_lookup) do {:ok, _member} -> {acc_inserted + 1, acc_failed, acc_errors} @@ -280,9 +296,13 @@ defmodule Mv.Membership.Import.MemberCSV do end # Processes a single row and creates member with custom field values - defp process_row(%{member: member_attrs, custom: custom_attrs}, line_number) do + defp process_row( + %{member: member_attrs, custom: custom_attrs}, + line_number, + custom_field_lookup + ) do # Prepare custom field values for Ash - custom_field_values = prepare_custom_field_values(custom_attrs) + custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup) # Create member with custom field values member_attrs_with_cf = @@ -314,53 +334,26 @@ defmodule Mv.Membership.Import.MemberCSV do end # Prepares custom field values from row map for Ash - defp prepare_custom_field_values(custom_attrs) when is_map(custom_attrs) do + defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do custom_attrs |> Enum.filter(fn {_id, value} -> value != nil && value != "" end) |> Enum.map(fn {custom_field_id_str, value} -> - # Load custom field to get value_type and ensure ID is correct - case load_custom_field_by_id(custom_field_id_str) do - {:ok, custom_field} -> - # Use the actual custom_field.id (UUID) from the database - %{ - "custom_field_id" => to_string(custom_field.id), - "value" => format_custom_field_value(value, custom_field.value_type) - } - - {:error, _} -> - # Skip if custom field not found + case Map.get(custom_field_lookup, custom_field_id_str) do + nil -> + # Custom field not found, skip nil + + %{id: custom_field_id, value_type: value_type} -> + %{ + "custom_field_id" => to_string(custom_field_id), + "value" => format_custom_field_value(value, value_type) + } end end) |> Enum.filter(&(&1 != nil)) end - defp prepare_custom_field_values(_), do: [] - - # Loads a custom field by ID (string or UUID) - defp load_custom_field_by_id(id) when is_binary(id) do - require Ash.Query - - try do - # Try to parse as UUID first - uuid_id = - case Ecto.UUID.cast(id) do - {:ok, uuid} -> uuid - :error -> id - end - - custom_field = - Mv.Membership.CustomField - |> Ash.Query.filter(id == ^uuid_id) - |> Ash.read_one!() - - {:ok, custom_field} - rescue - _ -> {:error, :not_found} - end - end - - defp load_custom_field_by_id(_), do: {:error, :invalid_id} + defp prepare_custom_field_values(_, _), do: [] # Formats a custom field value according to its type # Uses _union_type and _union_value format as expected by Ash diff --git a/test/mv/membership/import/member_csv_test.exs b/test/mv/membership/import/member_csv_test.exs index a38315f..356cc19 100644 --- a/test/mv/membership/import/member_csv_test.exs +++ b/test/mv/membership/import/member_csv_test.exs @@ -72,7 +72,7 @@ defmodule Mv.Membership.Import.MemberCSVTest do end end - describe "process_chunk/3" do + describe "process_chunk/4" do test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts" do chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}] column_map = %{email: 0} @@ -173,7 +173,12 @@ defmodule Mv.Membership.Import.MemberCSVTest do column_map = %{email: 0} custom_field_map = %{to_string(custom_field.id) => 1} - opts = [] + + custom_field_lookup = %{ + to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type} + } + + opts = [custom_field_lookup: custom_field_lookup] assert {:ok, chunk_result} = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)