diff --git a/lib/mv/membership/import/header_mapper.ex b/lib/mv/membership/import/header_mapper.ex new file mode 100644 index 0000000..4e4a77d --- /dev/null +++ b/lib/mv/membership/import/header_mapper.ex @@ -0,0 +1,396 @@ +defmodule Mv.Membership.Import.HeaderMapper do + @moduledoc """ + Maps CSV headers to canonical member fields and custom fields. + + Provides header normalization and mapping functionality for CSV imports. + Handles bilingual header variants (English/German) and custom field detection. + + ## Header Normalization + + Headers are normalized using the following rules: + - Trim whitespace + - Convert to lowercase + - Unicode normalization (ß → ss, ä → ae, ö → oe, ü → ue) + - Remove all whitespace (ensures "first name" == "firstname") + - Unify hyphen variants (en dash, minus sign → standard hyphen) + - Remove or unify punctuation (parentheses, slashes → spaces) + + ## Member Field Mapping + + Maps CSV headers to canonical member fields: + - `email` (required) + - `first_name` (optional) + - `last_name` (optional) + - `street` (optional) + - `postal_code` (optional) + - `city` (optional) + + Supports both English and German variants (e.g., "Email" / "E-Mail", "First Name" / "Vorname"). + + ## Custom Field Detection + + Custom fields are detected by matching normalized header names to custom field names. + Member fields have priority over custom fields (member field wins in case of collision). + + ## Examples + + iex> HeaderMapper.normalize_header(" E-Mail ") + "e-mail" + + iex> HeaderMapper.build_maps(["Email", "First Name"], []) + {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}} + + iex> HeaderMapper.build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}]) + {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}} + """ + + @type column_map :: %{atom() => non_neg_integer()} + @type custom_field_map :: %{String.t() => non_neg_integer()} + @type unknown_headers :: [String.t()] + + # Required member fields + @required_member_fields [:email] + + # Canonical member fields with their raw variants + # These will be normalized at runtime when building the lookup map + @member_field_variants_raw %{ + email: [ + "email", + "e-mail", + "e_mail", + "e mail", + "e-mail adresse", + "e-mail-adresse", + "mail" + ], + first_name: [ + "first name", + "firstname", + "vorname" + ], + last_name: [ + "last name", + "lastname", + "surname", + "nachname", + "familienname" + ], + street: [ + "street", + "address", + "strasse" + ], + postal_code: [ + "postal code", + "postal_code", + "zip", + "postcode", + "plz", + "postleitzahl" + ], + city: [ + "city", + "town", + "stadt", + "ort" + ] + } + + # Build reverse map: normalized_variant -> canonical_field + # Cached on first access for performance + defp normalized_to_canonical do + cached = Process.get({__MODULE__, :normalized_to_canonical}) + + if cached do + cached + else + map = build_normalized_to_canonical_map() + Process.put({__MODULE__, :normalized_to_canonical}, map) + map + end + end + + # Builds the normalized variant -> canonical field map + defp build_normalized_to_canonical_map do + @member_field_variants_raw + |> Enum.flat_map(&map_variants_to_normalized/1) + |> Map.new() + end + + # Maps a canonical field and its variants to normalized tuples + defp map_variants_to_normalized({canonical, variants}) do + Enum.map(variants, fn variant -> + {normalize_header(variant), canonical} + end) + end + + @doc """ + Normalizes a CSV header string for comparison. + + Applies the following transformations: + - Trim whitespace + - Convert to lowercase + - Unicode transliteration (ß → ss, ä → ae, ö → oe, ü → ue) + - Unify hyphen variants (en dash U+2013, minus sign U+2212 → standard hyphen) + - Remove or unify punctuation (parentheses, slashes → spaces) + - Remove all whitespace (ensures "first name" == "firstname") + - Final trim + + ## Examples + + iex> normalize_header(" E-Mail ") + "e-mail" + + iex> normalize_header("Straße") + "strasse" + + iex> normalize_header("E-Mail (privat)") + "e-mailprivat" + + iex> normalize_header("First Name") + "firstname" + + """ + @spec normalize_header(String.t()) :: String.t() + def normalize_header(header) when is_binary(header) do + header + |> String.trim() + |> String.downcase() + |> transliterate_unicode() + |> unify_hyphens() + |> normalize_punctuation() + |> compress_whitespace() + |> String.trim() + end + + def normalize_header(_), do: "" + + @doc """ + Builds column maps for member fields and custom fields from CSV headers. + + ## Parameters + + - `headers` - List of CSV header strings (in column order, 0-based indices) + - `custom_fields` - List of custom field maps/structs with at least `:id` and `:name` keys + + ## Returns + + - `{:ok, %{member: column_map, custom: custom_field_map, unknown: unknown_headers}}` on success + - `{:error, reason}` on error (missing required field, duplicate headers) + + ## Examples + + iex> build_maps(["Email", "First Name"], []) + {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}} + + iex> build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}]) + {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}} + + """ + @spec build_maps([String.t()], [map()]) :: + {:ok, %{member: column_map(), custom: custom_field_map(), unknown: unknown_headers()}} + | {:error, String.t()} + def build_maps(headers, custom_fields) when is_list(headers) and is_list(custom_fields) do + with {:ok, member_map, unknown_after_member} <- build_member_map(headers), + {:ok, custom_map, unknown_after_custom} <- + build_custom_field_map(headers, unknown_after_member, custom_fields, member_map) do + unknown = Enum.map(unknown_after_custom, &Enum.at(headers, &1)) + {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} + end + end + + # --- Private Functions --- + + # Transliterates German umlauts and special characters + defp transliterate_unicode(str) do + str + |> String.replace("ß", "ss") + |> String.replace("ä", "ae") + |> String.replace("ö", "oe") + |> String.replace("ü", "ue") + |> String.replace("Ä", "ae") + |> String.replace("Ö", "oe") + |> String.replace("Ü", "ue") + end + + # Unifies different hyphen variants to standard hyphen + defp unify_hyphens(str) do + str + # en dash + |> String.replace(<<0x2013::utf8>>, "-") + # em dash + |> String.replace(<<0x2014::utf8>>, "-") + # minus sign + |> String.replace(<<0x2212::utf8>>, "-") + end + + # Normalizes punctuation: parentheses, slashes, underscores become spaces + defp normalize_punctuation(str) do + str + |> String.replace("_", " ") + |> String.replace(~r/[()\[\]{}]/, " ") + |> String.replace(~r/[\/\\]/, " ") + end + + # Compresses multiple whitespace characters to single space, then removes all spaces + # This ensures "first name" and "firstname" normalize to the same value + defp compress_whitespace(str) do + str + |> String.replace(~r/\s+/, " ") + |> String.replace(" ", "") + end + + # Builds member field column map + defp build_member_map(headers) do + result = + headers + |> Enum.with_index() + |> Enum.reduce_while({%{}, []}, fn {header, index}, {acc_map, acc_unknown} -> + normalized = normalize_header(header) + + case process_member_header(header, index, normalized, acc_map, %{}) do + {:error, reason} -> + {:halt, {:error, reason}} + + {:ok, new_map, _} -> + {:cont, {new_map, acc_unknown}} + + {:unknown} -> + {:cont, {acc_map, [index | acc_unknown]}} + end + end) + + case result do + {:error, reason} -> + {:error, reason} + + {member_map, unknown_indices} -> + validate_required_fields(member_map, unknown_indices) + end + end + + # Processes a single header for member field mapping + defp process_member_header(_header, _index, normalized, acc_map, acc_seen) + when normalized == "" do + {:ok, acc_map, acc_seen} + end + + defp process_member_header(_header, index, normalized, acc_map, _acc_seen) do + case Map.get(normalized_to_canonical(), normalized) do + nil -> + {:unknown} + + canonical -> + if Map.has_key?(acc_map, canonical) do + {:error, "duplicate header for #{canonical} (normalized: #{normalized})"} + else + {:ok, Map.put(acc_map, canonical, index), %{}} + end + end + end + + # Validates that all required member fields are present + defp validate_required_fields(member_map, unknown_indices) do + missing_required = + @required_member_fields + |> Enum.filter(&(not Map.has_key?(member_map, &1))) + + if Enum.empty?(missing_required) do + {:ok, member_map, Enum.reverse(unknown_indices)} + else + missing_field = List.first(missing_required) + variants = Map.get(@member_field_variants_raw, missing_field, []) + accepted = Enum.join(variants, ", ") + + {:error, "Missing required header: #{missing_field} (accepted: #{accepted})"} + end + end + + # Builds custom field column map from unmatched headers + defp build_custom_field_map(headers, unknown_indices, custom_fields, _member_map) do + custom_field_lookup = build_custom_field_lookup(custom_fields) + + result = + unknown_indices + |> Enum.reduce_while({%{}, []}, fn index, {acc_map, acc_unknown} -> + header = Enum.at(headers, index) + normalized = normalize_header(header) + + case process_custom_field_header( + header, + index, + normalized, + custom_field_lookup, + acc_map, + %{} + ) do + {:error, reason} -> + {:halt, {:error, reason}} + + {:ok, new_map, _} -> + {:cont, {new_map, acc_unknown}} + + {:unknown} -> + {:cont, {acc_map, [index | acc_unknown]}} + end + end) + + case result do + {:error, reason} -> + {:error, reason} + + {custom_map, remaining_unknown} -> + {:ok, custom_map, Enum.reverse(remaining_unknown)} + end + end + + # Builds normalized custom field name -> id lookup map + defp build_custom_field_lookup(custom_fields) do + custom_fields + |> Enum.reduce(%{}, fn cf, acc -> + name = Map.get(cf, :name) || Map.get(cf, "name") + id = Map.get(cf, :id) || Map.get(cf, "id") + + if name && id do + normalized_name = normalize_header(name) + Map.put(acc, normalized_name, id) + else + acc + end + end) + end + + # Processes a single header for custom field mapping + defp process_custom_field_header( + _header, + _index, + normalized, + _custom_field_lookup, + acc_map, + _acc_seen + ) + when normalized == "" do + {:ok, acc_map, %{}} + end + + defp process_custom_field_header( + _header, + index, + normalized, + custom_field_lookup, + acc_map, + _acc_seen + ) do + if Map.has_key?(custom_field_lookup, normalized) do + custom_field_id = custom_field_lookup[normalized] + + if Map.has_key?(acc_map, custom_field_id) do + {:error, "duplicate custom field header (normalized: #{normalized})"} + else + {:ok, Map.put(acc_map, custom_field_id, index), %{}} + end + else + {:unknown} + end + end +end diff --git a/lib/mv/membership/import/member_csv.ex b/lib/mv/membership/import/member_csv.ex index 9e30a20..26756b4 100644 --- a/lib/mv/membership/import/member_csv.ex +++ b/lib/mv/membership/import/member_csv.ex @@ -2,6 +2,8 @@ defmodule Mv.Membership.Import.MemberCSV do @moduledoc """ Service module for importing members from CSV files. + require Ash.Query + This module provides the core API for CSV member import functionality: - `prepare/2` - Parses and validates CSV content, returns import state - `process_chunk/3` - Processes a chunk of rows and creates members @@ -61,6 +63,7 @@ defmodule Mv.Membership.Import.MemberCSV do chunks: list(list({pos_integer(), map()})), column_map: %{atom() => non_neg_integer()}, custom_field_map: %{String.t() => non_neg_integer()}, + custom_field_lookup: %{String.t() => %{id: String.t(), value_type: atom()}}, warnings: list(String.t()) } @@ -70,6 +73,9 @@ defmodule Mv.Membership.Import.MemberCSV do errors: list(Error.t()) } + alias Mv.Membership.Import.CsvParser + alias Mv.Membership.Import.HeaderMapper + @doc """ Prepares CSV content for import by parsing, mapping headers, and validating limits. @@ -104,12 +110,133 @@ defmodule Mv.Membership.Import.MemberCSV do """ @spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()} def prepare(file_content, opts \\ []) do - # TODO: Implement in Issue #3 (CSV Parsing) - # This is a skeleton implementation that will be filled in later - _ = {file_content, opts} + max_rows = Keyword.get(opts, :max_rows, 1000) + chunk_size = Keyword.get(opts, :chunk_size, 200) - # Placeholder return - will be replaced with actual implementation - {:error, "Not yet implemented"} + with {:ok, headers, rows} <- CsvParser.parse(file_content), + {:ok, custom_fields} <- load_custom_fields(), + {:ok, maps, warnings} <- build_header_maps(headers, custom_fields), + :ok <- validate_row_count(rows, max_rows) do + chunks = chunk_rows(rows, maps, chunk_size) + + # Build custom field lookup for efficient value processing + custom_field_lookup = build_custom_field_lookup(custom_fields) + + {:ok, + %{ + chunks: chunks, + column_map: maps.member, + custom_field_map: maps.custom, + custom_field_lookup: custom_field_lookup, + warnings: warnings + }} + end + end + + # Loads all custom fields from the database + defp load_custom_fields do + custom_fields = + Mv.Membership.CustomField + |> Ash.read!() + + {:ok, custom_fields} + rescue + e -> + {:error, "Failed to load custom fields: #{Exception.message(e)}"} + end + + # Builds custom field lookup map for efficient value processing + defp build_custom_field_lookup(custom_fields) do + custom_fields + |> Enum.reduce(%{}, fn cf, acc -> + id_str = to_string(cf.id) + Map.put(acc, id_str, %{id: cf.id, value_type: cf.value_type}) + end) + end + + # Builds header maps using HeaderMapper and collects warnings for unknown custom fields + defp build_header_maps(headers, custom_fields) do + # Convert custom fields to maps with id and name + custom_field_maps = + Enum.map(custom_fields, fn cf -> + %{id: to_string(cf.id), name: cf.name} + end) + + case HeaderMapper.build_maps(headers, custom_field_maps) do + {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} -> + # Build warnings for unknown custom field columns + warnings = + unknown + |> Enum.filter(fn header -> + # Check if it could be a custom field (not a known member field) + normalized = HeaderMapper.normalize_header(header) + # If it's not empty and not a member field, it might be a custom field + normalized != "" && not member_field?(normalized) + end) + |> Enum.map(fn header -> + "Unknown column '#{header}' will be ignored. " <> + "If this is a custom field, create it in Mila before importing." + end) + + {:ok, %{member: member_map, custom: custom_map}, warnings} + + {:error, reason} -> + {:error, reason} + end + end + + # Checks if a normalized header matches a member field + # Uses HeaderMapper's internal logic to check if header would map to a member field + defp member_field?(normalized) do + # Try to build maps with just this header - if it maps to a member field, it's a member field + case HeaderMapper.build_maps([normalized], []) do + {:ok, %{member: member_map}} -> + # If member_map is not empty, it's a member field + map_size(member_map) > 0 + + _ -> + false + end + end + + # Validates that row count doesn't exceed limit + defp validate_row_count(rows, max_rows) do + if length(rows) > max_rows do + {:error, "CSV file exceeds maximum row limit of #{max_rows} rows"} + else + :ok + end + end + + # Chunks rows and converts them to row maps using column maps + defp chunk_rows(rows, maps, chunk_size) do + rows + |> Enum.chunk_every(chunk_size) + |> Enum.map(fn chunk -> + Enum.map(chunk, fn {line_number, row_values} -> + row_map = build_row_map(row_values, maps) + {line_number, row_map} + end) + end) + end + + # Builds a row map from raw row values using column maps + defp build_row_map(row_values, maps) do + member_map = + maps.member + |> Enum.reduce(%{}, fn {field, index}, acc -> + value = Enum.at(row_values, index, "") + Map.put(acc, field, value) + end) + + custom_map = + maps.custom + |> Enum.reduce(%{}, fn {custom_field_id, index}, acc -> + value = Enum.at(row_values, index, "") + Map.put(acc, custom_field_id, value) + end) + + %{member: member_map, custom: custom_map} end @doc """ @@ -126,8 +253,9 @@ defmodule Mv.Membership.Import.MemberCSV do - `chunk_rows_with_lines` - List of tuples `{csv_line_number, row_map}` where: - `csv_line_number` - Physical line number in CSV (1-based) - - `row_map` - Map of column names to values - - `column_map` - Map of canonical field names (atoms) to column indices + - `row_map` - Map with `:member` and `:custom` keys containing field values + - `column_map` - Map of canonical field names (atoms) to column indices (for reference) + - `custom_field_map` - Map of custom field IDs (strings) to column indices (for reference) - `opts` - Optional keyword list for processing options ## Returns @@ -137,22 +265,212 @@ defmodule Mv.Membership.Import.MemberCSV do ## Examples - iex> chunk = [{2, %{"email" => "john@example.com"}}] + iex> chunk = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}] iex> column_map = %{email: 0} - iex> MemberCSV.process_chunk(chunk, column_map) + iex> custom_field_map = %{} + iex> MemberCSV.process_chunk(chunk, column_map, custom_field_map) {:ok, %{inserted: 1, failed: 0, errors: []}} """ @spec process_chunk( list({pos_integer(), map()}), %{atom() => non_neg_integer()}, + %{String.t() => non_neg_integer()}, keyword() ) :: {:ok, chunk_result()} | {:error, String.t()} - def process_chunk(chunk_rows_with_lines, column_map, opts \\ []) do - # TODO: Implement in Issue #6 (Persistence) - # This is a skeleton implementation that will be filled in later - _ = {chunk_rows_with_lines, column_map, opts} + def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do + custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{}) - # Placeholder return - will be replaced with actual implementation - {:ok, %{inserted: 0, failed: 0, errors: []}} + {inserted, failed, errors} = + Enum.reduce(chunk_rows_with_lines, {0, 0, []}, fn {line_number, row_map}, + {acc_inserted, acc_failed, acc_errors} -> + case process_row(row_map, line_number, custom_field_lookup) do + {:ok, _member} -> + {acc_inserted + 1, acc_failed, acc_errors} + + {:error, error} -> + {acc_inserted, acc_failed + 1, [error | acc_errors]} + end + end) + + {:ok, %{inserted: inserted, failed: failed, errors: Enum.reverse(errors)}} end + + # Processes a single row and creates member with custom field values + defp process_row( + %{member: member_attrs, custom: custom_attrs}, + line_number, + custom_field_lookup + ) do + # Prepare custom field values for Ash + custom_field_values = prepare_custom_field_values(custom_attrs, custom_field_lookup) + + # Create member with custom field values + member_attrs_with_cf = + member_attrs + |> Map.put(:custom_field_values, custom_field_values) + |> trim_string_values() + + # Only include custom_field_values if not empty + final_attrs = + if Enum.empty?(custom_field_values) do + Map.delete(member_attrs_with_cf, :custom_field_values) + else + member_attrs_with_cf + end + + case Mv.Membership.create_member(final_attrs) do + {:ok, member} -> + {:ok, member} + + {:error, %Ash.Error.Invalid{} = error} -> + {:error, format_ash_error(error, line_number)} + + {:error, error} -> + {:error, %Error{csv_line_number: line_number, field: nil, message: inspect(error)}} + end + rescue + e -> + {:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}} + end + + # Prepares custom field values from row map for Ash + defp prepare_custom_field_values(custom_attrs, custom_field_lookup) when is_map(custom_attrs) do + custom_attrs + |> Enum.filter(fn {_id, value} -> value != nil && value != "" end) + |> Enum.map(fn {custom_field_id_str, value} -> + case Map.get(custom_field_lookup, custom_field_id_str) do + nil -> + # Custom field not found, skip + nil + + %{id: custom_field_id, value_type: value_type} -> + %{ + "custom_field_id" => to_string(custom_field_id), + "value" => format_custom_field_value(value, value_type) + } + end + end) + |> Enum.filter(&(&1 != nil)) + end + + defp prepare_custom_field_values(_, _), do: [] + + # Formats a custom field value according to its type + # Uses _union_type and _union_value format as expected by Ash + defp format_custom_field_value(value, :string) when is_binary(value) do + %{"_union_type" => "string", "_union_value" => String.trim(value)} + end + + defp format_custom_field_value(value, :integer) when is_binary(value) do + case Integer.parse(value) do + {int_value, _} -> %{"_union_type" => "integer", "_union_value" => int_value} + :error -> %{"_union_type" => "string", "_union_value" => String.trim(value)} + end + end + + defp format_custom_field_value(value, :boolean) when is_binary(value) do + bool_value = + value + |> String.trim() + |> String.downcase() + |> case do + "true" -> true + "1" -> true + "yes" -> true + "ja" -> true + _ -> false + end + + %{"_union_type" => "boolean", "_union_value" => bool_value} + end + + defp format_custom_field_value(value, :date) when is_binary(value) do + case Date.from_iso8601(String.trim(value)) do + {:ok, date} -> %{"_union_type" => "date", "_union_value" => date} + {:error, _} -> %{"_union_type" => "string", "_union_value" => String.trim(value)} + end + end + + defp format_custom_field_value(value, :email) when is_binary(value) do + %{"_union_type" => "email", "_union_value" => String.trim(value)} + end + + defp format_custom_field_value(value, _type) when is_binary(value) do + # Default to string if type is unknown + %{"_union_type" => "string", "_union_value" => String.trim(value)} + end + + # Trims all string values in member attributes + defp trim_string_values(attrs) do + Enum.reduce(attrs, %{}, fn {key, value}, acc -> + trimmed_value = + if is_binary(value) do + String.trim(value) + else + value + end + + Map.put(acc, key, trimmed_value) + end) + end + + # Formats Ash errors into MemberCSV.Error structs + defp format_ash_error(%Ash.Error.Invalid{errors: errors}, line_number) do + # Try to find email-related errors first (for better error messages) + email_error = + Enum.find(errors, fn error -> + case error do + %{field: :email} -> true + _ -> false + end + end) + + case email_error || List.first(errors) do + %{field: field, message: message} when is_atom(field) -> + %Error{ + csv_line_number: line_number, + field: field, + message: format_error_message(message, field) + } + + %{message: message} -> + %Error{ + csv_line_number: line_number, + field: nil, + message: format_error_message(message, nil) + } + + _ -> + %Error{ + csv_line_number: line_number, + field: nil, + message: "Validation failed" + } + end + end + + # Formats error messages, handling common cases like email uniqueness + defp format_error_message(message, field) when is_binary(message) do + if email_uniqueness_error?(message, field) do + "email has already been taken" + else + message + end + end + + defp format_error_message(message, _field), do: to_string(message) + + # Checks if error message indicates email uniqueness constraint violation + defp email_uniqueness_error?(message, :email) do + message_lower = String.downcase(message) + + String.contains?(message_lower, "unique") or + String.contains?(message_lower, "constraint") or + String.contains?(message_lower, "duplicate") or + String.contains?(message_lower, "already been taken") or + String.contains?(message_lower, "already exists") or + String.contains?(message_lower, "violates unique constraint") + end + + defp email_uniqueness_error?(_message, _field), do: false end diff --git a/test/mv/membership/import/header_mapper_test.exs b/test/mv/membership/import/header_mapper_test.exs new file mode 100644 index 0000000..5e7efbd --- /dev/null +++ b/test/mv/membership/import/header_mapper_test.exs @@ -0,0 +1,244 @@ +defmodule Mv.Membership.Import.HeaderMapperTest do + use ExUnit.Case, async: true + + alias Mv.Membership.Import.HeaderMapper + + describe "normalize_header/1" do + test "trims whitespace" do + assert HeaderMapper.normalize_header(" email ") == "email" + end + + test "converts to lowercase" do + assert HeaderMapper.normalize_header("EMAIL") == "email" + assert HeaderMapper.normalize_header("E-Mail") == "e-mail" + end + + test "normalizes Unicode characters" do + # ß -> ss + assert HeaderMapper.normalize_header("Straße") == "strasse" + # Umlaute transliteration (ä -> ae, ö -> oe, ü -> ue) + assert HeaderMapper.normalize_header("Müller") == "mueller" + assert HeaderMapper.normalize_header("Köln") == "koeln" + assert HeaderMapper.normalize_header("Grün") == "gruen" + end + + test "compresses and removes whitespace" do + # Whitespace is removed entirely to ensure "first name" == "firstname" + assert HeaderMapper.normalize_header("first name") == "firstname" + assert HeaderMapper.normalize_header("email address") == "emailaddress" + end + + test "unifies hyphen variants" do + # Different Unicode hyphen characters should become standard hyphen + # en dash + assert HeaderMapper.normalize_header("E–Mail") == "e-mail" + # minus sign + assert HeaderMapper.normalize_header("E−Mail") == "e-mail" + # standard hyphen + assert HeaderMapper.normalize_header("E-Mail") == "e-mail" + end + + test "removes or unifies punctuation" do + # Parentheses, slashes, etc. are removed (whitespace is also removed) + assert HeaderMapper.normalize_header("E-Mail (privat)") == "e-mailprivat" + assert HeaderMapper.normalize_header("Telefon / Mobil") == "telefonmobil" + end + + test "handles empty strings" do + assert HeaderMapper.normalize_header("") == "" + assert HeaderMapper.normalize_header(" ") == "" + end + end + + describe "build_maps/2" do + test "maps English email variant correctly" do + headers = ["Email"] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert custom_map == %{} + assert unknown == [] + end + + test "maps German email variant correctly" do + headers = ["E-Mail"] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert custom_map == %{} + assert unknown == [] + end + + test "maps multiple member fields" do + headers = ["Email", "First Name", "Last Name"] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert member_map[:first_name] == 1 + assert member_map[:last_name] == 2 + assert custom_map == %{} + assert unknown == [] + end + + test "handles Unicode and whitespace in headers" do + headers = [" E-Mail ", "Straße", " Telefon / Mobil "] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert member_map[:street] == 1 + # "Telefon / Mobil" is not a known member field, so it should be unknown + assert length(unknown) == 1 + assert custom_map == %{} + end + + test "returns error when duplicate headers normalize to same field" do + headers = ["Email", "E-Mail"] + + assert {:error, reason} = HeaderMapper.build_maps(headers, []) + assert reason =~ "duplicate" + assert reason =~ "email" + end + + test "returns error when required field email is missing" do + headers = ["First Name", "Last Name"] + + assert {:error, reason} = HeaderMapper.build_maps(headers, []) + assert reason =~ "Missing required header" + assert reason =~ "email" + assert reason =~ "accepted" + end + + test "collects unknown columns" do + headers = ["Email", "FooBar", "UnknownColumn"] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert length(unknown) == 2 + assert "FooBar" in unknown or "foobar" in unknown + assert "UnknownColumn" in unknown or "unknowncolumn" in unknown + assert custom_map == %{} + end + + test "ignores empty headers after normalization" do + headers = ["Email", " ", ""] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert custom_map == %{} + assert unknown == [] + end + + test "maps custom field columns correctly" do + headers = ["Email", "Lieblingsfarbe"] + custom_fields = [%{id: "cf1", name: "Lieblingsfarbe"}] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, custom_fields) + + assert member_map[:email] == 0 + assert custom_map["cf1"] == 1 + assert unknown == [] + end + + test "custom field collision: member field wins" do + headers = ["Email"] + # Custom field with name "Email" should not override member field + custom_fields = [%{id: "cf1", name: "Email"}] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, custom_fields) + + assert member_map[:email] == 0 + # Custom field should not be in custom_map because member field has priority + assert custom_map == %{} + assert unknown == [] + end + + test "handles custom field with Unicode normalization" do + headers = ["Email", "Straße"] + custom_fields = [%{id: "cf1", name: "Straße"}] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, custom_fields) + + assert member_map[:email] == 0 + # "Straße" is a member field (street), so it should be in member_map, not custom_map + assert member_map[:street] == 1 + assert custom_map == %{} + assert unknown == [] + end + + test "handles unknown custom field columns" do + headers = ["Email", "UnknownCustomField"] + custom_fields = [%{id: "cf1", name: "KnownField"}] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, custom_fields) + + assert member_map[:email] == 0 + assert custom_map == %{} + # UnknownCustomField should be in unknown list + assert length(unknown) == 1 + end + + test "handles duplicate custom field names after normalization" do + headers = ["Email", "CustomField", "Custom Field"] + custom_fields = [%{id: "cf1", name: "CustomField"}] + + # Both "CustomField" and "Custom Field" normalize to the same, so this should error + assert {:error, reason} = HeaderMapper.build_maps(headers, custom_fields) + assert reason =~ "duplicate" + end + + test "maps all supported member fields" do + headers = [ + "Email", + "First Name", + "Last Name", + "Street", + "Postal Code", + "City" + ] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert member_map[:first_name] == 1 + assert member_map[:last_name] == 2 + assert member_map[:street] == 3 + assert member_map[:postal_code] == 4 + assert member_map[:city] == 5 + assert custom_map == %{} + assert unknown == [] + end + + test "maps German member field variants" do + headers = ["E-Mail", "Vorname", "Nachname", "Straße", "PLZ", "Stadt"] + + assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} = + HeaderMapper.build_maps(headers, []) + + assert member_map[:email] == 0 + assert member_map[:first_name] == 1 + assert member_map[:last_name] == 2 + assert member_map[:street] == 3 + assert member_map[:postal_code] == 4 + assert member_map[:city] == 5 + assert custom_map == %{} + assert unknown == [] + end + end +end diff --git a/test/mv/membership/import/member_csv_test.exs b/test/mv/membership/import/member_csv_test.exs index e918afd..356cc19 100644 --- a/test/mv/membership/import/member_csv_test.exs +++ b/test/mv/membership/import/member_csv_test.exs @@ -44,7 +44,6 @@ defmodule Mv.Membership.Import.MemberCSVTest do assert match?({:ok, _}, result) or match?({:error, _}, result) end - @tag :skip test "returns {:ok, import_state} on success" do file_content = "email\njohn@example.com" opts = [] @@ -56,6 +55,8 @@ defmodule Mv.Membership.Import.MemberCSVTest do assert Map.has_key?(import_state, :column_map) assert Map.has_key?(import_state, :custom_field_map) assert Map.has_key?(import_state, :warnings) + assert import_state.column_map[:email] == 0 + assert import_state.chunks != [] end test "returns {:error, reason} on failure" do @@ -71,24 +72,183 @@ defmodule Mv.Membership.Import.MemberCSVTest do end end - describe "process_chunk/3" do - test "function exists and accepts chunk_rows_with_lines, column_map, and opts" do - chunk_rows_with_lines = [{2, %{"email" => "john@example.com"}}] + describe "process_chunk/4" do + test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts" do + chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}] column_map = %{email: 0} + custom_field_map = %{} opts = [] # This will fail until the function is implemented - result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts) + result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) assert match?({:ok, _}, result) or match?({:error, _}, result) end - test "returns {:ok, chunk_result} on success" do - chunk_rows_with_lines = [{2, %{"email" => "john@example.com"}}] - column_map = %{email: 0} + test "creates member successfully with valid data" do + chunk_rows_with_lines = [ + {2, %{member: %{email: "john@example.com", first_name: "John"}, custom: %{}}} + ] + + column_map = %{email: 0, first_name: 1} + custom_field_map = %{} opts = [] assert {:ok, chunk_result} = - MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts) + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 1 + assert chunk_result.failed == 0 + assert chunk_result.errors == [] + + # Verify member was created + members = Mv.Membership.list_members!() + assert Enum.any?(members, &(&1.email == "john@example.com")) + end + + test "returns error for invalid email" do + chunk_rows_with_lines = [ + {2, %{member: %{email: "invalid-email"}, custom: %{}}} + ] + + column_map = %{email: 0} + custom_field_map = %{} + opts = [] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 0 + assert chunk_result.failed == 1 + assert length(chunk_result.errors) == 1 + + error = List.first(chunk_result.errors) + assert error.csv_line_number == 2 + assert error.field == :email + assert error.message =~ "email" + end + + test "returns error for duplicate email" do + # Create existing member first + {:ok, _existing} = + Mv.Membership.create_member(%{email: "duplicate@example.com", first_name: "Existing"}) + + chunk_rows_with_lines = [ + {2, %{member: %{email: "duplicate@example.com", first_name: "New"}, custom: %{}}} + ] + + column_map = %{email: 0, first_name: 1} + custom_field_map = %{} + opts = [] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 0 + assert chunk_result.failed == 1 + assert length(chunk_result.errors) == 1 + + error = List.first(chunk_result.errors) + assert error.csv_line_number == 2 + assert error.field == :email + assert error.message =~ "email" or error.message =~ "duplicate" or error.message =~ "unique" + end + + test "creates member with custom field values" do + # Create custom field first + {:ok, custom_field} = + Mv.Membership.CustomField + |> Ash.Changeset.for_create(:create, %{ + name: "Phone", + value_type: :string + }) + |> Ash.create() + + chunk_rows_with_lines = [ + {2, + %{ + member: %{email: "withcustom@example.com"}, + custom: %{to_string(custom_field.id) => "123-456-7890"} + }} + ] + + column_map = %{email: 0} + custom_field_map = %{to_string(custom_field.id) => 1} + + custom_field_lookup = %{ + to_string(custom_field.id) => %{id: custom_field.id, value_type: custom_field.value_type} + } + + opts = [custom_field_lookup: custom_field_lookup] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 1 + assert chunk_result.failed == 0 + + # Verify member and custom field value were created + members = Mv.Membership.list_members!() + member = Enum.find(members, &(&1.email == "withcustom@example.com")) + assert member != nil + + {:ok, member_with_cf} = Ash.load(member, :custom_field_values) + assert length(member_with_cf.custom_field_values) == 1 + cfv = List.first(member_with_cf.custom_field_values) + assert cfv.custom_field_id == custom_field.id + assert cfv.value.value == "123-456-7890" + end + + test "handles multiple rows with mixed success and failure" do + chunk_rows_with_lines = [ + {2, %{member: %{email: "valid1@example.com"}, custom: %{}}}, + {3, %{member: %{email: "invalid-email"}, custom: %{}}}, + {4, %{member: %{email: "valid2@example.com"}, custom: %{}}} + ] + + column_map = %{email: 0} + custom_field_map = %{} + opts = [] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 2 + assert chunk_result.failed == 1 + assert length(chunk_result.errors) == 1 + + error = List.first(chunk_result.errors) + assert error.csv_line_number == 3 + end + + test "preserves CSV line numbers in errors" do + chunk_rows_with_lines = [ + {5, %{member: %{email: "invalid"}, custom: %{}}}, + {10, %{member: %{email: "also-invalid"}, custom: %{}}} + ] + + column_map = %{email: 0} + custom_field_map = %{} + opts = [] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.failed == 2 + assert length(chunk_result.errors) == 2 + + line_numbers = Enum.map(chunk_result.errors, & &1.csv_line_number) + assert 5 in line_numbers + assert 10 in line_numbers + end + + test "returns {:ok, chunk_result} on success" do + chunk_rows_with_lines = [{2, %{member: %{email: "test@example.com"}, custom: %{}}}] + column_map = %{email: 0} + custom_field_map = %{} + opts = [] + + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) # Check that chunk_result contains expected fields assert Map.has_key?(chunk_result, :inserted) @@ -99,19 +259,23 @@ defmodule Mv.Membership.Import.MemberCSVTest do assert is_list(chunk_result.errors) end - test "returns {:error, reason} on failure" do + test "returns {:ok, _} with zero counts for empty chunk" do chunk_rows_with_lines = [] column_map = %{} + custom_field_map = %{} opts = [] - # This might return {:ok, _} with zero counts or {:error, _} - result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts) - assert match?({:ok, _}, result) or match?({:error, _}, result) + assert {:ok, chunk_result} = + MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts) + + assert chunk_result.inserted == 0 + assert chunk_result.failed == 0 + assert chunk_result.errors == [] end test "function has documentation" do # Check that @doc exists by reading the module - assert function_exported?(MemberCSV, :process_chunk, 3) + assert function_exported?(MemberCSV, :process_chunk, 4) end end