From 0673684cc154d1e283638f2bcd609d2ce2604504 Mon Sep 17 00:00:00 2001
From: carla <carla@local-it.org>
Date: Thu, 15 Jan 2026 16:11:02 +0100
Subject: [PATCH] test: adds tests for header normalization

---
 .../membership/import/header_mapper_test.exs  | 244 ++++++++++++++++++
 test/mv/membership/import/member_csv_test.exs | 185 ++++++++++++-
 2 files changed, 416 insertions(+), 13 deletions(-)
 create mode 100644 test/mv/membership/import/header_mapper_test.exs

diff --git a/test/mv/membership/import/header_mapper_test.exs b/test/mv/membership/import/header_mapper_test.exs
new file mode 100644
index 0000000..5e7efbd
--- /dev/null
+++ b/test/mv/membership/import/header_mapper_test.exs
@@ -0,0 +1,244 @@
+defmodule Mv.Membership.Import.HeaderMapperTest do
+  use ExUnit.Case, async: true
+
+  alias Mv.Membership.Import.HeaderMapper
+
+  describe "normalize_header/1" do
+    test "trims whitespace" do
+      assert HeaderMapper.normalize_header("  email  ") == "email"
+    end
+
+    test "converts to lowercase" do
+      assert HeaderMapper.normalize_header("EMAIL") == "email"
+      assert HeaderMapper.normalize_header("E-Mail") == "e-mail"
+    end
+
+    test "normalizes Unicode characters" do
+      # ß -> ss
+      assert HeaderMapper.normalize_header("Straße") == "strasse"
+      # Umlaute transliteration (ä -> ae, ö -> oe, ü -> ue)
+      assert HeaderMapper.normalize_header("Müller") == "mueller"
+      assert HeaderMapper.normalize_header("Köln") == "koeln"
+      assert HeaderMapper.normalize_header("Grün") == "gruen"
+    end
+
+    test "compresses and removes whitespace" do
+      # Whitespace is removed entirely to ensure "first name" == "firstname"
+      assert HeaderMapper.normalize_header("first  name") == "firstname"
+      assert HeaderMapper.normalize_header("email   address") == "emailaddress"
+    end
+
+    test "unifies hyphen variants" do
+      # Different Unicode hyphen characters should become standard hyphen
+      # en dash
+      assert HeaderMapper.normalize_header("E–Mail") == "e-mail"
+      # minus sign
+      assert HeaderMapper.normalize_header("E−Mail") == "e-mail"
+      # standard hyphen
+      assert HeaderMapper.normalize_header("E-Mail") == "e-mail"
+    end
+
+    test "removes or unifies punctuation" do
+      # Parentheses, slashes, etc. are removed (whitespace is also removed)
+      assert HeaderMapper.normalize_header("E-Mail (privat)") == "e-mailprivat"
+      assert HeaderMapper.normalize_header("Telefon / Mobil") == "telefonmobil"
+    end
+
+    test "handles empty strings" do
+      assert HeaderMapper.normalize_header("") == ""
+      assert HeaderMapper.normalize_header("   ") == ""
+    end
+  end
+
+  describe "build_maps/2" do
+    test "maps English email variant correctly" do
+      headers = ["Email"]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "maps German email variant correctly" do
+      headers = ["E-Mail"]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "maps multiple member fields" do
+      headers = ["Email", "First Name", "Last Name"]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert member_map[:first_name] == 1
+      assert member_map[:last_name] == 2
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "handles Unicode and whitespace in headers" do
+      headers = ["  E-Mail  ", "Straße", "  Telefon / Mobil  "]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert member_map[:street] == 1
+      # "Telefon / Mobil" is not a known member field, so it should be unknown
+      assert length(unknown) == 1
+      assert custom_map == %{}
+    end
+
+    test "returns error when duplicate headers normalize to same field" do
+      headers = ["Email", "E-Mail"]
+
+      assert {:error, reason} = HeaderMapper.build_maps(headers, [])
+      assert reason =~ "duplicate"
+      assert reason =~ "email"
+    end
+
+    test "returns error when required field email is missing" do
+      headers = ["First Name", "Last Name"]
+
+      assert {:error, reason} = HeaderMapper.build_maps(headers, [])
+      assert reason =~ "Missing required header"
+      assert reason =~ "email"
+      assert reason =~ "accepted"
+    end
+
+    test "collects unknown columns" do
+      headers = ["Email", "FooBar", "UnknownColumn"]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert length(unknown) == 2
+      assert "FooBar" in unknown or "foobar" in unknown
+      assert "UnknownColumn" in unknown or "unknowncolumn" in unknown
+      assert custom_map == %{}
+    end
+
+    test "ignores empty headers after normalization" do
+      headers = ["Email", "   ", ""]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "maps custom field columns correctly" do
+      headers = ["Email", "Lieblingsfarbe"]
+      custom_fields = [%{id: "cf1", name: "Lieblingsfarbe"}]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, custom_fields)
+
+      assert member_map[:email] == 0
+      assert custom_map["cf1"] == 1
+      assert unknown == []
+    end
+
+    test "custom field collision: member field wins" do
+      headers = ["Email"]
+      # Custom field with name "Email" should not override member field
+      custom_fields = [%{id: "cf1", name: "Email"}]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, custom_fields)
+
+      assert member_map[:email] == 0
+      # Custom field should not be in custom_map because member field has priority
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "handles custom field with Unicode normalization" do
+      headers = ["Email", "Straße"]
+      custom_fields = [%{id: "cf1", name: "Straße"}]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, custom_fields)
+
+      assert member_map[:email] == 0
+      # "Straße" is a member field (street), so it should be in member_map, not custom_map
+      assert member_map[:street] == 1
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "handles unknown custom field columns" do
+      headers = ["Email", "UnknownCustomField"]
+      custom_fields = [%{id: "cf1", name: "KnownField"}]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, custom_fields)
+
+      assert member_map[:email] == 0
+      assert custom_map == %{}
+      # UnknownCustomField should be in unknown list
+      assert length(unknown) == 1
+    end
+
+    test "handles duplicate custom field names after normalization" do
+      headers = ["Email", "CustomField", "Custom Field"]
+      custom_fields = [%{id: "cf1", name: "CustomField"}]
+
+      # Both "CustomField" and "Custom Field" normalize to the same, so this should error
+      assert {:error, reason} = HeaderMapper.build_maps(headers, custom_fields)
+      assert reason =~ "duplicate"
+    end
+
+    test "maps all supported member fields" do
+      headers = [
+        "Email",
+        "First Name",
+        "Last Name",
+        "Street",
+        "Postal Code",
+        "City"
+      ]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert member_map[:first_name] == 1
+      assert member_map[:last_name] == 2
+      assert member_map[:street] == 3
+      assert member_map[:postal_code] == 4
+      assert member_map[:city] == 5
+      assert custom_map == %{}
+      assert unknown == []
+    end
+
+    test "maps German member field variants" do
+      headers = ["E-Mail", "Vorname", "Nachname", "Straße", "PLZ", "Stadt"]
+
+      assert {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} =
+               HeaderMapper.build_maps(headers, [])
+
+      assert member_map[:email] == 0
+      assert member_map[:first_name] == 1
+      assert member_map[:last_name] == 2
+      assert member_map[:street] == 3
+      assert member_map[:postal_code] == 4
+      assert member_map[:city] == 5
+      assert custom_map == %{}
+      assert unknown == []
+    end
+  end
+end
diff --git a/test/mv/membership/import/member_csv_test.exs b/test/mv/membership/import/member_csv_test.exs
index e918afd..a38315f 100644
--- a/test/mv/membership/import/member_csv_test.exs
+++ b/test/mv/membership/import/member_csv_test.exs
@@ -44,7 +44,6 @@ defmodule Mv.Membership.Import.MemberCSVTest do
       assert match?({:ok, _}, result) or match?({:error, _}, result)
     end
 
-    @tag :skip
     test "returns {:ok, import_state} on success" do
       file_content = "email\njohn@example.com"
       opts = []
@@ -56,6 +55,8 @@ defmodule Mv.Membership.Import.MemberCSVTest do
       assert Map.has_key?(import_state, :column_map)
       assert Map.has_key?(import_state, :custom_field_map)
       assert Map.has_key?(import_state, :warnings)
+      assert import_state.column_map[:email] == 0
+      assert import_state.chunks != []
     end
 
     test "returns {:error, reason} on failure" do
@@ -72,23 +73,177 @@ defmodule Mv.Membership.Import.MemberCSVTest do
   end
 
   describe "process_chunk/3" do
-    test "function exists and accepts chunk_rows_with_lines, column_map, and opts" do
-      chunk_rows_with_lines = [{2, %{"email" => "john@example.com"}}]
+    test "function exists and accepts chunk_rows_with_lines, column_map, custom_field_map, and opts" do
+      chunk_rows_with_lines = [{2, %{member: %{email: "john@example.com"}, custom: %{}}}]
       column_map = %{email: 0}
+      custom_field_map = %{}
       opts = []
 
       # This will fail until the function is implemented
-      result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts)
+      result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
       assert match?({:ok, _}, result) or match?({:error, _}, result)
     end
 
-    test "returns {:ok, chunk_result} on success" do
-      chunk_rows_with_lines = [{2, %{"email" => "john@example.com"}}]
-      column_map = %{email: 0}
+    test "creates member successfully with valid data" do
+      chunk_rows_with_lines = [
+        {2, %{member: %{email: "john@example.com", first_name: "John"}, custom: %{}}}
+      ]
+
+      column_map = %{email: 0, first_name: 1}
+      custom_field_map = %{}
       opts = []
 
       assert {:ok, chunk_result} =
-               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts)
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 1
+      assert chunk_result.failed == 0
+      assert chunk_result.errors == []
+
+      # Verify member was created
+      members = Mv.Membership.list_members!()
+      assert Enum.any?(members, &(&1.email == "john@example.com"))
+    end
+
+    test "returns error for invalid email" do
+      chunk_rows_with_lines = [
+        {2, %{member: %{email: "invalid-email"}, custom: %{}}}
+      ]
+
+      column_map = %{email: 0}
+      custom_field_map = %{}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 0
+      assert chunk_result.failed == 1
+      assert length(chunk_result.errors) == 1
+
+      error = List.first(chunk_result.errors)
+      assert error.csv_line_number == 2
+      assert error.field == :email
+      assert error.message =~ "email"
+    end
+
+    test "returns error for duplicate email" do
+      # Create existing member first
+      {:ok, _existing} =
+        Mv.Membership.create_member(%{email: "duplicate@example.com", first_name: "Existing"})
+
+      chunk_rows_with_lines = [
+        {2, %{member: %{email: "duplicate@example.com", first_name: "New"}, custom: %{}}}
+      ]
+
+      column_map = %{email: 0, first_name: 1}
+      custom_field_map = %{}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 0
+      assert chunk_result.failed == 1
+      assert length(chunk_result.errors) == 1
+
+      error = List.first(chunk_result.errors)
+      assert error.csv_line_number == 2
+      assert error.field == :email
+      assert error.message =~ "email" or error.message =~ "duplicate" or error.message =~ "unique"
+    end
+
+    test "creates member with custom field values" do
+      # Create custom field first
+      {:ok, custom_field} =
+        Mv.Membership.CustomField
+        |> Ash.Changeset.for_create(:create, %{
+          name: "Phone",
+          value_type: :string
+        })
+        |> Ash.create()
+
+      chunk_rows_with_lines = [
+        {2,
+         %{
+           member: %{email: "withcustom@example.com"},
+           custom: %{to_string(custom_field.id) => "123-456-7890"}
+         }}
+      ]
+
+      column_map = %{email: 0}
+      custom_field_map = %{to_string(custom_field.id) => 1}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 1
+      assert chunk_result.failed == 0
+
+      # Verify member and custom field value were created
+      members = Mv.Membership.list_members!()
+      member = Enum.find(members, &(&1.email == "withcustom@example.com"))
+      assert member != nil
+
+      {:ok, member_with_cf} = Ash.load(member, :custom_field_values)
+      assert length(member_with_cf.custom_field_values) == 1
+      cfv = List.first(member_with_cf.custom_field_values)
+      assert cfv.custom_field_id == custom_field.id
+      assert cfv.value.value == "123-456-7890"
+    end
+
+    test "handles multiple rows with mixed success and failure" do
+      chunk_rows_with_lines = [
+        {2, %{member: %{email: "valid1@example.com"}, custom: %{}}},
+        {3, %{member: %{email: "invalid-email"}, custom: %{}}},
+        {4, %{member: %{email: "valid2@example.com"}, custom: %{}}}
+      ]
+
+      column_map = %{email: 0}
+      custom_field_map = %{}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 2
+      assert chunk_result.failed == 1
+      assert length(chunk_result.errors) == 1
+
+      error = List.first(chunk_result.errors)
+      assert error.csv_line_number == 3
+    end
+
+    test "preserves CSV line numbers in errors" do
+      chunk_rows_with_lines = [
+        {5, %{member: %{email: "invalid"}, custom: %{}}},
+        {10, %{member: %{email: "also-invalid"}, custom: %{}}}
+      ]
+
+      column_map = %{email: 0}
+      custom_field_map = %{}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.failed == 2
+      assert length(chunk_result.errors) == 2
+
+      line_numbers = Enum.map(chunk_result.errors, & &1.csv_line_number)
+      assert 5 in line_numbers
+      assert 10 in line_numbers
+    end
+
+    test "returns {:ok, chunk_result} on success" do
+      chunk_rows_with_lines = [{2, %{member: %{email: "test@example.com"}, custom: %{}}}]
+      column_map = %{email: 0}
+      custom_field_map = %{}
+      opts = []
+
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
 
       # Check that chunk_result contains expected fields
       assert Map.has_key?(chunk_result, :inserted)
@@ -99,19 +254,23 @@ defmodule Mv.Membership.Import.MemberCSVTest do
       assert is_list(chunk_result.errors)
     end
 
-    test "returns {:error, reason} on failure" do
+    test "returns {:ok, _} with zero counts for empty chunk" do
       chunk_rows_with_lines = []
       column_map = %{}
+      custom_field_map = %{}
       opts = []
 
-      # This might return {:ok, _} with zero counts or {:error, _}
-      result = MemberCSV.process_chunk(chunk_rows_with_lines, column_map, opts)
-      assert match?({:ok, _}, result) or match?({:error, _}, result)
+      assert {:ok, chunk_result} =
+               MemberCSV.process_chunk(chunk_rows_with_lines, column_map, custom_field_map, opts)
+
+      assert chunk_result.inserted == 0
+      assert chunk_result.failed == 0
+      assert chunk_result.errors == []
     end
 
     test "function has documentation" do
       # Check that @doc exists by reading the module
-      assert function_exported?(MemberCSV, :process_chunk, 3)
+      assert function_exported?(MemberCSV, :process_chunk, 4)
     end
   end