From 95c7bf7a15c21359497567efc735a4a6d9af8806 Mon Sep 17 00:00:00 2001 From: Moritz Date: Wed, 3 Jun 2026 02:01:09 +0200 Subject: [PATCH] feat(import): recognize group and fee-type columns and always ignore fee-status --- lib/mv/membership/import/header_mapper.ex | 97 +++++++++-- .../membership/import/header_mapper_test.exs | 164 ++++++++++++++++++ 2 files changed, 250 insertions(+), 11 deletions(-) diff --git a/lib/mv/membership/import/header_mapper.ex b/lib/mv/membership/import/header_mapper.ex index d96d96e..3047944 100644 --- a/lib/mv/membership/import/header_mapper.ex +++ b/lib/mv/membership/import/header_mapper.ex @@ -47,10 +47,10 @@ defmodule Mv.Membership.Import.HeaderMapper do "e-mail" iex> HeaderMapper.build_maps(["Email", "First Name"], []) - {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}} + {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}} iex> HeaderMapper.build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}]) - {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}} + {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}} """ @type column_map :: %{atom() => non_neg_integer()} @@ -60,6 +60,31 @@ defmodule Mv.Membership.Import.HeaderMapper do # Required member fields @required_member_fields [:email] + # Fee-status header variants that must never be imported (computed/read-only field). + # Stored already-normalized; checked before member, custom, groups, and fee-type mapping. + # Maintain this list when new locale translations for fee-status are added. + @ignored_normalized [ + "membershipfeestatus", + "mitgliedsbeitragsstatus", + "bezahlstatus" + ] + + # Normalized header variants for the groups column. The column is resolved to + # group associations during import; it is never a member or custom field. + @groups_column_normalized [ + "groups", + "gruppen", + "gruppe" + ] + + # Normalized header variants for the membership fee-type column. The column is + # resolved to a MembershipFeeType during import; it is never a member or custom field. + @fee_type_column_normalized [ + "membershipfeetype", + "feetype", + "beitragsart" + ] + # Canonical member fields with their raw variants # These will be normalized at runtime when building the lookup map @member_field_variants_raw %{ @@ -239,30 +264,79 @@ defmodule Mv.Membership.Import.HeaderMapper do ## Returns - - `{:ok, %{member: column_map, custom: custom_field_map, unknown: unknown_headers}}` on success + - `{:ok, %{member: column_map, custom: custom_field_map, unknown: unknown_headers, + ignored: [non_neg_integer], groups_column_index: non_neg_integer | nil, + fee_type_column_index: non_neg_integer | nil}}` on success - `{:error, reason}` on error (missing required field, duplicate headers) + The `ignored` list holds the indices of fee-status columns (computed/read-only), + which are never mapped to member or custom fields. + ## Examples iex> build_maps(["Email", "First Name"], []) - {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}} + {:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}} iex> build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}]) - {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}} + {:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}} """ @spec build_maps([String.t()], [map()]) :: - {:ok, %{member: column_map(), custom: custom_field_map(), unknown: unknown_headers()}} + {:ok, + %{ + member: column_map(), + custom: custom_field_map(), + unknown: unknown_headers(), + ignored: [non_neg_integer()], + groups_column_index: non_neg_integer() | nil, + fee_type_column_index: non_neg_integer() | nil + }} | {:error, String.t()} def build_maps(headers, custom_fields) when is_list(headers) and is_list(custom_fields) do - with {:ok, member_map, unknown_after_member} <- build_member_map(headers), + ignored = ignored_indices(headers) + groups_column_index = first_matching_index(headers, @groups_column_normalized) + fee_type_column_index = first_matching_index(headers, @fee_type_column_normalized) + + reserved = + [groups_column_index, fee_type_column_index | ignored] + |> Enum.reject(&is_nil/1) + |> MapSet.new() + + with {:ok, member_map, unknown_after_member} <- build_member_map(headers, reserved), {:ok, custom_map, unknown_after_custom} <- build_custom_field_map(headers, unknown_after_member, custom_fields, member_map) do unknown = Enum.map(unknown_after_custom, &Enum.at(headers, &1)) - {:ok, %{member: member_map, custom: custom_map, unknown: unknown}} + + {:ok, + %{ + member: member_map, + custom: custom_map, + unknown: unknown, + ignored: ignored, + groups_column_index: groups_column_index, + fee_type_column_index: fee_type_column_index + }} end end + # Returns the index of the first header whose normalized form is in `variants`, + # or nil if none match. + defp first_matching_index(headers, variants) do + headers + |> Enum.with_index() + |> Enum.find_value(fn {header, index} -> + if normalize_header(header) in variants, do: index + end) + end + + # Returns the column indices whose normalized header is in the fee-status ignore list. + defp ignored_indices(headers) do + headers + |> Enum.with_index() + |> Enum.filter(fn {header, _index} -> normalize_header(header) in @ignored_normalized end) + |> Enum.map(fn {_header, index} -> index end) + end + # --- Private Functions --- # Transliterates German umlauts and special characters @@ -304,13 +378,14 @@ defmodule Mv.Membership.Import.HeaderMapper do |> String.replace(" ", "") end - # Builds member field column map - defp build_member_map(headers) do + # Builds member field column map, skipping reserved (e.g. ignored) indices. + defp build_member_map(headers, reserved) do result = headers |> Enum.with_index() |> Enum.reduce_while({%{}, []}, fn {header, index}, {acc_map, acc_unknown} -> - normalized = normalize_header(header) + normalized = + if MapSet.member?(reserved, index), do: "", else: normalize_header(header) case process_member_header(header, index, normalized, acc_map, %{}) do {:error, reason} -> diff --git a/test/mv/membership/import/header_mapper_test.exs b/test/mv/membership/import/header_mapper_test.exs index 2f4fcad..f5519d8 100644 --- a/test/mv/membership/import/header_mapper_test.exs +++ b/test/mv/membership/import/header_mapper_test.exs @@ -1,5 +1,6 @@ defmodule Mv.Membership.Import.HeaderMapperTest do use ExUnit.Case, async: true + use ExUnitProperties alias Mv.Membership.Import.HeaderMapper @@ -272,4 +273,167 @@ defmodule Mv.Membership.Import.HeaderMapperTest do assert unknown == [] end end + + describe "build_maps/2 fee-status ignore list" do + test "places fee-status variants in ignored, not member or custom map" do + headers = ["email", "Bezahlstatus"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.member[:email] == 0 + assert result.custom == %{} + assert result.ignored == [1] + refute Map.has_key?(result.member, :bezahlstatus) + end + + test "ignores membership_fee_status snake-case variant" do + headers = ["email", "membership_fee_status"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.ignored == [1] + assert result.custom == %{} + end + + test "ignores German Mitgliedsbeitragsstatus variant" do + headers = ["email", "Mitgliedsbeitragsstatus"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.ignored == [1] + end + + test "fee-status takes priority over a same-named custom field" do + headers = ["email", "Bezahlstatus"] + custom_fields = [%{id: "cf1", name: "Bezahlstatus"}] + + assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields) + + assert result.ignored == [1] + assert result.custom == %{} + end + + test "result carries groups_column_index and fee_type_column_index keys" do + assert {:ok, result} = HeaderMapper.build_maps(["email"], []) + + assert Map.has_key?(result, :groups_column_index) + assert Map.has_key?(result, :fee_type_column_index) + end + end + + describe "build_maps/2 groups column detection" do + test "detects German Gruppen variant and excludes it from member/custom maps" do + headers = ["email", "Gruppen"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.groups_column_index == 1 + assert result.custom == %{} + assert result.unknown == [] + refute Map.has_key?(result.member, :gruppen) + end + + test "detects English Groups variant" do + headers = ["email", "Groups"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.groups_column_index == 1 + end + + test "detects singular Gruppe and lowercase groups variants" do + assert {:ok, %{groups_column_index: 1}} = HeaderMapper.build_maps(["email", "Gruppe"], []) + assert {:ok, %{groups_column_index: 1}} = HeaderMapper.build_maps(["email", "groups"], []) + end + + test "groups column takes priority over a same-named custom field" do + headers = ["email", "Gruppen"] + custom_fields = [%{id: "cf1", name: "Gruppen"}] + + assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields) + + assert result.groups_column_index == 1 + assert result.custom == %{} + end + + test "groups_column_index is nil when no groups column present" do + assert {:ok, %{groups_column_index: nil}} = HeaderMapper.build_maps(["email"], []) + end + end + + describe "build_maps/2 fee-type column detection" do + test "detects German Beitragsart variant and excludes it from member/custom maps" do + headers = ["email", "Beitragsart"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.fee_type_column_index == 1 + assert result.custom == %{} + assert result.unknown == [] + end + + test "detects English fee type variants" do + assert {:ok, %{fee_type_column_index: 1}} = + HeaderMapper.build_maps(["email", "Fee Type"], []) + + assert {:ok, %{fee_type_column_index: 1}} = + HeaderMapper.build_maps(["email", "fee type"], []) + + assert {:ok, %{fee_type_column_index: 1}} = + HeaderMapper.build_maps(["email", "fee_type"], []) + + assert {:ok, %{fee_type_column_index: 1}} = + HeaderMapper.build_maps(["email", "membership_fee_type"], []) + end + + test "fee-type column takes priority over a same-named custom field" do + headers = ["email", "Beitragsart"] + custom_fields = [%{id: "cf1", name: "Beitragsart"}] + + assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields) + + assert result.fee_type_column_index == 1 + assert result.custom == %{} + end + + test "fee_type_column_index is nil when no fee-type column present" do + assert {:ok, %{fee_type_column_index: nil}} = HeaderMapper.build_maps(["email"], []) + end + + test "detects groups and fee-type columns together" do + headers = ["email", "Gruppen", "Beitragsart"] + + assert {:ok, result} = HeaderMapper.build_maps(headers, []) + + assert result.groups_column_index == 1 + assert result.fee_type_column_index == 2 + assert result.member[:email] == 0 + assert result.custom == %{} + assert result.unknown == [] + end + end + + describe "build_maps/2 fee-status ignore property" do + property "every fee-status variant is ignored, never member or custom" do + check all( + variant <- + StreamData.member_of([ + "Membership Fee Status", + "membership_fee_status", + "Mitgliedsbeitragsstatus", + "Bezahlstatus", + " Bezahlstatus ", + "BEZAHLSTATUS" + ]) + ) do + custom_fields = [%{id: "cf1", name: variant}] + + assert {:ok, result} = HeaderMapper.build_maps(["email", variant], custom_fields) + + assert result.ignored == [1] + assert result.custom == %{} + refute Map.has_key?(result.member, :bezahlstatus) + end + end + end end