Merge remote-tracking branch 'origin/main' into issue/mitgliederverwaltung-508
Some checks failed
continuous-integration/drone/push Build is failing
continuous-integration/drone/promote/production Build is passing

This commit is contained in:
Simon 2026-06-04 09:53:27 +02:00
commit 6500dead95
Signed by: simon
GPG key ID: 40E7A58C4AA1EDB2
22 changed files with 2569 additions and 109 deletions

View file

@ -0,0 +1,27 @@
defmodule Mv.Membership.CustomFieldValueFormatterTest do
use ExUnit.Case, async: true
alias Mv.Membership.CustomFieldValueFormatter
describe "format_custom_field_value/2 for :date" do
test "formats an Ash.Union date value as ISO-8601" do
union = %Ash.Union{value: ~D[2024-03-15], type: :date}
assert CustomFieldValueFormatter.format_custom_field_value(union, %{value_type: :date}) ==
"2024-03-15"
end
test "formats a direct Date value as ISO-8601" do
assert CustomFieldValueFormatter.format_custom_field_value(~D[2024-03-15], %{
value_type: :date
}) == "2024-03-15"
end
test "formats an already-stored ISO-8601 string date as ISO-8601" do
union = %Ash.Union{value: "2024-03-15", type: :date}
assert CustomFieldValueFormatter.format_custom_field_value(union, %{value_type: :date}) ==
"2024-03-15"
end
end
end

View file

@ -0,0 +1,72 @@
defmodule Mv.Membership.Import.ColumnResolverQueryTest do
# async: false — attaches a global telemetry handler to inspect emitted SQL.
use Mv.DataCase, async: false
alias Mv.Membership.Import.ColumnResolver
setup do
%{actor: Mv.Helpers.SystemActor.get_system_actor()}
end
describe "create_or_find_group/3 group lookup is name-filtered (no full-table scan)" do
test "resolving a new name absent from the snapshot queries by name, not the whole table",
%{actor: actor} do
# Populate the table so a full-table read would be costly and observable.
for n <- 1..20, do: Mv.Fixtures.group_fixture(%{name: "Existing #{n}"})
queries =
capture_group_select_queries(fn ->
# The name is absent from the (empty) snapshot, forcing a DB lookup
# before the create attempt. That lookup must filter by name.
assert {:ok, group} = ColumnResolver.create_or_find_group("New One", [], actor)
assert group.name == "New One"
end)
# No SELECT against the groups table issued during resolution may be an
# unfiltered full-table scan. The pre-create existence check must filter by
# name (carry a WHERE predicate).
refute Enum.any?(queries, &unfiltered_groups_select?/1),
"expected no unfiltered groups table scan, got:\n#{Enum.join(queries, "\n")}"
end
end
defp capture_group_select_queries(fun) do
test_pid = self()
handler_id = "test-group-query-#{System.unique_integer([:positive])}"
:telemetry.attach(
handler_id,
[:mv, :repo, :query],
fn _event, _measurements, metadata, _config ->
sql = metadata[:query] || ""
if String.contains?(sql, "SELECT") and String.contains?(sql, "\"groups\"") do
send(test_pid, {:group_query, sql})
end
end,
nil
)
try do
fun.()
after
:telemetry.detach(handler_id)
end
collect_group_queries([])
end
defp collect_group_queries(acc) do
receive do
{:group_query, sql} -> collect_group_queries([sql | acc])
after
0 -> Enum.reverse(acc)
end
end
# An unfiltered groups SELECT reads the whole table: it selects FROM "groups"
# with no WHERE clause at all. A name-filtered lookup carries a WHERE predicate.
defp unfiltered_groups_select?(sql) do
String.contains?(sql, "FROM \"groups\"") and not String.contains?(sql, "WHERE")
end
end

View file

@ -0,0 +1,227 @@
defmodule Mv.Membership.Import.ColumnResolverTest do
use Mv.DataCase, async: true
use ExUnitProperties
alias Mv.Membership.Import.ColumnResolver
setup do
%{actor: Mv.Helpers.SystemActor.get_system_actor()}
end
defp fee_type_fixture(name, actor) do
{:ok, fee_type} =
Mv.MembershipFees.create_membership_fee_type(
%{name: name, amount: Decimal.new("10.00"), interval: :yearly},
actor: actor
)
fee_type
end
defp header_maps(overrides) do
Map.merge(
%{
member: %{email: 0},
custom: %{},
unknown: [],
ignored: [],
groups_column_index: nil,
fee_type_column_index: nil
},
overrides
)
end
describe "resolve/3 group classification" do
test "splits group names into found (existing) and to_create (missing)", %{actor: actor} do
existing = Mv.Fixtures.group_fixture(%{name: "Orchester"})
maps = header_maps(%{member: %{email: 0}, groups_column_index: 1})
rows = [
{2, ["a@example.com", "Orchester"]},
{3, ["b@example.com", "Neues Ensemble"]}
]
result = ColumnResolver.resolve(maps, rows, actor)
assert Enum.any?(result.groups_found, &(&1.name == "Orchester" and &1.id == existing.id))
assert "Neues Ensemble" in result.groups_to_create
refute "Orchester" in result.groups_to_create
end
test "groups_found and groups_to_create are empty when no groups column", %{actor: actor} do
maps = header_maps(%{})
rows = [{2, ["a@example.com"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.groups_found == []
assert result.groups_to_create == []
end
end
describe "resolve/3 preview rows" do
test "returns up to 3 preview rows", %{actor: actor} do
maps = header_maps(%{})
rows = [
{2, ["a@example.com"]},
{3, ["b@example.com"]},
{4, ["c@example.com"]},
{5, ["d@example.com"]}
]
result = ColumnResolver.resolve(maps, rows, actor)
assert length(result.preview_rows) == 3
assert result.preview_rows == [["a@example.com"], ["b@example.com"], ["c@example.com"]]
end
test "returns fewer preview rows when file has fewer data rows", %{actor: actor} do
maps = header_maps(%{})
rows = [{2, ["a@example.com"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.preview_rows == [["a@example.com"]]
end
end
describe "resolve/3 fee-type resolution" do
test "maps known fee-type names to their id by normalized name", %{actor: actor} do
standard = fee_type_fixture("Standard", actor)
maps = header_maps(%{fee_type_column_index: 1})
rows = [{2, ["a@example.com", "Standard"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.fee_type_map["standard"] == standard.id
assert result.fee_type_warnings == []
end
test "records a warning for an unknown fee-type name", %{actor: actor} do
maps = header_maps(%{fee_type_column_index: 1})
rows = [{2, ["a@example.com", "Nonexistent Type"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert "Nonexistent Type" in result.fee_type_warnings
end
test "sets has_empty_fee_type_cells? when a fee-type cell is blank", %{actor: actor} do
fee_type_fixture("Standard", actor)
maps = header_maps(%{fee_type_column_index: 1})
rows = [
{2, ["a@example.com", "Standard"]},
{3, ["b@example.com", " "]}
]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.has_empty_fee_type_cells? == true
end
test "has_empty_fee_type_cells? is false when all cells filled", %{actor: actor} do
fee_type_fixture("Standard", actor)
maps = header_maps(%{fee_type_column_index: 1})
rows = [{2, ["a@example.com", "Standard"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.has_empty_fee_type_cells? == false
end
test "fee-type resolution defaults are empty when no fee-type column", %{actor: actor} do
maps = header_maps(%{})
rows = [{2, ["a@example.com"]}]
result = ColumnResolver.resolve(maps, rows, actor)
assert result.fee_type_map == %{}
assert result.fee_type_warnings == []
assert result.has_empty_fee_type_cells? == false
end
end
describe "create_or_find_group/3" do
test "creates a new group when none exists", %{actor: actor} do
assert {:ok, group} = ColumnResolver.create_or_find_group("Brand New Group", [], actor)
assert group.name == "Brand New Group"
end
test "returns the existing group from the pre-fetched list without creating", %{actor: actor} do
existing = Mv.Fixtures.group_fixture(%{name: "Existing Group"})
before_count = length(Mv.Membership.list_groups!(actor: actor))
assert {:ok, group} =
ColumnResolver.create_or_find_group("Existing Group", [existing], actor)
assert group.id == existing.id
assert length(Mv.Membership.list_groups!(actor: actor)) == before_count
end
test "resolves to a group created concurrently after the snapshot was taken",
%{actor: actor} do
# Simulates a concurrent import session: the group name is absent from the
# caller's pre-fetched snapshot, but the group now exists in the DB. The
# resolver must link to the existing group, never error or duplicate it.
stale_snapshot = []
_concurrently_created = Mv.Fixtures.group_fixture(%{name: "Concurrent Group"})
before_count = length(Mv.Membership.list_groups!(actor: actor))
assert {:ok, group} =
ColumnResolver.create_or_find_group("Concurrent Group", stale_snapshot, actor)
assert group.name == "Concurrent Group"
assert length(Mv.Membership.list_groups!(actor: actor)) == before_count
end
property "is idempotent: same names never create duplicate groups", %{actor: actor} do
check all(
names <-
StreamData.list_of(
StreamData.string(:alphanumeric, min_length: 1, max_length: 20),
min_length: 1,
max_length: 5
),
max_runs: 25
) do
names = Enum.map(names, &("grp-" <> &1))
existing = Mv.Membership.list_groups!(actor: actor)
first_ids = resolve_all(names, existing, actor)
existing_after = Mv.Membership.list_groups!(actor: actor)
second_ids = resolve_all(names, existing_after, actor)
# Same name always resolves to the same group id across both passes.
assert first_ids == second_ids
# No duplicate groups exist for any of the names (case-insensitive).
all_groups = Mv.Membership.list_groups!(actor: actor)
for name <- Enum.uniq_by(names, &String.downcase/1) do
matching =
Enum.filter(all_groups, fn g ->
String.downcase(g.name) == String.downcase(name)
end)
assert length(matching) == 1
end
end
end
end
defp resolve_all(names, existing, actor) do
Enum.map(names, fn name ->
{:ok, group} = ColumnResolver.create_or_find_group(name, existing, actor)
{String.downcase(name), group.id}
end)
|> Map.new()
end
end

View file

@ -1,5 +1,6 @@
defmodule Mv.Membership.Import.HeaderMapperTest do
use ExUnit.Case, async: true
use ExUnitProperties
alias Mv.Membership.Import.HeaderMapper
@ -272,4 +273,167 @@ defmodule Mv.Membership.Import.HeaderMapperTest do
assert unknown == []
end
end
describe "build_maps/2 fee-status ignore list" do
test "places fee-status variants in ignored, not member or custom map" do
headers = ["email", "Bezahlstatus"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.member[:email] == 0
assert result.custom == %{}
assert result.ignored == [1]
refute Map.has_key?(result.member, :bezahlstatus)
end
test "ignores membership_fee_status snake-case variant" do
headers = ["email", "membership_fee_status"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.ignored == [1]
assert result.custom == %{}
end
test "ignores German Mitgliedsbeitragsstatus variant" do
headers = ["email", "Mitgliedsbeitragsstatus"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.ignored == [1]
end
test "fee-status takes priority over a same-named custom field" do
headers = ["email", "Bezahlstatus"]
custom_fields = [%{id: "cf1", name: "Bezahlstatus"}]
assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields)
assert result.ignored == [1]
assert result.custom == %{}
end
test "result carries groups_column_index and fee_type_column_index keys" do
assert {:ok, result} = HeaderMapper.build_maps(["email"], [])
assert Map.has_key?(result, :groups_column_index)
assert Map.has_key?(result, :fee_type_column_index)
end
end
describe "build_maps/2 groups column detection" do
test "detects German Gruppen variant and excludes it from member/custom maps" do
headers = ["email", "Gruppen"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.groups_column_index == 1
assert result.custom == %{}
assert result.unknown == []
refute Map.has_key?(result.member, :gruppen)
end
test "detects English Groups variant" do
headers = ["email", "Groups"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.groups_column_index == 1
end
test "detects singular Gruppe and lowercase groups variants" do
assert {:ok, %{groups_column_index: 1}} = HeaderMapper.build_maps(["email", "Gruppe"], [])
assert {:ok, %{groups_column_index: 1}} = HeaderMapper.build_maps(["email", "groups"], [])
end
test "groups column takes priority over a same-named custom field" do
headers = ["email", "Gruppen"]
custom_fields = [%{id: "cf1", name: "Gruppen"}]
assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields)
assert result.groups_column_index == 1
assert result.custom == %{}
end
test "groups_column_index is nil when no groups column present" do
assert {:ok, %{groups_column_index: nil}} = HeaderMapper.build_maps(["email"], [])
end
end
describe "build_maps/2 fee-type column detection" do
test "detects German Beitragsart variant and excludes it from member/custom maps" do
headers = ["email", "Beitragsart"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.fee_type_column_index == 1
assert result.custom == %{}
assert result.unknown == []
end
test "detects English fee type variants" do
assert {:ok, %{fee_type_column_index: 1}} =
HeaderMapper.build_maps(["email", "Fee Type"], [])
assert {:ok, %{fee_type_column_index: 1}} =
HeaderMapper.build_maps(["email", "fee type"], [])
assert {:ok, %{fee_type_column_index: 1}} =
HeaderMapper.build_maps(["email", "fee_type"], [])
assert {:ok, %{fee_type_column_index: 1}} =
HeaderMapper.build_maps(["email", "membership_fee_type"], [])
end
test "fee-type column takes priority over a same-named custom field" do
headers = ["email", "Beitragsart"]
custom_fields = [%{id: "cf1", name: "Beitragsart"}]
assert {:ok, result} = HeaderMapper.build_maps(headers, custom_fields)
assert result.fee_type_column_index == 1
assert result.custom == %{}
end
test "fee_type_column_index is nil when no fee-type column present" do
assert {:ok, %{fee_type_column_index: nil}} = HeaderMapper.build_maps(["email"], [])
end
test "detects groups and fee-type columns together" do
headers = ["email", "Gruppen", "Beitragsart"]
assert {:ok, result} = HeaderMapper.build_maps(headers, [])
assert result.groups_column_index == 1
assert result.fee_type_column_index == 2
assert result.member[:email] == 0
assert result.custom == %{}
assert result.unknown == []
end
end
describe "build_maps/2 fee-status ignore property" do
property "every fee-status variant is ignored, never member or custom" do
check all(
variant <-
StreamData.member_of([
"Membership Fee Status",
"membership_fee_status",
"Mitgliedsbeitragsstatus",
"Bezahlstatus",
" Bezahlstatus ",
"BEZAHLSTATUS"
])
) do
custom_fields = [%{id: "cf1", name: variant}]
assert {:ok, result} = HeaderMapper.build_maps(["email", variant], custom_fields)
assert result.ignored == [1]
assert result.custom == %{}
refute Map.has_key?(result.member, :bezahlstatus)
end
end
end
end

View file

@ -3,6 +3,83 @@ defmodule Mv.Membership.Import.ImportRunnerTest do
alias Mv.Membership.Import.ImportRunner
describe "carry_groups_forward/2" do
test "replaces import_state groups_found with the chunk's grown snapshot" do
import_state = %{groups_found: [%{id: "1", name: "A"}]}
chunk_result = %{groups_found: [%{id: "1", name: "A"}, %{id: "2", name: "B"}]}
assert ImportRunner.carry_groups_forward(import_state, chunk_result) == %{
groups_found: [%{id: "1", name: "A"}, %{id: "2", name: "B"}]
}
end
test "leaves import_state unchanged when the chunk result omits groups_found" do
import_state = %{groups_found: [%{id: "1", name: "A"}], other: :kept}
chunk_result = %{inserted: 1}
assert ImportRunner.carry_groups_forward(import_state, chunk_result) == import_state
end
end
describe "merge_progress/4 warning accumulation" do
test "deduplicates identical warnings across chunks instead of growing unbounded" do
progress = %{
inserted: 0,
failed: 0,
errors: [],
warnings: ["Fee type 'Ghost' not found; using the default fee type."],
status: :running,
current_chunk: 0,
total_chunks: 3
}
chunk_result = %{
inserted: 2,
failed: 0,
errors: [],
errors_truncated?: false,
warnings: [
"Fee type 'Ghost' not found; using the default fee type.",
"Fee type 'Ghost' not found; using the default fee type."
]
}
result = ImportRunner.merge_progress(progress, chunk_result, 0)
assert result.warnings == ["Fee type 'Ghost' not found; using the default fee type."]
end
test "preserves distinct warnings while collapsing duplicates" do
progress = %{
inserted: 0,
failed: 0,
errors: [],
warnings: ["Fee type 'A' not found; using the default fee type."],
status: :running,
current_chunk: 0,
total_chunks: 2
}
chunk_result = %{
inserted: 1,
failed: 0,
errors: [],
errors_truncated?: false,
warnings: [
"Fee type 'A' not found; using the default fee type.",
"Fee type 'B' not found; using the default fee type."
]
}
result = ImportRunner.merge_progress(progress, chunk_result, 0)
assert result.warnings == [
"Fee type 'A' not found; using the default fee type.",
"Fee type 'B' not found; using the default fee type."
]
end
end
describe "read_file_entry/2" do
test "returns {:ok, content} for a readable file" do
path =

View file

@ -1,5 +1,6 @@
defmodule Mv.Membership.Import.MemberCSVTest do
use Mv.DataCase, async: true
use ExUnitProperties
alias Mv.Membership.Import.MemberCSV
@ -899,4 +900,302 @@ defmodule Mv.Membership.Import.MemberCSVTest do
assert import_state.chunks != []
end
end
describe "prepare/2 column resolution integration" do
setup do
%{actor: Mv.Helpers.SystemActor.get_system_actor()}
end
test "exposes resolver output keys in import_state", %{actor: actor} do
csv_content = "email\njohn@example.com"
assert {:ok, import_state} = MemberCSV.prepare(csv_content, actor: actor)
assert Map.has_key?(import_state, :ignored)
assert Map.has_key?(import_state, :groups_to_create)
assert Map.has_key?(import_state, :fee_type_map)
assert Map.has_key?(import_state, :fee_type_warnings)
assert Map.has_key?(import_state, :has_empty_fee_type_cells?)
assert Map.has_key?(import_state, :preview_rows)
end
test "fee-status column is reported as ignored, not as a custom field", %{actor: actor} do
{:ok, _custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{name: "Bezahlstatus", value_type: :string})
|> Ash.create(actor: actor)
csv_content = "email;Bezahlstatus\njohn@example.com;paid"
assert {:ok, import_state} = MemberCSV.prepare(csv_content, actor: actor)
assert import_state.ignored == ["Bezahlstatus"]
assert import_state.custom_field_map == %{}
end
test "preview rows are limited to 3", %{actor: actor} do
csv_content = "email\na@example.com\nb@example.com\nc@example.com\nd@example.com"
assert {:ok, import_state} = MemberCSV.prepare(csv_content, actor: actor)
assert length(import_state.preview_rows) == 3
end
end
describe "process_chunk/4 fee-type assignment" do
setup do
actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, fee_type} =
Mv.MembershipFees.create_membership_fee_type(
%{name: "Premium", amount: Decimal.new("25.00"), interval: :yearly},
actor: actor
)
%{actor: actor, fee_type: fee_type}
end
test "sets membership_fee_type_id when fee-type cell matches a known type", %{
actor: actor,
fee_type: fee_type
} do
chunk = [
{2, %{member: %{email: "fee-known@example.com"}, custom: %{}, fee_type: "Premium"}}
]
opts = [
actor: actor,
fee_type_map: %{"premium" => fee_type.id}
]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
member =
Mv.Membership.list_members!(actor: actor)
|> Enum.find(&(&1.email == "fee-known@example.com"))
assert member.membership_fee_type_id == fee_type.id
end
test "adds a warning when the fee-type name is unknown", %{actor: actor} do
chunk = [
{2, %{member: %{email: "fee-unknown@example.com"}, custom: %{}, fee_type: "Ghost Type"}}
]
opts = [actor: actor, fee_type_map: %{}]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
assert Enum.any?(result.warnings, &(&1 =~ "Ghost Type"))
end
test "uses the default fee type when the fee-type cell is empty", %{
actor: actor,
fee_type: fee_type
} do
{:ok, settings} = Mv.Membership.get_settings()
{:ok, _settings} =
Mv.Membership.update_settings(
settings,
%{default_membership_fee_type_id: fee_type.id},
actor: actor
)
chunk = [{2, %{member: %{email: "fee-empty@example.com"}, custom: %{}, fee_type: ""}}]
opts = [actor: actor, fee_type_map: %{}]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
member =
Mv.Membership.list_members!(actor: actor)
|> Enum.find(&(&1.email == "fee-empty@example.com"))
# Default fee type assigned via SetDefaultMembershipFeeType.
assert member.membership_fee_type_id == fee_type.id
end
end
describe "process_chunk/4 group assignment" do
setup do
%{actor: Mv.Helpers.SystemActor.get_system_actor()}
end
defp group_names_for(email, actor) do
member =
Mv.Membership.list_members!(actor: actor)
|> Enum.find(&(&1.email == email))
member = Ash.load!(member, :groups, actor: actor)
member.groups |> Enum.map(& &1.name) |> Enum.sort()
end
test "assigns member to an existing group", %{actor: actor} do
existing = Mv.Fixtures.group_fixture(%{name: "Orchester"})
chunk = [
{2, %{member: %{email: "g-existing@example.com"}, custom: %{}, groups: "Orchester"}}
]
opts = [actor: actor, groups_found: [existing]]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
assert group_names_for("g-existing@example.com", actor) == ["Orchester"]
# No new group was created.
orchester = Enum.filter(Mv.Membership.list_groups!(actor: actor), &(&1.name == "Orchester"))
assert length(orchester) == 1
end
test "auto-creates an unknown group and assigns the member", %{actor: actor} do
chunk = [
{2, %{member: %{email: "g-new@example.com"}, custom: %{}, groups: "Frische Gruppe"}}
]
opts = [actor: actor, groups_found: []]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
assert group_names_for("g-new@example.com", actor) == ["Frische Gruppe"]
assert Enum.any?(Mv.Membership.list_groups!(actor: actor), &(&1.name == "Frische Gruppe"))
end
test "handles multiple comma-separated groups", %{actor: actor} do
chunk = [
{2, %{member: %{email: "g-multi@example.com"}, custom: %{}, groups: "Orchester, Chor"}}
]
opts = [actor: actor, groups_found: []]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
assert group_names_for("g-multi@example.com", actor) == ["Chor", "Orchester"]
end
test "does not re-read the group table once per row for a repeated novel name",
%{actor: actor} do
rows =
for i <- 1..10 do
{i + 1,
%{member: %{email: "g-nplus1-#{i}@example.com"}, custom: %{}, groups: "Shared Group"}}
end
group_read_count = Agent.start_link(fn -> 0 end) |> elem(1)
test_pid = self()
# process_chunk runs synchronously in this test process, so the telemetry
# handler (invoked in the query-executing process) sees self() == test_pid.
# Filtering on the pid keeps concurrent tests' group queries out of the count.
handler = fn _event, _measurements, metadata, _config ->
if self() == test_pid and metadata[:source] == "groups" and
is_binary(metadata[:query]) and String.starts_with?(metadata.query, "SELECT") do
Agent.update(group_read_count, &(&1 + 1))
end
end
handler_id = "test-group-read-counter-#{System.unique_integer([:positive])}"
:telemetry.attach(handler_id, [:mv, :repo, :query], handler, nil)
assert {:ok, %{inserted: 10}} =
MemberCSV.process_chunk(rows, %{email: 0}, %{}, actor: actor, groups_found: [])
reads = Agent.get(group_read_count, & &1)
:telemetry.detach(handler_id)
# The novel group is created on the first row and reused in memory for the
# remaining nine. Without accumulation each row triggers a fresh full-table
# read, scaling linearly with the row count.
assert reads <= 3,
"Expected the group table read at most a few times, got #{reads} reads for 10 rows (N+1)."
end
test "returns the grown group snapshot so later chunks skip the table read",
%{actor: actor} do
chunk1 = [
{2, %{member: %{email: "g-xchunk-1@example.com"}, custom: %{}, groups: "Shared X"}}
]
chunk2 = [
{3, %{member: %{email: "g-xchunk-2@example.com"}, custom: %{}, groups: "Shared X"}}
]
assert {:ok, result1} =
MemberCSV.process_chunk(chunk1, %{email: 0}, %{}, actor: actor, groups_found: [])
# The chunk result must expose the accumulated snapshot, including the group
# auto-created while processing this chunk, so the LiveView can thread it
# into the next chunk's opts.
assert is_list(result1.groups_found)
assert Enum.any?(result1.groups_found, &(&1.name == "Shared X"))
group_read_count = Agent.start_link(fn -> 0 end) |> elem(1)
test_pid = self()
handler = fn _event, _measurements, metadata, _config ->
if self() == test_pid and metadata[:source] == "groups" and
is_binary(metadata[:query]) and String.starts_with?(metadata.query, "SELECT") do
Agent.update(group_read_count, &(&1 + 1))
end
end
handler_id = "test-xchunk-group-read-#{System.unique_integer([:positive])}"
:telemetry.attach(handler_id, [:mv, :repo, :query], handler, nil)
assert {:ok, %{inserted: 1}} =
MemberCSV.process_chunk(chunk2, %{email: 0}, %{},
actor: actor,
groups_found: result1.groups_found
)
reads = Agent.get(group_read_count, & &1)
:telemetry.detach(handler_id)
# The second chunk receives the snapshot grown by the first, so the shared
# group resolves from memory without any full-table read.
assert reads == 0,
"Expected no group table read in the second chunk, got #{reads} (snapshot not threaded across chunks)."
end
test "empty groups cell leaves the member without group assignment", %{actor: actor} do
chunk = [{2, %{member: %{email: "g-empty@example.com"}, custom: %{}, groups: " "}}]
opts = [actor: actor, groups_found: []]
assert {:ok, result} = MemberCSV.process_chunk(chunk, %{email: 0}, %{}, opts)
assert result.inserted == 1
assert result.errors == []
assert group_names_for("g-empty@example.com", actor) == []
end
property "re-importing the same groups does not create duplicates", %{actor: actor} do
check all(
name <- StreamData.string(:alphanumeric, min_length: 1, max_length: 15),
max_runs: 15
) do
group_name = "dup-" <> name
email1 = "dup-#{System.unique_integer([:positive])}@example.com"
email2 = "dup-#{System.unique_integer([:positive])}@example.com"
opts = [actor: actor, groups_found: []]
chunk1 = [{2, %{member: %{email: email1}, custom: %{}, groups: group_name}}]
chunk2 = [{2, %{member: %{email: email2}, custom: %{}, groups: group_name}}]
assert {:ok, %{inserted: 1}} = MemberCSV.process_chunk(chunk1, %{email: 0}, %{}, opts)
assert {:ok, %{inserted: 1}} = MemberCSV.process_chunk(chunk2, %{email: 0}, %{}, opts)
matching =
Mv.Membership.list_groups!(actor: actor)
|> Enum.filter(&(String.downcase(&1.name) == String.downcase(group_name)))
assert length(matching) == 1
end
end
end
end

View file

@ -0,0 +1,104 @@
defmodule MvWeb.ImportTemplateControllerTest do
use MvWeb.ConnCase, async: true
setup %{conn: conn} do
actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, custom_field} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{name: "Lieblingsfarbe", value_type: :string})
|> Ash.create(actor: actor)
%{conn: conn, custom_field: custom_field}
end
describe "authenticated EN template" do
setup %{conn: conn} do
admin = Mv.Fixtures.user_with_role_fixture("admin")
%{conn: MvWeb.ConnCase.conn_with_password_user(conn, admin)}
end
test "returns CSV with English headers and current custom fields", %{conn: conn} do
conn = get(conn, ~p"/admin/import/template/en")
assert response_content_type(conn, :csv) =~ "text/csv"
body = response(conn, 200)
header = body |> String.split("\n") |> List.first()
assert header =~ "email"
# EN headers use the canonical English variant from HeaderMapper, not the
# underscore form, so the template stays faithful to the documented variant list.
assert header =~ "first name"
assert header =~ "last name"
refute header =~ "first_name"
assert header =~ "house number"
refute header =~ "house_number"
assert header =~ "Lieblingsfarbe"
assert get_resp_header(conn, "content-disposition")
|> Enum.any?(&(&1 =~ "member_import_en.csv"))
end
test "neutralizes formula-injection in a custom field header", %{conn: conn} do
actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, _} =
Mv.Membership.CustomField
|> Ash.Changeset.for_create(:create, %{
name: "=cmd|'/c calc'!A1",
value_type: :string
})
|> Ash.create(actor: actor)
conn = get(conn, ~p"/admin/import/template/en")
body = response(conn, 200)
header = body |> String.split("\n") |> List.first()
# The dangerous cell must be prefixed with a single quote so spreadsheet
# software does not evaluate it as a formula, matching the export writer.
refute header =~ ~r/(^|;)=cmd/
assert header =~ "'=cmd|'/c calc'!A1"
end
end
describe "authenticated DE template" do
setup %{conn: conn} do
admin = Mv.Fixtures.user_with_role_fixture("admin")
%{conn: MvWeb.ConnCase.conn_with_password_user(conn, admin)}
end
test "returns CSV with German headers and current custom fields", %{conn: conn} do
conn = get(conn, ~p"/admin/import/template/de")
body = response(conn, 200)
header = body |> String.split("\n") |> List.first()
assert header =~ "E-Mail"
assert header =~ "Vorname"
assert header =~ "Lieblingsfarbe"
assert get_resp_header(conn, "content-disposition")
|> Enum.any?(&(&1 =~ "member_import_de.csv"))
end
end
describe "authorization" do
@tag role: :unauthenticated
test "unauthenticated request does not receive a CSV", %{conn: conn} do
conn = get(conn, ~p"/admin/import/template/en")
refute conn.status == 200
refute get_resp_header(conn, "content-type") |> Enum.any?(&(&1 =~ "text/csv"))
refute to_string(conn.resp_body) =~ "email"
end
@tag role: :member
test "user without import permission is forbidden", %{conn: conn} do
conn = get(conn, ~p"/admin/import/template/en")
refute conn.status == 200
refute get_resp_header(conn, "content-type") |> Enum.any?(&(&1 =~ "text/csv"))
refute to_string(conn.resp_body) =~ "email"
end
end
end

View file

@ -0,0 +1,31 @@
defmodule MvWeb.ImportLive.ComponentsTest do
use ExUnit.Case, async: true
alias MvWeb.ImportLive.Components
describe "import_button_disabled?/2" do
@done_entry %{done?: true}
test "disables the Start Import button while the preview is displayed" do
# During :preview the upload entry is done, but re-clicking Start Import
# would re-run the upload processing and overwrite the current preview.
assert Components.import_button_disabled?(:preview, [@done_entry]) == true
end
test "disables the button while an import is running" do
assert Components.import_button_disabled?(:running, [@done_entry]) == true
end
test "disables the button when there are no upload entries" do
assert Components.import_button_disabled?(:idle, []) == true
end
test "disables the button while an upload entry is not yet done" do
assert Components.import_button_disabled?(:idle, [%{done?: false}]) == true
end
test "enables the button at idle with a completed upload" do
assert Components.import_button_disabled?(:idle, [@done_entry]) == false
end
end
end

View file

@ -27,6 +27,16 @@ defmodule MvWeb.ImportLiveTest do
defp submit_import(view), do: view |> form("#csv-upload-form", %{}) |> render_submit()
defp confirm_import(view),
do: view |> element("[data-testid='confirm-import-button']") |> render_click()
# Full flow: upload, enter preview (start), then confirm to begin processing.
defp run_full_import(view, csv_content, filename \\ "test_import.csv") do
upload_csv_file(view, csv_content, filename)
submit_import(view)
confirm_import(view)
end
defp wait_for_import_completion, do: Process.sleep(1000)
# ---------- Business logic: Authorization ----------
@ -56,8 +66,7 @@ defmodule MvWeb.ImportLiveTest do
|> File.read!()
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content)
submit_import(view)
run_full_import(view, csv_content)
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
@ -121,8 +130,7 @@ defmodule MvWeb.ImportLiveTest do
invalid_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content, "invalid_import.csv")
submit_import(view)
run_full_import(view, csv_content, "invalid_import.csv")
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
@ -141,8 +149,7 @@ defmodule MvWeb.ImportLiveTest do
invalid_rows =
for i <- 1..100, do: "Row#{i};Last#{i};;Country#{i};City#{i};Street#{i};12345\n"
upload_csv_file(view, header <> Enum.join(invalid_rows), "large_invalid.csv")
submit_import(view)
run_full_import(view, header <> Enum.join(invalid_rows), "large_invalid.csv")
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
@ -174,8 +181,7 @@ defmodule MvWeb.ImportLiveTest do
Path.join([__DIR__, "..", "..", "fixtures", "csv_with_bom_semicolon.csv"])
|> File.read!()
upload_csv_file(view, csv_content, "bom_import.csv")
submit_import(view)
run_full_import(view, csv_content, "bom_import.csv")
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
@ -193,8 +199,7 @@ defmodule MvWeb.ImportLiveTest do
Path.join([__DIR__, "..", "..", "fixtures", "csv_with_empty_lines.csv"])
|> File.read!()
upload_csv_file(view, csv_content, "empty_lines.csv")
submit_import(view)
run_full_import(view, csv_content, "empty_lines.csv")
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-error-list']")
@ -208,8 +213,7 @@ defmodule MvWeb.ImportLiveTest do
unknown_custom_field_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content, "unknown_custom.csv")
submit_import(view)
run_full_import(view, csv_content, "unknown_custom.csv")
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
@ -240,23 +244,41 @@ defmodule MvWeb.ImportLiveTest do
assert has_element?(view, "[data-testid='start-import-button']")
end
test "template links and file input are present", %{conn: conn} do
test "template links point to the dynamic import template routes", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
assert has_element?(view, "a[href='/admin/import/template/en']")
assert has_element?(view, "a[href='/admin/import/template/de']")
refute has_element?(view, "a[href*='/templates/member_import_en.csv']")
end
test "file input is present", %{conn: conn} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
assert has_element?(view, "a[href*='/templates/member_import_en.csv']")
assert has_element?(view, "a[href*='/templates/member_import_de.csv']")
assert has_element?(view, "label[for='csv_file']")
assert has_element?(view, "#csv_file_help")
assert has_element?(view, "[data-testid='csv-upload-form'] input[type='file']")
end
test "custom fields notice lists accepted groups and fee-type column names", %{conn: conn} do
{:ok, _view, html} = live(conn, ~p"/admin/import")
# Groups column variants (both EN and DE)
assert html =~ "Groups"
assert html =~ "Gruppen"
# Fee type column variants (both EN and DE)
assert html =~ "Beitragsart"
assert html =~ "Fee Type"
assert html =~ "fee type"
# Fee status is always ignored (named explicitly)
assert html =~ "Bezahlstatus"
end
test "after successful import, progress container has aria-live", %{conn: conn} do
csv_content =
Path.join([__DIR__, "..", "..", "fixtures", "valid_member_import.csv"])
|> File.read!()
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content)
submit_import(view)
run_full_import(view, csv_content)
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-progress-container']")
html = render(view)
@ -275,4 +297,187 @@ defmodule MvWeb.ImportLiveTest do
html = render(view)
assert html =~ "Failed to prepare"
end
describe "preview state machine" do
setup %{conn: conn} do
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn =
conn
|> MvWeb.ConnCase.conn_with_password_user(admin_user)
|> put_locale_en()
valid_csv =
Path.join([__DIR__, "..", "..", "fixtures", "valid_member_import.csv"])
|> File.read!()
{:ok, conn: conn, valid_csv: valid_csv}
end
test "start_import transitions to preview without processing", %{
conn: conn,
valid_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content)
submit_import(view)
# Preview is shown; no results panel yet because nothing was processed.
assert has_element?(view, "[data-testid='import-preview']")
refute has_element?(view, "[data-testid='import-results-panel']")
# No member was created during preview (read-only step).
system_actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, members} = Membership.list_members(actor: system_actor)
refute Enum.any?(
members,
&(&1.email in ["alice.smith@example.com", "bob.johnson@example.com"])
)
end
test "confirm_import starts processing and creates members", %{
conn: conn,
valid_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
run_full_import(view, csv_content)
wait_for_import_completion()
assert has_element?(view, "[data-testid='import-results-panel']")
system_actor = Mv.Helpers.SystemActor.get_system_actor()
{:ok, members} = Membership.list_members(actor: system_actor)
imported =
Enum.filter(
members,
&(&1.email in ["alice.smith@example.com", "bob.johnson@example.com"])
)
assert length(imported) == 2
end
test "cancel_import returns to idle and hides the preview", %{
conn: conn,
valid_csv: csv_content
} do
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv_content)
submit_import(view)
assert has_element?(view, "[data-testid='import-preview']")
view |> element("[data-testid='cancel-import-button']") |> render_click()
refute has_element?(view, "[data-testid='import-preview']")
refute has_element?(view, "[data-testid='import-results-panel']")
end
end
describe "preview contents" do
setup %{conn: conn} do
admin_user = Mv.Fixtures.user_with_role_fixture("admin")
conn =
conn
|> MvWeb.ConnCase.conn_with_password_user(admin_user)
|> put_locale_en()
{:ok, conn: conn}
end
test "shows the column mapping table with roles for each column", %{conn: conn} do
csv = "email;Gruppen;Beitragsart;Bezahlstatus;UnknownCol\na@e.com;Chor;Premium;paid;x"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
assert has_element?(view, "[data-testid='preview-mapping-table']")
html = render(view)
assert html =~ "email"
assert html =~ "Gruppen"
assert html =~ "Beitragsart"
assert html =~ "Bezahlstatus"
assert html =~ "UnknownCol"
end
test "lists every CSV column exactly once in the mapping table", %{conn: conn} do
headers = ["email", "Gruppen", "Beitragsart", "Bezahlstatus", "UnknownCol"]
csv = Enum.join(headers, ";") <> "\na@e.com;Chor;Premium;paid;x"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
# Count the data rows via their stable testid so the assertion is independent
# of how Phoenix renders class attributes or tr tags (§1.15).
html = render(view)
row_count =
html |> String.split(~s(data-testid="preview-column-row")) |> length() |> Kernel.-(1)
assert row_count == length(headers)
end
test "shows up to 3 sample data rows", %{conn: conn} do
csv = "email\nr1@e.com\nr2@e.com\nr3@e.com\nr4@e.com"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
html = render(view)
assert html =~ "r1@e.com"
assert html =~ "r2@e.com"
assert html =~ "r3@e.com"
refute html =~ "r4@e.com"
end
test "shows an auto-create notice for unknown group names", %{conn: conn} do
csv = "email;Gruppen\na@e.com;Ganz Neue Gruppe"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
assert has_element?(view, "[data-testid='preview-groups-notice']")
assert render(view) =~ "Ganz Neue Gruppe"
end
test "shows a warning and link for unknown fee-type names", %{conn: conn} do
csv = "email;Beitragsart\na@e.com;Phantom Tarif"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
assert has_element?(view, "[data-testid='preview-fee-type-warning']")
html = render(view)
assert html =~ "Phantom Tarif"
assert html =~ "/membership_fee_settings"
end
test "shows an info notice when fee-type cells are empty", %{conn: conn} do
csv = "email;Beitragsart\na@e.com;\nb@e.com;"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
assert has_element?(view, "[data-testid='preview-fee-type-info']")
end
test "shows a warning for unknown custom-field columns", %{conn: conn} do
csv = "email;TotallyUnknown\na@e.com;value"
{:ok, view, _html} = live(conn, ~p"/admin/import")
upload_csv_file(view, csv)
submit_import(view)
assert has_element?(view, "[data-testid='preview-unknown-warning']")
assert render(view) =~ "TotallyUnknown"
end
end
end