local-it/mitgliederverwaltung
local-it/mitgliederverwaltung
2
3
Fork 0
Code Issues 47 Pull requests 1 Projects 1 Releases 4 Packages 1 Activity Actions

feat(import): assign groups and fee types to imported members, creating missing groups

Browse source
This commit is contained in:
Moritz 2026-06-03 02:15:54 +02:00
parent a4a34cab3a
commit 00e1624ee4
6 changed files with 517 additions and 51 deletions
Download patch file Download diff file Expand all files Collapse all files

15
lib/mv/membership/import/header_mapper.ex
View file

@ -29,12 +29,21 @@ defmodule Mv.Membership.Import.HeaderMapper do
Supports English and German header variants (e.g. "Email" / "E-Mail", "Join Date" / "Beitrittsdatum").
## Special columns
- **groups** – Many-to-many relationship (through member_groups). Recognized via the
`groups_column_index` key (headers `Groups`, `Gruppen`, `Gruppe`). Comma-separated
names are resolved during processing; missing groups are auto-created.
- **membership_fee_type** – Recognized via the `fee_type_column_index` key (headers
`Fee Type`, `fee_type`, `membership_fee_type`, `Beitragsart`). Names are matched to
existing fee types; unknown names fall back to the default fee type.
## Fields not supported for import
- **membership_fee_status** – Computed (calculation from membership fee cycles). Not stored;
cannot be set via CSV. Export can include it.
- **groups** – Many-to-many relationship (through member_groups). Import would require
resolving group names/slugs to IDs and creating associations; not in current import scope.
cannot be set via CSV. Export can include it. Fee-status header variants
(`Membership Fee Status`, `Bezahlstatus`, `Mitgliedsbeitragsstatus`) are explicitly
placed in the `ignored` list and never mapped.
## Custom Field Detection

265
lib/mv/membership/import/member_csv.ex
View file

@ -6,7 +6,7 @@ defmodule Mv.Membership.Import.MemberCSV do
This module provides the core API for CSV member import functionality:
- `prepare/2` - Parses and validates CSV content, returns import state
- `process_chunk/3` - Processes a chunk of rows and creates members
- `process_chunk/4` - Processes a chunk of rows and creates members
## Error Handling
@ -22,10 +22,18 @@ defmodule Mv.Membership.Import.MemberCSV do
- `column_map` - Map of canonical field names to column indices
- `custom_field_map` - Map of custom field names to column indices
- `warnings` - List of warning messages (e.g., unknown custom field columns)
- `headers` - The raw CSV header row
- `ignored` - Header names of ignored (fee-status) columns
- `groups_column_index` / `fee_type_column_index` - Indices for resolved columns (or nil)
- `groups_found` / `groups_to_create` - Existing and to-be-created groups from the preview
- `fee_type_map` - Normalized fee-type name to id, for matched fee types
- `fee_type_warnings` - Unmatched fee-type names surfaced in the preview
- `has_empty_fee_type_cells?` - Whether any fee-type cell is blank (default applies)
- `preview_rows` - Up to 3 sample data rows for the mapping preview
## Chunk Results
The `chunk_result` returned by `process_chunk/3` contains:
The `chunk_result` returned by `process_chunk/4` contains:
- `inserted` - Number of successfully created members
- `failed` - Number of failed member creations
- `errors` - List of `%MemberCSV.Error{}` structs (capped at 50 per import)
@ -37,7 +45,9 @@ defmodule Mv.Membership.Import.MemberCSV do
# Process first chunk
chunk = Enum.at(import_state.chunks, 0)
{:ok, result} = MemberCSV.process_chunk(chunk, import_state.column_map)
{:ok, result} =
MemberCSV.process_chunk(chunk, import_state.column_map, import_state.custom_field_map, [])
"""
defmodule Error do
@ -66,16 +76,28 @@ defmodule Mv.Membership.Import.MemberCSV do
custom_field_lookup: %{
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
},
warnings: list(String.t())
warnings: list(String.t()),
headers: list(String.t()),
ignored: list(String.t()),
groups_column_index: non_neg_integer() | nil,
fee_type_column_index: non_neg_integer() | nil,
groups_found: list(%{id: String.t(), name: String.t()}),
groups_to_create: list(String.t()),
fee_type_map: %{String.t() => String.t()},
fee_type_warnings: list(String.t()),
has_empty_fee_type_cells?: boolean(),
preview_rows: list(list(String.t()))
}
@type chunk_result :: %{
inserted: non_neg_integer(),
failed: non_neg_integer(),
errors: list(Error.t()),
errors_truncated?: boolean()
errors_truncated?: boolean(),
warnings: list(String.t())
}
alias Mv.Membership.Import.ColumnResolver
alias Mv.Membership.Import.CsvParser
alias Mv.Membership.Import.HeaderMapper
@ -139,13 +161,27 @@ defmodule Mv.Membership.Import.MemberCSV do
# Build custom field lookup for efficient value processing
custom_field_lookup = build_custom_field_lookup(custom_fields)
# Resolve DB-backed columns (groups, fee types) read-only for the preview.
resolution = ColumnResolver.resolve(maps, rows, actor)
ignored_headers = Enum.map(maps.ignored, &Enum.at(headers, &1))
{:ok,
%{
chunks: chunks,
column_map: maps.member,
custom_field_map: maps.custom,
custom_field_lookup: custom_field_lookup,
warnings: warnings
warnings: warnings,
headers: headers,
ignored: ignored_headers,
groups_column_index: maps.groups_column_index,
fee_type_column_index: maps.fee_type_column_index,
groups_found: resolution.groups_found,
groups_to_create: resolution.groups_to_create,
fee_type_map: resolution.fee_type_map,
fee_type_warnings: resolution.fee_type_warnings,
has_empty_fee_type_cells?: resolution.has_empty_fee_type_cells?,
preview_rows: resolution.preview_rows
}}
end
end
@ -180,7 +216,7 @@ defmodule Mv.Membership.Import.MemberCSV do
end)
case HeaderMapper.build_maps(headers, custom_field_maps) do
{:ok, %{member: member_map, custom: custom_map, unknown: unknown}} ->
{:ok, %{unknown: unknown} = maps} ->
# Build warnings for unknown custom field columns
warnings =
unknown
@ -197,7 +233,7 @@ defmodule Mv.Membership.Import.MemberCSV do
)
end)
{:ok, %{member: member_map, custom: custom_map}, warnings}
{:ok, maps, warnings}
{:error, reason} ->
{:error, reason}
@ -250,9 +286,20 @@ defmodule Mv.Membership.Import.MemberCSV do
Map.put(acc, custom_field_id, value)
end)
%{member: member_map, custom: custom_map}
%{
member: member_map,
custom: custom_map,
fee_type: cell_at(row_tuple, tuple_size, maps.fee_type_column_index),
groups: cell_at(row_tuple, tuple_size, maps.groups_column_index)
}
end
# Returns the raw cell at the given index, or nil if the column is absent.
defp cell_at(_row_tuple, _size, nil), do: nil
defp cell_at(row_tuple, size, index) when index < size, do: elem(row_tuple, index)
defp cell_at(_row_tuple, _size, _index), do: ""
@doc """
Processes a chunk of CSV rows and creates members.
@ -268,12 +315,18 @@ defmodule Mv.Membership.Import.MemberCSV do
- `chunk_rows_with_lines` - List of tuples `{csv_line_number, row_map}` where:
- `csv_line_number` - Physical line number in CSV (1-based)
- `row_map` - Map with `:member` and `:custom` keys containing field values
- `column_map` - Map of canonical field names (atoms) to column indices (for reference)
- `custom_field_map` - Map of custom field IDs (strings) to column indices (for reference)
- `column_map` - Unused; kept for backward-compatible call sites. Field values are
read from each row's pre-built `:member`/`:custom` maps, not from this argument.
- `custom_field_map` - Unused; kept for backward-compatible call sites (see above).
- `opts` - Optional keyword list for processing options:
- `:custom_field_lookup` - Map of custom field IDs to metadata (default: `%{}`)
- `:existing_error_count` - Number of errors already collected in previous chunks (default: `0`)
- `:max_errors` - Maximum number of errors to collect per import overall (default: `50`)
- `:actor` - Actor used for all writes (default: the system actor)
- `:fee_type_map` - Map of normalized fee-type name to fee-type id, used to resolve
each row's fee-type cell (default: `%{}`)
- `:groups_found` - List of pre-fetched `Group` structs seeding in-memory group
resolution; the snapshot grows as groups are auto-created (default: `[]`)
## Error Capping
@ -312,27 +365,49 @@ defmodule Mv.Membership.Import.MemberCSV do
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
fee_type_map = Keyword.get(opts, :fee_type_map, %{})
groups_found = Keyword.get(opts, :groups_found, [])
{inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
{acc_inserted, acc_failed,
acc_errors, acc_error_count,
acc_truncated?} ->
base_row_opts = %{
custom_field_lookup: custom_field_lookup,
fee_type_map: fee_type_map,
actor: actor
}
{inserted, failed, errors, _collected_error_count, truncated?, warnings, _groups_acc} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false, [], groups_found}, fn {line_number,
row_map},
{acc_inserted,
acc_failed,
acc_errors,
acc_error_count,
acc_truncated?,
acc_warnings,
acc_groups} ->
current_error_count = existing_error_count + acc_error_count
row_opts = Map.put(base_row_opts, :groups_found, acc_groups)
case process_row(row_map, line_number, custom_field_lookup, actor) do
{:ok, _member} ->
update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
case process_row(row_map, line_number, row_opts) do
{:ok, _member, row_warnings, new_groups} ->
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?} =
update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
{:error, error} ->
handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?,
acc_warnings ++ row_warnings, new_groups}
{:error, error, new_groups} ->
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?} =
handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?, acc_warnings,
new_groups}
end
end)
@ -341,7 +416,8 @@ defmodule Mv.Membership.Import.MemberCSV do
inserted: inserted,
failed: failed,
errors: Enum.reverse(errors),
errors_truncated?: truncated?
errors_truncated?: truncated?,
warnings: warnings
}}
end
@ -505,18 +581,27 @@ defmodule Mv.Membership.Import.MemberCSV do
defp gettext_error_message(_), do: gettext("Email is invalid.")
# Processes a single row and creates member with custom field values
# Processes a single row and creates member with custom field values.
# On success returns {:ok, member, warnings, groups}; warnings carry non-fatal
# notices such as an unresolved fee-type name. The returned groups list is the
# accumulated in-memory group snapshot (seeded from the chunk, grown with any
# group created while linking this row) so later rows reuse it instead of
# re-reading the whole Group table per row.
defp process_row(
row_map,
line_number,
custom_field_lookup,
actor
%{
custom_field_lookup: custom_field_lookup,
fee_type_map: fee_type_map,
groups_found: groups_found,
actor: actor
} = _row_opts
) do
# Validate row before database insertion
case validate_row(row_map, line_number, []) do
{:error, error} ->
# Return validation error immediately, no DB insert attempted
{:error, error}
{:error, error, groups_found}
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
# Prepare custom field values for Ash
@ -524,20 +609,119 @@ defmodule Mv.Membership.Import.MemberCSV do
{:error, validation_errors} ->
# Custom field validation errors - return first error
first_error = List.first(validation_errors)
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error},
groups_found}
{:ok, custom_field_values} ->
create_member_with_custom_fields(
trimmed_member_attrs,
{fee_attrs, warnings} =
resolve_fee_type_attrs(Map.get(row_map, :fee_type), fee_type_map)
create_member_and_assign_groups(
Map.merge(trimmed_member_attrs, fee_attrs),
custom_field_values,
Map.get(row_map, :groups),
groups_found,
line_number,
actor
actor,
warnings
)
end
end
rescue
e ->
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)},
groups_found}
end
# Creates the member, then assigns groups as a post-creation step. A group
# assignment failure fails the row (the member was already created, but the
# row is reported as failed so the operator can act on it).
defp create_member_and_assign_groups(
member_attrs,
custom_field_values,
groups_cell,
groups_found,
line_number,
actor,
warnings
) do
case create_member_with_custom_fields(
member_attrs,
custom_field_values,
line_number,
actor,
warnings
) do
{:ok, member, member_warnings} ->
assign_groups(member, groups_cell, groups_found, line_number, actor, member_warnings)
{:error, error} ->
{:error, error, groups_found}
end
end
# Assigns the member to all groups listed in the cell, creating missing groups.
# Returns the (possibly grown) group snapshot so the caller can reuse it.
defp assign_groups(member, groups_cell, groups_found, line_number, actor, warnings) do
names = ColumnResolver.split_group_names(groups_cell)
Enum.reduce_while(names, {:ok, member, warnings, groups_found}, fn name,
{:ok, _m, _w, acc_groups} ->
case link_member_to_group(member, name, acc_groups, actor) do
{:ok, group} ->
{:cont, {:ok, member, warnings, add_group(acc_groups, group)}}
{:error, reason} ->
{:halt,
{:error,
%Error{
csv_line_number: line_number,
field: nil,
message: gettext("Group assignment failed: %{reason}", reason: inspect(reason))
}, acc_groups}}
end
end)
end
defp add_group(groups, group) do
if Enum.any?(groups, &(&1.id == group.id)), do: groups, else: [group | groups]
end
defp link_member_to_group(member, name, groups_found, actor) do
with {:ok, group} <- ColumnResolver.create_or_find_group(name, groups_found, actor),
{:ok, _member_group} <-
Mv.Membership.create_member_group(
%{member_id: member.id, group_id: group.id},
actor: actor
) do
{:ok, group}
end
end
# Resolves the fee-type cell into member attrs plus optional warnings.
# Empty cell -> default fee type (SetDefaultMembershipFeeType), no warning.
# Matched name -> membership_fee_type_id attr.
# Unmatched name -> no attr (default applies), warning naming the value.
defp resolve_fee_type_attrs(nil, _fee_type_map), do: {%{}, []}
defp resolve_fee_type_attrs(cell, fee_type_map) when is_binary(cell) do
trimmed = String.trim(cell)
if trimmed == "" do
{%{}, []}
else
case Map.get(fee_type_map, ColumnResolver.normalize_fee_type_name(trimmed)) do
nil ->
{%{},
[
gettext("Fee type '%{name}' not found; using the default fee type.", name: trimmed)
]}
fee_type_id ->
{%{membership_fee_type_id: fee_type_id}, []}
end
end
end
# Creates a member with custom field values, handling errors appropriately
@ -545,7 +729,8 @@ defmodule Mv.Membership.Import.MemberCSV do
trimmed_member_attrs,
custom_field_values,
line_number,
actor
actor,
warnings
) do
# Convert empty strings to nil for date fields so Ash accepts them
member_attrs = sanitize_date_fields(trimmed_member_attrs)
@ -565,7 +750,7 @@ defmodule Mv.Membership.Import.MemberCSV do
case Mv.Membership.create_member(final_attrs, actor: actor) do
{:ok, member} ->
{:ok, member}
{:ok, member, warnings}
{:error, %Ash.Error.Invalid{} = error} ->
# Extract email from final_attrs for better error messages