Merge branch 'main' into issue/mitgliederverwaltung-420
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone/promote/production Build is passing

Integrate current main (CSV import, GDPR join-form description, dependency and
tooling bumps) into the bulk-actions-dropdown feature. Gettext catalogs were
reconciled with mix gettext.extract --merge; the CHANGELOG Unreleased entries
of both sides were combined.
This commit is contained in:
Simon 2026-06-04 16:56:27 +02:00
commit 6a6099659b
48 changed files with 3541 additions and 148 deletions

View file

@ -4,13 +4,13 @@ defmodule Mv.Membership.CustomFieldValueFormatter do
Same logic as the member overview Formatter but without Gettext or web helpers,
so it can be used from the Membership context. For boolean: "Yes"/"No";
for date: European format (dd.mm.yyyy).
for date: ISO-8601 (YYYY-MM-DD) so exported values can be re-imported.
"""
@doc """
Formats a custom field value for plain text (e.g. CSV).
Handles nil, Ash.Union, JSONB map, and direct values. Uses custom_field.value_type
for typing. Boolean -> "Yes"/"No", Date -> dd.mm.yyyy.
for typing. Boolean -> "Yes"/"No", Date -> ISO-8601 (YYYY-MM-DD).
"""
def format_custom_field_value(nil, _custom_field), do: ""
@ -18,6 +18,10 @@ defmodule Mv.Membership.CustomFieldValueFormatter do
format_value_by_type(value, type, custom_field)
end
def format_custom_field_value(%Date{} = value, custom_field) do
format_value_by_type(value, :date, custom_field)
end
def format_custom_field_value(value, custom_field) when is_map(value) do
type = Map.get(value, "type") || Map.get(value, "_union_type")
val = Map.get(value, "value") || Map.get(value, "_union_value")
@ -41,12 +45,12 @@ defmodule Mv.Membership.CustomFieldValueFormatter do
defp format_value_by_type(value, :boolean, _), do: to_string(value)
defp format_value_by_type(%Date{} = date, :date, _) do
Calendar.strftime(date, "%d.%m.%Y")
Date.to_iso8601(date)
end
defp format_value_by_type(value, :date, _) when is_binary(value) do
case Date.from_iso8601(value) do
{:ok, date} -> Calendar.strftime(date, "%d.%m.%Y")
{:ok, date} -> Date.to_iso8601(date)
_ -> value
end
end

View file

@ -0,0 +1,258 @@
defmodule Mv.Membership.Import.ColumnResolver do
@moduledoc """
Read-only resolution of CSV import columns against the database.
Given the `HeaderMapper.build_maps/2` result, the raw numbered rows, and an
actor, `resolve/3` determines:
- which group names in the groups column already exist (`groups_found`) and
which would have to be created (`groups_to_create`);
- a small set of preview rows for the mapping preview UI.
No database writes happen here; the resolver only reads. Group creation and
member-group assignment happen during processing via `create_or_find_group/3`.
This module has no Phoenix or web dependencies.
"""
require Logger
alias Mv.Membership.Import.HeaderMapper
@preview_row_limit 3
@type numbered_row :: {pos_integer(), [String.t()]}
@type resolution :: %{
groups_found: [%{id: String.t(), name: String.t()}],
groups_to_create: [String.t()],
fee_type_map: %{String.t() => String.t()},
fee_type_warnings: [String.t()],
has_empty_fee_type_cells?: boolean(),
preview_rows: [[String.t()]]
}
@doc """
Resolves the group and fee-type columns of an import against the database and
extracts preview rows.
Returns a map with `:groups_found`, `:groups_to_create`, `:fee_type_map`,
`:fee_type_warnings`, `:has_empty_fee_type_cells?`, and `:preview_rows`.
"""
@spec resolve(map(), [numbered_row()], term()) :: resolution()
def resolve(header_maps, rows, actor) do
%{
groups_found: groups_found,
groups_to_create: groups_to_create
} = resolve_groups(header_maps, rows, actor)
%{
fee_type_map: fee_type_map,
fee_type_warnings: fee_type_warnings,
has_empty_fee_type_cells?: has_empty_fee_type_cells?
} = resolve_fee_types(header_maps, rows, actor)
%{
groups_found: groups_found,
groups_to_create: groups_to_create,
fee_type_map: fee_type_map,
fee_type_warnings: fee_type_warnings,
has_empty_fee_type_cells?: has_empty_fee_type_cells?,
preview_rows: preview_rows(rows)
}
end
defp resolve_groups(%{groups_column_index: nil}, _rows, _actor) do
%{groups_found: [], groups_to_create: []}
end
defp resolve_groups(%{groups_column_index: index}, rows, actor) do
existing_groups = list_groups(actor)
lookup = build_group_lookup(existing_groups)
names = unique_group_names(rows, index)
{found, to_create} =
Enum.reduce(names, {[], []}, fn name, {found, to_create} ->
case Map.get(lookup, normalize_name(name)) do
nil -> {found, [name | to_create]}
group -> {[%{id: group.id, name: group.name} | found], to_create}
end
end)
%{groups_found: Enum.reverse(found), groups_to_create: Enum.reverse(to_create)}
end
defp resolve_fee_types(%{fee_type_column_index: nil}, _rows, _actor) do
%{fee_type_map: %{}, fee_type_warnings: [], has_empty_fee_type_cells?: false}
end
defp resolve_fee_types(%{fee_type_column_index: index}, rows, actor) do
lookup = build_fee_type_lookup(actor)
cells = Enum.map(rows, fn {_line, values} -> Enum.at(values, index) end)
has_empty? = Enum.any?(cells, &blank?/1)
{fee_type_map, warnings} =
cells
|> Enum.reject(&blank?/1)
|> Enum.uniq_by(&normalize_fee_type_name/1)
|> Enum.reduce({%{}, []}, fn name, {map, warnings} ->
case Map.get(lookup, normalize_fee_type_name(name)) do
nil -> {map, [String.trim(name) | warnings]}
id -> {Map.put(map, normalize_fee_type_name(name), id), warnings}
end
end)
%{
fee_type_map: fee_type_map,
fee_type_warnings: Enum.reverse(warnings),
has_empty_fee_type_cells?: has_empty?
}
end
@doc """
Normalizes a fee-type name using the same rules as CSV header normalization
(trim, lowercase, transliterate, drop hyphens and whitespace).
"""
@spec normalize_fee_type_name(String.t() | nil) :: String.t()
def normalize_fee_type_name(name) when is_binary(name), do: HeaderMapper.normalize_header(name)
def normalize_fee_type_name(_), do: ""
defp build_fee_type_lookup(actor) do
actor
|> list_fee_types()
|> Enum.reduce(%{}, fn fee_type, acc ->
normalized = normalize_fee_type_name(fee_type.name)
if Map.has_key?(acc, normalized) do
Logger.warning(
"Multiple membership fee types normalize to #{inspect(normalized)}; using the first match for CSV import."
)
acc
else
Map.put(acc, normalized, fee_type.id)
end
end)
end
defp list_fee_types(actor) do
Mv.MembershipFees.list_membership_fee_types!(actor: actor)
end
defp blank?(nil), do: true
defp blank?(value) when is_binary(value), do: String.trim(value) == ""
defp blank?(_), do: false
@doc """
Finds an existing group by name (case-insensitive) or creates it.
Looks first in the pre-fetched `groups` list, then in the database (to catch
groups created earlier in the same import), and only creates a new group when
none is found. This keeps group resolution idempotent across re-imports.
"""
@spec create_or_find_group(String.t(), [Mv.Membership.Group.t()], term()) ::
{:ok, Mv.Membership.Group.t()} | {:error, term()}
def create_or_find_group(name, groups, actor) when is_binary(name) do
trimmed = String.trim(name)
normalized = normalize_name(trimmed)
case find_group_in_list(groups, normalized) do
nil -> find_or_create_group(trimmed, normalized, actor)
group -> {:ok, group}
end
end
defp find_group_in_list(groups, normalized) do
Enum.find(groups, fn group -> normalize_name(group.name) == normalized end)
end
defp find_or_create_group(trimmed, normalized, actor) do
case fetch_group_by_normalized_name(normalized, actor) do
nil -> create_group(trimmed, normalized, actor)
group -> {:ok, group}
end
end
# Normalizes the Ash code-interface return to a two-shape result.
#
# On a create failure the group may have been created concurrently by another
# import session between our read and our write (the DB unique index is the
# final arbiter, and the name validation is fail-open). Re-fetch by normalized
# name and link to the existing group rather than failing the row.
defp create_group(name, normalized, actor) do
case Mv.Membership.create_group(%{name: name}, actor: actor) do
{:ok, %Mv.Membership.Group{} = group} ->
{:ok, group}
{:error, reason} ->
case fetch_group_by_normalized_name(normalized, actor) do
nil -> {:error, reason}
group -> {:ok, group}
end
end
end
# Fetches a single group by case-insensitive name using a name-filtered query
# rather than reading the whole groups table. `normalized` is the trimmed,
# lower-cased name; the DB comparison uses LOWER(name) consistent with the
# Group resource's case-insensitive uniqueness constraint.
defp fetch_group_by_normalized_name(normalized, actor) do
require Ash.Query
Mv.Membership.Group
|> Ash.Query.filter(fragment("LOWER(?) = ?", name, ^normalized))
|> Ash.read(actor: actor, domain: Mv.Membership)
|> case do
{:ok, [group | _]} -> group
_ -> nil
end
end
@doc """
Splits a raw groups-cell value into trimmed, non-empty group names.
"""
@spec split_group_names(String.t() | nil) :: [String.t()]
def split_group_names(nil), do: []
def split_group_names(cell) when is_binary(cell) do
cell
|> String.split(",")
|> Enum.map(&String.trim/1)
|> Enum.reject(&(&1 == ""))
end
defp unique_group_names(rows, index) do
rows
|> Enum.flat_map(fn {_line, values} ->
values
|> Enum.at(index)
|> split_group_names()
end)
|> Enum.uniq_by(&normalize_name/1)
end
defp preview_rows(rows) do
rows
|> Enum.take(@preview_row_limit)
|> Enum.map(fn {_line, values} -> values end)
end
defp list_groups(actor) do
Mv.Membership.list_groups!(actor: actor)
end
defp build_group_lookup(groups) do
Enum.reduce(groups, %{}, fn group, acc ->
Map.put(acc, normalize_name(group.name), group)
end)
end
# Case-insensitive comparison consistent with the Group resource's
# case-insensitive name uniqueness.
defp normalize_name(name) when is_binary(name) do
name |> String.trim() |> String.downcase()
end
end

View file

@ -29,12 +29,21 @@ defmodule Mv.Membership.Import.HeaderMapper do
Supports English and German header variants (e.g. "Email" / "E-Mail", "Join Date" / "Beitrittsdatum").
## Special columns
- **groups** Many-to-many relationship (through member_groups). Recognized via the
`groups_column_index` key (headers `Groups`, `Gruppen`, `Gruppe`). Comma-separated
names are resolved during processing; missing groups are auto-created.
- **membership_fee_type** Recognized via the `fee_type_column_index` key (headers
`Fee Type`, `fee_type`, `membership_fee_type`, `Beitragsart`). Names are matched to
existing fee types; unknown names fall back to the default fee type.
## Fields not supported for import
- **membership_fee_status** Computed (calculation from membership fee cycles). Not stored;
cannot be set via CSV. Export can include it.
- **groups** Many-to-many relationship (through member_groups). Import would require
resolving group names/slugs to IDs and creating associations; not in current import scope.
cannot be set via CSV. Export can include it. Fee-status header variants
(`Membership Fee Status`, `Bezahlstatus`, `Mitgliedsbeitragsstatus`) are explicitly
placed in the `ignored` list and never mapped.
## Custom Field Detection
@ -47,10 +56,10 @@ defmodule Mv.Membership.Import.HeaderMapper do
"e-mail"
iex> HeaderMapper.build_maps(["Email", "First Name"], [])
{:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}}
{:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}}
iex> HeaderMapper.build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}])
{:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}}
{:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}}
"""
@type column_map :: %{atom() => non_neg_integer()}
@ -60,6 +69,33 @@ defmodule Mv.Membership.Import.HeaderMapper do
# Required member fields
@required_member_fields [:email]
# Fee-status header variants that must never be imported (computed/read-only field).
# Stored already-normalized; checked before member, custom, groups, and fee-type mapping.
# Maintain this list when new locale translations for fee-status are added.
@ignored_normalized [
"membershipfeestatus",
"mitgliedsbeitragsstatus",
"bezahlstatus",
# DE export label for membership_fee_start_date — system-managed, not importable
"startdatummitgliedsbeitrag"
]
# Normalized header variants for the groups column. The column is resolved to
# group associations during import; it is never a member or custom field.
@groups_column_normalized [
"groups",
"gruppen",
"gruppe"
]
# Normalized header variants for the membership fee-type column. The column is
# resolved to a MembershipFeeType during import; it is never a member or custom field.
@fee_type_column_normalized [
"membershipfeetype",
"feetype",
"beitragsart"
]
# Canonical member fields with their raw variants
# These will be normalized at runtime when building the lookup map
@member_field_variants_raw %{
@ -239,30 +275,79 @@ defmodule Mv.Membership.Import.HeaderMapper do
## Returns
- `{:ok, %{member: column_map, custom: custom_field_map, unknown: unknown_headers}}` on success
- `{:ok, %{member: column_map, custom: custom_field_map, unknown: unknown_headers,
ignored: [non_neg_integer], groups_column_index: non_neg_integer | nil,
fee_type_column_index: non_neg_integer | nil}}` on success
- `{:error, reason}` on error (missing required field, duplicate headers)
The `ignored` list holds the indices of fee-status columns (computed/read-only),
which are never mapped to member or custom fields.
## Examples
iex> build_maps(["Email", "First Name"], [])
{:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: []}}
{:ok, %{member: %{email: 0, first_name: 1}, custom: %{}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}}
iex> build_maps(["Email", "CustomField"], [%{id: "cf1", name: "CustomField"}])
{:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: []}}
{:ok, %{member: %{email: 0}, custom: %{"cf1" => 1}, unknown: [], ignored: [], groups_column_index: nil, fee_type_column_index: nil}}
"""
@spec build_maps([String.t()], [map()]) ::
{:ok, %{member: column_map(), custom: custom_field_map(), unknown: unknown_headers()}}
{:ok,
%{
member: column_map(),
custom: custom_field_map(),
unknown: unknown_headers(),
ignored: [non_neg_integer()],
groups_column_index: non_neg_integer() | nil,
fee_type_column_index: non_neg_integer() | nil
}}
| {:error, String.t()}
def build_maps(headers, custom_fields) when is_list(headers) and is_list(custom_fields) do
with {:ok, member_map, unknown_after_member} <- build_member_map(headers),
ignored = ignored_indices(headers)
groups_column_index = first_matching_index(headers, @groups_column_normalized)
fee_type_column_index = first_matching_index(headers, @fee_type_column_normalized)
reserved =
[groups_column_index, fee_type_column_index | ignored]
|> Enum.reject(&is_nil/1)
|> MapSet.new()
with {:ok, member_map, unknown_after_member} <- build_member_map(headers, reserved),
{:ok, custom_map, unknown_after_custom} <-
build_custom_field_map(headers, unknown_after_member, custom_fields, member_map) do
unknown = Enum.map(unknown_after_custom, &Enum.at(headers, &1))
{:ok, %{member: member_map, custom: custom_map, unknown: unknown}}
{:ok,
%{
member: member_map,
custom: custom_map,
unknown: unknown,
ignored: ignored,
groups_column_index: groups_column_index,
fee_type_column_index: fee_type_column_index
}}
end
end
# Returns the index of the first header whose normalized form is in `variants`,
# or nil if none match.
defp first_matching_index(headers, variants) do
headers
|> Enum.with_index()
|> Enum.find_value(fn {header, index} ->
if normalize_header(header) in variants, do: index
end)
end
# Returns the column indices whose normalized header is in the fee-status ignore list.
defp ignored_indices(headers) do
headers
|> Enum.with_index()
|> Enum.filter(fn {header, _index} -> normalize_header(header) in @ignored_normalized end)
|> Enum.map(fn {_header, index} -> index end)
end
# --- Private Functions ---
# Transliterates German umlauts and special characters
@ -304,13 +389,14 @@ defmodule Mv.Membership.Import.HeaderMapper do
|> String.replace(" ", "")
end
# Builds member field column map
defp build_member_map(headers) do
# Builds member field column map, skipping reserved (e.g. ignored) indices.
defp build_member_map(headers, reserved) do
result =
headers
|> Enum.with_index()
|> Enum.reduce_while({%{}, []}, fn {header, index}, {acc_map, acc_unknown} ->
normalized = normalize_header(header)
normalized =
if MapSet.member?(reserved, index), do: "", else: normalize_header(header)
case process_member_header(header, index, normalized, acc_map, %{}) do
{:error, reason} ->

View file

@ -80,7 +80,7 @@ defmodule Mv.Membership.Import.ImportRunner do
all_errors = progress.errors ++ chunk_result.errors
new_errors = Enum.take(all_errors, max_errors)
errors_truncated? = length(all_errors) > max_errors
new_warnings = progress.warnings ++ Map.get(chunk_result, :warnings, [])
new_warnings = Enum.uniq(progress.warnings ++ Map.get(chunk_result, :warnings, []))
chunks_processed = current_chunk_idx + 1
new_status = if chunks_processed >= progress.total_chunks, do: :done, else: :running
@ -97,6 +97,20 @@ defmodule Mv.Membership.Import.ImportRunner do
}
end
@doc """
Carries the in-memory group snapshot grown by a chunk back into `import_state`
so the next chunk reuses groups created earlier instead of re-reading the
Group table. When the chunk result omits `groups_found`, the state is returned
unchanged.
"""
@spec carry_groups_forward(map(), map()) :: map()
def carry_groups_forward(import_state, chunk_result) do
case Map.fetch(chunk_result, :groups_found) do
{:ok, groups_found} -> Map.put(import_state, :groups_found, groups_found)
:error -> import_state
end
end
@doc """
Returns the next action after processing a chunk: send the next chunk index or done.
"""

View file

@ -6,7 +6,7 @@ defmodule Mv.Membership.Import.MemberCSV do
This module provides the core API for CSV member import functionality:
- `prepare/2` - Parses and validates CSV content, returns import state
- `process_chunk/3` - Processes a chunk of rows and creates members
- `process_chunk/4` - Processes a chunk of rows and creates members
## Error Handling
@ -22,13 +22,24 @@ defmodule Mv.Membership.Import.MemberCSV do
- `column_map` - Map of canonical field names to column indices
- `custom_field_map` - Map of custom field names to column indices
- `warnings` - List of warning messages (e.g., unknown custom field columns)
- `headers` - The raw CSV header row
- `ignored` - Header names of ignored (fee-status) columns
- `groups_column_index` / `fee_type_column_index` - Indices for resolved columns (or nil)
- `groups_found` / `groups_to_create` - Existing and to-be-created groups from the preview
- `fee_type_map` - Normalized fee-type name to id, for matched fee types
- `fee_type_warnings` - Unmatched fee-type names surfaced in the preview
- `has_empty_fee_type_cells?` - Whether any fee-type cell is blank (default applies)
- `preview_rows` - Up to 3 sample data rows for the mapping preview
## Chunk Results
The `chunk_result` returned by `process_chunk/3` contains:
The `chunk_result` returned by `process_chunk/4` contains:
- `inserted` - Number of successfully created members
- `failed` - Number of failed member creations
- `errors` - List of `%MemberCSV.Error{}` structs (capped at 50 per import)
- `groups_found` - The in-memory group snapshot grown while processing this
chunk; thread it into the next chunk's `:groups_found` opt so groups created
in an earlier chunk are reused without re-reading the Group table
## Examples
@ -37,7 +48,9 @@ defmodule Mv.Membership.Import.MemberCSV do
# Process first chunk
chunk = Enum.at(import_state.chunks, 0)
{:ok, result} = MemberCSV.process_chunk(chunk, import_state.column_map)
{:ok, result} =
MemberCSV.process_chunk(chunk, import_state.column_map, import_state.custom_field_map, [])
"""
defmodule Error do
@ -66,16 +79,29 @@ defmodule Mv.Membership.Import.MemberCSV do
custom_field_lookup: %{
String.t() => %{id: String.t(), value_type: atom(), name: String.t()}
},
warnings: list(String.t())
warnings: list(String.t()),
headers: list(String.t()),
ignored: list(String.t()),
groups_column_index: non_neg_integer() | nil,
fee_type_column_index: non_neg_integer() | nil,
groups_found: list(%{id: String.t(), name: String.t()}),
groups_to_create: list(String.t()),
fee_type_map: %{String.t() => String.t()},
fee_type_warnings: list(String.t()),
has_empty_fee_type_cells?: boolean(),
preview_rows: list(list(String.t()))
}
@type chunk_result :: %{
inserted: non_neg_integer(),
failed: non_neg_integer(),
errors: list(Error.t()),
errors_truncated?: boolean()
errors_truncated?: boolean(),
warnings: list(String.t()),
groups_found: list(Mv.Membership.Group.t() | %{id: String.t(), name: String.t()})
}
alias Mv.Membership.Import.ColumnResolver
alias Mv.Membership.Import.CsvParser
alias Mv.Membership.Import.HeaderMapper
@ -139,13 +165,27 @@ defmodule Mv.Membership.Import.MemberCSV do
# Build custom field lookup for efficient value processing
custom_field_lookup = build_custom_field_lookup(custom_fields)
# Resolve DB-backed columns (groups, fee types) read-only for the preview.
resolution = ColumnResolver.resolve(maps, rows, actor)
ignored_headers = Enum.map(maps.ignored, &Enum.at(headers, &1))
{:ok,
%{
chunks: chunks,
column_map: maps.member,
custom_field_map: maps.custom,
custom_field_lookup: custom_field_lookup,
warnings: warnings
warnings: warnings,
headers: headers,
ignored: ignored_headers,
groups_column_index: maps.groups_column_index,
fee_type_column_index: maps.fee_type_column_index,
groups_found: resolution.groups_found,
groups_to_create: resolution.groups_to_create,
fee_type_map: resolution.fee_type_map,
fee_type_warnings: resolution.fee_type_warnings,
has_empty_fee_type_cells?: resolution.has_empty_fee_type_cells?,
preview_rows: resolution.preview_rows
}}
end
end
@ -180,7 +220,7 @@ defmodule Mv.Membership.Import.MemberCSV do
end)
case HeaderMapper.build_maps(headers, custom_field_maps) do
{:ok, %{member: member_map, custom: custom_map, unknown: unknown}} ->
{:ok, %{unknown: unknown} = maps} ->
# Build warnings for unknown custom field columns
warnings =
unknown
@ -197,7 +237,7 @@ defmodule Mv.Membership.Import.MemberCSV do
)
end)
{:ok, %{member: member_map, custom: custom_map}, warnings}
{:ok, maps, warnings}
{:error, reason} ->
{:error, reason}
@ -250,9 +290,20 @@ defmodule Mv.Membership.Import.MemberCSV do
Map.put(acc, custom_field_id, value)
end)
%{member: member_map, custom: custom_map}
%{
member: member_map,
custom: custom_map,
fee_type: cell_at(row_tuple, tuple_size, maps.fee_type_column_index),
groups: cell_at(row_tuple, tuple_size, maps.groups_column_index)
}
end
# Returns the raw cell at the given index, or nil if the column is absent.
defp cell_at(_row_tuple, _size, nil), do: nil
defp cell_at(row_tuple, size, index) when index < size, do: elem(row_tuple, index)
defp cell_at(_row_tuple, _size, _index), do: ""
@doc """
Processes a chunk of CSV rows and creates members.
@ -268,12 +319,18 @@ defmodule Mv.Membership.Import.MemberCSV do
- `chunk_rows_with_lines` - List of tuples `{csv_line_number, row_map}` where:
- `csv_line_number` - Physical line number in CSV (1-based)
- `row_map` - Map with `:member` and `:custom` keys containing field values
- `column_map` - Map of canonical field names (atoms) to column indices (for reference)
- `custom_field_map` - Map of custom field IDs (strings) to column indices (for reference)
- `column_map` - Unused; kept for backward-compatible call sites. Field values are
read from each row's pre-built `:member`/`:custom` maps, not from this argument.
- `custom_field_map` - Unused; kept for backward-compatible call sites (see above).
- `opts` - Optional keyword list for processing options:
- `:custom_field_lookup` - Map of custom field IDs to metadata (default: `%{}`)
- `:existing_error_count` - Number of errors already collected in previous chunks (default: `0`)
- `:max_errors` - Maximum number of errors to collect per import overall (default: `50`)
- `:actor` - Actor used for all writes (default: the system actor)
- `:fee_type_map` - Map of normalized fee-type name to fee-type id, used to resolve
each row's fee-type cell (default: `%{}`)
- `:groups_found` - List of pre-fetched `Group` structs seeding in-memory group
resolution; the snapshot grows as groups are auto-created (default: `[]`)
## Error Capping
@ -312,27 +369,49 @@ defmodule Mv.Membership.Import.MemberCSV do
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.get(opts, :actor, SystemActor.get_system_actor())
fee_type_map = Keyword.get(opts, :fee_type_map, %{})
groups_found = Keyword.get(opts, :groups_found, [])
{inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
{acc_inserted, acc_failed,
acc_errors, acc_error_count,
acc_truncated?} ->
base_row_opts = %{
custom_field_lookup: custom_field_lookup,
fee_type_map: fee_type_map,
actor: actor
}
{inserted, failed, errors, _collected_error_count, truncated?, warnings, groups_acc} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false, [], groups_found}, fn {line_number,
row_map},
{acc_inserted,
acc_failed,
acc_errors,
acc_error_count,
acc_truncated?,
acc_warnings,
acc_groups} ->
current_error_count = existing_error_count + acc_error_count
row_opts = Map.put(base_row_opts, :groups_found, acc_groups)
case process_row(row_map, line_number, custom_field_lookup, actor) do
{:ok, _member} ->
update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
case process_row(row_map, line_number, row_opts) do
{:ok, _member, row_warnings, new_groups} ->
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?} =
update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
{:error, error} ->
handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?,
acc_warnings ++ row_warnings, new_groups}
{:error, error, new_groups} ->
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?} =
handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
{new_inserted, new_failed, new_errors, new_error_count, new_truncated?, acc_warnings,
new_groups}
end
end)
@ -341,7 +420,9 @@ defmodule Mv.Membership.Import.MemberCSV do
inserted: inserted,
failed: failed,
errors: Enum.reverse(errors),
errors_truncated?: truncated?
errors_truncated?: truncated?,
warnings: warnings,
groups_found: groups_acc
}}
end
@ -505,18 +586,27 @@ defmodule Mv.Membership.Import.MemberCSV do
defp gettext_error_message(_), do: gettext("Email is invalid.")
# Processes a single row and creates member with custom field values
# Processes a single row and creates member with custom field values.
# On success returns {:ok, member, warnings, groups}; warnings carry non-fatal
# notices such as an unresolved fee-type name. The returned groups list is the
# accumulated in-memory group snapshot (seeded from the chunk, grown with any
# group created while linking this row) so later rows reuse it instead of
# re-reading the whole Group table per row.
defp process_row(
row_map,
line_number,
custom_field_lookup,
actor
%{
custom_field_lookup: custom_field_lookup,
fee_type_map: fee_type_map,
groups_found: groups_found,
actor: actor
} = _row_opts
) do
# Validate row before database insertion
case validate_row(row_map, line_number, []) do
{:error, error} ->
# Return validation error immediately, no DB insert attempted
{:error, error}
{:error, error, groups_found}
{:ok, %{member: trimmed_member_attrs, custom: custom_attrs}} ->
# Prepare custom field values for Ash
@ -524,20 +614,119 @@ defmodule Mv.Membership.Import.MemberCSV do
{:error, validation_errors} ->
# Custom field validation errors - return first error
first_error = List.first(validation_errors)
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error}}
{:error, %Error{csv_line_number: line_number, field: nil, message: first_error},
groups_found}
{:ok, custom_field_values} ->
create_member_with_custom_fields(
trimmed_member_attrs,
{fee_attrs, warnings} =
resolve_fee_type_attrs(Map.get(row_map, :fee_type), fee_type_map)
create_member_and_assign_groups(
Map.merge(trimmed_member_attrs, fee_attrs),
custom_field_values,
Map.get(row_map, :groups),
groups_found,
line_number,
actor
actor,
warnings
)
end
end
rescue
e ->
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)}}
{:error, %Error{csv_line_number: line_number, field: nil, message: Exception.message(e)},
groups_found}
end
# Creates the member, then assigns groups as a post-creation step. A group
# assignment failure fails the row (the member was already created, but the
# row is reported as failed so the operator can act on it).
defp create_member_and_assign_groups(
member_attrs,
custom_field_values,
groups_cell,
groups_found,
line_number,
actor,
warnings
) do
case create_member_with_custom_fields(
member_attrs,
custom_field_values,
line_number,
actor,
warnings
) do
{:ok, member, member_warnings} ->
assign_groups(member, groups_cell, groups_found, line_number, actor, member_warnings)
{:error, error} ->
{:error, error, groups_found}
end
end
# Assigns the member to all groups listed in the cell, creating missing groups.
# Returns the (possibly grown) group snapshot so the caller can reuse it.
defp assign_groups(member, groups_cell, groups_found, line_number, actor, warnings) do
names = ColumnResolver.split_group_names(groups_cell)
Enum.reduce_while(names, {:ok, member, warnings, groups_found}, fn name,
{:ok, _m, _w, acc_groups} ->
case link_member_to_group(member, name, acc_groups, actor) do
{:ok, group} ->
{:cont, {:ok, member, warnings, add_group(acc_groups, group)}}
{:error, reason} ->
{:halt,
{:error,
%Error{
csv_line_number: line_number,
field: nil,
message: gettext("Group assignment failed: %{reason}", reason: inspect(reason))
}, acc_groups}}
end
end)
end
defp add_group(groups, group) do
if Enum.any?(groups, &(&1.id == group.id)), do: groups, else: [group | groups]
end
defp link_member_to_group(member, name, groups_found, actor) do
with {:ok, group} <- ColumnResolver.create_or_find_group(name, groups_found, actor),
{:ok, _member_group} <-
Mv.Membership.create_member_group(
%{member_id: member.id, group_id: group.id},
actor: actor
) do
{:ok, group}
end
end
# Resolves the fee-type cell into member attrs plus optional warnings.
# Empty cell -> default fee type (SetDefaultMembershipFeeType), no warning.
# Matched name -> membership_fee_type_id attr.
# Unmatched name -> no attr (default applies), warning naming the value.
defp resolve_fee_type_attrs(nil, _fee_type_map), do: {%{}, []}
defp resolve_fee_type_attrs(cell, fee_type_map) when is_binary(cell) do
trimmed = String.trim(cell)
if trimmed == "" do
{%{}, []}
else
case Map.get(fee_type_map, ColumnResolver.normalize_fee_type_name(trimmed)) do
nil ->
{%{},
[
gettext("Fee type '%{name}' not found; using the default fee type.", name: trimmed)
]}
fee_type_id ->
{%{membership_fee_type_id: fee_type_id}, []}
end
end
end
# Creates a member with custom field values, handling errors appropriately
@ -545,7 +734,8 @@ defmodule Mv.Membership.Import.MemberCSV do
trimmed_member_attrs,
custom_field_values,
line_number,
actor
actor,
warnings
) do
# Convert empty strings to nil for date fields so Ash accepts them
member_attrs = sanitize_date_fields(trimmed_member_attrs)
@ -565,7 +755,7 @@ defmodule Mv.Membership.Import.MemberCSV do
case Mv.Membership.create_member(final_attrs, actor: actor) do
{:ok, member} ->
{:ok, member}
{:ok, member, warnings}
{:error, %Ash.Error.Invalid{} = error} ->
# Extract email from final_attrs for better error messages