fat: adds csv import live view to settings

This commit is contained in:
carla 2026-01-20 10:05:40 +01:00 committed by Moritz
parent bf9e47b257
commit 092fd99d48
Signed by: moritz
GPG key ID: 1020A035E5DD0824
8 changed files with 1098 additions and 30 deletions

View file

@ -79,6 +79,22 @@ defmodule Mv.Membership.Import.MemberCSV do
use Gettext, backend: MvWeb.Gettext
# Configuration constants
@default_max_errors 50
@default_chunk_size 200
@default_max_rows 1000
# Known member field names (normalized) for efficient lookup
# These match the canonical fields in HeaderMapper
@known_member_fields [
"email",
"firstname",
"lastname",
"street",
"postalcode",
"city"
]
@doc """
Prepares CSV content for import by parsing, mapping headers, and validating limits.
@ -113,8 +129,8 @@ defmodule Mv.Membership.Import.MemberCSV do
"""
@spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()}
def prepare(file_content, opts \\ []) do
max_rows = Keyword.get(opts, :max_rows, 1000)
chunk_size = Keyword.get(opts, :chunk_size, 200)
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
with {:ok, headers, rows} <- CsvParser.parse(file_content),
{:ok, custom_fields} <- load_custom_fields(),
@ -189,19 +205,13 @@ defmodule Mv.Membership.Import.MemberCSV do
end
# Checks if a normalized header matches a member field
# Uses HeaderMapper's internal logic to check if header would map to a member field
defp member_field?(normalized) do
# Try to build maps with just this header - if it maps to a member field, it's a member field
case HeaderMapper.build_maps([normalized], []) do
{:ok, %{member: member_map}} ->
# If member_map is not empty, it's a member field
map_size(member_map) > 0
_ ->
false
end
# Uses direct lookup for better performance (avoids calling build_maps/2)
defp member_field?(normalized) when is_binary(normalized) do
normalized in @known_member_fields
end
defp member_field?(_), do: false
# Validates that row count doesn't exceed limit
defp validate_row_count(rows, max_rows) do
if length(rows) > max_rows do
@ -299,18 +309,29 @@ defmodule Mv.Membership.Import.MemberCSV do
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, 50)
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.get(opts, :actor)
{inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map}, acc ->
current_error_count = existing_error_count + elem(acc, 3)
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
{acc_inserted, acc_failed,
acc_errors, acc_error_count,
acc_truncated?} ->
current_error_count = existing_error_count + acc_error_count
case process_row(row_map, line_number, custom_field_lookup) do
case process_row(row_map, line_number, custom_field_lookup, actor) do
{:ok, _member} ->
update_inserted(acc)
update_inserted(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
)
{:error, error} ->
handle_row_error(acc, error, current_error_count, max_errors)
handle_row_error(
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
error,
current_error_count,
max_errors
)
end
end)
@ -487,7 +508,8 @@ defmodule Mv.Membership.Import.MemberCSV do
defp process_row(
row_map,
line_number,
custom_field_lookup
custom_field_lookup,
actor
) do
# Validate row before database insertion
case validate_row(row_map, line_number, []) do
@ -512,10 +534,7 @@ defmodule Mv.Membership.Import.MemberCSV do
member_attrs_with_cf
end
# Use system_actor for CSV imports (systemic operation)
system_actor = Mv.Helpers.SystemActor.get_system_actor()
case Mv.Membership.create_member(final_attrs, actor: system_actor) do
case Mv.Membership.create_member(final_attrs, actor: actor) do
{:ok, member} ->
{:ok, member}