fat: adds csv import live view to settings

This commit is contained in:
carla 2026-01-20 10:05:40 +01:00
parent d599439357
commit aaf9c7127d
8 changed files with 1094 additions and 25 deletions

View file

@ -79,6 +79,22 @@ defmodule Mv.Membership.Import.MemberCSV do
use Gettext, backend: MvWeb.Gettext
# Configuration constants
@default_max_errors 50
@default_chunk_size 200
@default_max_rows 1000
# Known member field names (normalized) for efficient lookup
# These match the canonical fields in HeaderMapper
@known_member_fields [
"email",
"firstname",
"lastname",
"street",
"postalcode",
"city"
]
@doc """
Prepares CSV content for import by parsing, mapping headers, and validating limits.
@ -113,8 +129,8 @@ defmodule Mv.Membership.Import.MemberCSV do
"""
@spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()}
def prepare(file_content, opts \\ []) do
max_rows = Keyword.get(opts, :max_rows, 1000)
chunk_size = Keyword.get(opts, :chunk_size, 200)
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
with {:ok, headers, rows} <- CsvParser.parse(file_content),
{:ok, custom_fields} <- load_custom_fields(),
@ -189,19 +205,13 @@ defmodule Mv.Membership.Import.MemberCSV do
end
# Checks if a normalized header matches a member field
# Uses HeaderMapper's internal logic to check if header would map to a member field
defp member_field?(normalized) do
# Try to build maps with just this header - if it maps to a member field, it's a member field
case HeaderMapper.build_maps([normalized], []) do
{:ok, %{member: member_map}} ->
# If member_map is not empty, it's a member field
map_size(member_map) > 0
_ ->
false
end
# Uses direct lookup for better performance (avoids calling build_maps/2)
defp member_field?(normalized) when is_binary(normalized) do
normalized in @known_member_fields
end
defp member_field?(_), do: false
# Validates that row count doesn't exceed limit
defp validate_row_count(rows, max_rows) do
if length(rows) > max_rows do
@ -299,14 +309,17 @@ defmodule Mv.Membership.Import.MemberCSV do
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
max_errors = Keyword.get(opts, :max_errors, 50)
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
actor = Keyword.get(opts, :actor)
{inserted, failed, errors, _collected_error_count, truncated?} =
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?} ->
{acc_inserted, acc_failed,
acc_errors, acc_error_count,
acc_truncated?} ->
current_error_count = existing_error_count + acc_error_count
case process_row(row_map, line_number, custom_field_lookup) do
case process_row(row_map, line_number, custom_field_lookup, actor) do
{:ok, _member} ->
{acc_inserted + 1, acc_failed, acc_errors, acc_error_count, acc_truncated?}
@ -325,7 +338,13 @@ defmodule Mv.Membership.Import.MemberCSV do
end
end)
{:ok, %{inserted: inserted, failed: failed, errors: Enum.reverse(errors), errors_truncated?: truncated?}}
{:ok,
%{
inserted: inserted,
failed: failed,
errors: Enum.reverse(errors),
errors_truncated?: truncated?
}}
end
@doc """
@ -444,7 +463,8 @@ defmodule Mv.Membership.Import.MemberCSV do
defp process_row(
row_map,
line_number,
custom_field_lookup
custom_field_lookup,
actor
) do
# Validate row before database insertion
case validate_row(row_map, line_number, []) do
@ -469,7 +489,7 @@ defmodule Mv.Membership.Import.MemberCSV do
member_attrs_with_cf
end
case Mv.Membership.create_member(final_attrs) do
case Mv.Membership.create_member(final_attrs, actor: actor) do
{:ok, member} ->
{:ok, member}