fat: adds csv import live view to settings
This commit is contained in:
parent
bf9e47b257
commit
092fd99d48
8 changed files with 1098 additions and 30 deletions
|
|
@ -79,6 +79,22 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
|
||||
use Gettext, backend: MvWeb.Gettext
|
||||
|
||||
# Configuration constants
|
||||
@default_max_errors 50
|
||||
@default_chunk_size 200
|
||||
@default_max_rows 1000
|
||||
|
||||
# Known member field names (normalized) for efficient lookup
|
||||
# These match the canonical fields in HeaderMapper
|
||||
@known_member_fields [
|
||||
"email",
|
||||
"firstname",
|
||||
"lastname",
|
||||
"street",
|
||||
"postalcode",
|
||||
"city"
|
||||
]
|
||||
|
||||
@doc """
|
||||
Prepares CSV content for import by parsing, mapping headers, and validating limits.
|
||||
|
||||
|
|
@ -113,8 +129,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
"""
|
||||
@spec prepare(String.t(), keyword()) :: {:ok, import_state()} | {:error, String.t()}
|
||||
def prepare(file_content, opts \\ []) do
|
||||
max_rows = Keyword.get(opts, :max_rows, 1000)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, 200)
|
||||
max_rows = Keyword.get(opts, :max_rows, @default_max_rows)
|
||||
chunk_size = Keyword.get(opts, :chunk_size, @default_chunk_size)
|
||||
|
||||
with {:ok, headers, rows} <- CsvParser.parse(file_content),
|
||||
{:ok, custom_fields} <- load_custom_fields(),
|
||||
|
|
@ -189,19 +205,13 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
end
|
||||
|
||||
# Checks if a normalized header matches a member field
|
||||
# Uses HeaderMapper's internal logic to check if header would map to a member field
|
||||
defp member_field?(normalized) do
|
||||
# Try to build maps with just this header - if it maps to a member field, it's a member field
|
||||
case HeaderMapper.build_maps([normalized], []) do
|
||||
{:ok, %{member: member_map}} ->
|
||||
# If member_map is not empty, it's a member field
|
||||
map_size(member_map) > 0
|
||||
|
||||
_ ->
|
||||
false
|
||||
end
|
||||
# Uses direct lookup for better performance (avoids calling build_maps/2)
|
||||
defp member_field?(normalized) when is_binary(normalized) do
|
||||
normalized in @known_member_fields
|
||||
end
|
||||
|
||||
defp member_field?(_), do: false
|
||||
|
||||
# Validates that row count doesn't exceed limit
|
||||
defp validate_row_count(rows, max_rows) do
|
||||
if length(rows) > max_rows do
|
||||
|
|
@ -299,18 +309,29 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
def process_chunk(chunk_rows_with_lines, _column_map, _custom_field_map, opts \\ []) do
|
||||
custom_field_lookup = Keyword.get(opts, :custom_field_lookup, %{})
|
||||
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
|
||||
max_errors = Keyword.get(opts, :max_errors, 50)
|
||||
max_errors = Keyword.get(opts, :max_errors, @default_max_errors)
|
||||
actor = Keyword.get(opts, :actor)
|
||||
|
||||
{inserted, failed, errors, _collected_error_count, truncated?} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map}, acc ->
|
||||
current_error_count = existing_error_count + elem(acc, 3)
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
|
||||
{acc_inserted, acc_failed,
|
||||
acc_errors, acc_error_count,
|
||||
acc_truncated?} ->
|
||||
current_error_count = existing_error_count + acc_error_count
|
||||
|
||||
case process_row(row_map, line_number, custom_field_lookup) do
|
||||
case process_row(row_map, line_number, custom_field_lookup, actor) do
|
||||
{:ok, _member} ->
|
||||
update_inserted(acc)
|
||||
update_inserted(
|
||||
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?}
|
||||
)
|
||||
|
||||
{:error, error} ->
|
||||
handle_row_error(acc, error, current_error_count, max_errors)
|
||||
handle_row_error(
|
||||
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?},
|
||||
error,
|
||||
current_error_count,
|
||||
max_errors
|
||||
)
|
||||
end
|
||||
end)
|
||||
|
||||
|
|
@ -487,7 +508,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
defp process_row(
|
||||
row_map,
|
||||
line_number,
|
||||
custom_field_lookup
|
||||
custom_field_lookup,
|
||||
actor
|
||||
) do
|
||||
# Validate row before database insertion
|
||||
case validate_row(row_map, line_number, []) do
|
||||
|
|
@ -512,10 +534,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
member_attrs_with_cf
|
||||
end
|
||||
|
||||
# Use system_actor for CSV imports (systemic operation)
|
||||
system_actor = Mv.Helpers.SystemActor.get_system_actor()
|
||||
|
||||
case Mv.Membership.create_member(final_attrs, actor: system_actor) do
|
||||
case Mv.Membership.create_member(final_attrs, actor: actor) do
|
||||
{:ok, member} ->
|
||||
{:ok, member}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue