Implements error capping #356
1 changed files with 14 additions and 12 deletions
|
|
@ -70,7 +70,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
@type chunk_result :: %{
|
||||
inserted: non_neg_integer(),
|
||||
failed: non_neg_integer(),
|
||||
errors: list(Error.t())
|
||||
errors: list(Error.t()),
|
||||
errors_truncated?: boolean()
|
||||
}
|
||||
|
||||
alias Mv.Membership.Import.CsvParser
|
||||
|
|
@ -269,6 +270,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
- No additional errors are collected in the `errors` list
|
||||
- Processing continues for all rows
|
||||
- The `failed` count continues to increment correctly for all failed rows
|
||||
- The `errors_truncated?` flag is set to `true` to indicate that additional errors were suppressed
|
||||
|
||||
## Returns
|
||||
|
||||
|
|
@ -286,7 +288,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
iex> chunk = [{2, %{member: %{email: "invalid"}, custom: %{}}}]
|
||||
iex> opts = [existing_error_count: 25, max_errors: 50]
|
||||
iex> MemberCSV.process_chunk(chunk, %{}, %{}, opts)
|
||||
{:ok, %{inserted: 0, failed: 1, errors: [%Error{}]}}
|
||||
{:ok, %{inserted: 0, failed: 1, errors: [%Error{}], errors_truncated?: false}}
|
||||
"""
|
||||
@spec process_chunk(
|
||||
list({pos_integer(), map()}),
|
||||
|
|
@ -299,31 +301,31 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
existing_error_count = Keyword.get(opts, :existing_error_count, 0)
|
||||
max_errors = Keyword.get(opts, :max_errors, 50)
|
||||
|
||||
{inserted, failed, errors} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, []}, fn {line_number, row_map},
|
||||
{acc_inserted, acc_failed, acc_errors} ->
|
||||
current_error_count = existing_error_count + length(acc_errors)
|
||||
{inserted, failed, errors, _collected_error_count, truncated?} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false}, fn {line_number, row_map},
|
||||
{acc_inserted, acc_failed, acc_errors, acc_error_count, acc_truncated?} ->
|
||||
current_error_count = existing_error_count + acc_error_count
|
||||
|
||||
case process_row(row_map, line_number, custom_field_lookup) do
|
||||
{:ok, _member} ->
|
||||
{acc_inserted + 1, acc_failed, acc_errors}
|
||||
{acc_inserted + 1, acc_failed, acc_errors, acc_error_count, acc_truncated?}
|
||||
|
||||
{:error, error} ->
|
||||
new_acc_failed = acc_failed + 1
|
||||
|
||||
# Only collect errors if under limit
|
||||
new_acc_errors =
|
||||
{new_acc_errors, new_error_count, new_truncated?} =
|
||||
if current_error_count < max_errors do
|
||||
[error | acc_errors]
|
||||
{[error | acc_errors], acc_error_count + 1, acc_truncated?}
|
||||
else
|
||||
acc_errors
|
||||
{acc_errors, acc_error_count, true}
|
||||
end
|
||||
|
||||
{acc_inserted, new_acc_failed, new_acc_errors}
|
||||
{acc_inserted, new_acc_failed, new_acc_errors, new_error_count, new_truncated?}
|
||||
end
|
||||
end)
|
||||
|
||||
{:ok, %{inserted: inserted, failed: failed, errors: Enum.reverse(errors)}}
|
||||
{:ok, %{inserted: inserted, failed: failed, errors: Enum.reverse(errors), errors_truncated?: truncated?}}
|
||||
end
|
||||
|
||||
@doc """
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue