perf(import): reuse auto-created groups across import chunks
This commit is contained in:
parent
68a1a9530a
commit
118b9f8d57
5 changed files with 90 additions and 3 deletions
|
|
@ -37,6 +37,9 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
- `inserted` - Number of successfully created members
|
||||
- `failed` - Number of failed member creations
|
||||
- `errors` - List of `%MemberCSV.Error{}` structs (capped at 50 per import)
|
||||
- `groups_found` - The in-memory group snapshot grown while processing this
|
||||
chunk; thread it into the next chunk's `:groups_found` opt so groups created
|
||||
in an earlier chunk are reused without re-reading the Group table
|
||||
|
||||
## Examples
|
||||
|
||||
|
|
@ -94,7 +97,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
failed: non_neg_integer(),
|
||||
errors: list(Error.t()),
|
||||
errors_truncated?: boolean(),
|
||||
warnings: list(String.t())
|
||||
warnings: list(String.t()),
|
||||
groups_found: list(Mv.Membership.Group.t() | %{id: String.t(), name: String.t()})
|
||||
}
|
||||
|
||||
alias Mv.Membership.Import.ColumnResolver
|
||||
|
|
@ -374,7 +378,7 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
actor: actor
|
||||
}
|
||||
|
||||
{inserted, failed, errors, _collected_error_count, truncated?, warnings, _groups_acc} =
|
||||
{inserted, failed, errors, _collected_error_count, truncated?, warnings, groups_acc} =
|
||||
Enum.reduce(chunk_rows_with_lines, {0, 0, [], 0, false, [], groups_found}, fn {line_number,
|
||||
row_map},
|
||||
{acc_inserted,
|
||||
|
|
@ -417,7 +421,8 @@ defmodule Mv.Membership.Import.MemberCSV do
|
|||
failed: failed,
|
||||
errors: Enum.reverse(errors),
|
||||
errors_truncated?: truncated?,
|
||||
warnings: warnings
|
||||
warnings: warnings,
|
||||
groups_found: groups_acc
|
||||
}}
|
||||
end
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue